cojson 0.20.1 → 0.20.3

package/src/localNode.ts

@@ -39,7 +39,7 @@ import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromS
 import { expectGroup } from "./typeUtils/expectGroup.js";
 import { canBeBranched } from "./coValueCore/branching.js";
 import { connectedPeers } from "./streamUtils.js";
-import { emptyKnownState } from "./knownState.js";
+import { CoValueKnownState, emptyKnownState } from "./knownState.js";
 
 /** A `LocalNode` represents a local view of a set of loaded `CoValue`s, from the perspective of a particular account (or primitive cryptographic agent).
 
@@ -87,7 +87,7 @@ export class LocalNode {
       return;
     }
 
-    this.garbageCollector = new GarbageCollector(this.coValues);
+    this.garbageCollector = new GarbageCollector(this);
   }
 
   setStorage(storage: StorageAPI) {
@@ -123,7 +123,13 @@ export class LocalNode {
       return false;
     }
 
-    return coValue.loadingState !== "unknown";
+    const state = coValue.loadingState;
+    // garbageCollected and onlyKnownState shells don't have actual content loaded
+    return (
+      state !== "unknown" &&
+      state !== "garbageCollected" &&
+      state !== "onlyKnownState"
+    );
   }
 
   getCoValue(id: RawCoID) {
@@ -143,11 +149,64 @@ export class LocalNode {
     return this.coValues.values();
   }
 
+  /**
+   * Simple delete of a CoValue from memory.
+   * Used for testing and forced cleanup scenarios.
+   * @internal
+   */
   internalDeleteCoValue(id: RawCoID) {
     this.coValues.delete(id);
     this.storage?.onCoValueUnmounted(id);
   }
 
+  /**
+   * Unmount a CoValue from memory, keeping a shell with cached knownState.
+   * This enables accurate LOAD requests during peer reconciliation.
+   *
+   * @returns true if the coValue was successfully unmounted, false otherwise
+   */
+  internalUnmountCoValue(id: RawCoID): boolean {
+    const coValue = this.coValues.get(id);
+
+    if (!coValue) {
+      return false;
+    }
+
+    if (coValue.listeners.size > 0) {
+      // The coValue is still in use
+      return false;
+    }
+
+    for (const dependant of coValue.dependant) {
+      if (this.hasCoValue(dependant)) {
+        // Another in-memory coValue depends on this coValue
+        return false;
+      }
+    }
+
+    if (!this.syncManager.isSyncedToServerPeers(id)) {
+      return false;
+    }
+
+    // Cache the knownState before replacing
+    const lastKnownState = coValue.knownState();
+
+    // Handle counter: decrement old coValue's state
+    coValue.decrementLoadingStateCounter();
+
+    // Create new shell CoValueCore in garbageCollected state
+    const shell = new CoValueCore(id, this);
+    shell.setGarbageCollectedState(lastKnownState);
+
+    // Single map update (replacing old with shell)
+    this.coValues.set(id, shell);
+
+    // Notify storage
+    this.storage?.onCoValueUnmounted(id);
+
+    return true;
+  }
+
   getCurrentAccountOrAgentID(): RawAccountID | AgentID {
     return accountOrAgentIDfromSessionID(this.currentSessionID);
   }
@@ -449,7 +508,9 @@ export class LocalNode {
 
       if (
         coValue.loadingState === "unknown" ||
-        coValue.loadingState === "unavailable"
+        coValue.loadingState === "unavailable" ||
+        coValue.loadingState === "garbageCollected" ||
+        coValue.loadingState === "onlyKnownState"
       ) {
         const peers = this.syncManager.getServerPeers(id, skipLoadingFromPeer);
 

package/src/permissions.ts

@@ -235,6 +235,7 @@ function determineValidTransactionsForGroup(
   const writeOnlyKeys: Record<RawAccountID | AgentID, KeyID> = {};
   const writeKeys = new Set<string>();
   const memberRoleResolver = new MemberRoleResolver();
+  const isGroup = coValue.isGroup();
 
   for (const transaction of coValue.verifiedTransactions) {
     const transactor = transaction.author;
@@ -247,6 +248,11 @@ function determineValidTransactionsForGroup(
     const tx = transaction.tx;
 
     if (tx.privacy === "private") {
+      if (isGroup) {
+        transaction.markInvalid("Can't make private transactions in groups");
+        continue;
+      }
+
       if (transactorRole === "admin") {
         transaction.markValid();
         continue;

package/src/sync.ts

@@ -401,16 +401,18 @@ export class SyncManager {
     };
 
     for (const coValue of this.local.allCoValues()) {
-      if (!coValue.isAvailable()) {
-        // If the coValue is unavailable and we never tried this peer
-        // we try to load it from the peer
-        if (!peer.loadRequestSent.has(coValue.id)) {
-          peer.sendLoadRequest(coValue, "low-priority");
-        }
-      } else {
-        // Build the list of coValues ordered by dependency
-        // so we can send the load message in the correct order
+      if (coValue.isAvailable()) {
+        // In memory - build ordered list for dependency-aware sending
         buildOrderedCoValueList(coValue);
+      } else if (coValue.loadingState === "unknown") {
+        // Skip unknown CoValues - we never tried to load them, so don't
+        // restore a subscription we never had. This prevents loading
+        // content for CoValues we don't actually care about.
+        continue;
+      } else if (!peer.loadRequestSent.has(coValue.id)) {
+        // For garbageCollected/onlyKnownState: knownState() returns lastKnownState
+        // For unavailable/loading/errored: knownState() returns empty state
+        peer.sendLoadRequest(coValue, "low-priority");
       }
 
       // Fill the missing known states with empty known states

package/src/tests/coValueCore.loadFromStorage.test.ts

@@ -470,6 +470,114 @@ describe("CoValueCore.loadFromStorage", () => {
     });
   });
 
+  describe("when state is garbageCollected", () => {
+    test("should load from storage even if storage state is not unknown", () => {
+      const { state, node, header, id } = setup();
+      const loadSpy = vi.fn();
+      const storage = createMockStorage({ load: loadSpy });
+      node.setStorage(storage);
+
+      // First, simulate that storage was previously accessed and marked available
+      state.markFoundInPeer("storage", state.loadingState);
+
+      // Then set the CoValue to garbageCollected state (simulating GC)
+      // This is what happens when a GC'd CoValueCore shell is created
+      state.setGarbageCollectedState({
+        id,
+        header: true,
+        sessions: {},
+      });
+
+      // Verify we're in garbageCollected state
+      expect(state.loadingState).toBe("garbageCollected");
+
+      // Now call loadFromStorage - it should proceed to load
+      state.loadFromStorage();
+
+      // Should have called storage.load because we need full content
+      expect(loadSpy).toHaveBeenCalledTimes(1);
+    });
+
+    test("should load from storage when garbageCollected and storage state is unknown", () => {
+      const { state, node, id } = setup();
+      const loadSpy = vi.fn();
+      const storage = createMockStorage({ load: loadSpy });
+      node.setStorage(storage);
+
+      // Set the CoValue to garbageCollected state
+      state.setGarbageCollectedState({
+        id,
+        header: true,
+        sessions: {},
+      });
+
+      expect(state.loadingState).toBe("garbageCollected");
+      expect(state.getLoadingStateForPeer("storage")).toBe("unknown");
+
+      state.loadFromStorage();
+
+      expect(loadSpy).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe("when state is onlyKnownState", () => {
+    test("should load from storage to get full content", () => {
+      const { state, node, id } = setup();
+      const loadSpy = vi.fn();
+      const storage = createMockStorage({
+        load: loadSpy,
+        loadKnownState: (id, callback) => {
+          // Simulate storage finding knownState
+          callback({
+            id,
+            header: true,
+            sessions: { session1: 5 },
+          });
+        },
+      });
+      node.setStorage(storage);
+
+      // First, call getKnownStateFromStorage to set onlyKnownState
+      state.getKnownStateFromStorage((knownState) => {
+        expect(knownState).toBeDefined();
+      });
+
+      // Verify we're in onlyKnownState
+      expect(state.loadingState).toBe("onlyKnownState");
+
+      // Now call loadFromStorage - it should proceed to load full content
+      state.loadFromStorage();
+
+      expect(loadSpy).toHaveBeenCalledTimes(1);
+    });
+
+    test("should load from storage when onlyKnownState and storage state is unknown", () => {
+      const { state, node, id } = setup();
+      const loadSpy = vi.fn();
+      const storage = createMockStorage({
+        load: loadSpy,
+        loadKnownState: (id, callback) => {
+          callback({
+            id,
+            header: true,
+            sessions: {},
+          });
+        },
+      });
+      node.setStorage(storage);
+
+      // Set onlyKnownState via getKnownStateFromStorage
+      state.getKnownStateFromStorage(() => {});
+
+      expect(state.loadingState).toBe("onlyKnownState");
+      expect(state.getLoadingStateForPeer("storage")).toBe("unknown");
+
+      state.loadFromStorage();
+
+      expect(loadSpy).toHaveBeenCalledTimes(1);
+    });
+  });
+
   describe("edge cases and integration", () => {
     test("should handle transition from unknown to pending to available", async () => {
       const { state, node, header } = setup();

package/src/tests/group.parentGroupCache.test.ts

@@ -140,7 +140,7 @@ describe("Parent Group Cache", () => {
             value: "revoked",
           },
         ],
-        "private",
+        "trusting",
         undefined,
         t2,
       );
@@ -153,7 +153,7 @@ describe("Parent Group Cache", () => {
             value: "extend",
           },
         ],
-        "private",
+        "trusting",
         undefined,
         t1,
       );

package/src/tests/knownState.lazyLoading.test.ts

@@ -222,4 +222,56 @@ describe("CoValueCore.getKnownStateFromStorage", () => {
 
     expect(doneSpy).toHaveBeenCalledWith(undefined);
   });
+
+  test("sets onlyKnownState and caches lastKnownState when storage returns data", () => {
+    const { node, coValue, id } = setup();
+    const storageKnownState = {
+      id,
+      header: true,
+      sessions: { session1: 5 },
+    };
+    const loadKnownStateSpy = vi.fn((id, callback) => {
+      callback(storageKnownState);
+    });
+    const storage = createMockStorage({ loadKnownState: loadKnownStateSpy });
+    node.setStorage(storage);
+
+    // Initially unknown
+    expect(coValue.loadingState).toBe("unknown");
+
+    const doneSpy = vi.fn();
+    coValue.getKnownStateFromStorage(doneSpy);
+
+    // After storage returns data, should be onlyKnownState
+    expect(coValue.loadingState).toBe("onlyKnownState");
+
+    // knownState() should return the cached lastKnownState
+    expect(coValue.knownState()).toEqual(storageKnownState);
+  });
+
+  test("returns cached lastKnownState on subsequent calls without hitting storage", () => {
+    const { node, coValue, id } = setup();
+    const storageKnownState = {
+      id,
+      header: true,
+      sessions: { session1: 5 },
+    };
+    const loadKnownStateSpy = vi.fn((id, callback) => {
+      callback(storageKnownState);
+    });
+    const storage = createMockStorage({ loadKnownState: loadKnownStateSpy });
+    node.setStorage(storage);
+
+    // First call - hits storage
+    const doneSpy1 = vi.fn();
+    coValue.getKnownStateFromStorage(doneSpy1);
+    expect(loadKnownStateSpy).toHaveBeenCalledTimes(1);
+    expect(doneSpy1).toHaveBeenCalledWith(storageKnownState);
+
+    // Second call - should use cached lastKnownState, not hit storage
+    const doneSpy2 = vi.fn();
+    coValue.getKnownStateFromStorage(doneSpy2);
+    expect(loadKnownStateSpy).toHaveBeenCalledTimes(1); // Still 1, not 2
+    expect(doneSpy2).toHaveBeenCalledWith(storageKnownState);
+  });
 });

package/src/tests/permissions.test.ts

@@ -1,4 +1,4 @@
-import { expect, test, vi } from "vitest";
+import { beforeEach, describe, expect, test, vi } from "vitest";
 import { expectMap } from "../coValue.js";
 import { ControlledAgent } from "../coValues/account.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
@@ -13,6 +13,8 @@ import {
   loadCoValueOrFail,
   newGroup,
   newGroupHighLevel,
+  setupTestAccount,
+  setupTestNode,
   waitFor,
 } from "./testUtils.js";
 import { Role } from "../permissions.js";
@@ -2727,3 +2729,118 @@ test("Can revoke read permission from 'everyone'", async () => {
   // Verify the new account cannot read after revocation
   expect(childContent2.get("foo")).toEqual("bar");
 });
+
+describe("Private transactions in groups", () => {
+  beforeEach(async () => {
+    setupTestNode({ isSyncServer: true });
+  });
+
+  test("Admins can make private transactions directly in accounts", async () => {
+    const alice = await setupTestAccount();
+
+    const group = alice.node.createGroup();
+    const map = group.createMap();
+
+    // Count valid transactions before attempting private transaction
+    const validTransactionsBefore = alice.account.core.getValidTransactions();
+    const countBefore = validTransactionsBefore.length;
+
+    alice.account.set("root", map.id, "private");
+
+    const validTransactions = alice.account.core.getValidTransactions();
+    expect(validTransactions).toHaveLength(countBefore + 1);
+
+    expect(alice.account.get("root")).toEqual(map.id);
+    expect(alice.account.core.getCurrentReadKey()).not.toBeUndefined();
+  });
+
+  test("Admins cannot make private transactions directly in groups", async () => {
+    const alice = await setupTestAccount();
+
+    const group = alice.node.createGroup();
+    const map = group.createMap();
+
+    // Count valid transactions before attempting private transaction
+    const validTransactionsBefore = group.core.getValidTransactions();
+    const countBefore = validTransactionsBefore.length;
+
+    // Attempt to make a private transaction directly on the group
+    group.set("root", map.id, "private");
+
+    // Verify the transaction is marked invalid (count should not increase)
+    const validTransactions = group.core.getValidTransactions();
+    expect(validTransactions).toHaveLength(countBefore);
+
+    // Verify the change didn't take effect
+    expect(group.get("root")).toBeUndefined();
+  });
+
+  test("Writers cannot make private transactions directly in groups", async () => {
+    const alice = await setupTestAccount({ connected: true });
+    const bob = await setupTestAccount({ connected: true });
+
+    const group = alice.node.createGroup();
+    const map = group.createMap();
+
+    group.addMember(bob.account, "writer");
+
+    // Count valid transactions before attempting private transaction
+    const validTransactionsBefore = group.core.getValidTransactions();
+    const countBefore = validTransactionsBefore.length;
+
+    const groupAsBob = await loadCoValueOrFail(bob.node, group.id);
+
+    // Attempt to make a private transaction directly on the group
+    groupAsBob.set("root", map.id, "private");
+
+    // Verify the transaction is marked invalid (count should not increase)
+    const validTransactions = groupAsBob.core.getValidTransactions();
+    expect(validTransactions).toHaveLength(countBefore);
+
+    // Verify the change didn't take effect
+    expect(groupAsBob.get("root")).toBeUndefined();
+  });
+
+  test("Managers cannot make private transactions directly in groups", async () => {
+    const alice = await setupTestAccount({ connected: true });
+    const bob = await setupTestAccount({ connected: true });
+
+    const group = alice.node.createGroup();
+    const map = group.createMap();
+
+    group.addMember(bob.account, "manager");
+
+    // Count valid transactions before attempting private transaction
+    const validTransactionsBefore = group.core.getValidTransactions();
+    const countBefore = validTransactionsBefore.length;
+
+    const groupAsBob = await loadCoValueOrFail(bob.node, group.id);
+
+    // Attempt to make a private transaction directly on the group
+    groupAsBob.set("root", map.id, "private");
+
+    // Verify the transaction is marked invalid (count should not increase)
+    const validTransactions = groupAsBob.core.getValidTransactions();
+    expect(validTransactions).toHaveLength(countBefore);
+
+    // Verify the change didn't take effect
+    expect(groupAsBob.get("root")).toBeUndefined();
+  });
+
+  test("A Group with a private transaction can be loaded without issues", async () => {
+    const alice = await setupTestAccount({ connected: true });
+    const bob = await setupTestAccount({ connected: true });
+
+    const group = alice.node.createGroup();
+    const map = group.createMap();
+
+    // Attempt to make a private transaction directly on the group
+    group.set("root", map.id, "private");
+    group.addMember(bob.account, "reader");
+
+    const groupAsBob = await loadCoValueOrFail(bob.node, group.id);
+
+    expect(groupAsBob.myRole()).toBe("reader");
+    expect(groupAsBob.core.getCurrentReadKey()).not.toBeUndefined();
+  });
+});

package/src/tests/sync.garbageCollection.test.ts

@@ -1,5 +1,6 @@
 import { beforeEach, describe, expect, test } from "vitest";
 
+import { expectMap } from "../coValue";
 import { setGarbageCollectorMaxAge } from "../config";
 import {
   SyncMessagesLog,
@@ -177,6 +178,8 @@ describe("sync after the garbage collector has run", () => {
     const mapOnServer = await loadCoValueOrFail(jazzCloud.node, map.id);
     expect(mapOnServer.get("hello")).toEqual("updated");
 
+    // With garbageCollected shells, client uses cached knownState (header/1)
+    // which is more accurate than asking storage (which returns empty)
     expect(
       SyncMessagesLog.getMessages({
         Group: group.core,
@@ -184,14 +187,14 @@ describe("sync after the garbage collector has run", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
-        "client -> server | LOAD Map sessions: empty",
+        "client -> server | LOAD Map sessions: header/1",
         "client -> server | LOAD Group sessions: header/3",
         "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
         "client -> server | CONTENT Group header: true new: After: 0 New: 3",
         "client -> storage | CONTENT Map header: true new: After: 0 New: 1",
         "client -> server | CONTENT Map header: true new: After: 0 New: 1",
-        "server -> storage | LOAD Map sessions: empty",
-        "storage -> server | KNOWN Map sessions: empty",
+        "server -> storage | GET_KNOWN_STATE Map",
+        "storage -> server | GET_KNOWN_STATE_RESULT Map sessions: empty",
         "server -> client | KNOWN Map sessions: empty",
         "server -> storage | GET_KNOWN_STATE Group",
         "storage -> server | GET_KNOWN_STATE_RESULT Group sessions: empty",
@@ -203,4 +206,113 @@ describe("sync after the garbage collector has run", () => {
       ]
     `);
   });
+
+  test("knownStateWithStreaming returns lastKnownState for garbageCollected CoValues", async () => {
+    // This test verifies that knownStateWithStreaming() returns the cached lastKnownState
+    // for garbage-collected CoValues, not an empty state. This is important for peer
+    // reconciliation where we want to send the last known state to minimize data transfer.
+
+    const client = setupTestNode();
+    client.addStorage({ ourName: "client" });
+    client.node.enableGarbageCollector();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // Sync to server
+    client.connectToSyncServer();
+    await client.node.syncManager.waitForAllCoValuesSync();
+
+    // Capture known state before GC
+    const originalKnownState = map.core.knownState();
+    const originalKnownStateWithStreaming = map.core.knownStateWithStreaming();
+
+    // For available CoValues, both should be equal (no streaming in progress)
+    expect(originalKnownState).toEqual(originalKnownStateWithStreaming);
+    expect(originalKnownState.header).toBe(true);
+    expect(Object.values(originalKnownState.sessions)[0]).toBe(1);
+
+    // Disconnect before GC
+    client.disconnect();
+
+    // Run GC to create garbageCollected shell
+    client.node.garbageCollector?.collect();
+    client.node.garbageCollector?.collect();
+
+    const gcCoValue = client.node.getCoValue(map.id);
+    expect(gcCoValue.loadingState).toBe("garbageCollected");
+
+    // Key assertion: knownStateWithStreaming() should return lastKnownState, not empty state
+    const gcKnownState = gcCoValue.knownState();
+    const gcKnownStateWithStreaming = gcCoValue.knownStateWithStreaming();
+
+    // Both should equal the original known state (the cached lastKnownState)
+    expect(gcKnownState).toEqual(originalKnownState);
+    expect(gcKnownStateWithStreaming).toEqual(originalKnownState);
+
+    // Specifically verify it's NOT an empty state
+    expect(gcKnownStateWithStreaming.header).toBe(true);
+    expect(
+      Object.keys(gcKnownStateWithStreaming.sessions).length,
+    ).toBeGreaterThan(0);
+  });
+
+  test("garbageCollected CoValues read from verified content after reload", async () => {
+    // This test verifies that after reloading a GC'd CoValue:
+    // 1. lastKnownState is cleared
+    // 2. knownState() returns data from verified content (not cached)
+    // We prove this by adding a transaction after reload and verifying knownState() updates
+
+    const client = setupTestNode();
+    client.addStorage({ ourName: "client" });
+    client.node.enableGarbageCollector();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // Sync to server
+    client.connectToSyncServer();
+    await client.node.syncManager.waitForAllCoValuesSync();
+
+    // Capture known state before GC (has 1 transaction)
+    const originalKnownState = map.core.knownState();
+    const originalSessionCount = Object.values(originalKnownState.sessions)[0];
+    expect(originalSessionCount).toBe(1);
+
+    // Disconnect before GC
+    client.disconnect();
+
+    // Run GC to create garbageCollected shell
+    client.node.garbageCollector?.collect();
+    client.node.garbageCollector?.collect();
+
+    const gcMap = client.node.getCoValue(map.id);
+    expect(gcMap.loadingState).toBe("garbageCollected");
+
+    // Verify knownState() returns lastKnownState (still shows 1 transaction)
+    expect(gcMap.knownState()).toEqual(originalKnownState);
+
+    // Reconnect and reload
+    client.connectToSyncServer();
+    const reloadedCore = await client.node.loadCoValueCore(map.id);
+
+    // Verify CoValue is now available
+    expect(reloadedCore.loadingState).toBe("available");
+    expect(reloadedCore.isAvailable()).toBe(true);
+
+    // At this point, knownState() should be reading from verified content
+    // To prove this, we add a new transaction and verify knownState() updates
+    const reloadedContent = expectMap(reloadedCore.getCurrentContent());
+    reloadedContent.set("hello", "updated locally", "trusting");
+
+    // Verify knownState() now shows 2 transactions
+    // This proves we're reading from verified content, not cached lastKnownState
+    const newKnownState = reloadedCore.knownState();
+    const newSessionCount = Object.values(newKnownState.sessions)[0];
+
+    expect(newSessionCount).toBe(2);
+    expect(newKnownState).not.toEqual(originalKnownState);
+  });
 });