cojson 0.20.1 → 0.20.2

package/src/coValueCore/coValueCore.ts

@@ -282,6 +282,15 @@ export class CoValueCore {
       }
   >();
 
+  // Tracks why we have lastKnownState (separate from loadingStatuses)
+  // - "garbageCollected": was in memory, got GC'd
+  // - "onlyKnownState": checked storage, found knownState, but didn't load full content
+  #lastKnownStateSource?: "garbageCollected" | "onlyKnownState";
+
+  // Cache the knownState when transitioning to garbageCollected/onlyKnownState
+  // Used during peer reconciliation to send accurate LOAD requests
+  #lastKnownState?: CoValueKnownState;
+
   // cached state and listeners
   private _cachedContent?: RawCoValue;
   readonly listeners: Set<(core: CoValueCore, unsub: () => void) => void> =
@@ -308,14 +317,26 @@ export class CoValueCore {
   get loadingState() {
     if (this.verified) {
       return "available";
-    } else if (this.loadingStatuses.size === 0) {
-      return "unknown";
     }
 
+    // Check for pending peers FIRST - loading takes priority over other states
     for (const peer of this.loadingStatuses.values()) {
       if (peer.type === "pending") {
         return "loading";
-      } else if (peer.type === "unknown") {
+      }
+    }
+
+    // Check for lastKnownStateSource (garbageCollected or onlyKnownState)
+    if (this.#lastKnownStateSource) {
+      return this.#lastKnownStateSource;
+    }
+
+    if (this.loadingStatuses.size === 0) {
+      return "unknown";
+    }
+
+    for (const peer of this.loadingStatuses.values()) {
+      if (peer.type === "unknown") {
         return "unknown";
       }
     }
@@ -429,32 +450,22 @@ export class CoValueCore {
   }
 
   /**
-   * Removes the CoValue from memory.
+   * Removes the CoValue content from memory but keeps a shell with cached knownState.
+   * This enables accurate LOAD requests during peer reconciliation.
    *
    * @returns true if the coValue was successfully unmounted, false otherwise
    */
   unmount(): boolean {
-    if (this.listeners.size > 0) {
-      // The coValue is still in use
-      return false;
-    }
-
-    for (const dependant of this.dependant) {
-      if (this.node.hasCoValue(dependant)) {
-        // Another in-memory coValue depends on this coValue
-        return false;
-      }
-    }
-
-    if (!this.node.syncManager.isSyncedToServerPeers(this.id)) {
-      return false;
-    }
+    return this.node.internalUnmountCoValue(this.id);
+  }
 
+  /**
+   * Decrements the counter for the current loading state.
+   * Used during unmount to properly track state transitions.
+   * @internal
+   */
+  decrementLoadingStateCounter() {
     this.counter.add(-1, { state: this.loadingState });
-
-    this.node.internalDeleteCoValue(this.id);
-
-    return true;
   }
 
   markNotFoundInPeer(peerId: PeerID) {
@@ -470,6 +481,35 @@ export class CoValueCore {
     this.scheduleNotifyUpdate();
   }
 
+  /**
+   * Clean up cached state when CoValue becomes available.
+   * Called after the CoValue transitions from garbageCollected/onlyKnownState to available.
+   */
+  private cleanupLastKnownState() {
+    // Clear both fields - in-memory verified state is now authoritative
+    this.#lastKnownStateSource = undefined;
+    this.#lastKnownState = undefined;
+  }
+
+  /**
+   * Initialize this CoValueCore as a garbageCollected shell.
+   * Called when creating a replacement CoValueCore after unmounting.
+   */
+  setGarbageCollectedState(knownState: CoValueKnownState) {
+    // Only set garbageCollected state if storage is active
+    // Without storage, we can't reload the CoValue anyway
+    if (!this.node.storage) {
+      return;
+    }
+
+    // Transition counter from 'unknown' (set by constructor) to 'garbageCollected'
+    // previousState will be 'unknown', newState will be 'garbageCollected'
+    const previousState = this.loadingState;
+    this.#lastKnownStateSource = "garbageCollected";
+    this.#lastKnownState = knownState;
+    this.updateCounter(previousState);
+  }
+
   missingDependencies = new Set<RawCoID>();
 
   isCircularDependency(dependency: CoValueCore) {
@@ -576,6 +616,10 @@ export class CoValueCore {
       header,
       new SessionMap(this.id, this.node.crypto, streamingKnownState),
     );
+    // Clean up if transitioning from garbageCollected/onlyKnownState
+    if (this.isAvailable()) {
+      this.cleanupLastKnownState();
+    }
 
     return true;
   }
@@ -606,10 +650,11 @@ export class CoValueCore {
    * Used to correctly manage the content & subscriptions during the content streaming process
    */
   knownStateWithStreaming(): CoValueKnownState {
-    return (
-      this.verified?.immutableKnownStateWithStreaming() ??
-      emptyKnownState(this.id)
-    );
+    if (this.verified) {
+      return this.verified.immutableKnownStateWithStreaming();
+    }
+
+    return this.knownState();
   }
 
   /**
@@ -618,9 +663,22 @@ export class CoValueCore {
    * The return value identity is going to be stable as long as the CoValue is not modified.
    *
    * On change the knownState is invalidated and a new object is returned.
+   *
+   * For garbageCollected/onlyKnownState CoValues, returns the cached knownState.
    */
   knownState(): CoValueKnownState {
-    return this.verified?.immutableKnownState() ?? emptyKnownState(this.id);
+    // 1. If we have verified content in memory, use that (authoritative)
+    if (this.verified) {
+      return this.verified.immutableKnownState();
+    }
+
+    // 2. If we have last known state (GC'd or onlyKnownState), use that
+    if (this.#lastKnownState) {
+      return this.#lastKnownState;
+    }
+
+    // 3. Fallback to empty state (truly unknown CoValue)
+    return emptyKnownState(this.id);
   }
 
   /**
@@ -1891,7 +1949,16 @@ export class CoValueCore {
       return;
     }
 
-    if (currentState !== "unknown") {
+    // Check if we need to load from storage:
+    // - If storage state is not unknown (already tried), AND
+    // - Overall state is not garbageCollected/onlyKnownState (which need full content)
+    // Then return early
+    const overallState = this.loadingState;
+    if (
+      currentState !== "unknown" &&
+      overallState !== "garbageCollected" &&
+      overallState !== "onlyKnownState"
+    ) {
       done?.(currentState === "available");
       return;
     }
@@ -1916,7 +1983,8 @@ export class CoValueCore {
    * Lazily load only the knownState from storage without loading full transaction data.
    * This is useful for checking if a peer needs new content before committing to a full load.
    *
-   * Caching and deduplication are handled at the storage layer.
+   * If found in storage, marks the CoValue as onlyKnownState and caches the knownState.
+   * This enables accurate LOAD requests during peer reconciliation.
    *
    * @param done - Callback with the storage knownState, or undefined if not found in storage
    */
@@ -1928,14 +1996,26 @@ export class CoValueCore {
       return;
     }
 
-    // If already available in memory, return the current knownState
-    if (this.isAvailable()) {
-      done(this.knownState());
+    // If we already have knowledge about this CoValue (in memory or cached), return it
+    // knownState() returns verified state, lastKnownState, or empty state
+    const knownState = this.knownState();
+    if (knownState.header) {
+      done(knownState);
       return;
     }
 
     // Delegate to storage - caching is handled at storage level
-    this.node.storage.loadKnownState(this.id, done);
+    this.node.storage.loadKnownState(this.id, (knownState) => {
+      // The coValue could become available in the meantime.
+      if (knownState && !this.isAvailable()) {
+        // Cache the knownState and mark as onlyKnownState
+        const previousState = this.loadingState;
+        this.#lastKnownStateSource = "onlyKnownState";
+        this.#lastKnownState = knownState;
+        this.updateCounter(previousState);
+      }
+      done(knownState);
+    });
   }
 
   loadFromPeers(peers: PeerState[], mode?: LoadMode) {

package/src/localNode.ts

@@ -39,7 +39,7 @@ import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromS
 import { expectGroup } from "./typeUtils/expectGroup.js";
 import { canBeBranched } from "./coValueCore/branching.js";
 import { connectedPeers } from "./streamUtils.js";
-import { emptyKnownState } from "./knownState.js";
+import { CoValueKnownState, emptyKnownState } from "./knownState.js";
 
 /** A `LocalNode` represents a local view of a set of loaded `CoValue`s, from the perspective of a particular account (or primitive cryptographic agent).
 
@@ -87,7 +87,7 @@ export class LocalNode {
       return;
     }
 
-    this.garbageCollector = new GarbageCollector(this.coValues);
+    this.garbageCollector = new GarbageCollector(this);
   }
 
   setStorage(storage: StorageAPI) {
@@ -123,7 +123,13 @@ export class LocalNode {
       return false;
     }
 
-    return coValue.loadingState !== "unknown";
+    const state = coValue.loadingState;
+    // garbageCollected and onlyKnownState shells don't have actual content loaded
+    return (
+      state !== "unknown" &&
+      state !== "garbageCollected" &&
+      state !== "onlyKnownState"
+    );
   }
 
   getCoValue(id: RawCoID) {
@@ -143,11 +149,64 @@ export class LocalNode {
     return this.coValues.values();
   }
 
+  /**
+   * Simple delete of a CoValue from memory.
+   * Used for testing and forced cleanup scenarios.
+   * @internal
+   */
   internalDeleteCoValue(id: RawCoID) {
     this.coValues.delete(id);
     this.storage?.onCoValueUnmounted(id);
   }
 
+  /**
+   * Unmount a CoValue from memory, keeping a shell with cached knownState.
+   * This enables accurate LOAD requests during peer reconciliation.
+   *
+   * @returns true if the coValue was successfully unmounted, false otherwise
+   */
+  internalUnmountCoValue(id: RawCoID): boolean {
+    const coValue = this.coValues.get(id);
+
+    if (!coValue) {
+      return false;
+    }
+
+    if (coValue.listeners.size > 0) {
+      // The coValue is still in use
+      return false;
+    }
+
+    for (const dependant of coValue.dependant) {
+      if (this.hasCoValue(dependant)) {
+        // Another in-memory coValue depends on this coValue
+        return false;
+      }
+    }
+
+    if (!this.syncManager.isSyncedToServerPeers(id)) {
+      return false;
+    }
+
+    // Cache the knownState before replacing
+    const lastKnownState = coValue.knownState();
+
+    // Handle counter: decrement old coValue's state
+    coValue.decrementLoadingStateCounter();
+
+    // Create new shell CoValueCore in garbageCollected state
+    const shell = new CoValueCore(id, this);
+    shell.setGarbageCollectedState(lastKnownState);
+
+    // Single map update (replacing old with shell)
+    this.coValues.set(id, shell);
+
+    // Notify storage
+    this.storage?.onCoValueUnmounted(id);
+
+    return true;
+  }
+
   getCurrentAccountOrAgentID(): RawAccountID | AgentID {
     return accountOrAgentIDfromSessionID(this.currentSessionID);
   }
@@ -449,7 +508,9 @@ export class LocalNode {
 
       if (
         coValue.loadingState === "unknown" ||
-        coValue.loadingState === "unavailable"
+        coValue.loadingState === "unavailable" ||
+        coValue.loadingState === "garbageCollected" ||
+        coValue.loadingState === "onlyKnownState"
       ) {
         const peers = this.syncManager.getServerPeers(id, skipLoadingFromPeer);
 

package/src/sync.ts

@@ -401,16 +401,18 @@ export class SyncManager {
     };
 
     for (const coValue of this.local.allCoValues()) {
-      if (!coValue.isAvailable()) {
-        // If the coValue is unavailable and we never tried this peer
-        // we try to load it from the peer
-        if (!peer.loadRequestSent.has(coValue.id)) {
-          peer.sendLoadRequest(coValue, "low-priority");
-        }
-      } else {
-        // Build the list of coValues ordered by dependency
-        // so we can send the load message in the correct order
+      if (coValue.isAvailable()) {
+        // In memory - build ordered list for dependency-aware sending
         buildOrderedCoValueList(coValue);
+      } else if (coValue.loadingState === "unknown") {
+        // Skip unknown CoValues - we never tried to load them, so don't
+        // restore a subscription we never had. This prevents loading
+        // content for CoValues we don't actually care about.
+        continue;
+      } else if (!peer.loadRequestSent.has(coValue.id)) {
+        // For garbageCollected/onlyKnownState: knownState() returns lastKnownState
+        // For unavailable/loading/errored: knownState() returns empty state
+        peer.sendLoadRequest(coValue, "low-priority");
       }
 
       // Fill the missing known states with empty known states

package/src/tests/coValueCore.loadFromStorage.test.ts

@@ -470,6 +470,114 @@ describe("CoValueCore.loadFromStorage", () => {
     });
   });
 
+  describe("when state is garbageCollected", () => {
+    test("should load from storage even if storage state is not unknown", () => {
+      const { state, node, header, id } = setup();
+      const loadSpy = vi.fn();
+      const storage = createMockStorage({ load: loadSpy });
+      node.setStorage(storage);
+
+      // First, simulate that storage was previously accessed and marked available
+      state.markFoundInPeer("storage", state.loadingState);
+
+      // Then set the CoValue to garbageCollected state (simulating GC)
+      // This is what happens when a GC'd CoValueCore shell is created
+      state.setGarbageCollectedState({
+        id,
+        header: true,
+        sessions: {},
+      });
+
+      // Verify we're in garbageCollected state
+      expect(state.loadingState).toBe("garbageCollected");
+
+      // Now call loadFromStorage - it should proceed to load
+      state.loadFromStorage();
+
+      // Should have called storage.load because we need full content
+      expect(loadSpy).toHaveBeenCalledTimes(1);
+    });
+
+    test("should load from storage when garbageCollected and storage state is unknown", () => {
+      const { state, node, id } = setup();
+      const loadSpy = vi.fn();
+      const storage = createMockStorage({ load: loadSpy });
+      node.setStorage(storage);
+
+      // Set the CoValue to garbageCollected state
+      state.setGarbageCollectedState({
+        id,
+        header: true,
+        sessions: {},
+      });
+
+      expect(state.loadingState).toBe("garbageCollected");
+      expect(state.getLoadingStateForPeer("storage")).toBe("unknown");
+
+      state.loadFromStorage();
+
+      expect(loadSpy).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe("when state is onlyKnownState", () => {
+    test("should load from storage to get full content", () => {
+      const { state, node, id } = setup();
+      const loadSpy = vi.fn();
+      const storage = createMockStorage({
+        load: loadSpy,
+        loadKnownState: (id, callback) => {
+          // Simulate storage finding knownState
+          callback({
+            id,
+            header: true,
+            sessions: { session1: 5 },
+          });
+        },
+      });
+      node.setStorage(storage);
+
+      // First, call getKnownStateFromStorage to set onlyKnownState
+      state.getKnownStateFromStorage((knownState) => {
+        expect(knownState).toBeDefined();
+      });
+
+      // Verify we're in onlyKnownState
+      expect(state.loadingState).toBe("onlyKnownState");
+
+      // Now call loadFromStorage - it should proceed to load full content
+      state.loadFromStorage();
+
+      expect(loadSpy).toHaveBeenCalledTimes(1);
+    });
+
+    test("should load from storage when onlyKnownState and storage state is unknown", () => {
+      const { state, node, id } = setup();
+      const loadSpy = vi.fn();
+      const storage = createMockStorage({
+        load: loadSpy,
+        loadKnownState: (id, callback) => {
+          callback({
+            id,
+            header: true,
+            sessions: {},
+          });
+        },
+      });
+      node.setStorage(storage);
+
+      // Set onlyKnownState via getKnownStateFromStorage
+      state.getKnownStateFromStorage(() => {});
+
+      expect(state.loadingState).toBe("onlyKnownState");
+      expect(state.getLoadingStateForPeer("storage")).toBe("unknown");
+
+      state.loadFromStorage();
+
+      expect(loadSpy).toHaveBeenCalledTimes(1);
+    });
+  });
+
   describe("edge cases and integration", () => {
     test("should handle transition from unknown to pending to available", async () => {
       const { state, node, header } = setup();

package/src/tests/knownState.lazyLoading.test.ts

@@ -222,4 +222,56 @@ describe("CoValueCore.getKnownStateFromStorage", () => {
 
     expect(doneSpy).toHaveBeenCalledWith(undefined);
   });
+
+  test("sets onlyKnownState and caches lastKnownState when storage returns data", () => {
+    const { node, coValue, id } = setup();
+    const storageKnownState = {
+      id,
+      header: true,
+      sessions: { session1: 5 },
+    };
+    const loadKnownStateSpy = vi.fn((id, callback) => {
+      callback(storageKnownState);
+    });
+    const storage = createMockStorage({ loadKnownState: loadKnownStateSpy });
+    node.setStorage(storage);
+
+    // Initially unknown
+    expect(coValue.loadingState).toBe("unknown");
+
+    const doneSpy = vi.fn();
+    coValue.getKnownStateFromStorage(doneSpy);
+
+    // After storage returns data, should be onlyKnownState
+    expect(coValue.loadingState).toBe("onlyKnownState");
+
+    // knownState() should return the cached lastKnownState
+    expect(coValue.knownState()).toEqual(storageKnownState);
+  });
+
+  test("returns cached lastKnownState on subsequent calls without hitting storage", () => {
+    const { node, coValue, id } = setup();
+    const storageKnownState = {
+      id,
+      header: true,
+      sessions: { session1: 5 },
+    };
+    const loadKnownStateSpy = vi.fn((id, callback) => {
+      callback(storageKnownState);
+    });
+    const storage = createMockStorage({ loadKnownState: loadKnownStateSpy });
+    node.setStorage(storage);
+
+    // First call - hits storage
+    const doneSpy1 = vi.fn();
+    coValue.getKnownStateFromStorage(doneSpy1);
+    expect(loadKnownStateSpy).toHaveBeenCalledTimes(1);
+    expect(doneSpy1).toHaveBeenCalledWith(storageKnownState);
+
+    // Second call - should use cached lastKnownState, not hit storage
+    const doneSpy2 = vi.fn();
+    coValue.getKnownStateFromStorage(doneSpy2);
+    expect(loadKnownStateSpy).toHaveBeenCalledTimes(1); // Still 1, not 2
+    expect(doneSpy2).toHaveBeenCalledWith(storageKnownState);
+  });
 });

package/src/tests/sync.garbageCollection.test.ts

@@ -1,5 +1,6 @@
 import { beforeEach, describe, expect, test } from "vitest";
 
+import { expectMap } from "../coValue";
 import { setGarbageCollectorMaxAge } from "../config";
 import {
   SyncMessagesLog,
@@ -177,6 +178,8 @@ describe("sync after the garbage collector has run", () => {
     const mapOnServer = await loadCoValueOrFail(jazzCloud.node, map.id);
     expect(mapOnServer.get("hello")).toEqual("updated");
 
+    // With garbageCollected shells, client uses cached knownState (header/1)
+    // which is more accurate than asking storage (which returns empty)
     expect(
       SyncMessagesLog.getMessages({
         Group: group.core,
@@ -184,14 +187,14 @@ describe("sync after the garbage collector has run", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
-        "client -> server | LOAD Map sessions: empty",
+        "client -> server | LOAD Map sessions: header/1",
         "client -> server | LOAD Group sessions: header/3",
         "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
         "client -> server | CONTENT Group header: true new: After: 0 New: 3",
         "client -> storage | CONTENT Map header: true new: After: 0 New: 1",
         "client -> server | CONTENT Map header: true new: After: 0 New: 1",
-        "server -> storage | LOAD Map sessions: empty",
-        "storage -> server | KNOWN Map sessions: empty",
+        "server -> storage | GET_KNOWN_STATE Map",
+        "storage -> server | GET_KNOWN_STATE_RESULT Map sessions: empty",
         "server -> client | KNOWN Map sessions: empty",
         "server -> storage | GET_KNOWN_STATE Group",
         "storage -> server | GET_KNOWN_STATE_RESULT Group sessions: empty",
@@ -203,4 +206,113 @@ describe("sync after the garbage collector has run", () => {
       ]
     `);
   });
+
+  test("knownStateWithStreaming returns lastKnownState for garbageCollected CoValues", async () => {
+    // This test verifies that knownStateWithStreaming() returns the cached lastKnownState
+    // for garbage-collected CoValues, not an empty state. This is important for peer
+    // reconciliation where we want to send the last known state to minimize data transfer.
+
+    const client = setupTestNode();
+    client.addStorage({ ourName: "client" });
+    client.node.enableGarbageCollector();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // Sync to server
+    client.connectToSyncServer();
+    await client.node.syncManager.waitForAllCoValuesSync();
+
+    // Capture known state before GC
+    const originalKnownState = map.core.knownState();
+    const originalKnownStateWithStreaming = map.core.knownStateWithStreaming();
+
+    // For available CoValues, both should be equal (no streaming in progress)
+    expect(originalKnownState).toEqual(originalKnownStateWithStreaming);
+    expect(originalKnownState.header).toBe(true);
+    expect(Object.values(originalKnownState.sessions)[0]).toBe(1);
+
+    // Disconnect before GC
+    client.disconnect();
+
+    // Run GC to create garbageCollected shell
+    client.node.garbageCollector?.collect();
+    client.node.garbageCollector?.collect();
+
+    const gcCoValue = client.node.getCoValue(map.id);
+    expect(gcCoValue.loadingState).toBe("garbageCollected");
+
+    // Key assertion: knownStateWithStreaming() should return lastKnownState, not empty state
+    const gcKnownState = gcCoValue.knownState();
+    const gcKnownStateWithStreaming = gcCoValue.knownStateWithStreaming();
+
+    // Both should equal the original known state (the cached lastKnownState)
+    expect(gcKnownState).toEqual(originalKnownState);
+    expect(gcKnownStateWithStreaming).toEqual(originalKnownState);
+
+    // Specifically verify it's NOT an empty state
+    expect(gcKnownStateWithStreaming.header).toBe(true);
+    expect(
+      Object.keys(gcKnownStateWithStreaming.sessions).length,
+    ).toBeGreaterThan(0);
+  });
+
+  test("garbageCollected CoValues read from verified content after reload", async () => {
+    // This test verifies that after reloading a GC'd CoValue:
+    // 1. lastKnownState is cleared
+    // 2. knownState() returns data from verified content (not cached)
+    // We prove this by adding a transaction after reload and verifying knownState() updates
+
+    const client = setupTestNode();
+    client.addStorage({ ourName: "client" });
+    client.node.enableGarbageCollector();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // Sync to server
+    client.connectToSyncServer();
+    await client.node.syncManager.waitForAllCoValuesSync();
+
+    // Capture known state before GC (has 1 transaction)
+    const originalKnownState = map.core.knownState();
+    const originalSessionCount = Object.values(originalKnownState.sessions)[0];
+    expect(originalSessionCount).toBe(1);
+
+    // Disconnect before GC
+    client.disconnect();
+
+    // Run GC to create garbageCollected shell
+    client.node.garbageCollector?.collect();
+    client.node.garbageCollector?.collect();
+
+    const gcMap = client.node.getCoValue(map.id);
+    expect(gcMap.loadingState).toBe("garbageCollected");
+
+    // Verify knownState() returns lastKnownState (still shows 1 transaction)
+    expect(gcMap.knownState()).toEqual(originalKnownState);
+
+    // Reconnect and reload
+    client.connectToSyncServer();
+    const reloadedCore = await client.node.loadCoValueCore(map.id);
+
+    // Verify CoValue is now available
+    expect(reloadedCore.loadingState).toBe("available");
+    expect(reloadedCore.isAvailable()).toBe(true);
+
+    // At this point, knownState() should be reading from verified content
+    // To prove this, we add a new transaction and verify knownState() updates
+    const reloadedContent = expectMap(reloadedCore.getCurrentContent());
+    reloadedContent.set("hello", "updated locally", "trusting");
+
+    // Verify knownState() now shows 2 transactions
+    // This proves we're reading from verified content, not cached lastKnownState
+    const newKnownState = reloadedCore.knownState();
+    const newSessionCount = Object.values(newKnownState.sessions)[0];
+
+    expect(newSessionCount).toBe(2);
+    expect(newKnownState).not.toEqual(originalKnownState);
+  });
 });