cojson 0.20.18 → 0.20.19

package/src/storage/storageSync.ts

@@ -300,8 +300,18 @@ export class StorageApiSync implements StorageAPI {
     pushCallback(contentMessage);
   }
 
-  store(msg: NewContentMessage, correctionCallback: CorrectionCallback) {
-    return this.storeSingle(msg, correctionCallback);
+  store(
+    msg: NewContentMessage,
+    correctionCallback: CorrectionCallback,
+    done?: () => void,
+  ) {
+    const success = this.storeSingle(msg, correctionCallback);
+
+    if (success) {
+      done?.();
+    }
+
+    return success;
   }
 
   /**

package/src/storage/types.ts

@@ -60,7 +60,19 @@ export interface StorageAPI {
     callback: (data: NewContentMessage) => void,
     done?: (found: boolean) => void,
   ): void;
-  store(data: NewContentMessage, handleCorrection: CorrectionCallback): void;
+  /**
+   * Stores the content. `done` is invoked only after the content is durably
+   * stored (including any correction round-trip). It is NOT invoked when the
+   * write fails or is dropped — callers rely on that to detect content that
+   * was sent to peers but never persisted. Depending on the implementation,
+   * done may fire synchronously before store() returns (sync storage) or
+   * later (async queue).
+   */
+  store(
+    data: NewContentMessage,
+    handleCorrection: CorrectionCallback,
+    done?: () => void,
+  ): void;
 
   streamingQueue?: StorageStreamingQueue;
 

package/src/sync.ts

@@ -53,6 +53,11 @@ export type KnownStateMessage = {
   asDependencyOf?: RawCoID;
 } & CoValueKnownState;
 
+export type LocalStoreDurabilityListener = (
+  hasPending: boolean,
+  sessionID: SessionID,
+) => void;
+
 export type NewContentMessage = {
   action: "content";
   id: RawCoID;
@@ -805,6 +810,15 @@ export class SyncManager {
   }
 
   trySendToPeer(peer: PeerState, msg: SyncMessage) {
+    if (msg.action === "content") {
+      // Content leaves the node from several paths (local-transaction sync,
+      // reconnection reconciliation, corrections). This is the single choke
+      // point for all of them, so the durability window is opened here: if any
+      // local store is still pending when content goes out, the session must
+      // be marked unsafe to reuse until storage drains.
+      this.openLocalStoreDurabilityWindow();
+    }
+
     return peer.pushOutgoingMessage(msg);
   }
 
@@ -1424,10 +1438,62 @@ export class SyncManager {
   syncLocalTransaction = this.syncQueue.syncTransaction;
   trackDirtyCoValues = this.syncQueue.trackDirtyCoValues;
 
+  /**
+   * Tracks locally-created content that has been handed to async storage but
+   * is not yet durably stored. If such content is also sent to a peer and the
+   * process crashes before the write completes, local storage ends up behind
+   * what the server received for this session — reusing the session after
+   * restart would then fork its hash chain. The listener lets the platform
+   * layer mark the session as unsafe to reuse while that window is open.
+   *
+   * With synchronous storage the store completes inline, the counter is back
+   * to 0 before any send, and the listener never fires.
+   *
+   * Exceptions thrown by the listener are caught and logged so they can't
+   * disrupt the sync/store flow.
+   */
+  onLocalStoreDurabilityChange?: LocalStoreDurabilityListener;
+  private pendingLocalStores = 0;
+  private localStoreDurabilityWindowOpen = false;
+
+  private emitLocalStoreDurabilityChange(hasPending: boolean) {
+    try {
+      this.onLocalStoreDurabilityChange?.(
+        hasPending,
+        this.local.currentSessionID,
+      );
+    } catch (err) {
+      logger.error("Error in onLocalStoreDurabilityChange listener", { err });
+    }
+  }
+
+  private handleLocalStoreDone = () => {
+    this.pendingLocalStores--;
+
+    if (this.pendingLocalStores === 0 && this.localStoreDurabilityWindowOpen) {
+      this.localStoreDurabilityWindowOpen = false;
+      this.emitLocalStoreDurabilityChange(false);
+    }
+  };
+
+  private openLocalStoreDurabilityWindow() {
+    if (this.pendingLocalStores > 0 && !this.localStoreDurabilityWindowOpen) {
+      this.localStoreDurabilityWindowOpen = true;
+      this.emitLocalStoreDurabilityChange(true);
+    }
+  }
+
   syncContent(content: NewContentMessage) {
     const coValue = this.local.getCoValue(content.id);
 
-    this.storeContent(content);
+    if (this.local.storage) {
+      // A per-message done callback is used instead of storage.waitForSync
+      // because waitForSync resolves on known-state coverage (which can also
+      // happen on deletion/erasure) and allocates a promise + subscription per
+      // wait — done fires only on a durable write of exactly this content.
+      this.pendingLocalStores++;
+      this.storeContent(content, this.handleLocalStoreDone);
+    }
 
     this.trackSyncState(coValue.id);
 
@@ -1498,7 +1564,7 @@ export class SyncManager {
     }
   }
 
-  private storeContent(content: NewContentMessage) {
+  private storeContent(content: NewContentMessage, onStored?: () => void) {
     const storage = this.local.storage;
 
     if (!storage) return;
@@ -1513,22 +1579,26 @@ export class SyncManager {
 
     // Try to store the content as-is for performance
     // In case that some transactions are missing, a correction will be requested, but it's an edge case
-    storage.store(content, (correction) => {
-      if (!value.verified) {
-        logger.error(
-          "Correction requested for a CoValue with no verified content",
-          {
-            id: content.id,
-            content: getContenDebugInfo(content),
-            correction,
-            state: value.loadingState,
-          },
-        );
-        return undefined;
-      }
+    storage.store(
+      content,
+      (correction) => {
+        if (!value.verified) {
+          logger.error(
+            "Correction requested for a CoValue with no verified content",
+            {
+              id: content.id,
+              content: getContenDebugInfo(content),
+              correction,
+              state: value.loadingState,
+            },
+          );
+          return undefined;
+        }
 
-      return value.newContentSince(correction);
-    });
+        return value.newContentSince(correction);
+      },
+      onStored,
+    );
   }
 
   /**

package/src/tests/SQLiteClientAsync.test.ts

@@ -2,6 +2,15 @@ import { beforeEach, describe, expect, test } from "vitest";
 import { getDbPath } from "./testStorage.js";
 import { setupTestNode } from "./testUtils.js";
 import { DBClientInterfaceAsync } from "../exports.js";
+import type { Transaction } from "../coValueCore/verifiedState.js";
+
+function makeTrustingTransaction(changes: string) {
+  return {
+    privacy: "trusting",
+    madeAt: 0,
+    changes,
+  } as unknown as Transaction;
+}
 
 describe("SQLiteClientAsync", () => {
   describe("transaction", () => {
@@ -50,18 +59,12 @@ describe("SQLiteClientAsync", () => {
           signature: `signature_z0`,
         });
       });
-      // Second transaction fails (duplicate primary key)
+      // Second transaction fails
       await expect(
-        dbClient.transaction(async (tx) => {
-          return tx.addSignatureAfter({
-            sessionRowID: 0,
-            idx: 0,
-            signature: `signature_z0`,
-          });
+        dbClient.transaction(async () => {
+          throw new Error("transaction failed");
         }),
-      ).rejects.toThrow(
-        /UNIQUE constraint failed: signatureAfter\.ses, signatureAfter\.idx/,
-      );
+      ).rejects.toThrow("transaction failed");
       // Third transaction succeeds
       await dbClient.transaction(async (tx) => {
         return tx.addSignatureAfter({
@@ -71,5 +74,47 @@ describe("SQLiteClientAsync", () => {
         });
       });
     });
+
+    test("addTransaction overwrites an orphan row at the same (ses, idx)", async () => {
+      // Simulates a row left behind by an interrupted write transaction:
+      // the session row was rolled back but the transaction row persisted.
+      await dbClient.transaction(async (tx) => {
+        return tx.addTransaction(1, 0, makeTrustingTransaction("orphan"));
+      });
+
+      await expect(
+        dbClient.transaction(async (tx) => {
+          return tx.addTransaction(1, 0, makeTrustingTransaction("recovered"));
+        }),
+      ).resolves.not.toThrow();
+
+      const txs = await dbClient.getNewTransactionInSession(1, 0, 0);
+      expect(txs).toHaveLength(1);
+      expect(txs[0]?.tx).toEqual(makeTrustingTransaction("recovered"));
+    });
+
+    test("addSignatureAfter overwrites an orphan row at the same (ses, idx)", async () => {
+      await dbClient.transaction(async (tx) => {
+        return tx.addSignatureAfter({
+          sessionRowID: 1,
+          idx: 0,
+          signature: "signature_zorphan",
+        });
+      });
+
+      await expect(
+        dbClient.transaction(async (tx) => {
+          return tx.addSignatureAfter({
+            sessionRowID: 1,
+            idx: 0,
+            signature: "signature_zrecovered",
+          });
+        }),
+      ).resolves.not.toThrow();
+
+      const signatures = await dbClient.getSignatures(1, 0);
+      expect(signatures).toHaveLength(1);
+      expect(signatures[0]?.signature).toBe("signature_zrecovered");
+    });
   });
 });

package/src/tests/StoreQueue.test.ts

@@ -82,11 +82,13 @@ describe("StoreQueue", () => {
         1,
         data1,
         mockCorrectionCallback,
+        undefined,
       );
       expect(mockCallback).toHaveBeenNthCalledWith(
         2,
         data2,
         mockCorrectionCallback,
+        undefined,
       );
     });
 
@@ -192,6 +194,7 @@ describe("StoreQueue", () => {
       expect(mockCallback).toHaveBeenLastCalledWith(
         data3,
         mockCorrectionCallback,
+        undefined,
       );
     });
 
@@ -267,4 +270,21 @@ describe("StoreQueue", () => {
       expect(count).toBe(entries);
     });
   });
+
+  describe("done callbacks", () => {
+    test("passes each entry's done callback to the processing callback", async () => {
+      const queue = new StoreQueue();
+      const done1 = vi.fn();
+
+      queue.push(createMockNewContentMessage("co_z1"), () => undefined, done1);
+      queue.push(createMockNewContentMessage("co_z2"), () => undefined);
+
+      const seen: Array<(() => void) | undefined> = [];
+      await queue.processQueue(async (_data, _correction, done) => {
+        seen.push(done);
+      });
+
+      expect(seen).toEqual([done1, undefined]);
+    });
+  });
 });

package/src/tests/sync.localStoreDurability.test.ts

@@ -0,0 +1,357 @@
+import { beforeEach, describe, expect, test, vi } from "vitest";
+import { WasmCrypto } from "../crypto/WasmCrypto.js";
+import { LocalNode } from "../localNode";
+import {
+  SyncMessagesLog,
+  getSyncServerConnectedPeer,
+  loadCoValueOrFail,
+  setupTestNode,
+  waitFor,
+} from "./testUtils";
+import { getDbPath, registerStorageCleanupRunner } from "./testStorage";
+
+const Crypto = await WasmCrypto.create();
+
+let jazzCloud: ReturnType<typeof setupTestNode>;
+
+beforeEach(async () => {
+  registerStorageCleanupRunner();
+  SyncMessagesLog.clear();
+  jazzCloud = setupTestNode({ isSyncServer: true });
+});
+
+describe("StorageAPI.store done callback", () => {
+  test("async storage invokes done after the write completes", async () => {
+    const client = setupTestNode();
+    const group = client.node.createGroup();
+    // Attach storage after creating the group so our manual store is the only write
+    const { storage } = await client.addAsyncStorage();
+
+    const content = group.core.newContentSince(undefined)!;
+    const done = vi.fn();
+
+    storage.store(content[0]!, () => undefined, done);
+
+    expect(done).not.toHaveBeenCalled();
+    await waitFor(() => expect(done).toHaveBeenCalledTimes(1));
+  });
+
+  test("sync storage invokes done synchronously", async () => {
+    const client = setupTestNode();
+    const group = client.node.createGroup();
+    const { storage } = client.addStorage();
+
+    const content = group.core.newContentSince(undefined)!;
+    const done = vi.fn();
+
+    storage.store(content[0]!, () => undefined, done);
+
+    expect(done).toHaveBeenCalledTimes(1);
+  });
+
+  test("async storage invokes done after a successful correction round-trip", async () => {
+    const client = setupTestNode();
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("a", 1, "trusting");
+    const { storage } = await client.addAsyncStorage();
+
+    const fullContent = map.core.newContentSince(undefined)!;
+    // Content without the header, stored into empty storage → triggers a correction
+    const withoutHeader = map.core.newContentSince({
+      id: map.id,
+      header: true,
+      sessions: {},
+    })!;
+    const done = vi.fn();
+
+    storage.store(withoutHeader[0]!, () => fullContent, done);
+
+    await waitFor(() => expect(done).toHaveBeenCalledTimes(1));
+  });
+});
+
+describe("local store durability window", () => {
+  test("opens before the first peer send and closes when async storage drains", async () => {
+    const client = setupTestNode();
+    await client.addAsyncStorage();
+    const { peerState } = client.connectToSyncServer();
+
+    // Instrument the actual outgoing push: the window must open before any
+    // content message physically leaves the node
+    let contentSent = false;
+    const originalPush = peerState.pushOutgoingMessage.bind(peerState);
+    vi.spyOn(peerState, "pushOutgoingMessage").mockImplementation((msg) => {
+      if (msg.action === "content") {
+        contentSent = true;
+      }
+      return originalPush(msg);
+    });
+
+    const events: Array<{ hasPending: boolean; contentSentAlready: boolean }> =
+      [];
+    client.node.syncManager.onLocalStoreDurabilityChange = (hasPending) => {
+      events.push({ hasPending, contentSentAlready: contentSent });
+    };
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await waitFor(() => expect(events.some((e) => !e.hasPending)).toBe(true));
+
+    // The window opened BEFORE any content was sent to a peer
+    expect(events[0]).toEqual({ hasPending: true, contentSentAlready: false });
+    // ...and eventually closed once storage drained
+    expect(events.at(-1)!.hasPending).toBe(false);
+  });
+
+  test("reopens for a new local edit after the previous window closed", async () => {
+    const client = setupTestNode();
+    await client.addAsyncStorage();
+    client.connectToSyncServer();
+
+    const events: boolean[] = [];
+    client.node.syncManager.onLocalStoreDurabilityChange = (hasPending) => {
+      events.push(hasPending);
+    };
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // First window: opened and closed once storage drained
+    await waitFor(() => expect(events).toEqual([true, false]));
+
+    // A second local edit reopens the window and it closes again on drain
+    map.set("second", "batch", "trusting");
+
+    await waitFor(() => expect(events).toEqual([true, false, true, false]));
+  });
+
+  test("reports the node's current session ID", async () => {
+    const client = setupTestNode();
+    await client.addAsyncStorage();
+    client.connectToSyncServer();
+
+    const sessions: string[] = [];
+    client.node.syncManager.onLocalStoreDurabilityChange = (
+      _hasPending,
+      sessionID,
+    ) => {
+      sessions.push(sessionID);
+    };
+
+    const group = client.node.createGroup();
+    group.createMap().set("hello", "world", "trusting");
+
+    await waitFor(() => expect(sessions.length).toBeGreaterThan(0));
+    expect(sessions.every((s) => s === client.node.currentSessionID)).toBe(
+      true,
+    );
+  });
+
+  test("does not fire with synchronous storage", async () => {
+    const client = setupTestNode();
+    client.addStorage();
+    client.connectToSyncServer();
+
+    const listener = vi.fn();
+    client.node.syncManager.onLocalStoreDurabilityChange = listener;
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await map.core.waitForSync();
+    expect(listener).not.toHaveBeenCalled();
+  });
+
+  test("does not fire when there are no peers to send to", async () => {
+    const client = setupTestNode(); // not connected
+    await client.addAsyncStorage();
+
+    const listener = vi.fn();
+    client.node.syncManager.onLocalStoreDurabilityChange = listener;
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await client.node.syncManager.waitForStorageSync(map.id);
+    expect(listener).not.toHaveBeenCalled();
+  });
+
+  test("remote content does not open the window on the receiving node", async () => {
+    const server = setupTestNode({ isSyncServer: true });
+    await server.addAsyncStorage({ ourName: "server" });
+
+    const listener = vi.fn();
+    server.node.syncManager.onLocalStoreDurabilityChange = listener;
+
+    const client = setupTestNode({ connected: true });
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await map.core.waitForSync();
+    expect(listener).not.toHaveBeenCalled();
+  });
+
+  test("window stays open when storage writes never complete (crash window)", async () => {
+    const client = setupTestNode();
+    const { storage } = await client.addAsyncStorage();
+    const { peerState } = client.connectToSyncServer();
+
+    // Simulate the crash window: storage accepts writes but never completes them
+    storage.store = async () => {};
+
+    const events: boolean[] = [];
+    client.node.syncManager.onLocalStoreDurabilityChange = (hasPending) =>
+      events.push(hasPending);
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // Wait for the content to reach the server (peer sync only) — storage sync
+    // never resolves in this scenario since store() is stubbed to a no-op, so
+    // we can't use map.core.waitForSync() here, which also waits on storage.
+    await client.node.syncManager.waitForSyncWithPeer(
+      peerState.id,
+      map.id,
+      5000,
+    );
+    expect(events).toEqual([true]); // opened, never closed
+  });
+
+  test("opens when pending content is sent through reconnection reconciliation", async () => {
+    const client = setupTestNode();
+    const { storage } = await client.addAsyncStorage();
+
+    // Writes never complete: the local edit below stays pending forever
+    storage.store = async () => {};
+
+    const events: boolean[] = [];
+    client.node.syncManager.onLocalStoreDurabilityChange = (hasPending) =>
+      events.push(hasPending);
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // Not connected yet: nothing was sent, so the window must not be open
+    await new Promise((resolve) => setTimeout(resolve, 0));
+    expect(events).toEqual([]);
+
+    // Connecting sends the pending content through reconciliation
+    // (sendNewContent), not through syncContent — the window must still open
+    const { peerState } = client.connectToSyncServer();
+    await client.node.syncManager.waitForSyncWithPeer(
+      peerState.id,
+      map.id,
+      5000,
+    );
+
+    expect(events).toEqual([true]);
+  });
+});
+
+describe("LocalNode creation options", () => {
+  test("withNewlyCreatedAccount wires onLocalStoreDurabilityChange into the node's syncManager", async () => {
+    const listener = vi.fn();
+
+    const { node } = await LocalNode.withNewlyCreatedAccount({
+      creationProps: { name: "test" },
+      crypto: Crypto,
+      peers: [],
+      onLocalStoreDurabilityChange: listener,
+    });
+
+    expect(node.syncManager.onLocalStoreDurabilityChange).toBe(listener);
+    await node.gracefulShutdown();
+  });
+
+  test("withLoadedAccount wires onLocalStoreDurabilityChange", async () => {
+    const listener = vi.fn();
+
+    const created = await LocalNode.withNewlyCreatedAccount({
+      creationProps: { name: "test" },
+      crypto: Crypto,
+      peers: [],
+    });
+
+    const { peer } = getSyncServerConnectedPeer({
+      peerId: created.accountID,
+    });
+
+    // Sync the account to the server so it can be loaded
+    created.node.syncManager.addPeer(peer);
+    await created.node.syncManager.waitForAllCoValuesSync();
+
+    const { peer: peer2 } = getSyncServerConnectedPeer({
+      peerId: "loadingNode",
+    });
+
+    const node = await LocalNode.withLoadedAccount({
+      accountID: created.accountID,
+      accountSecret: created.accountSecret,
+      sessionID: undefined,
+      peers: [peer2],
+      crypto: Crypto,
+      onLocalStoreDurabilityChange: listener,
+    });
+
+    expect(node.syncManager.onLocalStoreDurabilityChange).toBe(listener);
+    await node.gracefulShutdown();
+    await created.node.gracefulShutdown();
+  });
+});
+
+describe("crash recovery with a fresh session", () => {
+  test("restarting with a new session over the same storage recovers and syncs cleanly", async () => {
+    const client = setupTestNode();
+    const dbPath = getDbPath();
+    const { storage } = await client.addAsyncStorage({ filename: dbPath });
+    const { peerState } = client.connectToSyncServer();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+    await map.core.waitForSync();
+
+    // Enter the crash window: further writes reach the server but not storage
+    storage.store = async () => {};
+    map.set("crashed", "yes", "trusting");
+    await client.node.syncManager.waitForSyncWithPeer(
+      peerState.id,
+      map.id,
+      5000,
+    );
+
+    // "Crash": abandon the node without flushing (no graceful shutdown), then
+    // restart with the SAME agent, SAME storage file, but a NEW session — which
+    // is exactly what the session providers do when the durability marker is set.
+    client.disconnect();
+    const restarted = setupTestNode({ secret: client.node.agentSecret });
+    await restarted.addAsyncStorage({ filename: dbPath });
+    restarted.connectToSyncServer();
+
+    // The fresh session is load-bearing: reusing the crashed session could fork its hash chain
+    expect(restarted.node.currentSessionID).not.toBe(
+      client.node.currentSessionID,
+    );
+
+    // The crashed transaction comes back down from the server
+    const mapOnRestart = await loadCoValueOrFail(restarted.node, map.id);
+    await waitFor(() => expect(mapOnRestart.get("crashed")).toEqual("yes"));
+
+    // New writes from the fresh session sync cleanly — no signature rejection
+    mapOnRestart.set("recovered", "yes", "trusting");
+    await mapOnRestart.core.waitForSync();
+
+    const mapOnServer = await loadCoValueOrFail(jazzCloud.node, map.id);
+    expect(mapOnServer.get("recovered")).toEqual("yes");
+    expect(mapOnServer.get("crashed")).toEqual("yes");
+  });
+});