cojson 0.20.7 → 0.20.9

package/src/permissions.ts

@@ -10,7 +10,7 @@ import {
   isInheritableRole,
   isSelfExtension,
 } from "./coValues/group.js";
-import { KeyID } from "./crypto/crypto.js";
+import { KeyID, SealerID } from "./crypto/crypto.js";
 import {
   AgentID,
   ParentGroupReference,
@@ -273,6 +273,7 @@ function determineValidTransactionsForGroup(
     const change = changes[0] as
       | MapOpPayload<RawAccountID | AgentID | Everyone, Role>
       | MapOpPayload<"readKey", JsonValue>
+      | MapOpPayload<"groupSealer", SealerID>
       | MapOpPayload<"profile", CoID<RawProfile>>
       | MapOpPayload<"root", CoID<RawCoMap>>
       | MapOpPayload<`parent_${CoID<RawGroup>}`, CoID<RawGroup>>
@@ -294,6 +295,14 @@ function determineValidTransactionsForGroup(
         continue;
       }
 
+      transaction.markValid();
+      continue;
+    } else if (change.key === "groupSealer") {
+      if (!canAdmin(transactorRole)) {
+        transaction.markInvalid("Only admins can set groupSealer");
+        continue;
+      }
+
       transaction.markValid();
       continue;
     } else if (change.key === "profile") {
@@ -314,7 +323,8 @@ function determineValidTransactionsForGroup(
       continue;
     } else if (
       isKeyForKeyField(change.key) ||
-      isKeyForAccountField(change.key)
+      isKeyForAccountField(change.key) ||
+      isKeySealedForGroupField(change.key)
     ) {
       if (
         transactorRole !== "admin" &&
@@ -596,6 +606,12 @@ export function isKeyForAccountField(
   );
 }
 
+export function isKeySealedForGroupField(
+  co: string,
+): co is `${KeyID}_sealedFor_${SealerID}` {
+  return co.startsWith("key_") && co.includes("_sealedFor_sealer");
+}
+
 function isParentExtension(key: string): key is `parent_${CoID<RawGroup>}` {
   return key.startsWith("parent_");
 }
@@ -605,15 +621,22 @@ function isChildExtension(key: string): key is `child_${CoID<RawGroup>}` {
 }
 
 function isOwnWriteKeyRevelation(
-  key: `${KeyID}_for_${string}`,
+  key: `${KeyID}_for_${string}` | `${KeyID}_sealedFor_${SealerID}`,
   memberKey: RawAccountID | AgentID,
   writeOnlyKeys: Record<RawAccountID | AgentID, KeyID>,
-): key is `${KeyID}_for_${RawAccountID | AgentID}` {
-  if (Object.keys(writeOnlyKeys).length === 0) {
-    return false;
+): key is
+  | `${KeyID}_for_${RawAccountID | AgentID}`
+  | `${KeyID}_sealedFor_${SealerID}` {
+  let i = key.indexOf("_for_");
+  if (i === -1) {
+    i = key.indexOf("_sealedFor_");
   }
 
-  const keyID = key.slice(0, key.indexOf("_for_"));
+  const keyID = key.slice(0, i);
+
+  if (!keyID) {
+    return false;
+  }
 
   return writeOnlyKeys[memberKey] === keyID;
 }

package/src/storage/sqliteAsync/client.ts

@@ -38,15 +38,138 @@ export function getErrorMessage(error: unknown) {
   return error instanceof Error ? error.message : "Unknown error";
 }
 
-export class SQLiteClientAsync
-  implements DBClientInterfaceAsync, DBTransactionInterfaceAsync
-{
+/**
+ * Executes storage operations inside a single DB transaction.
+ */
+export class SQLiteTransactionAsync implements DBTransactionInterfaceAsync {
+  constructor(private readonly tx: SQLiteDatabaseDriverAsync) {}
+
+  async getSingleCoValueSession(
+    coValueRowId: number,
+    sessionID: SessionID,
+  ): Promise<StoredSessionRow | undefined> {
+    return this.tx.get<StoredSessionRow>(
+      "SELECT * FROM sessions WHERE coValue = ? AND sessionID = ?",
+      [coValueRowId, sessionID],
+    );
+  }
+
+  async markCoValueAsDeleted(id: RawCoID): Promise<void> {
+    await this.tx.run(
+      `INSERT INTO deletedCoValues (coValueID) VALUES (?) ON CONFLICT(coValueID) DO NOTHING`,
+      [id],
+    );
+  }
+
+  async addSessionUpdate({
+    sessionUpdate,
+  }: {
+    sessionUpdate: SessionRow;
+    sessionRow?: StoredSessionRow;
+  }): Promise<number> {
+    const result = await this.tx.get<{ rowID: number }>(
+      `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
+                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
+                            RETURNING rowID`,
+      [
+        sessionUpdate.coValue,
+        sessionUpdate.sessionID,
+        sessionUpdate.lastIdx,
+        sessionUpdate.lastSignature,
+        sessionUpdate.bytesSinceLastSignature,
+      ],
+    );
+
+    if (!result) {
+      throw new Error("Failed to add session update");
+    }
+
+    return result.rowID;
+  }
+
+  async addTransaction(
+    sessionRowID: number,
+    nextIdx: number,
+    newTransaction: Transaction,
+  ): Promise<void> {
+    await this.tx.run(
+      "INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)",
+      [sessionRowID, nextIdx, JSON.stringify(newTransaction)],
+    );
+  }
+
+  async addSignatureAfter({
+    sessionRowID,
+    idx,
+    signature,
+  }: {
+    sessionRowID: number;
+    idx: number;
+    signature: Signature;
+  }): Promise<void> {
+    await this.tx.run(
+      "INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)",
+      [sessionRowID, idx, signature],
+    );
+  }
+
+  async deleteCoValueContent(
+    coValueRow: Pick<StoredCoValueRow, "rowID" | "id">,
+  ): Promise<void> {
+    await this.tx.run(
+      `DELETE FROM transactions
+       WHERE ses IN (
+         SELECT rowID FROM sessions
+         WHERE coValue = ?
+           AND sessionID NOT LIKE '%$'
+       )`,
+      [coValueRow.rowID],
+    );
+
+    await this.tx.run(
+      `DELETE FROM signatureAfter
+       WHERE ses IN (
+         SELECT rowID FROM sessions
+         WHERE coValue = ?
+           AND sessionID NOT LIKE '%$'
+       )`,
+      [coValueRow.rowID],
+    );
+
+    await this.tx.run(
+      `DELETE FROM sessions
+       WHERE coValue = ?
+         AND sessionID NOT LIKE '%$'`,
+      [coValueRow.rowID],
+    );
+
+    await this.tx.run(
+      `INSERT INTO deletedCoValues (coValueID, status) VALUES (?, ?)
+       ON CONFLICT(coValueID) DO UPDATE SET status=?`,
+      [
+        coValueRow.id,
+        DeletedCoValueDeletionStatus.Done,
+        DeletedCoValueDeletionStatus.Done,
+      ],
+    );
+  }
+}
+
+export class SQLiteClientAsync implements DBClientInterfaceAsync {
   private readonly db: SQLiteDatabaseDriverAsync;
+  /** Serialize transactions to avoid SQLITE_BUSY errors */
+  private txQueue = Promise.resolve() as Promise<unknown>;
 
   constructor(db: SQLiteDatabaseDriverAsync) {
     this.db = db;
   }
 
+  private enqueueTx<T>(fn: () => Promise<T>): Promise<T> {
+    const next = this.txQueue.then(fn, fn);
+    this.txQueue = next;
+    return next;
+  }
+
   async getCoValue(coValueId: RawCoID): Promise<StoredCoValueRow | undefined> {
     const coValueRow = await this.db.get<RawCoValueRow & { rowID: number }>(
       "SELECT * FROM coValues WHERE id = ?",
@@ -80,16 +203,6 @@ export class SQLiteClientAsync
     );
   }
 
-  async getSingleCoValueSession(
-    coValueRowId: number,
-    sessionID: SessionID,
-  ): Promise<StoredSessionRow | undefined> {
-    return this.db.get<StoredSessionRow>(
-      "SELECT * FROM sessions WHERE coValue = ? AND sessionID = ?",
-      [coValueRowId, sessionID],
-    );
-  }
-
   async getNewTransactionInSession(
     sessionRowId: number,
     fromIdx: number,
@@ -151,15 +264,6 @@ export class SQLiteClientAsync
     return result.rowID;
   }
 
-  async markCoValueAsDeleted(id: RawCoID) {
-    // Work queue entry. Table only stores the coValueID.
-    // Idempotent by design.
-    await this.db.run(
-      `INSERT INTO deletedCoValues (coValueID) VALUES (?) ON CONFLICT(coValueID) DO NOTHING`,
-      [id],
-    );
-  }
-
   async eraseCoValueButKeepTombstone(coValueId: RawCoID) {
     const coValueRow = await this.db.get<RawCoValueRow & { rowID: number }>(
       "SELECT * FROM coValues WHERE id = ?",
@@ -171,112 +275,29 @@ export class SQLiteClientAsync
       return;
     }
 
-    await this.transaction(async () => {
-      await this.db.run(
-        `DELETE FROM transactions
-       WHERE ses IN (
-         SELECT rowID FROM sessions
-         WHERE coValue = ?
-           AND sessionID NOT LIKE '%$'
-       )`,
-        [coValueRow.rowID],
-      );
-
-      await this.db.run(
-        `DELETE FROM signatureAfter
-       WHERE ses IN (
-         SELECT rowID FROM sessions
-         WHERE coValue = ?
-           AND sessionID NOT LIKE '%$'
-       )`,
-        [coValueRow.rowID],
-      );
-
-      await this.db.run(
-        `DELETE FROM sessions
-       WHERE coValue = ?
-         AND sessionID NOT LIKE '%$'`,
-        [coValueRow.rowID],
-      );
-
-      await this.db.run(
-        `INSERT INTO deletedCoValues (coValueID, status) VALUES (?, ?)
-       ON CONFLICT(coValueID) DO UPDATE SET status=?`,
-        [
-          coValueId,
-          DeletedCoValueDeletionStatus.Done,
-          DeletedCoValueDeletionStatus.Done,
-        ],
-      );
+    await this.transaction(async (tx) => {
+      await tx.deleteCoValueContent(coValueRow);
     });
   }
 
   async getAllCoValuesWaitingForDelete(): Promise<RawCoID[]> {
     const rows = await this.db.query<DeletedCoValueQueueRow>(
       `SELECT coValueID as id
-       FROM deletedCoValues
-       WHERE status = ?`,
+         FROM deletedCoValues
+         WHERE status = ?`,
       [DeletedCoValueDeletionStatus.Pending],
     );
     return rows.map((r) => r.id);
   }
 
-  async addSessionUpdate({
-    sessionUpdate,
-  }: {
-    sessionUpdate: SessionRow;
-  }): Promise<number> {
-    const result = await this.db.get<{ rowID: number }>(
-      `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
-                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
-                            RETURNING rowID`,
-      [
-        sessionUpdate.coValue,
-        sessionUpdate.sessionID,
-        sessionUpdate.lastIdx,
-        sessionUpdate.lastSignature,
-        sessionUpdate.bytesSinceLastSignature,
-      ],
-    );
-
-    if (!result) {
-      throw new Error("Failed to add session update");
-    }
-
-    return result.rowID;
-  }
-
-  addTransaction(
-    sessionRowID: number,
-    nextIdx: number,
-    newTransaction: Transaction,
-  ) {
-    this.db.run("INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)", [
-      sessionRowID,
-      nextIdx,
-      JSON.stringify(newTransaction),
-    ]);
-  }
-
-  async addSignatureAfter({
-    sessionRowID,
-    idx,
-    signature,
-  }: {
-    sessionRowID: number;
-    idx: number;
-    signature: Signature;
-  }) {
-    this.db.run(
-      "INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)",
-      [sessionRowID, idx, signature],
-    );
-  }
-
   async transaction(
     operationsCallback: (tx: DBTransactionInterfaceAsync) => Promise<unknown>,
-  ) {
-    return this.db.transaction(() => operationsCallback(this));
+  ): Promise<unknown> {
+    return this.enqueueTx(() =>
+      this.db.transaction((tx) =>
+        operationsCallback(new SQLiteTransactionAsync(tx)),
+      ),
+    );
   }
 
   async getUnsyncedCoValueIDs(): Promise<RawCoID[]> {

package/src/storage/sqliteAsync/types.ts

@@ -3,7 +3,9 @@ export interface SQLiteDatabaseDriverAsync {
   run(sql: string, params: unknown[]): Promise<void>;
   query<T>(sql: string, params: unknown[]): Promise<T[]>;
   get<T>(sql: string, params: unknown[]): Promise<T | undefined>;
-  transaction(callback: () => unknown): Promise<unknown>;
+  transaction(
+    callback: (tx: SQLiteDatabaseDriverAsync) => unknown,
+  ): Promise<unknown>;
   closeDb(): Promise<unknown>;
   getMigrationVersion?(): Promise<number>;
   saveMigrationVersion?(version: number): Promise<void>;

package/src/storage/types.ts

@@ -172,6 +172,10 @@ export interface DBTransactionInterfaceAsync {
     idx: number;
     signature: Signature;
   }): Promise<unknown>;
+
+  deleteCoValueContent(
+    coValueRow: Pick<StoredCoValueRow, "rowID" | "id">,
+  ): Promise<unknown>;
 }
 
 export interface DBClientInterfaceAsync {

package/src/sync.ts

@@ -626,6 +626,13 @@ export class SyncManager {
           action: "known",
           ...storageKnownState,
         });
+
+        // Subscribe to server peers (e.g., core) to receive future updates.
+        // Even though we responded with KNOWN (client has everything), we need
+        // to establish a subscription so that updates from core flow to us.
+        const serverPeers = this.getServerPeers(msg.id, peer.id);
+        coValue.loadFromPeers(serverPeers);
+
         return;
       }
 
@@ -888,6 +895,8 @@ export class SyncManager {
 
     let wasAlreadyDeleted = coValue.isDeleted;
 
+    const knownState = coValue.knownState();
+
     /**
      * The coValue is in memory, load the transactions from the content message
      */
@@ -901,7 +910,7 @@ export class SyncManager {
 
       const newTransactions = getNewTransactionsFromContentMessage(
         newContentForSession,
-        coValue.knownState(),
+        knownState,
         sessionID,
       );
 

package/src/tests/CojsonMessageChannel.test.ts

@@ -57,9 +57,9 @@ describe("CojsonMessageChannel", () => {
     ).toMatchInlineSnapshot(`
       [
         "server -> client | LOAD Map sessions: empty",
-        "client -> server | CONTENT Group header: true new: After: 0 New: 5",
+        "client -> server | CONTENT Group header: true new: After: 0 New: 6",
         "client -> server | CONTENT Map header: true new: After: 0 New: 1",
-        "server -> client | KNOWN Group sessions: header/5",
+        "server -> client | KNOWN Group sessions: header/6",
         "server -> client | KNOWN Map sessions: header/1",
       ]
     `);

package/src/tests/SQLiteClientAsync.test.ts

@@ -0,0 +1,75 @@
+import { beforeEach, describe, expect, test } from "vitest";
+import { getDbPath } from "./testStorage.js";
+import { setupTestNode } from "./testUtils.js";
+import { DBClientInterfaceAsync } from "../exports.js";
+
+describe("SQLiteClientAsync", () => {
+  describe("transaction", () => {
+    let dbClient: DBClientInterfaceAsync;
+
+    beforeEach(async () => {
+      const node = setupTestNode();
+      const { storage } = await node.addAsyncStorage({
+        ourName: "test",
+        storageName: "test-storage",
+        filename: getDbPath(),
+      });
+      // @ts-expect-error - dbClient is private
+      dbClient = storage.dbClient;
+    });
+
+    test("serializes concurrent transactions to avoid SQLITE_BUSY errors", async () => {
+      const times = Array.from({ length: 10 });
+      await Promise.all(
+        times.map(async (_, i) => {
+          return dbClient.transaction(async (tx) => {
+            // Sleep between 0 and 100ms to force interleaving
+            await new Promise((r) => setTimeout(r, Math.random() * 100));
+            return tx.addSignatureAfter({
+              sessionRowID: 0,
+              idx: i,
+              signature: `signature_z${i}`,
+            });
+          });
+        }),
+      );
+
+      const signatures = await dbClient.getSignatures(0, 0);
+      expect(signatures.length).toBe(10);
+      signatures.forEach(async ({ signature }, i) => {
+        expect(signature).toBe(`signature_z${i}`);
+      });
+    });
+
+    test("continues to serialize transactions even if one fails", async () => {
+      // First transaction succeeds
+      await dbClient.transaction(async (tx) => {
+        return tx.addSignatureAfter({
+          sessionRowID: 0,
+          idx: 0,
+          signature: `signature_z0`,
+        });
+      });
+      // Second transaction fails (duplicate primary key)
+      await expect(
+        dbClient.transaction(async (tx) => {
+          return tx.addSignatureAfter({
+            sessionRowID: 0,
+            idx: 0,
+            signature: `signature_z0`,
+          });
+        }),
+      ).rejects.toThrow(
+        /UNIQUE constraint failed: signatureAfter\.ses, signatureAfter\.idx/,
+      );
+      // Third transaction succeeds
+      await dbClient.transaction(async (tx) => {
+        return tx.addSignatureAfter({
+          sessionRowID: 0,
+          idx: 1,
+          signature: `signature_z1`,
+        });
+      });
+    });
+  });
+});

package/src/tests/StorageApiAsync.test.ts

@@ -945,15 +945,10 @@ describe("StorageApiAsync", () => {
         return true;
       });
 
-      let releaseBarrier!: () => void;
-      const barrier = new Promise<void>((resolve) => {
-        releaseBarrier = resolve;
-      });
-
-      let firstTxStartedResolve!: () => void;
-      const firstTxStarted = new Promise<void>((resolve) => {
-        firstTxStartedResolve = resolve;
-      });
+      const { promise: barrier, resolve: releaseBarrier } =
+        Promise.withResolvers<void>();
+      const { promise: firstTxStarted, resolve: firstTxStartedResolve } =
+        Promise.withResolvers<void>();
 
       // @ts-expect-error - dbClient is private
       const dbClient = storage.dbClient;

package/src/tests/SyncStateManager.test.ts

@@ -307,8 +307,8 @@ describe("SyncStateManager", () => {
       [
         "server -> client | CONTENT Map header: true new: After: 0 New: 1",
         "client -> server | LOAD Group sessions: empty",
-        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
-        "client -> server | KNOWN Group sessions: header/3",
+        "server -> client | CONTENT Group header: true new: After: 0 New: 4",
+        "client -> server | KNOWN Group sessions: header/4",
         "client -> server | KNOWN Map sessions: header/1",
       ]
     `);

package/src/tests/WasmCrypto.test.ts

@@ -183,7 +183,7 @@ describe("WasmCrypto", () => {
     const mapInOtherSession = await loadCoValueOrFail(session2.node, map.id);
 
     const transferredMeta = JSON.parse(
-      mapInOtherSession.core.verified.sessions.get(client.node.currentSessionID)
+      mapInOtherSession.core.verified.getSession(client.node.currentSessionID)
         ?.transactions[0]?.meta!,
     );
 
@@ -193,28 +193,4 @@ describe("WasmCrypto", () => {
       },
     });
   });
-
-  it("fails to verify signatures without a signer ID", async () => {
-    const agentSecret = wasmCrypto.newRandomAgentSecret();
-    const sessionID = wasmCrypto.newRandomSessionID(
-      wasmCrypto.getAgentID(agentSecret),
-    );
-
-    const sessionLog = wasmCrypto.createSessionLog("co_z12345678", sessionID);
-    expect(() =>
-      sessionLog.tryAdd(
-        [
-          {
-            privacy: "trusting",
-            changes: JSON.stringify([
-              { op: "set", key: "count", value: 1 },
-            ]) as Stringified<JsonValue[]>,
-            madeAt: Date.now(),
-          },
-        ],
-        "signature_z12345678",
-        false,
-      ),
-    ).toThrow(expect.stringContaining("Signature verification failed"));
-  });
 });

package/src/tests/coList.test.ts

@@ -210,6 +210,7 @@ test("init the list correctly", () => {
     "universe",
     "hello",
   ]);
+  expect(content.core.verified.header.createdAt).toBeDefined();
 });
 
 test("Items prepended to start appear with latest first", () => {
@@ -904,6 +905,8 @@ describe("CoList Branching", () => {
     // Client1 adds items to the branch
     aliceBranch.append("eggs", undefined, "trusting");
 
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
     // Client2 loads the branch from a different session
     const branchOnClient2 = await loadCoValueOrFail(
       client2.node,
@@ -917,11 +920,11 @@ describe("CoList Branching", () => {
       "trusting",
     );
 
-    // Merge the branch back to source
     branchOnClient2.core.mergeBranch();
 
-    // Wait for sync
-    await groceryList.core.waitForSync();
+    // Wait for all coValues to sync on both nodes
+    await client2.node.syncManager.waitForAllCoValuesSync();
+    await client1.node.syncManager.waitForAllCoValuesSync();
 
     // Source list should contain the final state
     expect(groceryList.toJSON()).toEqual(["milk", "eggs", "cheese"]);
@@ -960,6 +963,8 @@ describe("CoList Branching", () => {
     // Client2 adds different items to second branch
     bobBranch.append("eggs", undefined, "trusting");
 
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
     // Client2 loads first branch and modifies it
     const aliceBranchOnClient2 = await loadCoValueOrFail(
       client2.node,
@@ -975,8 +980,9 @@ describe("CoList Branching", () => {
 
     bobBranch.core.mergeBranch();
 
-    // Wait for sync
-    await groceryList.core.waitForSync();
+    // Wait for all coValues to sync on both nodes
+    await client2.node.syncManager.waitForAllCoValuesSync();
+    await client1.node.syncManager.waitForAllCoValuesSync();
 
     // Source list should contain all changes
     expect(groceryList.toJSON()).toMatchInlineSnapshot(`
@@ -1017,3 +1023,31 @@ test("the list should rebuild when the group permissions change", async () => {
   expect(listOnBob.version).toEqual(1);
   expect(listOnBob.totalValidTransactions).toEqual(1);
 });
+
+test("items appended after a losing init transaction are preserved", async () => {
+  const alice = setupTestNode({ connected: true });
+  const bob = setupTestNode({ connected: true });
+
+  const group = alice.node.createGroup();
+  group.addMember("everyone", "writer");
+
+  const list = group.createList(
+    ["alice-init"],
+    undefined,
+    "trusting",
+    undefined,
+    { fww: "init" },
+  );
+
+  await new Promise((resolve) => setTimeout(resolve, 5));
+
+  const listOnBob = await loadCoValueOrFail(bob.node, list.id);
+
+  listOnBob.appendItems(["bob-init"], undefined, "trusting", { fww: "init" });
+  listOnBob.appendItems(["bob-update"], undefined, "trusting");
+
+  await waitFor(() => {
+    expect(listOnBob.toJSON()).toEqual(["alice-init", "bob-update"]);
+    expect(list.toJSON()).toEqual(["alice-init", "bob-update"]);
+  });
+});