cojson-storage-indexeddb 0.8.34 → 0.8.35

package/src/syncManager.ts

@@ -0,0 +1,336 @@
+import {
+  CojsonInternalTypes,
+  MAX_RECOMMENDED_TX_SIZE,
+  OutgoingSyncQueue,
+  SessionID,
+  SyncMessage,
+  cojsonInternals,
+  emptyKnownState,
+} from "cojson";
+import { IDBClient, StoredSessionRow } from "./idbClient";
+import { SyncPromise } from "./syncPromises.js";
+import { collectNewTxs, getDependedOnCoValues } from "./syncUtils";
+import NewContentMessage = CojsonInternalTypes.NewContentMessage;
+import KnownStateMessage = CojsonInternalTypes.KnownStateMessage;
+import RawCoID = CojsonInternalTypes.RawCoID;
+
+type OutputMessageMap = Record<
+  RawCoID,
+  { knownMessage: KnownStateMessage; contentMessages?: NewContentMessage[] }
+>;
+
+export class SyncManager {
+  private readonly toLocalNode: OutgoingSyncQueue;
+  private readonly idbClient: IDBClient;
+
+  constructor(idbClient: IDBClient, toLocalNode: OutgoingSyncQueue) {
+    this.toLocalNode = toLocalNode;
+    this.idbClient = idbClient;
+  }
+
+  async handleSyncMessage(msg: SyncMessage) {
+    switch (msg.action) {
+      case "load":
+        await this.handleLoad(msg);
+        break;
+      case "content":
+        await this.handleContent(msg);
+        break;
+      case "known":
+        await this.handleKnown(msg);
+        break;
+      case "done":
+        await this.handleDone(msg);
+        break;
+    }
+  }
+
+  async handleSessionUpdate({
+    sessionRow,
+    peerKnownState,
+    newContentMessages,
+  }: {
+    sessionRow: StoredSessionRow;
+    peerKnownState: CojsonInternalTypes.CoValueKnownState;
+    newContentMessages: CojsonInternalTypes.NewContentMessage[];
+  }) {
+    if (
+      sessionRow.lastIdx <= (peerKnownState.sessions[sessionRow.sessionID] || 0)
+    )
+      return;
+
+    const firstNewTxIdx = peerKnownState.sessions[sessionRow.sessionID] || 0;
+    const signaturesAndIdxs = await this.idbClient.getSignatures(
+      sessionRow,
+      firstNewTxIdx,
+    );
+    const newTxsInSession = await this.idbClient.getNewTransactionInSession(
+      sessionRow,
+      firstNewTxIdx,
+    );
+
+    collectNewTxs(
+      newTxsInSession,
+      newContentMessages,
+      sessionRow,
+      signaturesAndIdxs,
+      peerKnownState,
+      firstNewTxIdx,
+    );
+  }
+
+  async sendNewContent(
+    coValueKnownState: CojsonInternalTypes.CoValueKnownState,
+  ): Promise<void> {
+    const outputMessages: OutputMessageMap =
+      await this.collectCoValueData(coValueKnownState);
+
+    // reverse it to send the top level id the last in the order
+    const collectedMessages = Object.values(outputMessages).reverse();
+    collectedMessages.forEach(({ knownMessage, contentMessages }) => {
+      this.sendStateMessage(knownMessage);
+
+      contentMessages?.length &&
+        contentMessages.forEach((msg) => this.sendStateMessage(msg));
+    });
+  }
+
+  private async collectCoValueData(
+    peerKnownState: CojsonInternalTypes.CoValueKnownState,
+    messageMap: OutputMessageMap = {},
+    asDependencyOf?: CojsonInternalTypes.RawCoID,
+  ) {
+    if (messageMap[peerKnownState.id]) {
+      return messageMap;
+    }
+
+    const coValueRow = await this.idbClient.getCoValue(peerKnownState.id);
+
+    if (!coValueRow) {
+      const emptyKnownMessage: KnownStateMessage = {
+        action: "known",
+        ...emptyKnownState(peerKnownState.id),
+      };
+      asDependencyOf && (emptyKnownMessage.asDependencyOf = asDependencyOf);
+      messageMap[peerKnownState.id] = { knownMessage: emptyKnownMessage };
+      return messageMap;
+    }
+
+    const allCoValueSessions = await this.idbClient.getCoValueSessions(
+      coValueRow.rowID,
+    );
+
+    const newCoValueKnownState: CojsonInternalTypes.CoValueKnownState = {
+      id: coValueRow.id,
+      header: true,
+      sessions: {},
+    };
+
+    const newContentMessages: CojsonInternalTypes.NewContentMessage[] = [
+      {
+        action: "content",
+        id: coValueRow.id,
+        header: coValueRow.header,
+        new: {},
+        priority: cojsonInternals.getPriorityFromHeader(coValueRow.header),
+      },
+    ];
+
+    await Promise.all(
+      allCoValueSessions.map((sessionRow) => {
+        newCoValueKnownState.sessions[sessionRow.sessionID] =
+          sessionRow.lastIdx;
+        // Collect new sessions data into newContentMessages
+        return this.handleSessionUpdate({
+          sessionRow,
+          peerKnownState,
+          newContentMessages,
+        });
+      }),
+    );
+
+    const dependedOnCoValuesList = getDependedOnCoValues({
+      coValueRow,
+      newContentMessages,
+    });
+
+    const knownMessage: KnownStateMessage = {
+      action: "known",
+      ...newCoValueKnownState,
+    };
+    asDependencyOf && (knownMessage.asDependencyOf = asDependencyOf);
+    messageMap[newCoValueKnownState.id] = {
+      knownMessage: knownMessage,
+      contentMessages: newContentMessages,
+    };
+
+    await Promise.all(
+      dependedOnCoValuesList.map((dependedOnCoValue) =>
+        this.collectCoValueData(
+          {
+            id: dependedOnCoValue,
+            header: false,
+            sessions: {},
+          },
+          messageMap,
+          asDependencyOf || coValueRow.id,
+        ),
+      ),
+    );
+
+    return messageMap;
+  }
+
+  handleLoad(msg: CojsonInternalTypes.LoadMessage) {
+    return this.sendNewContent(msg);
+  }
+
+  async handleContent(
+    msg: CojsonInternalTypes.NewContentMessage,
+  ): Promise<void | unknown> {
+    const coValueRow = await this.idbClient.getCoValue(msg.id);
+
+    // We have no info about coValue header
+    const invalidAssumptionOnHeaderPresence = !msg.header && !coValueRow;
+
+    if (invalidAssumptionOnHeaderPresence) {
+      return this.sendStateMessage({
+        action: "known",
+        id: msg.id,
+        header: false,
+        sessions: {},
+        isCorrection: true,
+      });
+    }
+
+    const storedCoValueRowID: number = coValueRow
+      ? coValueRow.rowID
+      : await this.idbClient.addCoValue(msg);
+
+    const allOurSessionsEntries =
+      await this.idbClient.getCoValueSessions(storedCoValueRowID);
+
+    const allOurSessions: {
+      [sessionID: SessionID]: StoredSessionRow;
+    } = Object.fromEntries(
+      allOurSessionsEntries.map((row) => [row.sessionID, row]),
+    );
+
+    const ourKnown: CojsonInternalTypes.CoValueKnownState = {
+      id: msg.id,
+      header: true,
+      sessions: {},
+    };
+    let invalidAssumptions = false;
+
+    await Promise.all(
+      (Object.keys(msg.new) as SessionID[]).map((sessionID) => {
+        const sessionRow = allOurSessions[sessionID];
+        if (sessionRow) {
+          ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
+        }
+
+        if ((sessionRow?.lastIdx || 0) < (msg.new[sessionID]?.after || 0)) {
+          invalidAssumptions = true;
+        } else {
+          return this.putNewTxs(msg, sessionID, sessionRow, storedCoValueRowID);
+        }
+      }),
+    );
+
+    if (invalidAssumptions) {
+      this.sendStateMessage({
+        action: "known",
+        ...ourKnown,
+        isCorrection: invalidAssumptions,
+      });
+    }
+  }
+
+  private async putNewTxs(
+    msg: CojsonInternalTypes.NewContentMessage,
+    sessionID: SessionID,
+    sessionRow: StoredSessionRow | undefined,
+    storedCoValueRowID: number,
+  ) {
+    const newTransactions = msg.new[sessionID]?.newTransactions || [];
+
+    const actuallyNewOffset =
+      (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
+
+    const actuallyNewTransactions = newTransactions.slice(actuallyNewOffset);
+
+    let newBytesSinceLastSignature =
+      (sessionRow?.bytesSinceLastSignature || 0) +
+      actuallyNewTransactions.reduce(
+        (sum, tx) =>
+          sum +
+          (tx.privacy === "private"
+            ? tx.encryptedChanges.length
+            : tx.changes.length),
+        0,
+      );
+
+    const newLastIdx =
+      (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
+
+    let shouldWriteSignature = false;
+
+    if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+      shouldWriteSignature = true;
+      newBytesSinceLastSignature = 0;
+    }
+
+    const nextIdx = sessionRow?.lastIdx || 0;
+
+    const sessionUpdate = {
+      coValue: storedCoValueRowID,
+      sessionID: sessionID,
+      lastIdx: newLastIdx,
+      lastSignature: msg.new[sessionID]!.lastSignature,
+      bytesSinceLastSignature: newBytesSinceLastSignature,
+    };
+
+    const sessionRowID: number = await this.idbClient.addSessionUpdate(
+      sessionRow,
+      sessionUpdate,
+    );
+
+    let maybePutRequest;
+    if (shouldWriteSignature) {
+      maybePutRequest = this.idbClient.addSignatureAfter({
+        sessionRowID,
+        idx: newLastIdx - 1,
+        signature: msg.new[sessionID]!.lastSignature,
+      });
+    } else {
+      maybePutRequest = SyncPromise.resolve();
+    }
+
+    return maybePutRequest.then(() =>
+      Promise.all(
+        actuallyNewTransactions.map((newTransaction, i) => {
+          return this.idbClient.addTransaction(
+            sessionRowID,
+            nextIdx + i,
+            newTransaction,
+          );
+        }),
+      ),
+    );
+  }
+
+  handleKnown(_msg: KnownStateMessage) {
+    // We don't intend to use IndexedDB itself as a synchronisation mechanism, so we can ignore the known messages
+  }
+
+  async sendStateMessage(msg: any): Promise<unknown> {
+    return this.toLocalNode
+      .push(msg)
+      .catch((e) =>
+        console.error(`Error sending ${msg.action} state, id ${msg.id}`, e),
+      );
+  }
+
+  handleDone(_msg: CojsonInternalTypes.DoneMessage) {}
+}

package/src/syncUtils.ts

@@ -0,0 +1,146 @@
+import {
+  CojsonInternalTypes,
+  JsonValue,
+  SessionID,
+  Stringified,
+  cojsonInternals,
+} from "cojson";
+import {
+  SignatureAfterRow,
+  StoredCoValueRow,
+  StoredSessionRow,
+  TransactionRow,
+} from "./idbClient";
+
+export function collectNewTxs(
+  newTxsInSession: TransactionRow[],
+  newContentMessages: CojsonInternalTypes.NewContentMessage[],
+  sessionRow: StoredSessionRow,
+  signaturesAndIdxs: SignatureAfterRow[],
+  theirKnown: CojsonInternalTypes.CoValueKnownState,
+  firstNewTxIdx: number,
+) {
+  let idx = firstNewTxIdx;
+
+  for (const tx of newTxsInSession) {
+    let sessionEntry =
+      newContentMessages[newContentMessages.length - 1]!.new[
+        sessionRow.sessionID
+      ];
+    if (!sessionEntry) {
+      sessionEntry = {
+        after: idx,
+        lastSignature: "WILL_BE_REPLACED" as CojsonInternalTypes.Signature,
+        newTransactions: [],
+      };
+      newContentMessages[newContentMessages.length - 1]!.new[
+        sessionRow.sessionID
+      ] = sessionEntry;
+    }
+
+    sessionEntry.newTransactions.push(tx.tx);
+
+    if (signaturesAndIdxs[0] && idx === signaturesAndIdxs[0].idx) {
+      sessionEntry.lastSignature = signaturesAndIdxs[0].signature;
+      signaturesAndIdxs.shift();
+      newContentMessages.push({
+        action: "content",
+        id: theirKnown.id,
+        new: {},
+        priority: cojsonInternals.getPriorityFromHeader(undefined),
+      });
+    } else if (idx === firstNewTxIdx + newTxsInSession.length - 1) {
+      sessionEntry.lastSignature = sessionRow.lastSignature;
+    }
+    idx += 1;
+  }
+}
+
+export function getDependedOnCoValues({
+  coValueRow,
+  newContentMessages,
+}: {
+  coValueRow: StoredCoValueRow;
+  newContentMessages: CojsonInternalTypes.NewContentMessage[];
+}) {
+  return coValueRow.header.ruleset.type === "group"
+    ? getGroupDependedOnCoValues(newContentMessages)
+    : coValueRow.header.ruleset.type === "ownedByGroup"
+      ? getOwnedByGroupDependedOnCoValues(coValueRow, newContentMessages)
+      : [];
+}
+
+function getGroupDependedOnCoValues(
+  newContentMessages: CojsonInternalTypes.NewContentMessage[],
+) {
+  const keys: CojsonInternalTypes.RawCoID[] = [];
+
+  /**
+   * Collect all the signing keys inside the transactions to list all the
+   * dependencies required to correctly access the CoValue.
+   */
+  for (const piece of newContentMessages) {
+    for (const sessionEntry of Object.values(piece.new)) {
+      for (const tx of sessionEntry.newTransactions) {
+        if (tx.privacy !== "trusting") continue;
+
+        const changes = safeParseChanges(tx.changes);
+        for (const change of changes) {
+          if (
+            change &&
+            typeof change === "object" &&
+            "op" in change &&
+            change.op === "set" &&
+            "key" in change &&
+            change.key
+          ) {
+            const key = cojsonInternals.getGroupDependentKey(change.key);
+
+            if (key) {
+              keys.push(key);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return keys;
+}
+
+function getOwnedByGroupDependedOnCoValues(
+  coValueRow: StoredCoValueRow,
+  newContentMessages: CojsonInternalTypes.NewContentMessage[],
+) {
+  if (coValueRow.header.ruleset.type !== "ownedByGroup") return [];
+
+  const keys: CojsonInternalTypes.RawCoID[] = [coValueRow.header.ruleset.group];
+
+  /**
+   * Collect all the signing keys inside the transactions to list all the
+   * dependencies required to correctly access the CoValue.
+   */
+  for (const piece of newContentMessages) {
+    for (const sessionID of Object.keys(piece.new) as SessionID[]) {
+      const accountId =
+        cojsonInternals.accountOrAgentIDfromSessionID(sessionID);
+
+      if (
+        cojsonInternals.isAccountID(accountId) &&
+        accountId !== coValueRow.id
+      ) {
+        keys.push(accountId);
+      }
+    }
+  }
+
+  return keys;
+}
+
+function safeParseChanges(changes: Stringified<JsonValue[]>) {
+  try {
+    return cojsonInternals.parseJSON(changes);
+  } catch (e) {
+    return [];
+  }
+}

package/src/tests/fixtureMessages.ts

@@ -0,0 +1,95 @@
+export const fixtures = {
+  co_zKwG8NyfZ8GXqcjDHY4NS3SbU2m: {
+    getContent: ({ after = 0 }: { after?: number }) => ({
+      action: "content",
+      id: "co_zKwG8NyfZ8GXqcjDHY4NS3SbU2m",
+      header: {
+        type: "comap",
+        ruleset: {
+          type: "group",
+          initialAdmin:
+            "sealer_zRKetKBH6tdGP8poA2rV9JDejXqTyAmpusCT4jRcXa4m/signer_z6bcctDRiWxtgmuqLRR6rVhM54DA3xJ2pWCEs6DVf4PSy",
+        },
+        meta: {
+          type: "account",
+        },
+        createdAt: null,
+        uniqueness: null,
+      },
+      new: {
+        "sealer_zRKetKBH6tdGP8poA2rV9JDejXqTyAmpusCT4jRcXa4m/signer_z6bcctDRiWxtgmuqLRR6rVhM54DA3xJ2pWCEs6DVf4PSy_session_zbcBS6rHy8kA":
+          {
+            after,
+            lastSignature:
+              "signature_z2kcFHUPe1qGFYDY4ayvvFR2unFc4jeYph93nSCSjZYS14vnGN4uAw7pKZx1PEhwnspJcDizMRbLaFC8v13i6S79A",
+            newTransactions: [
+              {
+                privacy: "trusting",
+                madeAt: 1732368535089,
+                changes:
+                  '[{"key":"sealer_zRKetKBH6tdGP8poA2rV9JDejXqTyAmpusCT4jRcXa4m/signer_z6bcctDRiWxtgmuqLRR6rVhM54DA3xJ2pWCEs6DVf4PSy","op":"set","value":"admin"}]',
+              },
+              {
+                privacy: "trusting",
+                madeAt: 1732368535096,
+                changes:
+                  '[{"key":"key_z2YMuLXEfXG44Z2jGk_for_sealer_zRKetKBH6tdGP8poA2rV9JDejXqTyAmpusCT4jRcXa4m/signer_z6bcctDRiWxtgmuqLRR6rVhM54DA3xJ2pWCEs6DVf4PSy","op":"set","value":"sealed_UAIpJTby8EovZW6WPtAqdaczA2_r6PEWRBuEtLN93-Dh9xDJFaGUNTXK1Cck61tjvA3GoGn9EyQdNN2fU6tnmWP2M09a83dG41Q=="}]',
+              },
+              {
+                privacy: "trusting",
+                madeAt: 1732368535099,
+                changes:
+                  '[{"key":"readKey","op":"set","value":"key_z2YMuLXEfXG44Z2jGk"}]',
+              },
+            ],
+          },
+        "sealer_zRKetKBH6tdGP8poA2rV9JDejXqTyAmpusCT4jRcXa4m/signer_z6bcctDRiWxtgmuqLRR6rVhM54DA3xJ2pWCEs6DVf4PSy_session_zXgW54i2cCNA":
+          {
+            after,
+            lastSignature:
+              "signature_z5FsinkJCpqZfozVBkEMSchCQarsAjvMYpWN4d227PZtqCiM7KRBNukND3B25Q73idBLdY2MsghbmYFz5JHXk3d4D",
+            newTransactions: [
+              {
+                privacy: "trusting",
+                madeAt: 1732368535113,
+                changes:
+                  '[{"key":"profile","op":"set","value":"co_zMKhQJs5rAeGjta3JX2qEdBS6hS"}]',
+              },
+            ],
+          },
+      },
+      priority: 0,
+    }),
+    known: {
+      action: "known",
+      id: "co_zKwG8NyfZ8GXqcjDHY4NS3SbU2m",
+      header: true,
+      sessions: {
+        "sealer_zRKetKBH6tdGP8poA2rV9JDejXqTyAmpusCT4jRcXa4m/signer_z6bcctDRiWxtgmuqLRR6rVhM54DA3xJ2pWCEs6DVf4PSy_session_zbcBS6rHy8kA": 3,
+        "sealer_zRKetKBH6tdGP8poA2rV9JDejXqTyAmpusCT4jRcXa4m/signer_z6bcctDRiWxtgmuqLRR6rVhM54DA3xJ2pWCEs6DVf4PSy_session_zXgW54i2cCNA": 1,
+      },
+    },
+    sessionRecords: [
+      {
+        bytesSinceLastSignature: 479,
+        coValue: 2,
+        lastIdx: 3,
+        lastSignature:
+          "signature_z2kcFHUPe1qGFYDY4ayvvFR2unFc4jeYph93nSCSjZYS14vnGN4uAw7pKZx1PEhwnspJcDizMRbLaFC8v13i6S79A",
+        rowID: 2,
+        sessionID:
+          "sealer_zRKetKBH6tdGP8poA2rV9JDejXqTyAmpusCT4jRcXa4m/signer_z6bcctDRiWxtgmuqLRR6rVhM54DA3xJ2pWCEs6DVf4PSy_session_zbcBS6rHy8kA",
+      },
+      {
+        bytesSinceLastSignature: 71,
+        coValue: 2,
+        lastIdx: 1,
+        lastSignature:
+          "signature_z5FsinkJCpqZfozVBkEMSchCQarsAjvMYpWN4d227PZtqCiM7KRBNukND3B25Q73idBLdY2MsghbmYFz5JHXk3d4D",
+        rowID: 3,
+        sessionID:
+          "sealer_zRKetKBH6tdGP8poA2rV9JDejXqTyAmpusCT4jRcXa4m/signer_z6bcctDRiWxtgmuqLRR6rVhM54DA3xJ2pWCEs6DVf4PSy_session_zXgW54i2cCNA",
+      },
+    ],
+  },
+};