cojson 0.15.8 → 0.15.9

package/src/storage/storageSync.ts

@@ -0,0 +1,343 @@
+import { UpDownCounter, metrics } from "@opentelemetry/api";
+import {
+  CoValueCore,
+  MAX_RECOMMENDED_TX_SIZE,
+  RawCoID,
+  type SessionID,
+  type StorageAPI,
+} from "../exports.js";
+import { getPriorityFromHeader } from "../priority.js";
+import {
+  CoValueKnownState,
+  NewContentMessage,
+  emptyKnownState,
+} from "../sync.js";
+import { StorageKnownState } from "./knownState.js";
+import { collectNewTxs, getDependedOnCoValues } from "./syncUtils.js";
+import type {
+  DBClientInterfaceSync,
+  SignatureAfterRow,
+  StoredCoValueRow,
+  StoredSessionRow,
+} from "./types.js";
+
+export class StorageApiSync implements StorageAPI {
+  private streamingCounter: UpDownCounter;
+
+  private readonly dbClient: DBClientInterfaceSync;
+  private loadedCoValues = new Set<RawCoID>();
+
+  constructor(dbClient: DBClientInterfaceSync) {
+    this.dbClient = dbClient;
+    this.streamingCounter = metrics
+      .getMeter("cojson")
+      .createUpDownCounter(`jazz.storage.streaming`, {
+        description: "Number of streaming coValues",
+        unit: "1",
+      });
+    this.streamingCounter.add(0);
+  }
+
+  knwonStates = new StorageKnownState();
+
+  getKnownState(id: string): CoValueKnownState {
+    return this.knwonStates.getKnownState(id);
+  }
+
+  async load(
+    id: string,
+    callback: (data: NewContentMessage) => void,
+    done: (found: boolean) => void,
+  ) {
+    await this.loadCoValue(id, callback, done);
+  }
+
+  async loadCoValue(
+    id: string,
+    callback: (data: NewContentMessage) => void,
+    done?: (found: boolean) => void,
+  ) {
+    const coValueRow = this.dbClient.getCoValue(id);
+
+    if (!coValueRow) {
+      done?.(false);
+      return;
+    }
+
+    const allCoValueSessions = this.dbClient.getCoValueSessions(
+      coValueRow.rowID,
+    );
+
+    const signaturesBySession = new Map<
+      SessionID,
+      Pick<SignatureAfterRow, "idx" | "signature">[]
+    >();
+
+    let contentStreaming = false;
+    for (const sessionRow of allCoValueSessions) {
+      const signatures = this.dbClient.getSignatures(sessionRow.rowID, 0);
+
+      if (signatures.length > 0) {
+        contentStreaming = true;
+        signaturesBySession.set(sessionRow.sessionID, signatures);
+      }
+    }
+
+    const knownState = this.knwonStates.getKnownState(coValueRow.id);
+
+    for (const sessionRow of allCoValueSessions) {
+      knownState.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
+    }
+
+    this.loadedCoValues.add(coValueRow.id);
+
+    let contentMessage = {
+      action: "content",
+      id: coValueRow.id,
+      header: coValueRow.header,
+      new: {},
+      priority: getPriorityFromHeader(coValueRow.header),
+    } as NewContentMessage;
+
+    if (contentStreaming) {
+      this.streamingCounter.add(1);
+      contentMessage.expectContentUntil = knownState["sessions"];
+    }
+
+    for (const sessionRow of allCoValueSessions) {
+      const signatures = signaturesBySession.get(sessionRow.sessionID) || [];
+
+      let idx = 0;
+
+      signatures.push({
+        idx: sessionRow.lastIdx,
+        signature: sessionRow.lastSignature,
+      });
+
+      for (const signature of signatures) {
+        const newTxsInSession = this.dbClient.getNewTransactionInSession(
+          sessionRow.rowID,
+          idx,
+          signature.idx,
+        );
+
+        collectNewTxs({
+          newTxsInSession,
+          contentMessage,
+          sessionRow,
+          firstNewTxIdx: idx,
+          signature: signature.signature,
+        });
+
+        idx = signature.idx + 1;
+
+        if (signatures.length > 1) {
+          this.pushContentWithDependencies(
+            coValueRow,
+            contentMessage,
+            callback,
+          );
+          contentMessage = {
+            action: "content",
+            id: coValueRow.id,
+            header: coValueRow.header,
+            new: {},
+            priority: getPriorityFromHeader(coValueRow.header),
+          } satisfies NewContentMessage;
+
+          // Introduce a delay to not block the main thread
+          // for the entire content processing
+          await new Promise((resolve) => setTimeout(resolve));
+        }
+      }
+    }
+
+    const hasNewContent = Object.keys(contentMessage.new).length > 0;
+
+    // If there is no new content but steaming is not active, it's the case for a coValue with the header but no transactions
+    // For streaming the push has already been done in the loop above
+    if (hasNewContent || !contentStreaming) {
+      this.pushContentWithDependencies(coValueRow, contentMessage, callback);
+    }
+
+    if (contentStreaming) {
+      this.streamingCounter.add(-1);
+    }
+
+    this.knwonStates.handleUpdate(coValueRow.id, knownState);
+    done?.(true);
+  }
+
+  async pushContentWithDependencies(
+    coValueRow: StoredCoValueRow,
+    contentMessage: NewContentMessage,
+    pushCallback: (data: NewContentMessage) => void,
+  ) {
+    const dependedOnCoValuesList = getDependedOnCoValues(
+      coValueRow.header,
+      contentMessage,
+    );
+
+    for (const dependedOnCoValue of dependedOnCoValuesList) {
+      if (this.loadedCoValues.has(dependedOnCoValue)) {
+        continue;
+      }
+
+      this.loadCoValue(dependedOnCoValue, pushCallback);
+    }
+
+    pushCallback(contentMessage);
+  }
+
+  store(
+    msgs: NewContentMessage[],
+    correctionCallback: (data: CoValueKnownState) => void,
+  ) {
+    for (const msg of msgs) {
+      const success = this.storeSingle(msg, correctionCallback);
+
+      if (!success) {
+        return false;
+      }
+    }
+  }
+
+  private storeSingle(
+    msg: NewContentMessage,
+    correctionCallback: (data: CoValueKnownState) => void,
+  ): boolean {
+    const id = msg.id;
+    const coValueRow = this.dbClient.getCoValue(id);
+
+    // We have no info about coValue header
+    const invalidAssumptionOnHeaderPresence = !msg.header && !coValueRow;
+
+    if (invalidAssumptionOnHeaderPresence) {
+      const knownState = emptyKnownState(id as RawCoID);
+      correctionCallback(knownState);
+
+      this.knwonStates.setKnownState(id, knownState);
+
+      return false;
+    }
+
+    const storedCoValueRowID: number = coValueRow
+      ? coValueRow.rowID
+      : this.dbClient.addCoValue(msg);
+
+    const knownState = this.knwonStates.getKnownState(id);
+    knownState.header = true;
+
+    let invalidAssumptions = false;
+
+    for (const sessionID of Object.keys(msg.new) as SessionID[]) {
+      this.dbClient.transaction(() => {
+        const sessionRow = this.dbClient.getSingleCoValueSession(
+          storedCoValueRowID,
+          sessionID,
+        );
+
+        if (sessionRow) {
+          knownState.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
+        }
+
+        if ((sessionRow?.lastIdx || 0) < (msg.new[sessionID]?.after || 0)) {
+          invalidAssumptions = true;
+        } else {
+          const newLastIdx = this.putNewTxs(
+            msg,
+            sessionID,
+            sessionRow,
+            storedCoValueRowID,
+          );
+          knownState.sessions[sessionID] = newLastIdx;
+        }
+      });
+    }
+
+    this.knwonStates.handleUpdate(id, knownState);
+
+    if (invalidAssumptions) {
+      correctionCallback(knownState);
+      return false;
+    }
+
+    return true;
+  }
+
+  private putNewTxs(
+    msg: NewContentMessage,
+    sessionID: SessionID,
+    sessionRow: StoredSessionRow | undefined,
+    storedCoValueRowID: number,
+  ) {
+    const newTransactions = msg.new[sessionID]?.newTransactions || [];
+
+    const actuallyNewOffset =
+      (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
+
+    const actuallyNewTransactions = newTransactions.slice(actuallyNewOffset);
+
+    if (actuallyNewTransactions.length === 0) {
+      return sessionRow?.lastIdx || 0;
+    }
+
+    let newBytesSinceLastSignature =
+      (sessionRow?.bytesSinceLastSignature || 0) +
+      actuallyNewTransactions.reduce(
+        (sum, tx) =>
+          sum +
+          (tx.privacy === "private"
+            ? tx.encryptedChanges.length
+            : tx.changes.length),
+        0,
+      );
+
+    const newLastIdx =
+      (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
+
+    let shouldWriteSignature = false;
+
+    if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+      shouldWriteSignature = true;
+      newBytesSinceLastSignature = 0;
+    }
+
+    const nextIdx = sessionRow?.lastIdx || 0;
+
+    if (!msg.new[sessionID]) throw new Error("Session ID not found");
+
+    const sessionUpdate = {
+      coValue: storedCoValueRowID,
+      sessionID,
+      lastIdx: newLastIdx,
+      lastSignature: msg.new[sessionID].lastSignature,
+      bytesSinceLastSignature: newBytesSinceLastSignature,
+    };
+
+    const sessionRowID: number = this.dbClient.addSessionUpdate({
+      sessionUpdate,
+      sessionRow,
+    });
+
+    if (shouldWriteSignature) {
+      this.dbClient.addSignatureAfter({
+        sessionRowID,
+        idx: newLastIdx - 1,
+        signature: msg.new[sessionID].lastSignature,
+      });
+    }
+
+    actuallyNewTransactions.map((newTransaction, i) =>
+      this.dbClient.addTransaction(sessionRowID, nextIdx + i, newTransaction),
+    );
+
+    return newLastIdx;
+  }
+
+  waitForSync(id: string, coValue: CoValueCore) {
+    return this.knwonStates.waitForSync(id, coValue);
+  }
+
+  close() {}
+}

package/src/storage/syncUtils.ts

@@ -0,0 +1,50 @@
+import { getDependedOnCoValuesFromRawData } from "../coValueCore/utils.js";
+import type { CoValueHeader } from "../coValueCore/verifiedState.js";
+import type { Signature } from "../crypto/crypto.js";
+import type { SessionID } from "../exports.js";
+import type { NewContentMessage } from "../sync.js";
+import type { StoredSessionRow, TransactionRow } from "./types.js";
+
+export function collectNewTxs({
+  newTxsInSession,
+  contentMessage,
+  sessionRow,
+  firstNewTxIdx,
+  signature,
+}: {
+  newTxsInSession: TransactionRow[];
+  contentMessage: NewContentMessage;
+  sessionRow: StoredSessionRow;
+  signature: Signature;
+  firstNewTxIdx: number;
+}) {
+  let sessionEntry = contentMessage.new[sessionRow.sessionID];
+
+  if (!sessionEntry) {
+    sessionEntry = {
+      after: firstNewTxIdx,
+      lastSignature: signature,
+      newTransactions: [],
+    };
+    contentMessage.new[sessionRow.sessionID] = sessionEntry;
+  } else {
+    sessionEntry.lastSignature = signature;
+  }
+
+  for (const tx of newTxsInSession) {
+    sessionEntry.newTransactions.push(tx.tx);
+  }
+}
+
+export function getDependedOnCoValues(
+  header: CoValueHeader,
+  contentMessage: NewContentMessage,
+) {
+  const id = contentMessage.id;
+  const sessionIDs = Object.keys(contentMessage.new) as SessionID[];
+  const transactions = Object.values(contentMessage.new).map(
+    (entry) => entry.newTransactions,
+  );
+
+  return getDependedOnCoValuesFromRawData(id, header, sessionIDs, transactions);
+}

package/src/storage/types.ts

@@ -0,0 +1,162 @@
+import type {
+  CoValueHeader,
+  Transaction,
+} from "../coValueCore/verifiedState.js";
+import { Signature } from "../crypto/crypto.js";
+import type { CoValueCore, RawCoID, SessionID } from "../exports.js";
+import { CoValueKnownState, NewContentMessage } from "../sync.js";
+
+/**
+ * The StorageAPI is the interface that the StorageSync and StorageAsync classes implement.
+ *
+ * It uses callbacks instead of promises to have no overhead when using the StorageSync and less overhead when using the StorageAsync.
+ */
+export interface StorageAPI {
+  load(
+    id: string,
+    // This callback is fired when data is found, might be called multiple times if the content requires streaming (e.g when loading files)
+    callback: (data: NewContentMessage) => void,
+    done?: (found: boolean) => void,
+  ): void;
+  store(
+    data: NewContentMessage[] | undefined,
+    handleCorrection: (correction: CoValueKnownState) => void,
+  ): void;
+
+  getKnownState(id: string): CoValueKnownState;
+
+  waitForSync(id: string, coValue: CoValueCore): Promise<void>;
+
+  close(): void;
+}
+
+export type CoValueRow = {
+  id: RawCoID;
+  header: CoValueHeader;
+};
+
+export type StoredCoValueRow = CoValueRow & { rowID: number };
+
+export type SessionRow = {
+  coValue: number;
+  sessionID: SessionID;
+  lastIdx: number;
+  lastSignature: Signature;
+  bytesSinceLastSignature?: number;
+};
+
+export type StoredSessionRow = SessionRow & { rowID: number };
+
+export type TransactionRow = {
+  ses: number;
+  idx: number;
+  tx: Transaction;
+};
+
+export type SignatureAfterRow = {
+  ses: number;
+  idx: number;
+  signature: Signature;
+};
+
+export interface DBClientInterfaceAsync {
+  getCoValue(
+    coValueId: string,
+  ): Promise<StoredCoValueRow | undefined> | undefined;
+
+  getCoValueSessions(coValueRowId: number): Promise<StoredSessionRow[]>;
+
+  getSingleCoValueSession(
+    coValueRowId: number,
+    sessionID: SessionID,
+  ): Promise<StoredSessionRow | undefined>;
+
+  getNewTransactionInSession(
+    sessionRowId: number,
+    fromIdx: number,
+    toIdx: number,
+  ): Promise<TransactionRow[]>;
+
+  getSignatures(
+    sessionRowId: number,
+    firstNewTxIdx: number,
+  ): Promise<SignatureAfterRow[]>;
+
+  addCoValue(msg: NewContentMessage): Promise<number>;
+
+  addSessionUpdate({
+    sessionUpdate,
+    sessionRow,
+  }: {
+    sessionUpdate: SessionRow;
+    sessionRow?: StoredSessionRow;
+  }): Promise<number>;
+
+  addTransaction(
+    sessionRowID: number,
+    idx: number,
+    newTransaction: Transaction,
+  ): Promise<number> | undefined | unknown;
+
+  addSignatureAfter({
+    sessionRowID,
+    idx,
+    signature,
+  }: {
+    sessionRowID: number;
+    idx: number;
+    signature: Signature;
+  }): Promise<unknown>;
+
+  transaction(callback: () => unknown): Promise<unknown>;
+}
+
+export interface DBClientInterfaceSync {
+  getCoValue(coValueId: string): StoredCoValueRow | undefined;
+
+  getCoValueSessions(coValueRowId: number): StoredSessionRow[];
+
+  getSingleCoValueSession(
+    coValueRowId: number,
+    sessionID: SessionID,
+  ): StoredSessionRow | undefined;
+
+  getNewTransactionInSession(
+    sessionRowId: number,
+    fromIdx: number,
+    toIdx: number,
+  ): TransactionRow[];
+
+  getSignatures(
+    sessionRowId: number,
+    firstNewTxIdx: number,
+  ): Pick<SignatureAfterRow, "idx" | "signature">[];
+
+  addCoValue(msg: NewContentMessage): number;
+
+  addSessionUpdate({
+    sessionUpdate,
+    sessionRow,
+  }: {
+    sessionUpdate: SessionRow;
+    sessionRow?: StoredSessionRow;
+  }): number;
+
+  addTransaction(
+    sessionRowID: number,
+    idx: number,
+    newTransaction: Transaction,
+  ): number | undefined | unknown;
+
+  addSignatureAfter({
+    sessionRowID,
+    idx,
+    signature,
+  }: {
+    sessionRowID: number;
+    idx: number;
+    signature: Signature;
+  }): number | undefined | unknown;
+
+  transaction(callback: () => unknown): unknown;
+}

package/src/streamUtils.ts

@@ -1,6 +1,11 @@
-import { Channel } from "queueueue";
-import { Peer, PeerID, SyncMessage } from "./sync.js";
-export { Channel } from "queueueue";
+import {
+  DisconnectedError,
+  IncomingPeerChannel,
+  OutgoingPeerChannel,
+  Peer,
+  PeerID,
+  SyncMessage,
+} from "./sync.js";
 
 export function connectedPeers(
   peer1id: PeerID,
@@ -8,40 +13,77 @@ export function connectedPeers(
   {
     peer1role = "client",
     peer2role = "client",
-    crashOnClose = false,
   }: {
     peer1role?: Peer["role"];
     peer2role?: Peer["role"];
-    crashOnClose?: boolean;
   } = {},
 ): [Peer, Peer] {
-  const [from1to2Rx, from1to2Tx] = newQueuePair();
-  const [from2to1Rx, from2to1Tx] = newQueuePair();
+  const from1to2 = new ConnectedPeerChannel();
+  const from2to1 = new ConnectedPeerChannel();
 
   const peer2AsPeer: Peer = {
     id: peer2id,
-    incoming: from2to1Rx,
-    outgoing: from1to2Tx,
+    incoming: from2to1,
+    outgoing: from1to2,
     role: peer2role,
-    crashOnClose: crashOnClose,
   };
 
   const peer1AsPeer: Peer = {
     id: peer1id,
-    incoming: from1to2Rx,
-    outgoing: from2to1Tx,
+    incoming: from1to2,
+    outgoing: from2to1,
     role: peer1role,
-    crashOnClose: crashOnClose,
   };
 
   return [peer1AsPeer, peer2AsPeer];
 }
 
-export function newQueuePair(): [
-  AsyncIterable<SyncMessage>,
-  Channel<SyncMessage>,
-] {
-  const channel = new Channel<SyncMessage>();
+export function newQueuePair(): [ConnectedPeerChannel, ConnectedPeerChannel] {
+  const channel = new ConnectedPeerChannel();
 
-  return [channel.wrap(), channel];
+  return [channel, channel];
+}
+
+export class ConnectedPeerChannel
+  implements IncomingPeerChannel, OutgoingPeerChannel
+{
+  buffer: (SyncMessage | DisconnectedError)[] = [];
+
+  push(msg: SyncMessage | DisconnectedError) {
+    if (!this.listeners.size) {
+      this.buffer.push(msg);
+      return;
+    }
+
+    for (const listener of this.listeners) {
+      listener(msg);
+    }
+  }
+
+  close() {
+    this.closed = true;
+    for (const listener of this.closeListeners) {
+      listener();
+    }
+    this.closeListeners.clear();
+    this.listeners.clear();
+  }
+
+  listeners = new Set<(msg: SyncMessage | DisconnectedError) => void>();
+  onMessage(callback: (msg: SyncMessage | DisconnectedError) => void) {
+    if (this.buffer.length) {
+      for (const msg of this.buffer) {
+        callback(msg);
+      }
+      this.buffer = [];
+    }
+
+    this.listeners.add(callback);
+  }
+
+  closed = false;
+  closeListeners = new Set<() => void>();
+  onClose(callback: () => void) {
+    this.closeListeners.add(callback);
+  }
 }