cojson 0.15.8 → 0.15.9

package/src/localNode.ts

@@ -3,7 +3,6 @@ import { CoID } from "./coValue.js";
 import { RawCoValue } from "./coValue.js";
 import {
   AvailableCoValueCore,
-  CO_VALUE_LOADING_CONFIG,
   CoValueCore,
   idforHeader,
 } from "./coValueCore/coValueCore.js";
@@ -31,9 +30,11 @@ import {
   RawGroup,
   secretSeedFromInviteSecret,
 } from "./coValues/group.js";
+import { CO_VALUE_LOADING_CONFIG } from "./config.js";
 import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
 import { logger } from "./logger.js";
+import { StorageAPI } from "./storage/index.js";
 import { Peer, PeerID, SyncManager } from "./sync.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
@@ -64,6 +65,8 @@ export class LocalNode {
 
   crashed: Error | undefined = undefined;
 
+  storage?: StorageAPI;
+
   /** @category 3. Low-level */
   constructor(
     agentSecret: AgentSecret,
@@ -75,6 +78,10 @@ export class LocalNode {
     this.crypto = crypto;
   }
 
+  setStorage(storage: StorageAPI) {
+    this.storage = storage;
+  }
+
   getCoValue(id: RawCoID) {
     let entry = this.coValues.get(id);
 
@@ -142,6 +149,7 @@ export class LocalNode {
     crypto: CryptoProvider;
     initialAgentSecret?: AgentSecret;
     peersToLoadFrom?: Peer[];
+    storage?: StorageAPI;
   }): RawAccount {
     const {
       crypto,
@@ -160,6 +168,10 @@ export class LocalNode {
       crypto,
     );
 
+    if (opts.storage) {
+      node.setStorage(opts.storage);
+    }
+
     for (const peer of peersToLoadFrom) {
       node.syncManager.addPeer(peer);
     }
@@ -198,12 +210,14 @@ export class LocalNode {
     migration,
     crypto,
     initialAgentSecret = crypto.newRandomAgentSecret(),
+    storage,
   }: {
     creationProps: { name: string };
     peersToLoadFrom?: Peer[];
     migration?: RawAccountMigration<AccountMeta>;
     crypto: CryptoProvider;
     initialAgentSecret?: AgentSecret;
+    storage?: StorageAPI;
   }): Promise<{
     node: LocalNode;
     accountID: RawAccountID;
@@ -214,6 +228,7 @@ export class LocalNode {
       crypto,
       initialAgentSecret,
       peersToLoadFrom,
+      storage,
     });
     const node = account.core.node;
 
@@ -234,7 +249,7 @@ export class LocalNode {
       throw new Error("Must set account profile in initial migration");
     }
 
-    if (node.syncManager.hasStoragePeers()) {
+    if (node.storage) {
       await Promise.all([
         node.syncManager.waitForStorageSync(account.id),
         node.syncManager.waitForStorageSync(profileId),
@@ -257,6 +272,7 @@ export class LocalNode {
     peersToLoadFrom,
     crypto,
     migration,
+    storage,
   }: {
     accountID: RawAccountID;
     accountSecret: AgentSecret;
@@ -264,6 +280,7 @@ export class LocalNode {
     peersToLoadFrom: Peer[];
     crypto: CryptoProvider;
     migration?: RawAccountMigration<AccountMeta>;
+    storage?: StorageAPI;
   }): Promise<LocalNode> {
     try {
       const node = new LocalNode(
@@ -272,6 +289,10 @@ export class LocalNode {
         crypto,
       );
 
+      if (storage) {
+        node.setStorage(storage);
+      }
+
       for (const peer of peersToLoadFrom) {
         node.syncManager.addPeer(peer);
       }
@@ -354,23 +375,16 @@ export class LocalNode {
         coValue.loadingState === "unknown" ||
         coValue.loadingState === "unavailable"
       ) {
-        const peers =
-          this.syncManager.getServerAndStoragePeers(skipLoadingFromPeer);
+        const peers = this.syncManager.getServerPeers(skipLoadingFromPeer);
 
-        if (peers.length === 0) {
+        if (!this.storage && peers.length === 0) {
           return coValue;
         }
 
-        coValue.loadFromPeers(peers).catch((e) => {
-          logger.error("Error loading from peers", {
-            id,
-            err: e,
-          });
-        });
+        coValue.load(peers);
       }
 
       const result = await coValue.waitForAvailableOrUnavailable();
-
       if (
         result.isAvailable() ||
         skipRetry ||
@@ -714,6 +728,7 @@ export class LocalNode {
   }
 
   gracefulShutdown() {
+    this.storage?.close();
     this.syncManager.gracefulShutdown();
   }
 }

package/src/storage/StoreQueue.ts

@@ -0,0 +1,56 @@
+import { LinkedList } from "../PriorityBasedMessageQueue.js";
+import { logger } from "../logger.js";
+import { CoValueKnownState, NewContentMessage } from "../sync.js";
+
+type StoreQueueEntry = {
+  data: NewContentMessage[];
+  correctionCallback: (data: CoValueKnownState) => void;
+};
+
+export class StoreQueue {
+  private queue = new LinkedList<StoreQueueEntry>();
+
+  public push(
+    data: NewContentMessage[],
+    correctionCallback: (data: CoValueKnownState) => void,
+  ) {
+    this.queue.push({ data, correctionCallback });
+  }
+
+  public pull() {
+    return this.queue.shift();
+  }
+
+  processing = false;
+
+  async processQueue(
+    callback: (
+      data: NewContentMessage[],
+      correctionCallback: (data: CoValueKnownState) => void,
+    ) => Promise<void>,
+  ) {
+    if (this.processing) {
+      return;
+    }
+
+    this.processing = true;
+
+    let entry: StoreQueueEntry | undefined;
+
+    while ((entry = this.pull())) {
+      const { data, correctionCallback } = entry;
+
+      try {
+        await callback(data, correctionCallback);
+      } catch (err) {
+        logger.error("Error processing message in store queue", { err });
+      }
+    }
+
+    this.processing = false;
+  }
+
+  drain() {
+    while (this.pull()) {}
+  }
+}

package/src/storage/index.ts

@@ -0,0 +1,5 @@
+export * from "./types.js";
+export * from "./sqlite/index.js";
+export * from "./sqliteAsync/index.js";
+export * from "./storageSync.js";
+export * from "./storageAsync.js";

package/src/storage/knownState.ts

@@ -0,0 +1,88 @@
+import { getIsUploaded } from "../SyncStateManager.js";
+import { type CoValueCore } from "../exports.js";
+import { RawCoID } from "../ids.js";
+import { CoValueKnownState, emptyKnownState } from "../sync.js";
+
+/**
+ * Track how much data we have stored inside our storage
+ * and provides the API to wait for the data to be fully stored.
+ */
+export class StorageKnownState {
+  knwonStates = new Map<string, CoValueKnownState>();
+
+  getKnownState(id: string): CoValueKnownState {
+    const knownState = this.knwonStates.get(id);
+
+    if (!knownState) {
+      const empty = emptyKnownState(id as RawCoID);
+      this.knwonStates.set(id, empty);
+      return empty;
+    }
+
+    return knownState;
+  }
+
+  setKnownState(id: string, knownState: CoValueKnownState) {
+    this.knwonStates.set(id, knownState);
+  }
+
+  handleUpdate(id: string, knownState: CoValueKnownState) {
+    const requests = this.waitForSyncRequests.get(id);
+
+    if (!requests) {
+      return;
+    }
+
+    for (const request of requests) {
+      if (isInSync(request.knownState, knownState)) {
+        request.resolve();
+        requests.delete(request);
+      }
+    }
+  }
+
+  waitForSyncRequests = new Map<
+    string,
+    Set<{
+      knownState: CoValueKnownState;
+      resolve: (value: void) => void;
+    }>
+  >();
+
+  waitForSync(id: string, coValue: CoValueCore) {
+    const initialKnownState = coValue.knownState();
+    if (isInSync(initialKnownState, this.getKnownState(id))) {
+      return Promise.resolve();
+    }
+
+    const requests = this.waitForSyncRequests.get(id) || new Set();
+    this.waitForSyncRequests.set(id, requests);
+
+    return new Promise<void>((resolve) => {
+      const unsubscribe = coValue.subscribe((coValue) => {
+        req.knownState = coValue.knownState();
+        this.handleUpdate(id, this.getKnownState(id));
+      }, false);
+
+      const handleResolve = () => {
+        resolve();
+        unsubscribe();
+      };
+
+      const req = { knownState: initialKnownState, resolve: handleResolve };
+
+      requests.add(req);
+    });
+  }
+}
+
+function isInSync(
+  knownState: CoValueKnownState,
+  knownStateFromStorage: CoValueKnownState,
+) {
+  if (!knownStateFromStorage.header && knownState.header) {
+    return false;
+  }
+
+  return getIsUploaded(knownState.sessions, knownStateFromStorage.sessions);
+}

package/src/storage/sqlite/client.ts

@@ -0,0 +1,180 @@
+import type {
+  CoValueHeader,
+  Transaction,
+} from "../../coValueCore/verifiedState.js";
+import type { Signature } from "../../crypto/crypto.js";
+import type { RawCoID, SessionID } from "../../exports.js";
+import { logger } from "../../logger.js";
+import type { NewContentMessage } from "../../sync.js";
+import type {
+  DBClientInterfaceSync,
+  SessionRow,
+  SignatureAfterRow,
+  StoredCoValueRow,
+  StoredSessionRow,
+  TransactionRow,
+} from "../types.js";
+import type { SQLiteDatabaseDriver } from "./types.js";
+
+export type RawCoValueRow = {
+  id: RawCoID;
+  header: string;
+};
+
+export type RawTransactionRow = {
+  ses: number;
+  idx: number;
+  tx: string;
+};
+
+export function getErrorMessage(error: unknown) {
+  return error instanceof Error ? error.message : "Unknown error";
+}
+
+export class SQLiteClient implements DBClientInterfaceSync {
+  private readonly db: SQLiteDatabaseDriver;
+
+  constructor(db: SQLiteDatabaseDriver) {
+    this.db = db;
+  }
+
+  getCoValue(coValueId: RawCoID): StoredCoValueRow | undefined {
+    const coValueRow = this.db.get<RawCoValueRow & { rowID: number }>(
+      "SELECT * FROM coValues WHERE id = ?",
+      [coValueId],
+    );
+
+    if (!coValueRow) return;
+
+    try {
+      const parsedHeader = (coValueRow?.header &&
+        JSON.parse(coValueRow.header)) as CoValueHeader;
+
+      return {
+        ...coValueRow,
+        header: parsedHeader,
+      };
+    } catch (e) {
+      const headerValue = coValueRow?.header ?? "";
+      logger.warn(`Invalid JSON in header: ${headerValue}`, {
+        id: coValueId,
+        err: e,
+      });
+      return;
+    }
+  }
+
+  getCoValueSessions(coValueRowId: number): StoredSessionRow[] {
+    return this.db.query<StoredSessionRow>(
+      "SELECT * FROM sessions WHERE coValue = ?",
+      [coValueRowId],
+    ) as StoredSessionRow[];
+  }
+
+  getSingleCoValueSession(
+    coValueRowId: number,
+    sessionID: SessionID,
+  ): StoredSessionRow | undefined {
+    return this.db.get<StoredSessionRow>(
+      "SELECT * FROM sessions WHERE coValue = ? AND sessionID = ?",
+      [coValueRowId, sessionID],
+    );
+  }
+
+  getNewTransactionInSession(
+    sessionRowId: number,
+    fromIdx: number,
+    toIdx: number,
+  ): TransactionRow[] {
+    const txs = this.db.query<RawTransactionRow>(
+      "SELECT * FROM transactions WHERE ses = ? AND idx >= ? AND idx <= ?",
+      [sessionRowId, fromIdx, toIdx],
+    ) as RawTransactionRow[];
+
+    try {
+      return txs.map((transactionRow) => ({
+        ...transactionRow,
+        tx: JSON.parse(transactionRow.tx) as Transaction,
+      }));
+    } catch (e) {
+      logger.warn("Invalid JSON in transaction", { err: e });
+      return [];
+    }
+  }
+
+  getSignatures(
+    sessionRowId: number,
+    firstNewTxIdx: number,
+  ): SignatureAfterRow[] {
+    return this.db.query<SignatureAfterRow>(
+      "SELECT * FROM signatureAfter WHERE ses = ? AND idx >= ?",
+      [sessionRowId, firstNewTxIdx],
+    ) as SignatureAfterRow[];
+  }
+
+  addCoValue(msg: NewContentMessage): number {
+    const result = this.db.get<{ rowID: number }>(
+      "INSERT INTO coValues (id, header) VALUES (?, ?) RETURNING rowID",
+      [msg.id, JSON.stringify(msg.header)],
+    );
+
+    if (!result) {
+      throw new Error("Failed to add coValue");
+    }
+
+    return result.rowID;
+  }
+
+  addSessionUpdate({
+    sessionUpdate,
+  }: {
+    sessionUpdate: SessionRow;
+  }): number {
+    const result = this.db.get<{ rowID: number }>(
+      `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
+                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
+                            RETURNING rowID`,
+      [
+        sessionUpdate.coValue,
+        sessionUpdate.sessionID,
+        sessionUpdate.lastIdx,
+        sessionUpdate.lastSignature,
+        sessionUpdate.bytesSinceLastSignature,
+      ],
+    );
+
+    if (!result) {
+      throw new Error("Failed to add session update");
+    }
+
+    return result.rowID;
+  }
+
+  addTransaction(
+    sessionRowID: number,
+    nextIdx: number,
+    newTransaction: Transaction,
+  ) {
+    this.db.run("INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)", [
+      sessionRowID,
+      nextIdx,
+      JSON.stringify(newTransaction),
+    ]);
+  }
+
+  addSignatureAfter({
+    sessionRowID,
+    idx,
+    signature,
+  }: { sessionRowID: number; idx: number; signature: Signature }) {
+    this.db.run(
+      "INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)",
+      [sessionRowID, idx, signature],
+    );
+  }
+
+  transaction(operationsCallback: () => unknown) {
+    this.db.transaction(operationsCallback);
+    return undefined;
+  }
+}

package/src/storage/sqlite/index.ts

@@ -0,0 +1,19 @@
+import { StorageApiSync } from "../storageSync.js";
+import { SQLiteClient } from "./client.js";
+import { getSQLiteMigrationQueries } from "./sqliteMigrations.js";
+import type { SQLiteDatabaseDriver } from "./types.js";
+
+export type { SQLiteDatabaseDriver };
+
+export function getSqliteStorage(db: SQLiteDatabaseDriver) {
+  const rows = db.query<{ user_version: string }>("PRAGMA user_version", []);
+  const userVersion = Number(rows[0]?.user_version) ?? 0;
+
+  const migrations = getSQLiteMigrationQueries(userVersion);
+
+  for (const migration of migrations) {
+    db.run(migration, []);
+  }
+
+  return new StorageApiSync(new SQLiteClient(db));
+}

package/src/storage/sqlite/sqliteMigrations.ts

@@ -0,0 +1,44 @@
+export const migrations: Record<number, string[]> = {
+  1: [
+    `CREATE TABLE IF NOT EXISTS transactions (
+      ses INTEGER,
+      idx INTEGER,
+      tx TEXT NOT NULL,
+      PRIMARY KEY (ses, idx)
+    ) WITHOUT ROWID;`,
+    `CREATE TABLE IF NOT EXISTS sessions (
+      rowID INTEGER PRIMARY KEY,
+      coValue INTEGER NOT NULL,
+      sessionID TEXT NOT NULL,
+      lastIdx INTEGER,
+      lastSignature TEXT,
+      UNIQUE (sessionID, coValue)
+    );`,
+    "CREATE INDEX IF NOT EXISTS sessionsByCoValue ON sessions (coValue);",
+    `CREATE TABLE IF NOT EXISTS coValues (
+      rowID INTEGER PRIMARY KEY,
+      id TEXT NOT NULL UNIQUE,
+      header TEXT NOT NULL UNIQUE
+    );`,
+    "CREATE INDEX IF NOT EXISTS coValuesByID ON coValues (id);",
+    "PRAGMA user_version = 1;",
+  ],
+  3: [
+    `CREATE TABLE IF NOT EXISTS signatureAfter (
+      ses INTEGER,
+      idx INTEGER,
+      signature TEXT NOT NULL,
+      PRIMARY KEY (ses, idx)
+    ) WITHOUT ROWID;`,
+    "ALTER TABLE sessions ADD COLUMN bytesSinceLastSignature INTEGER;",
+    "PRAGMA user_version = 3;",
+  ],
+};
+
+export function getSQLiteMigrationQueries(version: number): string[] {
+  return Object.keys(migrations)
+    .map((k) => Number.parseInt(k, 10))
+    .filter((v) => v > version)
+    .sort((a, b) => a - b)
+    .flatMap((v) => migrations[v] ?? []);
+}

package/src/storage/sqlite/types.ts

@@ -0,0 +1,7 @@
+export interface SQLiteDatabaseDriver {
+  run(sql: string, params: unknown[]): void;
+  get<T>(sql: string, params: unknown[]): T | undefined;
+  query<T>(sql: string, params: unknown[]): T[];
+  transaction(callback: () => unknown): void;
+  closeDb(): void;
+}