cojson 0.15.7 → 0.15.9

package/src/storage/sqliteAsync/client.ts

@@ -0,0 +1,179 @@
+import type {
+  CoValueHeader,
+  Transaction,
+} from "../../coValueCore/verifiedState.js";
+import type { Signature } from "../../crypto/crypto.js";
+import type { RawCoID, SessionID } from "../../exports.js";
+import { logger } from "../../logger.js";
+import type { NewContentMessage } from "../../sync.js";
+import type {
+  DBClientInterfaceAsync,
+  SessionRow,
+  SignatureAfterRow,
+  StoredCoValueRow,
+  StoredSessionRow,
+  TransactionRow,
+} from "../types.js";
+import type { SQLiteDatabaseDriverAsync } from "./types.js";
+
+export type RawCoValueRow = {
+  id: RawCoID;
+  header: string;
+};
+
+export type RawTransactionRow = {
+  ses: number;
+  idx: number;
+  tx: string;
+};
+
+export function getErrorMessage(error: unknown) {
+  return error instanceof Error ? error.message : "Unknown error";
+}
+
+export class SQLiteClientAsync implements DBClientInterfaceAsync {
+  private readonly db: SQLiteDatabaseDriverAsync;
+
+  constructor(db: SQLiteDatabaseDriverAsync) {
+    this.db = db;
+  }
+
+  async getCoValue(coValueId: RawCoID): Promise<StoredCoValueRow | undefined> {
+    const coValueRow = await this.db.get<RawCoValueRow & { rowID: number }>(
+      "SELECT * FROM coValues WHERE id = ?",
+      [coValueId],
+    );
+
+    if (!coValueRow) return;
+
+    try {
+      const parsedHeader = (coValueRow?.header &&
+        JSON.parse(coValueRow.header)) as CoValueHeader;
+
+      return {
+        ...coValueRow,
+        header: parsedHeader,
+      };
+    } catch (e) {
+      const headerValue = coValueRow?.header ?? "";
+      logger.warn(`Invalid JSON in header: ${headerValue}`, {
+        id: coValueId,
+        err: e,
+      });
+      return;
+    }
+  }
+
+  async getCoValueSessions(coValueRowId: number): Promise<StoredSessionRow[]> {
+    return this.db.query<StoredSessionRow>(
+      "SELECT * FROM sessions WHERE coValue = ?",
+      [coValueRowId],
+    );
+  }
+
+  async getSingleCoValueSession(
+    coValueRowId: number,
+    sessionID: SessionID,
+  ): Promise<StoredSessionRow | undefined> {
+    return this.db.get<StoredSessionRow>(
+      "SELECT * FROM sessions WHERE coValue = ? AND sessionID = ?",
+      [coValueRowId, sessionID],
+    );
+  }
+
+  async getNewTransactionInSession(
+    sessionRowId: number,
+    fromIdx: number,
+    toIdx: number,
+  ): Promise<TransactionRow[]> {
+    const txs = await this.db.query<RawTransactionRow>(
+      "SELECT * FROM transactions WHERE ses = ? AND idx >= ? AND idx <= ?",
+      [sessionRowId, fromIdx, toIdx],
+    );
+
+    try {
+      return txs.map((transactionRow) => ({
+        ...transactionRow,
+        tx: JSON.parse(transactionRow.tx) as Transaction,
+      }));
+    } catch (e) {
+      logger.warn("Invalid JSON in transaction", { err: e });
+      return [];
+    }
+  }
+
+  async getSignatures(
+    sessionRowId: number,
+    firstNewTxIdx: number,
+  ): Promise<SignatureAfterRow[]> {
+    return this.db.query<SignatureAfterRow>(
+      "SELECT * FROM signatureAfter WHERE ses = ? AND idx >= ?",
+      [sessionRowId, firstNewTxIdx],
+    );
+  }
+
+  async addCoValue(msg: NewContentMessage): Promise<number> {
+    const result = await this.db.get<{ rowID: number }>(
+      "INSERT INTO coValues (id, header) VALUES (?, ?) RETURNING rowID",
+      [msg.id, JSON.stringify(msg.header)],
+    );
+
+    if (!result) {
+      throw new Error("Failed to add coValue");
+    }
+
+    return result.rowID;
+  }
+
+  async addSessionUpdate({
+    sessionUpdate,
+  }: {
+    sessionUpdate: SessionRow;
+  }): Promise<number> {
+    const result = await this.db.get<{ rowID: number }>(
+      `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
+                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
+                            RETURNING rowID`,
+      [
+        sessionUpdate.coValue,
+        sessionUpdate.sessionID,
+        sessionUpdate.lastIdx,
+        sessionUpdate.lastSignature,
+        sessionUpdate.bytesSinceLastSignature,
+      ],
+    );
+
+    if (!result) {
+      throw new Error("Failed to add session update");
+    }
+
+    return result.rowID;
+  }
+
+  addTransaction(
+    sessionRowID: number,
+    nextIdx: number,
+    newTransaction: Transaction,
+  ) {
+    this.db.run("INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)", [
+      sessionRowID,
+      nextIdx,
+      JSON.stringify(newTransaction),
+    ]);
+  }
+
+  async addSignatureAfter({
+    sessionRowID,
+    idx,
+    signature,
+  }: { sessionRowID: number; idx: number; signature: Signature }) {
+    this.db.run(
+      "INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)",
+      [sessionRowID, idx, signature],
+    );
+  }
+
+  async transaction(operationsCallback: () => unknown) {
+    return this.db.transaction(operationsCallback);
+  }
+}

package/src/storage/sqliteAsync/index.ts

@@ -0,0 +1,25 @@
+export { SQLiteClientAsync } from "./client.js";
+export type { SQLiteDatabaseDriverAsync } from "./types.js";
+
+import { getSQLiteMigrationQueries } from "../sqlite/sqliteMigrations.js";
+import { StorageApiAsync } from "../storageAsync.js";
+import { SQLiteClientAsync } from "./client.js";
+import type { SQLiteDatabaseDriverAsync } from "./types.js";
+
+export async function getSqliteStorageAsync(db: SQLiteDatabaseDriverAsync) {
+  await db.initialize();
+
+  const rows = await db.query<{ user_version: string }>(
+    "PRAGMA user_version",
+    [],
+  );
+  const userVersion = Number(rows[0]?.user_version) ?? 0;
+
+  const migrations = getSQLiteMigrationQueries(userVersion);
+
+  for (const migration of migrations) {
+    await db.run(migration, []);
+  }
+
+  return new StorageApiAsync(new SQLiteClientAsync(db));
+}

package/src/storage/sqliteAsync/types.ts

@@ -0,0 +1,8 @@
+export interface SQLiteDatabaseDriverAsync {
+  initialize(): Promise<void>;
+  run(sql: string, params: unknown[]): Promise<void>;
+  query<T>(sql: string, params: unknown[]): Promise<T[]>;
+  get<T>(sql: string, params: unknown[]): Promise<T | undefined>;
+  transaction(callback: () => unknown): Promise<unknown>;
+  closeDb(): Promise<unknown>;
+}

package/src/storage/storageAsync.ts

@@ -0,0 +1,367 @@
+import { LinkedList } from "../PriorityBasedMessageQueue.js";
+import {
+  type CoValueCore,
+  MAX_RECOMMENDED_TX_SIZE,
+  type RawCoID,
+  type SessionID,
+  type StorageAPI,
+} from "../exports.js";
+import { getPriorityFromHeader } from "../priority.js";
+import {
+  CoValueKnownState,
+  NewContentMessage,
+  emptyKnownState,
+} from "../sync.js";
+import { StoreQueue } from "./StoreQueue.js";
+import { StorageKnownState } from "./knownState.js";
+import { collectNewTxs, getDependedOnCoValues } from "./syncUtils.js";
+import type {
+  DBClientInterfaceAsync,
+  SignatureAfterRow,
+  StoredCoValueRow,
+  StoredSessionRow,
+} from "./types.js";
+
+export class StorageApiAsync implements StorageAPI {
+  private readonly dbClient: DBClientInterfaceAsync;
+
+  private loadedCoValues = new Set<RawCoID>();
+
+  constructor(dbClient: DBClientInterfaceAsync) {
+    this.dbClient = dbClient;
+  }
+
+  knwonStates = new StorageKnownState();
+
+  getKnownState(id: string): CoValueKnownState {
+    return this.knwonStates.getKnownState(id);
+  }
+
+  async load(
+    id: string,
+    callback: (data: NewContentMessage) => void,
+    done: (found: boolean) => void,
+  ) {
+    await this.loadCoValue(id, callback, done);
+  }
+
+  async loadCoValue(
+    id: string,
+    callback: (data: NewContentMessage) => void,
+    done: (found: boolean) => void,
+  ) {
+    const coValueRow = await this.dbClient.getCoValue(id);
+
+    if (!coValueRow) {
+      done?.(false);
+      return;
+    }
+
+    const allCoValueSessions = await this.dbClient.getCoValueSessions(
+      coValueRow.rowID,
+    );
+
+    const signaturesBySession = new Map<
+      SessionID,
+      Pick<SignatureAfterRow, "idx" | "signature">[]
+    >();
+
+    let contentStreaming = false;
+
+    await Promise.all(
+      allCoValueSessions.map(async (sessionRow) => {
+        const signatures = await this.dbClient.getSignatures(
+          sessionRow.rowID,
+          0,
+        );
+
+        if (signatures.length > 0) {
+          contentStreaming = true;
+          signaturesBySession.set(sessionRow.sessionID, signatures);
+        }
+      }),
+    );
+
+    const knownState = this.knwonStates.getKnownState(coValueRow.id);
+
+    for (const sessionRow of allCoValueSessions) {
+      knownState.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
+    }
+
+    this.loadedCoValues.add(coValueRow.id);
+
+    let contentMessage = {
+      action: "content",
+      id: coValueRow.id,
+      header: coValueRow.header,
+      new: {},
+      priority: getPriorityFromHeader(coValueRow.header),
+    } as NewContentMessage;
+
+    if (contentStreaming) {
+      contentMessage.expectContentUntil = knownState["sessions"];
+    }
+
+    for (const sessionRow of allCoValueSessions) {
+      const signatures = signaturesBySession.get(sessionRow.sessionID) || [];
+
+      let idx = 0;
+
+      signatures.push({
+        idx: sessionRow.lastIdx,
+        signature: sessionRow.lastSignature,
+      });
+
+      for (const signature of signatures) {
+        const newTxsInSession = await this.dbClient.getNewTransactionInSession(
+          sessionRow.rowID,
+          idx,
+          signature.idx,
+        );
+
+        collectNewTxs({
+          newTxsInSession,
+          contentMessage,
+          sessionRow,
+          firstNewTxIdx: idx,
+          signature: signature.signature,
+        });
+
+        idx = signature.idx + 1;
+
+        if (signatures.length > 1) {
+          // Having more than one signature means that the content needs streaming
+          // So we start pushing the content to the client, and start a new content message
+          await this.pushContentWithDependencies(
+            coValueRow,
+            contentMessage,
+            callback,
+          );
+          contentMessage = {
+            action: "content",
+            id: coValueRow.id,
+            header: coValueRow.header,
+            new: {},
+            priority: getPriorityFromHeader(coValueRow.header),
+          } satisfies NewContentMessage;
+        }
+      }
+    }
+
+    const hasNewContent = Object.keys(contentMessage.new).length > 0;
+
+    // If there is no new content but steaming is not active, it's the case for a coValue with the header but no transactions
+    // For streaming the push has already been done in the loop above
+    if (hasNewContent || !contentStreaming) {
+      await this.pushContentWithDependencies(
+        coValueRow,
+        contentMessage,
+        callback,
+      );
+    }
+
+    this.knwonStates.handleUpdate(coValueRow.id, knownState);
+    done?.(true);
+  }
+
+  async pushContentWithDependencies(
+    coValueRow: StoredCoValueRow,
+    contentMessage: NewContentMessage,
+    pushCallback: (data: NewContentMessage) => void,
+  ) {
+    const dependedOnCoValuesList = getDependedOnCoValues(
+      coValueRow.header,
+      contentMessage,
+    );
+
+    const promises = [];
+
+    for (const dependedOnCoValue of dependedOnCoValuesList) {
+      if (this.loadedCoValues.has(dependedOnCoValue)) {
+        continue;
+      }
+
+      promises.push(
+        new Promise((resolve) => {
+          this.loadCoValue(dependedOnCoValue, pushCallback, resolve);
+        }),
+      );
+    }
+
+    await Promise.all(promises);
+
+    pushCallback(contentMessage);
+  }
+
+  storeQueue = new StoreQueue();
+
+  async store(
+    msgs: NewContentMessage[],
+    correctionCallback: (data: CoValueKnownState) => void,
+  ) {
+    /**
+     * The store operations must be done one by one, because we can't start a new transaction when there
+     * is already a transaction open.
+     */
+    this.storeQueue.push(msgs, correctionCallback);
+
+    this.storeQueue.processQueue(async (data, correctionCallback) => {
+      for (const msg of data) {
+        const success = await this.storeSingle(msg, correctionCallback);
+
+        if (!success) {
+          // Stop processing the messages for this entry, because the data is out of sync with storage
+          // and the other transactions will be rejected anyway.
+          break;
+        }
+      }
+    });
+  }
+
+  private async storeSingle(
+    msg: NewContentMessage,
+    correctionCallback: (data: CoValueKnownState) => void,
+  ): Promise<boolean> {
+    const id = msg.id;
+    const coValueRow = await this.dbClient.getCoValue(id);
+
+    // We have no info about coValue header
+    const invalidAssumptionOnHeaderPresence = !msg.header && !coValueRow;
+
+    if (invalidAssumptionOnHeaderPresence) {
+      const knownState = emptyKnownState(id as RawCoID);
+      this.knwonStates.setKnownState(id, knownState);
+
+      correctionCallback(knownState);
+      return false;
+    }
+
+    const storedCoValueRowID: number = coValueRow
+      ? coValueRow.rowID
+      : await this.dbClient.addCoValue(msg);
+
+    const knownState = this.knwonStates.getKnownState(id);
+    knownState.header = true;
+
+    let invalidAssumptions = false;
+
+    for (const sessionID of Object.keys(msg.new) as SessionID[]) {
+      await this.dbClient.transaction(async () => {
+        const sessionRow = await this.dbClient.getSingleCoValueSession(
+          storedCoValueRowID,
+          sessionID,
+        );
+
+        if (sessionRow) {
+          knownState.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
+        }
+
+        const lastIdx = sessionRow?.lastIdx || 0;
+        const after = msg.new[sessionID]?.after || 0;
+
+        if (lastIdx < after) {
+          knownState.sessions[sessionID] = lastIdx;
+          invalidAssumptions = true;
+        } else {
+          const newLastIdx = await this.putNewTxs(
+            msg,
+            sessionID,
+            sessionRow,
+            storedCoValueRowID,
+          );
+          knownState.sessions[sessionID] = newLastIdx;
+        }
+      });
+    }
+
+    this.knwonStates.handleUpdate(id, knownState);
+
+    if (invalidAssumptions) {
+      correctionCallback(knownState);
+      return false;
+    }
+
+    return true;
+  }
+
+  private async putNewTxs(
+    msg: NewContentMessage,
+    sessionID: SessionID,
+    sessionRow: StoredSessionRow | undefined,
+    storedCoValueRowID: number,
+  ) {
+    const newTransactions = msg.new[sessionID]?.newTransactions || [];
+
+    const actuallyNewOffset =
+      (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
+
+    const actuallyNewTransactions = newTransactions.slice(actuallyNewOffset);
+
+    if (actuallyNewTransactions.length === 0) {
+      return sessionRow?.lastIdx || 0;
+    }
+
+    let newBytesSinceLastSignature =
+      (sessionRow?.bytesSinceLastSignature || 0) +
+      actuallyNewTransactions.reduce(
+        (sum, tx) =>
+          sum +
+          (tx.privacy === "private"
+            ? tx.encryptedChanges.length
+            : tx.changes.length),
+        0,
+      );
+
+    const newLastIdx =
+      (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
+
+    let shouldWriteSignature = false;
+
+    if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+      shouldWriteSignature = true;
+      newBytesSinceLastSignature = 0;
+    }
+
+    const nextIdx = sessionRow?.lastIdx || 0;
+
+    if (!msg.new[sessionID]) throw new Error("Session ID not found");
+
+    const sessionUpdate = {
+      coValue: storedCoValueRowID,
+      sessionID,
+      lastIdx: newLastIdx,
+      lastSignature: msg.new[sessionID].lastSignature,
+      bytesSinceLastSignature: newBytesSinceLastSignature,
+    };
+
+    const sessionRowID: number = await this.dbClient.addSessionUpdate({
+      sessionUpdate,
+      sessionRow,
+    });
+
+    if (shouldWriteSignature) {
+      await this.dbClient.addSignatureAfter({
+        sessionRowID,
+        idx: newLastIdx - 1,
+        signature: msg.new[sessionID].lastSignature,
+      });
+    }
+
+    await Promise.all(
+      actuallyNewTransactions.map((newTransaction, i) =>
+        this.dbClient.addTransaction(sessionRowID, nextIdx + i, newTransaction),
+      ),
+    );
+
+    return newLastIdx;
+  }
+
+  waitForSync(id: string, coValue: CoValueCore) {
+    return this.knwonStates.waitForSync(id, coValue);
+  }
+
+  close() {
+    // Drain the store queue
+    this.storeQueue.drain();
+  }
+}