cojson-storage-sqlite 0.8.34 → 0.8.36

package/src/sqliteClient.ts

@@ -0,0 +1,153 @@
+import { Database as DatabaseT } from "better-sqlite3";
+import { CojsonInternalTypes, OutgoingSyncQueue, SessionID } from "cojson";
+import RawCoID = CojsonInternalTypes.RawCoID;
+import Signature = CojsonInternalTypes.Signature;
+import Transaction = CojsonInternalTypes.Transaction;
+import {
+  DBClientInterface,
+  SessionRow,
+  SignatureAfterRow,
+  StoredCoValueRow,
+  StoredSessionRow,
+  TransactionRow,
+} from "cojson-storage";
+
+export type RawCoValueRow = {
+  id: CojsonInternalTypes.RawCoID;
+  header: string;
+};
+
+export type RawTransactionRow = {
+  ses: number;
+  idx: number;
+  tx: string;
+};
+
+export class SQLiteClient implements DBClientInterface {
+  private readonly db: DatabaseT;
+  private readonly toLocalNode: OutgoingSyncQueue;
+
+  constructor(db: DatabaseT, toLocalNode: OutgoingSyncQueue) {
+    this.db = db;
+    this.toLocalNode = toLocalNode;
+  }
+
+  getCoValue(coValueId: RawCoID): StoredCoValueRow | undefined {
+    const coValueRow = this.db
+      .prepare(`SELECT * FROM coValues WHERE id = ?`)
+      .get(coValueId) as RawCoValueRow & { rowID: number };
+
+    if (!coValueRow) return;
+
+    try {
+      const parsedHeader = (coValueRow?.header &&
+        JSON.parse(coValueRow.header)) as CojsonInternalTypes.CoValueHeader;
+
+      return {
+        ...coValueRow,
+        header: parsedHeader,
+      };
+    } catch (e) {
+      console.warn(coValueId, "Invalid JSON in header", e, coValueRow?.header);
+      return;
+    }
+  }
+
+  getCoValueSessions(coValueRowId: number): StoredSessionRow[] {
+    return this.db
+      .prepare<number>(`SELECT * FROM sessions WHERE coValue = ?`)
+      .all(coValueRowId) as StoredSessionRow[];
+  }
+
+  getNewTransactionInSession(
+    sessionRowId: number,
+    firstNewTxIdx: number,
+  ): TransactionRow[] {
+    const txs = this.db
+      .prepare<[number, number]>(
+        `SELECT * FROM transactions WHERE ses = ? AND idx >= ?`,
+      )
+      .all(sessionRowId, firstNewTxIdx) as RawTransactionRow[];
+
+    try {
+      return txs.map((transactionRow) => ({
+        ...transactionRow,
+        tx: JSON.parse(transactionRow.tx) as Transaction,
+      }));
+    } catch (e) {
+      console.warn("Invalid JSON in transaction", e);
+      return [];
+    }
+  }
+
+  getSignatures(
+    sessionRowId: number,
+    firstNewTxIdx: number,
+  ): SignatureAfterRow[] {
+    return this.db
+      .prepare<[number, number]>(
+        `SELECT * FROM signatureAfter WHERE ses = ? AND idx >= ?`,
+      )
+      .all(sessionRowId, firstNewTxIdx) as SignatureAfterRow[];
+  }
+
+  addCoValue(msg: CojsonInternalTypes.NewContentMessage): number {
+    return this.db
+      .prepare<[CojsonInternalTypes.RawCoID, string]>(
+        `INSERT INTO coValues (id, header) VALUES (?, ?)`,
+      )
+      .run(msg.id, JSON.stringify(msg.header)).lastInsertRowid as number;
+  }
+
+  addSessionUpdate({
+    sessionUpdate,
+    sessionRow,
+  }: {
+    sessionUpdate: SessionRow;
+    sessionRow?: StoredSessionRow;
+  }): number {
+    return (
+      this.db
+        .prepare<[number, string, number, string, number | undefined]>(
+          `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature, bytesSinceLastSignature) VALUES (?, ?, ?, ?, ?)
+                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature, bytesSinceLastSignature=excluded.bytesSinceLastSignature
+                            RETURNING rowID`,
+        )
+        .get(
+          sessionUpdate.coValue,
+          sessionUpdate.sessionID,
+          sessionUpdate.lastIdx,
+          sessionUpdate.lastSignature,
+          sessionUpdate.bytesSinceLastSignature,
+        ) as { rowID: number }
+    ).rowID;
+  }
+
+  addTransaction(
+    sessionRowID: number,
+    nextIdx: number,
+    newTransaction: Transaction,
+  ) {
+    this.db
+      .prepare<[number, number, string]>(
+        `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`,
+      )
+      .run(sessionRowID, nextIdx, JSON.stringify(newTransaction));
+  }
+
+  addSignatureAfter({
+    sessionRowID,
+    idx,
+    signature,
+  }: { sessionRowID: number; idx: number; signature: Signature }) {
+    this.db
+      .prepare<[number, number, string]>(
+        `INSERT INTO signatureAfter (ses, idx, signature) VALUES (?, ?, ?)`,
+      )
+      .run(sessionRowID, idx, signature);
+  }
+
+  unitOfWork(operationsCallback: () => any[]) {
+    this.db.transaction(operationsCallback)();
+  }
+}

package/src/sqliteNode.ts

@@ -0,0 +1,181 @@
+import Database, { Database as DatabaseT } from "better-sqlite3";
+import {
+  IncomingSyncStream,
+  OutgoingSyncQueue,
+  Peer,
+  cojsonInternals,
+} from "cojson";
+import { SyncManager, TransactionRow } from "cojson-storage";
+import { SQLiteClient } from "./sqliteClient.js";
+
+export class SQLiteNode {
+  private readonly syncManager: SyncManager;
+  private readonly dbClient: SQLiteClient;
+
+  constructor(
+    db: DatabaseT,
+    fromLocalNode: IncomingSyncStream,
+    toLocalNode: OutgoingSyncQueue,
+  ) {
+    this.dbClient = new SQLiteClient(db, toLocalNode);
+    this.syncManager = new SyncManager(this.dbClient, toLocalNode);
+
+    const processMessages = async () => {
+      for await (const msg of fromLocalNode) {
+        try {
+          if (msg === "Disconnected" || msg === "PingTimeout") {
+            throw new Error("Unexpected Disconnected message");
+          }
+          await this.syncManager.handleSyncMessage(msg);
+        } catch (e) {
+          console.error(
+            new Error(
+              `Error reading from localNode, handling msg\n\n${JSON.stringify(
+                msg,
+                (k, v) =>
+                  k === "changes" || k === "encryptedChanges"
+                    ? v.slice(0, 20) + "..."
+                    : v,
+              )}`,
+              { cause: e },
+            ),
+          );
+        }
+      }
+    };
+
+    processMessages().catch((e) =>
+      console.error("Error in processMessages in sqlite", e),
+    );
+  }
+
+  static async asPeer({
+    filename,
+    trace,
+    localNodeName = "local",
+  }: {
+    filename: string;
+    trace?: boolean;
+    localNodeName?: string;
+  }): Promise<Peer> {
+    const [localNodeAsPeer, storageAsPeer] = cojsonInternals.connectedPeers(
+      localNodeName,
+      "storage",
+      { peer1role: "client", peer2role: "storage", trace, crashOnClose: true },
+    );
+
+    await SQLiteNode.open(
+      filename,
+      localNodeAsPeer.incoming,
+      localNodeAsPeer.outgoing,
+    );
+
+    return { ...storageAsPeer, priority: 100 };
+  }
+
+  static async open(
+    filename: string,
+    fromLocalNode: IncomingSyncStream,
+    toLocalNode: OutgoingSyncQueue,
+  ) {
+    const db = Database(filename);
+    db.pragma("journal_mode = WAL");
+
+    const oldVersion = (
+      db.pragma("user_version") as [{ user_version: number }]
+    )[0].user_version as number;
+
+    console.log("DB version", oldVersion);
+
+    if (oldVersion === 0) {
+      console.log("Migration 0 -> 1: Basic schema");
+      db.prepare(
+        `CREATE TABLE IF NOT EXISTS transactions (
+                    ses INTEGER,
+                    idx INTEGER,
+                    tx TEXT NOT NULL,
+                    PRIMARY KEY (ses, idx)
+                ) WITHOUT ROWID;`,
+      ).run();
+
+      db.prepare(
+        `CREATE TABLE IF NOT EXISTS sessions (
+                    rowID INTEGER PRIMARY KEY,
+                    coValue INTEGER NOT NULL,
+                    sessionID TEXT NOT NULL,
+                    lastIdx INTEGER,
+                    lastSignature TEXT,
+                    UNIQUE (sessionID, coValue)
+                );`,
+      ).run();
+
+      db.prepare(
+        `CREATE INDEX IF NOT EXISTS sessionsByCoValue ON sessions (coValue);`,
+      ).run();
+
+      db.prepare(
+        `CREATE TABLE IF NOT EXISTS coValues (
+                    rowID INTEGER PRIMARY KEY,
+                    id TEXT NOT NULL UNIQUE,
+                    header TEXT NOT NULL UNIQUE
+                );`,
+      ).run();
+
+      db.prepare(
+        `CREATE INDEX IF NOT EXISTS coValuesByID ON coValues (id);`,
+      ).run();
+
+      db.pragma("user_version = 1");
+      console.log("Migration 0 -> 1: Basic schema - done");
+    }
+
+    if (oldVersion <= 1) {
+      // fix embarrassing off-by-one error for transaction indices
+      console.log(
+        "Migration 1 -> 2: Fix off-by-one error for transaction indices",
+      );
+
+      const txs = db
+        .prepare(`SELECT * FROM transactions`)
+        .all() as TransactionRow[];
+
+      for (const tx of txs) {
+        db.prepare(`DELETE FROM transactions WHERE ses = ? AND idx = ?`).run(
+          tx.ses,
+          tx.idx,
+        );
+        tx.idx -= 1;
+        db.prepare(
+          `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`,
+        ).run(tx.ses, tx.idx, tx.tx);
+      }
+
+      db.pragma("user_version = 2");
+      console.log(
+        "Migration 1 -> 2: Fix off-by-one error for transaction indices - done",
+      );
+    }
+
+    if (oldVersion <= 2) {
+      console.log("Migration 2 -> 3: Add signatureAfter");
+
+      db.prepare(
+        `CREATE TABLE IF NOT EXISTS signatureAfter (
+                    ses INTEGER,
+                    idx INTEGER,
+                    signature TEXT NOT NULL,
+                    PRIMARY KEY (ses, idx)
+                ) WITHOUT ROWID;`,
+      ).run();
+
+      db.prepare(
+        `ALTER TABLE sessions ADD COLUMN bytesSinceLastSignature INTEGER;`,
+      ).run();
+
+      db.pragma("user_version = 3");
+      console.log("Migration 2 -> 3: Add signatureAfter - done!!");
+    }
+
+    return new SQLiteNode(db, fromLocalNode, toLocalNode);
+  }
+}