cojson-storage-sqlite 0.1.1

package/.eslintrc.cjs

@@ -0,0 +1,17 @@
+module.exports = {
+    extends: [
+        'eslint:recommended',
+        'plugin:@typescript-eslint/recommended',
+    ],
+    parser: '@typescript-eslint/parser',
+    plugins: ['@typescript-eslint'],
+    parserOptions: {
+        project: "./tsconfig.json",
+    },
+    root: true,
+    rules: {
+        "no-unused-vars": "off",
+        "@typescript-eslint/no-unused-vars": ["error", { "argsIgnorePattern": "^_", "varsIgnorePattern": "^_" }],
+        // "@typescript-eslint/no-floating-promises": "error",
+    },
+};

package/dist/index.d.ts

@@ -0,0 +1,21 @@
+import { SyncMessage, Peer, CojsonInternalTypes } from "cojson";
+import { ReadableStream, WritableStream, ReadableStreamDefaultReader, WritableStreamDefaultWriter } from "isomorphic-streams";
+import { Database as DatabaseT } from "better-sqlite3";
+export declare class SQLiteStorage {
+    fromLocalNode: ReadableStreamDefaultReader<SyncMessage>;
+    toLocalNode: WritableStreamDefaultWriter<SyncMessage>;
+    db: DatabaseT;
+    constructor(db: DatabaseT, fromLocalNode: ReadableStream<SyncMessage>, toLocalNode: WritableStream<SyncMessage>);
+    static asPeer({ filename, trace, localNodeName, }: {
+        filename: string;
+        trace?: boolean;
+        localNodeName?: string;
+    }): Promise<Peer>;
+    static open(filename: string, fromLocalNode: ReadableStream<SyncMessage>, toLocalNode: WritableStream<SyncMessage>): Promise<SQLiteStorage>;
+    handleSyncMessage(msg: SyncMessage): Promise<void>;
+    sendNewContentAfter(theirKnown: CojsonInternalTypes.CoValueKnownState, asDependencyOf?: CojsonInternalTypes.RawCoID): Promise<void>;
+    handleLoad(msg: CojsonInternalTypes.LoadMessage): Promise<void>;
+    handleContent(msg: CojsonInternalTypes.NewContentMessage): Promise<void>;
+    handleKnown(msg: CojsonInternalTypes.KnownStateMessage): Promise<void>;
+    handleDone(_msg: CojsonInternalTypes.DoneMessage): void;
+}

package/dist/index.js

@@ -0,0 +1,221 @@
+import { cojsonInternals,
+// CojsonInternalTypes,
+// SessionID,
+ } from "cojson";
+import Database from "better-sqlite3";
+export class SQLiteStorage {
+    constructor(db, fromLocalNode, toLocalNode) {
+        this.db = db;
+        this.fromLocalNode = fromLocalNode.getReader();
+        this.toLocalNode = toLocalNode.getWriter();
+        (async () => {
+            let done = false;
+            while (!done) {
+                const result = await this.fromLocalNode.read();
+                done = result.done;
+                if (result.value) {
+                    this.handleSyncMessage(result.value);
+                }
+            }
+        })();
+    }
+    static async asPeer({ filename, trace, localNodeName = "local", }) {
+        const [localNodeAsPeer, storageAsPeer] = cojsonInternals.connectedPeers(localNodeName, "storage", { peer1role: "client", peer2role: "server", trace });
+        await SQLiteStorage.open(filename, localNodeAsPeer.incoming, localNodeAsPeer.outgoing);
+        return storageAsPeer;
+    }
+    static async open(filename, fromLocalNode, toLocalNode) {
+        const db = Database(filename);
+        db.pragma("journal_mode = WAL");
+        db.prepare(`CREATE TABLE IF NOT EXISTS transactions (
+                ses INTEGER,
+                idx INTEGER,
+                tx TEXT NOT NULL ,
+                PRIMARY KEY (ses, idx)
+            ) WITHOUT ROWID;`).run();
+        db.prepare(`CREATE TABLE IF NOT EXISTS sessions (
+                rowID INTEGER PRIMARY KEY,
+                coValue INTEGER NOT NULL,
+                sessionID TEXT NOT NULL,
+                lastIdx INTEGER,
+                lastSignature TEXT,
+                UNIQUE (sessionID, coValue)
+            );`).run();
+        db.prepare(`CREATE INDEX IF NOT EXISTS sessionsByCoValue ON sessions (coValue);`).run();
+        db.prepare(`CREATE TABLE IF NOT EXISTS coValues (
+                rowID INTEGER PRIMARY KEY,
+                id TEXT NOT NULL UNIQUE,
+                header TEXT NOT NULL UNIQUE
+            );`).run();
+        db.prepare(`CREATE INDEX IF NOT EXISTS coValuesByID ON coValues (id);`).run();
+        return new SQLiteStorage(db, fromLocalNode, toLocalNode);
+    }
+    async handleSyncMessage(msg) {
+        switch (msg.action) {
+            case "load":
+                await this.handleLoad(msg);
+                break;
+            case "content":
+                await this.handleContent(msg);
+                break;
+            case "known":
+                await this.handleKnown(msg);
+                break;
+            case "done":
+                await this.handleDone(msg);
+                break;
+        }
+    }
+    async sendNewContentAfter(theirKnown, asDependencyOf) {
+        const coValueRow = (await this.db
+            .prepare(`SELECT * FROM coValues WHERE id = ?`)
+            .get(theirKnown.id));
+        const allOurSessions = coValueRow
+            ? this.db
+                .prepare(`SELECT * FROM sessions WHERE coValue = ?`)
+                .all(coValueRow.rowID)
+            : [];
+        const ourKnown = {
+            id: theirKnown.id,
+            header: !!coValueRow,
+            sessions: {},
+        };
+        const parsedHeader = (coValueRow?.header &&
+            JSON.parse(coValueRow.header));
+        const newContent = {
+            action: "content",
+            id: theirKnown.id,
+            header: theirKnown.header ? undefined : parsedHeader,
+            new: {},
+        };
+        for (const sessionRow of allOurSessions) {
+            ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
+            if (sessionRow.lastIdx >
+                (theirKnown.sessions[sessionRow.sessionID] || 0)) {
+                const firstNewTxIdx = theirKnown.sessions[sessionRow.sessionID] || 0;
+                const newTxInSession = this.db
+                    .prepare(`SELECT * FROM transactions WHERE ses = ? AND idx > ?`)
+                    .all(sessionRow.rowID, firstNewTxIdx);
+                newContent.new[sessionRow.sessionID] = {
+                    after: firstNewTxIdx,
+                    lastSignature: sessionRow.lastSignature,
+                    newTransactions: newTxInSession.map((row) => JSON.parse(row.tx)),
+                };
+            }
+        }
+        const dependedOnCoValues = parsedHeader?.ruleset.type === "group"
+            ? Object.values(newContent.new).flatMap((sessionEntry) => sessionEntry.newTransactions.flatMap((tx) => {
+                if (tx.privacy !== "trusting")
+                    return [];
+                return tx.changes
+                    .map((change) => change &&
+                    typeof change === "object" &&
+                    "op" in change &&
+                    change.op === "set" &&
+                    "key" in change &&
+                    change.key)
+                    .filter((key) => typeof key === "string" &&
+                    key.startsWith("co_"));
+            }))
+            : parsedHeader?.ruleset.type === "ownedByGroup"
+                ? [parsedHeader?.ruleset.group]
+                : [];
+        for (const dependedOnCoValue of dependedOnCoValues) {
+            await this.sendNewContentAfter({ id: dependedOnCoValue, header: false, sessions: {} }, asDependencyOf || theirKnown.id);
+        }
+        await this.toLocalNode.write({
+            action: "known",
+            ...ourKnown,
+            asDependencyOf,
+        });
+        if (newContent.header || Object.keys(newContent.new).length > 0) {
+            await this.toLocalNode.write(newContent);
+        }
+    }
+    handleLoad(msg) {
+        return this.sendNewContentAfter(msg);
+    }
+    async handleContent(msg) {
+        let storedCoValueRowID = this.db
+            .prepare(`SELECT rowID FROM coValues WHERE id = ?`)
+            .get(msg.id)?.rowID;
+        if (storedCoValueRowID === undefined) {
+            const header = msg.header;
+            if (!header) {
+                console.error("Expected to be sent header first");
+                await this.toLocalNode.write({
+                    action: "known",
+                    id: msg.id,
+                    header: false,
+                    sessions: {},
+                    isCorrection: true,
+                });
+                return;
+            }
+            storedCoValueRowID = this.db
+                .prepare(`INSERT INTO coValues (id, header) VALUES (?, ?)`)
+                .run(msg.id, JSON.stringify(header)).lastInsertRowid;
+        }
+        const ourKnown = {
+            id: msg.id,
+            header: true,
+            sessions: {},
+        };
+        let invalidAssumptions = false;
+        this.db.transaction(() => {
+            const allOurSessions = this.db
+                .prepare(`SELECT * FROM sessions WHERE coValue = ?`)
+                .all(storedCoValueRowID).reduce((acc, row) => {
+                acc[row.sessionID] = row;
+                return acc;
+            }, {});
+            for (const sessionID of Object.keys(msg.new)) {
+                const sessionRow = allOurSessions[sessionID];
+                if (sessionRow) {
+                    ourKnown.sessions[sessionRow.sessionID] =
+                        sessionRow.lastIdx;
+                }
+                if ((sessionRow?.lastIdx || 0) <
+                    (msg.new[sessionID]?.after || 0)) {
+                    invalidAssumptions = true;
+                }
+                else {
+                    const newTransactions = msg.new[sessionID]?.newTransactions || [];
+                    const actuallyNewOffset = (sessionRow?.lastIdx || 0) -
+                        (msg.new[sessionID]?.after || 0);
+                    const actuallyNewTransactions = newTransactions.slice(actuallyNewOffset);
+                    let nextIdx = sessionRow?.lastIdx || 0;
+                    const sessionUpdate = {
+                        coValue: storedCoValueRowID,
+                        sessionID: sessionID,
+                        lastIdx: (sessionRow?.lastIdx || 0) +
+                            actuallyNewTransactions.length,
+                        lastSignature: msg.new[sessionID].lastSignature,
+                    };
+                    const sessionRowID = this.db
+                        .prepare(`INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature) VALUES (?, ?, ?, ?)
+                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature`)
+                        .run(sessionUpdate.coValue, sessionUpdate.sessionID, sessionUpdate.lastIdx, sessionUpdate.lastSignature).lastInsertRowid;
+                    for (const newTransaction of actuallyNewTransactions) {
+                        nextIdx++;
+                        this.db
+                            .prepare(`INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`)
+                            .run(sessionRowID, nextIdx, JSON.stringify(newTransaction));
+                    }
+                }
+            }
+        })();
+        if (invalidAssumptions) {
+            await this.toLocalNode.write({
+                action: "known",
+                ...ourKnown,
+                isCorrection: invalidAssumptions,
+            });
+        }
+    }
+    handleKnown(msg) {
+        return this.sendNewContentAfter(msg);
+    }
+    handleDone(_msg) { }
+}
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACH,eAAe;AAKf,uBAAuB;AACvB,aAAa;EAChB,MAAM,QAAQ,CAAC;AAQhB,OAAO,QAAmC,MAAM,gBAAgB,CAAC;AAyBjE,MAAM,OAAO,aAAa;IAKtB,YACI,EAAa,EACb,aAA0C,EAC1C,WAAwC;QAExC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC,SAAS,EAAE,CAAC;QAC/C,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC,SAAS,EAAE,CAAC;QAE3C,CAAC,KAAK,IAAI,EAAE;YACR,IAAI,IAAI,GAAG,KAAK,CAAC;YACjB,OAAO,CAAC,IAAI,EAAE;gBACV,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;gBAC/C,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;gBAEnB,IAAI,MAAM,CAAC,KAAK,EAAE;oBACd,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;iBACxC;aACJ;QACL,CAAC,CAAC,EAAE,CAAC;IACT,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAChB,QAAQ,EACR,KAAK,EACL,aAAa,GAAG,OAAO,GAK1B;QACG,MAAM,CAAC,eAAe,EAAE,aAAa,CAAC,GAAG,eAAe,CAAC,cAAc,CACnE,aAAa,EACb,SAAS,EACT,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,KAAK,EAAE,CACtD,CAAC;QAEF,MAAM,aAAa,CAAC,IAAI,CACpB,QAAQ,EACR,eAAe,CAAC,QAAQ,EACxB,eAAe,CAAC,QAAQ,CAC3B,CAAC;QAEF,OAAO,aAAa,CAAC;IACzB,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,IAAI,CACb,QAAgB,EAChB,aAA0C,EAC1C,WAAwC;QAExC,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAC9B,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QAEhC,EAAE,CAAC,OAAO,CACN;;;;;6BAKiB,CACpB,CAAC,GAAG,EAAE,CAAC;QAER,EAAE,CAAC,OAAO,CACN;;;;;;;eAOG,CACN,CAAC,GAAG,EAAE,CAAC;QAER,EAAE,CAAC,OAAO,CACN,qEAAqE,CACxE,CAAC,GAAG,EAAE,CAAC;QAER,EAAE,CAAC,OAAO,CACN;;;;eAIG,CACN,CAAC,GAAG,EAAE,CAAC;QAER,EAAE,CAAC,OAAO,CACN,2DAA2D,CAC9D,CAAC,GAAG,EAAE,CAAC;QAER,OAAO,IAAI,aAAa,CAAC,EAAE,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,GAAgB;QACpC,QAAQ,GAAG,CAAC,MAAM,EAAE;YAChB,KAAK,MAAM;gBACP,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC3B,MAAM;YACV,KAAK,SAAS;gBACV,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBAC9B,MAAM;YACV,KAAK,OAAO;gBACR,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBAC5B,MAAM;YACV,KAAK,MAAM;gBACP,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC3B,MAAM;SACb;IACL,CAAC;IAED,KAAK,CAAC,mBAAmB,CACrB,UAAiD,EACjD,cAA4C;QAE5C,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE;aAC5B,OAAO,CAAC,qCAAqC,CAAC;aAC9C,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,CAAiC,CAAC;QAEzD,MAAM,cAAc,GAAG,UAAU;YAC7B,CAAC,CAAE,IAAI,CAAC,EAAE;iBACH,OAAO,CAAS,0CAA0C,CAAC;iBAC3D,GAAG,CAAC,UAAU,CAAC,KAAK,CAAwB;YACnD,CAAC,CAAC,EAAE,CAAC;QAET,MAAM,QAAQ,GAA0C;YACpD,EAAE,EAAE,UAAU,CAAC,EAAE;YACjB,MAAM,EAAE,CAAC,CAAC,UAAU;YACpB,QAAQ,EAAE,EAAE;SACf,CAAC;QAEF,MAAM,YAAY,GAAG,CAAC,UAAU,EAAE,MAAM;YACpC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAElB,CAAC;QAEhB,MAAM,UAAU,GAA0C;YACtD,MAAM,EAAE,SAAS;YACjB,EAAE,EAAE,UAAU,CAAC,EAAE;YACjB,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY;YACpD,GAAG,EAAE,EAAE;SACV,CAAC;QAEF,KAAK,MAAM,UAAU,IAAI,cAAc,EAAE;YACrC,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG,UAAU,CAAC,OAAO,CAAC;YAE7D,IACI,UAAU,CAAC,OAAO;gBAClB,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,EAClD;gBACE,MAAM,aAAa,GACf,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAEnD,MAAM,cAAc,GAAG,IAAI,CAAC,EAAE;qBACzB,OAAO,CACJ,sDAAsD,CACzD;qBACA,GAAG,CAAC,UAAU,CAAC,KAAK,EAAE,aAAa,CAAqB,CAAC;gBAE9D,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,GAAG;oBACnC,KAAK,EAAE,aAAa;oBACpB,aAAa,EAAE,UAAU,CAAC,aAAa;oBACvC,eAAe,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CACxC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CACrB;iBACJ,CAAC;aACL;SACJ;QAED,MAAM,kBAAkB,GACpB,YAAY,EAAE,OAAO,CAAC,IAAI,KAAK,OAAO;YAClC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,YAAY,EAAE,EAAE,CACnD,YAAY,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,EAAE;gBACxC,IAAI,EAAE,CAAC,OAAO,KAAK,UAAU;oBAAE,OAAO,EAAE,CAAC;gBACzC,OAAO,EAAE,CAAC,OAAO;qBACZ,GAAG,CACA,CAAC,MAAM,EAAE,EAAE,CACP,MAAM;oBACN,OAAO,MAAM,KAAK,QAAQ;oBAC1B,IAAI,IAAI,MAAM;oBACd,MAAM,CAAC,EAAE,KAAK,KAAK;oBACnB,KAAK,IAAI,MAAM;oBACf,MAAM,CAAC,GAAG,CACjB;qBACA,MAAM,CACH,CAAC,GAAG,EAAsC,EAAE,CACxC,OAAO,GAAG,KAAK,QAAQ;oBACvB,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAC5B,CAAC;YACV,CAAC,CAAC,CACL;YACH,CAAC,CAAC,YAAY,EAAE,OAAO,CAAC,IAAI,KAAK,cAAc;gBAC/C,CAAC,CAAC,CAAC,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC;gBAC/B,CAAC,CAAC,EAAE,CAAC;QAEb,KAAK,MAAM,iBAAiB,IAAI,kBAAkB,EAAE;YAChD,MAAM,IAAI,CAAC,mBAAmB,CAC1B,EAAE,EAAE,EAAE,iBAAiB,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,EACtD,cAAc,IAAI,UAAU,CAAC,EAAE,CAClC,CAAC;SACL;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;YACzB,MAAM,EAAE,OAAO;YACf,GAAG,QAAQ;YACX,cAAc;SACjB,CAAC,CAAC;QAEH,IAAI,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE;YAC7D,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;SAC5C;IACL,CAAC;IAED,UAAU,CAAC,GAAoC;QAC3C,OAAO,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,GAA0C;QAC1D,IAAI,kBAAkB,GAClB,IAAI,CAAC,EAAE;aACF,OAAO,CAAU,yCAAyC,CAAC;aAC3D,GAAG,CAAC,GAAG,CAAC,EAAE,CAClB,EAAE,KAAK,CAAC;QAET,IAAI,kBAAkB,KAAK,SAAS,EAAE;YAClC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC;YAC1B,IAAI,CAAC,MAAM,EAAE;gBACT,OAAO,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;gBAClD,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;oBACzB,MAAM,EAAE,OAAO;oBACf,EAAE,EAAE,GAAG,CAAC,EAAE;oBACV,MAAM,EAAE,KAAK;oBACb,QAAQ,EAAE,EAAE;oBACZ,YAAY,EAAE,IAAI;iBACrB,CAAC,CAAC;gBACH,OAAO;aACV;YAED,kBAAkB,GAAG,IAAI,CAAC,EAAE;iBACvB,OAAO,CACJ,iDAAiD,CACpD;iBACA,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,eAAyB,CAAC;SACtE;QAED,MAAM,QAAQ,GAA0C;YACpD,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,EAAE;SACf,CAAC;QACF,IAAI,kBAAkB,GAAG,KAAK,CAAC;QAE/B,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YACrB,MAAM,cAAc,GAChB,IAAI,CAAC,EAAE;iBACF,OAAO,CAAS,0CAA0C,CAAC;iBAC3D,GAAG,CAAC,kBAAmB,CAC/B,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBAClB,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;gBACzB,OAAO,GAAG,CAAC;YACf,CAAC,EAAE,EAA+C,CAAC,CAAC;YAEpD,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAgB,EAAE;gBACzD,MAAM,UAAU,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;gBAC7C,IAAI,UAAU,EAAE;oBACZ,QAAQ,CAAC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC;wBACnC,UAAU,CAAC,OAAO,CAAC;iBAC1B;gBAED,IACI,CAAC,UAAU,EAAE,OAAO,IAAI,CAAC,CAAC;oBAC1B,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,EAClC;oBACE,kBAAkB,GAAG,IAAI,CAAC;iBAC7B;qBAAM;oBACH,MAAM,eAAe,GACjB,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,eAAe,IAAI,EAAE,CAAC;oBAE9C,MAAM,iBAAiB,GACnB,CAAC,UAAU,EAAE,OAAO,IAAI,CAAC,CAAC;wBAC1B,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;oBACrC,MAAM,uBAAuB,GACzB,eAAe,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC;oBAE7C,IAAI,OAAO,GAAG,UAAU,EAAE,OAAO,IAAI,CAAC,CAAC;oBAEvC,MAAM,aAAa,GAAG;wBAClB,OAAO,EAAE,kBAAmB;wBAC5B,SAAS,EAAE,SAAS;wBACpB,OAAO,EACH,CAAC,UAAU,EAAE,OAAO,IAAI,CAAC,CAAC;4BAC1B,uBAAuB,CAAC,MAAM;wBAClC,aAAa,EAAE,GAAG,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC,aAAa;qBACnD,CAAC;oBAEF,MAAM,YAAY,GAAG,IAAI,CAAC,EAAE;yBACvB,OAAO,CACJ;yIAC6G,CAChH;yBACA,GAAG,CACA,aAAa,CAAC,OAAO,EACrB,aAAa,CAAC,SAAS,EACvB,aAAa,CAAC,OAAO,EACrB,aAAa,CAAC,aAAa,CAC9B,CAAC,eAAyB,CAAC;oBAEhC,KAAK,MAAM,cAAc,IAAI,uBAAuB,EAAE;wBAClD,OAAO,EAAE,CAAC;wBACV,IAAI,CAAC,EAAE;6BACF,OAAO,CACJ,0DAA0D,CAC7D;6BACA,GAAG,CACA,YAAY,EACZ,OAAO,EACP,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,CACjC,CAAC;qBACT;iBACJ;aACJ;QACL,CAAC,CAAC,EAAE,CAAC;QAEL,IAAI,kBAAkB,EAAE;YACpB,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC;gBACzB,MAAM,EAAE,OAAO;gBACf,GAAG,QAAQ;gBACX,YAAY,EAAE,kBAAkB;aACnC,CAAC,CAAC;SACN;IACL,CAAC;IAED,WAAW,CAAC,GAA0C;QAClD,OAAO,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,UAAU,CAAC,IAAqC,IAAG,CAAC;CACvD"}

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "cojson-storage-sqlite",
+  "type": "module",
+  "version": "0.1.1",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "license": "MIT",
+  "dependencies": {
+    "better-sqlite3": "^8.5.2",
+    "cojson": "^0.1.4",
+    "typescript": "^5.1.6"
+  },
+  "scripts": {
+    "lint": "eslint src/**/*.ts",
+    "build": "npm run lint && rm -rf ./dist && tsc --declaration --sourceMap --outDir dist",
+    "prepublishOnly": "npm run build"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "^7.6.4"
+  },
+  "gitHead": "65a7a66c159ae23cfebfa9aa5c3e173194810573"
+}

package/src/index.ts

@@ -0,0 +1,383 @@
+import {
+    cojsonInternals,
+    SyncMessage,
+    Peer,
+    CojsonInternalTypes,
+    SessionID,
+    // CojsonInternalTypes,
+    // SessionID,
+} from "cojson";
+import {
+    ReadableStream,
+    WritableStream,
+    ReadableStreamDefaultReader,
+    WritableStreamDefaultWriter,
+} from "isomorphic-streams";
+
+import Database, { Database as DatabaseT } from "better-sqlite3";
+import { RawCoID } from "cojson/dist/ids";
+
+type CoValueRow = {
+    id: CojsonInternalTypes.RawCoID;
+    header: string;
+};
+
+type StoredCoValueRow = CoValueRow & { rowID: number };
+
+type SessionRow = {
+    coValue: number;
+    sessionID: SessionID;
+    lastIdx: number;
+    lastSignature: CojsonInternalTypes.Signature;
+};
+
+type StoredSessionRow = SessionRow & { rowID: number };
+
+type TransactionRow = {
+    ses: number;
+    idx: number;
+    tx: string;
+};
+
+export class SQLiteStorage {
+    fromLocalNode!: ReadableStreamDefaultReader<SyncMessage>;
+    toLocalNode: WritableStreamDefaultWriter<SyncMessage>;
+    db: DatabaseT;
+
+    constructor(
+        db: DatabaseT,
+        fromLocalNode: ReadableStream<SyncMessage>,
+        toLocalNode: WritableStream<SyncMessage>
+    ) {
+        this.db = db;
+        this.fromLocalNode = fromLocalNode.getReader();
+        this.toLocalNode = toLocalNode.getWriter();
+
+        (async () => {
+            let done = false;
+            while (!done) {
+                const result = await this.fromLocalNode.read();
+                done = result.done;
+
+                if (result.value) {
+                    this.handleSyncMessage(result.value);
+                }
+            }
+        })();
+    }
+
+    static async asPeer({
+        filename,
+        trace,
+        localNodeName = "local",
+    }: {
+        filename: string;
+        trace?: boolean;
+        localNodeName?: string;
+    }): Promise<Peer> {
+        const [localNodeAsPeer, storageAsPeer] = cojsonInternals.connectedPeers(
+            localNodeName,
+            "storage",
+            { peer1role: "client", peer2role: "server", trace }
+        );
+
+        await SQLiteStorage.open(
+            filename,
+            localNodeAsPeer.incoming,
+            localNodeAsPeer.outgoing
+        );
+
+        return storageAsPeer;
+    }
+
+    static async open(
+        filename: string,
+        fromLocalNode: ReadableStream<SyncMessage>,
+        toLocalNode: WritableStream<SyncMessage>
+    ) {
+        const db = Database(filename);
+        db.pragma("journal_mode = WAL");
+
+        db.prepare(
+            `CREATE TABLE IF NOT EXISTS transactions (
+                ses INTEGER,
+                idx INTEGER,
+                tx TEXT NOT NULL ,
+                PRIMARY KEY (ses, idx)
+            ) WITHOUT ROWID;`
+        ).run();
+
+        db.prepare(
+            `CREATE TABLE IF NOT EXISTS sessions (
+                rowID INTEGER PRIMARY KEY,
+                coValue INTEGER NOT NULL,
+                sessionID TEXT NOT NULL,
+                lastIdx INTEGER,
+                lastSignature TEXT,
+                UNIQUE (sessionID, coValue)
+            );`
+        ).run();
+
+        db.prepare(
+            `CREATE INDEX IF NOT EXISTS sessionsByCoValue ON sessions (coValue);`
+        ).run();
+
+        db.prepare(
+            `CREATE TABLE IF NOT EXISTS coValues (
+                rowID INTEGER PRIMARY KEY,
+                id TEXT NOT NULL UNIQUE,
+                header TEXT NOT NULL UNIQUE
+            );`
+        ).run();
+
+        db.prepare(
+            `CREATE INDEX IF NOT EXISTS coValuesByID ON coValues (id);`
+        ).run();
+
+        return new SQLiteStorage(db, fromLocalNode, toLocalNode);
+    }
+
+    async handleSyncMessage(msg: SyncMessage) {
+        switch (msg.action) {
+            case "load":
+                await this.handleLoad(msg);
+                break;
+            case "content":
+                await this.handleContent(msg);
+                break;
+            case "known":
+                await this.handleKnown(msg);
+                break;
+            case "done":
+                await this.handleDone(msg);
+                break;
+        }
+    }
+
+    async sendNewContentAfter(
+        theirKnown: CojsonInternalTypes.CoValueKnownState,
+        asDependencyOf?: CojsonInternalTypes.RawCoID
+    ) {
+        const coValueRow = (await this.db
+            .prepare(`SELECT * FROM coValues WHERE id = ?`)
+            .get(theirKnown.id)) as StoredCoValueRow | undefined;
+
+        const allOurSessions = coValueRow
+            ? (this.db
+                  .prepare<number>(`SELECT * FROM sessions WHERE coValue = ?`)
+                  .all(coValueRow.rowID) as StoredSessionRow[])
+            : [];
+
+        const ourKnown: CojsonInternalTypes.CoValueKnownState = {
+            id: theirKnown.id,
+            header: !!coValueRow,
+            sessions: {},
+        };
+
+        const parsedHeader = (coValueRow?.header &&
+            JSON.parse(coValueRow.header)) as
+            | CojsonInternalTypes.CoValueHeader
+            | undefined;
+
+        const newContent: CojsonInternalTypes.NewContentMessage = {
+            action: "content",
+            id: theirKnown.id,
+            header: theirKnown.header ? undefined : parsedHeader,
+            new: {},
+        };
+
+        for (const sessionRow of allOurSessions) {
+            ourKnown.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
+
+            if (
+                sessionRow.lastIdx >
+                (theirKnown.sessions[sessionRow.sessionID] || 0)
+            ) {
+                const firstNewTxIdx =
+                    theirKnown.sessions[sessionRow.sessionID] || 0;
+
+                const newTxInSession = this.db
+                    .prepare<[number, number]>(
+                        `SELECT * FROM transactions WHERE ses = ? AND idx > ?`
+                    )
+                    .all(sessionRow.rowID, firstNewTxIdx) as TransactionRow[];
+
+                newContent.new[sessionRow.sessionID] = {
+                    after: firstNewTxIdx,
+                    lastSignature: sessionRow.lastSignature,
+                    newTransactions: newTxInSession.map((row) =>
+                        JSON.parse(row.tx)
+                    ),
+                };
+            }
+        }
+
+        const dependedOnCoValues =
+            parsedHeader?.ruleset.type === "group"
+                ? Object.values(newContent.new).flatMap((sessionEntry) =>
+                      sessionEntry.newTransactions.flatMap((tx) => {
+                          if (tx.privacy !== "trusting") return [];
+                          return tx.changes
+                              .map(
+                                  (change) =>
+                                      change &&
+                                      typeof change === "object" &&
+                                      "op" in change &&
+                                      change.op === "set" &&
+                                      "key" in change &&
+                                      change.key
+                              )
+                              .filter(
+                                  (key): key is CojsonInternalTypes.RawCoID =>
+                                      typeof key === "string" &&
+                                      key.startsWith("co_")
+                              );
+                      })
+                  )
+                : parsedHeader?.ruleset.type === "ownedByGroup"
+                ? [parsedHeader?.ruleset.group]
+                : [];
+
+        for (const dependedOnCoValue of dependedOnCoValues) {
+            await this.sendNewContentAfter(
+                { id: dependedOnCoValue, header: false, sessions: {} },
+                asDependencyOf || theirKnown.id
+            );
+        }
+
+        await this.toLocalNode.write({
+            action: "known",
+            ...ourKnown,
+            asDependencyOf,
+        });
+
+        if (newContent.header || Object.keys(newContent.new).length > 0) {
+            await this.toLocalNode.write(newContent);
+        }
+    }
+
+    handleLoad(msg: CojsonInternalTypes.LoadMessage) {
+        return this.sendNewContentAfter(msg);
+    }
+
+    async handleContent(msg: CojsonInternalTypes.NewContentMessage) {
+        let storedCoValueRowID = (
+            this.db
+                .prepare<RawCoID>(`SELECT rowID FROM coValues WHERE id = ?`)
+                .get(msg.id) as StoredCoValueRow | undefined
+        )?.rowID;
+
+        if (storedCoValueRowID === undefined) {
+            const header = msg.header;
+            if (!header) {
+                console.error("Expected to be sent header first");
+                await this.toLocalNode.write({
+                    action: "known",
+                    id: msg.id,
+                    header: false,
+                    sessions: {},
+                    isCorrection: true,
+                });
+                return;
+            }
+
+            storedCoValueRowID = this.db
+                .prepare<[RawCoID, string]>(
+                    `INSERT INTO coValues (id, header) VALUES (?, ?)`
+                )
+                .run(msg.id, JSON.stringify(header)).lastInsertRowid as number;
+        }
+
+        const ourKnown: CojsonInternalTypes.CoValueKnownState = {
+            id: msg.id,
+            header: true,
+            sessions: {},
+        };
+        let invalidAssumptions = false;
+
+        this.db.transaction(() => {
+            const allOurSessions = (
+                this.db
+                    .prepare<number>(`SELECT * FROM sessions WHERE coValue = ?`)
+                    .all(storedCoValueRowID!) as StoredSessionRow[]
+            ).reduce((acc, row) => {
+                acc[row.sessionID] = row;
+                return acc;
+            }, {} as { [sessionID: string]: StoredSessionRow });
+
+            for (const sessionID of Object.keys(msg.new) as SessionID[]) {
+                const sessionRow = allOurSessions[sessionID];
+                if (sessionRow) {
+                    ourKnown.sessions[sessionRow.sessionID] =
+                        sessionRow.lastIdx;
+                }
+
+                if (
+                    (sessionRow?.lastIdx || 0) <
+                    (msg.new[sessionID]?.after || 0)
+                ) {
+                    invalidAssumptions = true;
+                } else {
+                    const newTransactions =
+                        msg.new[sessionID]?.newTransactions || [];
+
+                    const actuallyNewOffset =
+                        (sessionRow?.lastIdx || 0) -
+                        (msg.new[sessionID]?.after || 0);
+                    const actuallyNewTransactions =
+                        newTransactions.slice(actuallyNewOffset);
+
+                    let nextIdx = sessionRow?.lastIdx || 0;
+
+                    const sessionUpdate = {
+                        coValue: storedCoValueRowID!,
+                        sessionID: sessionID,
+                        lastIdx:
+                            (sessionRow?.lastIdx || 0) +
+                            actuallyNewTransactions.length,
+                        lastSignature: msg.new[sessionID]!.lastSignature,
+                    };
+
+                    const sessionRowID = this.db
+                        .prepare<[number, string, number, string]>(
+                            `INSERT INTO sessions (coValue, sessionID, lastIdx, lastSignature) VALUES (?, ?, ?, ?)
+                            ON CONFLICT(coValue, sessionID) DO UPDATE SET lastIdx=excluded.lastIdx, lastSignature=excluded.lastSignature`
+                        )
+                        .run(
+                            sessionUpdate.coValue,
+                            sessionUpdate.sessionID,
+                            sessionUpdate.lastIdx,
+                            sessionUpdate.lastSignature
+                        ).lastInsertRowid as number;
+
+                    for (const newTransaction of actuallyNewTransactions) {
+                        nextIdx++;
+                        this.db
+                            .prepare<[number, number, string]>(
+                                `INSERT INTO transactions (ses, idx, tx) VALUES (?, ?, ?)`
+                            )
+                            .run(
+                                sessionRowID,
+                                nextIdx,
+                                JSON.stringify(newTransaction)
+                            );
+                    }
+                }
+            }
+        })();
+
+        if (invalidAssumptions) {
+            await this.toLocalNode.write({
+                action: "known",
+                ...ourKnown,
+                isCorrection: invalidAssumptions,
+            });
+        }
+    }
+
+    handleKnown(msg: CojsonInternalTypes.KnownStateMessage) {
+        return this.sendNewContentAfter(msg);
+    }
+
+    handleDone(_msg: CojsonInternalTypes.DoneMessage) {}
+}

package/tsconfig.json

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "lib": ["ESNext"],
+    "module": "esnext",
+    "target": "ES2020",
+    "moduleResolution": "bundler",
+    "moduleDetection": "force",
+    "strict": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "noUncheckedIndexedAccess": true,
+    "esModuleInterop": true,
+  },
+  "include": ["./src/**/*"],
+}