@syncular/server 0.1.3 → 0.2.0

package/dist/sqlite-storage.js

@@ -0,0 +1,477 @@
+/**
+ * SQLite storage via `bun:sqlite` (dev-speed, dependency-free).
+ *
+ * Scope fanout is index-first (REVISE B2): both the commit log and the
+ * current-row table carry a (table, variable, value) inverted index; reads
+ * select candidates from the index and verify the full multi-variable
+ * match against the stored scope map — never a log scan.
+ */
+import { Database } from 'bun:sqlite';
+import { syncError } from './errors.js';
+import { commitWindowPageSql, deleteRowSql, layoutsOf, migratePayload, parseLayouts, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
+import { matchesEffective } from './scopes.js';
+import { collectCommitWindowPage, deserializePushResult, SQLITE_DDL, serializePushResult, toStoredRow, } from './sqlite-dialect.js';
+class SqliteTransaction {
+    #storage;
+    #partition;
+    #open = true;
+    constructor(storage, partition) {
+        this.#storage = storage;
+        this.#partition = partition;
+        storage.db.exec('BEGIN IMMEDIATE');
+    }
+    #assertOpen() {
+        if (!this.#open)
+            throw new Error('transaction already finished');
+    }
+    getRow(table, rowId) {
+        this.#assertOpen();
+        return this.#storage.getRow(this.#partition, table, rowId);
+    }
+    async upsertRow(table, row) {
+        this.#assertOpen();
+        this.#storage.writeRow(this.#partition, table, row);
+    }
+    async deleteRow(table, rowId) {
+        this.#assertOpen();
+        const db = this.#storage.db;
+        db.query(deleteRowSql(this.#storage.table(table), 'sqlite')).run(this.#partition, rowId);
+        db.query('DELETE FROM sync_row_scopes WHERE partition=? AND tbl=? AND row_id=?').run(this.#partition, table, rowId);
+        // §5.9.4: a deleted row references no blobs.
+        db.query('DELETE FROM sync_blob_refs WHERE partition=? AND tbl=? AND row_id=?').run(this.#partition, table, rowId);
+    }
+    async setBlobRefs(table, rowId, blobIds) {
+        this.#assertOpen();
+        const db = this.#storage.db;
+        // Replace the row's reference set atomically (§5.9.4).
+        db.query('DELETE FROM sync_blob_refs WHERE partition=? AND tbl=? AND row_id=?').run(this.#partition, table, rowId);
+        for (const blobId of blobIds) {
+            db.query('INSERT OR IGNORE INTO sync_blob_refs(partition, tbl, row_id, blob_id) VALUES (?,?,?,?)').run(this.#partition, table, rowId, blobId);
+        }
+    }
+    async appendCommit(commit) {
+        this.#assertOpen();
+        const db = this.#storage.db;
+        const p = this.#partition;
+        db.query('INSERT OR IGNORE INTO sync_partitions(partition) VALUES (?)').run(p);
+        const row = db
+            .query('SELECT max_commit_seq FROM sync_partitions WHERE partition=?')
+            .get(p);
+        const commitSeq = (row?.max_commit_seq ?? 0) + 1;
+        db.query('UPDATE sync_partitions SET max_commit_seq=? WHERE partition=?').run(commitSeq, p);
+        db.query('INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms) VALUES (?,?,?,?,?,?)').run(p, commitSeq, commit.clientId, commit.clientCommitId, commit.actorId, commit.createdAtMs);
+        commit.changes.forEach((change, idx) => {
+            db.query('INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload) VALUES (?,?,?,?,?,?,?,?,?)').run(p, commitSeq, idx, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion ?? null, JSON.stringify(change.scopes), change.payload ?? null);
+            for (const [variable, value] of Object.entries(change.scopes)) {
+                db.query('INSERT OR IGNORE INTO sync_change_scopes(partition, tbl, var, value, commit_seq) VALUES (?,?,?,?,?)').run(p, change.table, variable, value, commitSeq);
+            }
+        });
+        return commitSeq;
+    }
+    async putPushResult(clientId, clientCommitId, result) {
+        this.#assertOpen();
+        this.#storage.db
+            .query('INSERT OR REPLACE INTO sync_push_results(partition, client_id, client_commit_id, result) VALUES (?,?,?,?)')
+            .run(this.#partition, clientId, clientCommitId, serializePushResult(result));
+    }
+    async commit() {
+        this.#assertOpen();
+        this.#open = false;
+        this.#storage.db.exec('COMMIT');
+    }
+    async rollback() {
+        if (!this.#open)
+            return;
+        this.#open = false;
+        this.#storage.db.exec('ROLLBACK');
+    }
+}
+export class SqliteServerStorage {
+    db;
+    /** Set by `ensureSchema`: app-table lookup for the relational row store. */
+    #tables;
+    #schemaVersion;
+    constructor(db = ':memory:') {
+        this.db = typeof db === 'string' ? new Database(db) : db;
+        this.db.exec(SQLITE_DDL);
+    }
+    /** Resolve a table's compiled schema; row operations require `ensureSchema`. */
+    table(name) {
+        const table = this.#tables?.get(name);
+        if (table === undefined) {
+            throw new Error(`unknown table ${JSON.stringify(name)} — ensureSchema(schema) must run before row operations`);
+        }
+        return table;
+    }
+    async ensureSchema(schema) {
+        // Memoized fast path: same instance, same schema version.
+        if (this.#schemaVersion === schema.version)
+            return;
+        this.db.exec(SCHEMA_META_DDL_SQLITE);
+        const marker = this.db
+            .query('SELECT schema_version, layouts FROM sync_schema_meta WHERE id=1')
+            .get();
+        if (marker !== null && marker.schema_version > schema.version) {
+            throw new Error(`stored schema version ${marker.schema_version} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`);
+        }
+        if (marker === null || marker.schema_version < schema.version) {
+            // Introspect existing app tables, then apply the migration subset
+            // (CREATE TABLE / ADD COLUMN / CREATE INDEX) to reach `schema`, then
+            // rewrite stored rows (payload re-encode for layout changes, and/or
+            // projection backfill for flipped-on materialization). One
+            // transaction: a failed bump leaves no half-state.
+            const layouts = parseLayouts(marker?.layouts);
+            const existing = new Map();
+            for (const table of schema.tables.values()) {
+                const columns = this.db
+                    .query(`PRAGMA table_info("${table.name.replaceAll('"', '""')}")`)
+                    .all();
+                if (columns.length > 0) {
+                    existing.set(table.name, new Set(columns.map((c) => c.name)));
+                }
+            }
+            this.db.exec('BEGIN IMMEDIATE');
+            try {
+                for (const statement of schemaDdl(schema, existing, 'sqlite')) {
+                    this.db.exec(statement);
+                }
+                for (const table of schema.tables.values()) {
+                    const oldLayout = layouts[table.name];
+                    const plan = rewritePlan(table, oldLayout, existing.get(table.name));
+                    if (!plan.migrate && !plan.backfill)
+                        continue;
+                    this.#rewriteRows(table, plan.migrate ? oldLayout : undefined);
+                }
+                this.db
+                    .query('INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts')
+                    .run(schema.version, layoutsOf(schema));
+                this.db.exec('COMMIT');
+            }
+            catch (error) {
+                this.db.exec('ROLLBACK');
+                throw error;
+            }
+        }
+        this.#tables = schema.tables;
+        this.#schemaVersion = schema.version;
+    }
+    /**
+     * Migration rewrite (DESIGN "optional materialization"): keyset-paged walk
+     * of a row table; when `oldLayout` is given every payload re-encodes under
+     * the current columns, and the projection (when materialized) refreshes
+     * from the payload either way.
+     */
+    #rewriteRows(table, oldLayout) {
+        const select = selectRowsForRewriteSql(table, 'sqlite');
+        const update = this.db.query(rewriteRowSql(table, 'sqlite'));
+        const BATCH = 500;
+        let afterPartition = '';
+        let afterRowId = '';
+        for (;;) {
+            const rows = this.db
+                .query(select)
+                .all(afterPartition, afterRowId, BATCH);
+            if (rows.length === 0)
+                break;
+            for (const row of rows) {
+                const payload = oldLayout !== undefined
+                    ? migratePayload(oldLayout, table, row.payload)
+                    : row.payload;
+                update.run(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite'));
+            }
+            const last = rows[rows.length - 1];
+            if (last === undefined || rows.length < BATCH)
+                break;
+            afterPartition = last.partition;
+            afterRowId = last.row_id;
+        }
+    }
+    async begin(partition) {
+        return new SqliteTransaction(this, partition);
+    }
+    /** Internal: write a row + refresh its scope-index entries. */
+    writeRow(partition, table, row) {
+        const compiled = this.table(table);
+        this.db
+            .query(upsertSql(compiled, 'sqlite'))
+            .run(...upsertValues(compiled, partition, row, 'sqlite'));
+        this.db
+            .query('DELETE FROM sync_row_scopes WHERE partition=? AND tbl=? AND row_id=?')
+            .run(partition, table, row.rowId);
+        for (const [variable, value] of Object.entries(row.scopes)) {
+            this.db
+                .query('INSERT OR IGNORE INTO sync_row_scopes(partition, tbl, var, value, row_id) VALUES (?,?,?,?,?)')
+                .run(partition, table, variable, value, row.rowId);
+        }
+    }
+    async getMaxCommitSeq(partition) {
+        const row = this.db
+            .query('SELECT max_commit_seq FROM sync_partitions WHERE partition=?')
+            .get(partition);
+        return row?.max_commit_seq ?? 0;
+    }
+    async getHorizonSeq(partition) {
+        const row = this.db
+            .query('SELECT horizon_seq FROM sync_partitions WHERE partition=?')
+            .get(partition);
+        return row?.horizon_seq ?? 0;
+    }
+    async setHorizonSeq(partition, seq) {
+        this.db
+            .query('INSERT OR IGNORE INTO sync_partitions(partition) VALUES (?)')
+            .run(partition);
+        this.db
+            .query('UPDATE sync_partitions SET horizon_seq=? WHERE partition=?')
+            .run(seq, partition);
+    }
+    async pruneCommitsThrough(partition, seq) {
+        const removed = this.db
+            .query('DELETE FROM sync_commits WHERE partition=? AND commit_seq<=?')
+            .run(partition, seq);
+        this.db
+            .query('DELETE FROM sync_changes WHERE partition=? AND commit_seq<=?')
+            .run(partition, seq);
+        this.db
+            .query('DELETE FROM sync_change_scopes WHERE partition=? AND commit_seq<=?')
+            .run(partition, seq);
+        return Number(removed.changes);
+    }
+    async getCommitSeqBefore(partition, createdBeforeMs) {
+        const row = this.db
+            .query('SELECT max(commit_seq) AS seq FROM sync_commits WHERE partition=? AND created_at_ms<?')
+            .get(partition, createdBeforeMs);
+        return row?.seq ?? 0;
+    }
+    async getRow(partition, table, rowId) {
+        const record = this.db
+            .query(selectRowSql(this.table(table), 'sqlite'))
+            .get(partition, rowId);
+        return record === null ? undefined : toStoredRow(record);
+    }
+    async getPushResult(partition, clientId, clientCommitId) {
+        const record = this.db
+            .query('SELECT result FROM sync_push_results WHERE partition=? AND client_id=? AND client_commit_id=?')
+            .get(partition, clientId, clientCommitId);
+        if (record === null)
+            return undefined;
+        try {
+            return deserializePushResult(record.result);
+        }
+        catch {
+            throw syncError('sync.idempotency_cache_miss', 'persisted push result unreadable (§6.3)');
+        }
+    }
+    async readCommitWindow(partition, query) {
+        const variables = Object.keys(query.scopeFilter).sort();
+        const firstVariable = variables[0];
+        if (firstVariable === undefined)
+            return [];
+        const firstValues = query.scopeFilter[firstVariable] ?? [];
+        if (firstValues.length === 0)
+            return [];
+        // Candidates via the inverted index (one variable) LEFT JOINed to the
+        // commit meta + the table's changes — one statement per page, never two
+        // per candidate (see `commitWindowPageSql`). Exact multi-variable
+        // verification against the stored scope map in `collectCommitWindowPage`.
+        const sql = commitWindowPageSql(firstValues.length, 'sqlite');
+        const commits = [];
+        let deliveredChanges = 0;
+        let afterSeq = query.afterSeq;
+        const batchSize = Math.max(64, query.limitChanges);
+        while (deliveredChanges < query.limitChanges) {
+            const records = this.db
+                .query(sql)
+                .all(partition, query.table, firstVariable, ...firstValues, afterSeq, query.throughSeq, batchSize, partition, partition, query.table);
+            if (records.length === 0)
+                break;
+            const page = collectCommitWindowPage(records, query.scopeFilter, query.limitChanges - deliveredChanges);
+            commits.push(...page.commits);
+            deliveredChanges += page.delivered;
+            afterSeq = page.lastSeq;
+            if (page.candidateCount < batchSize)
+                break;
+        }
+        return commits;
+    }
+    async scanRows(partition, query) {
+        const variables = Object.keys(query.scopeFilter).sort();
+        const firstVariable = variables[0];
+        if (firstVariable === undefined)
+            return [];
+        const firstValues = query.scopeFilter[firstVariable] ?? [];
+        if (firstValues.length === 0)
+            return [];
+        // Candidates via the inverted index LEFT JOINed to the row table — one
+        // statement per page, never one per row (see `scanRowPageSql`). Exact
+        // multi-variable verification against the stored scope map below.
+        const sql = scanRowPageSql(this.table(query.table), firstValues.length, 'sqlite');
+        const rows = [];
+        let afterRowId = query.afterRowId ?? '';
+        const batchSize = Math.max(64, query.limit);
+        while (rows.length < query.limit) {
+            const records = this.db
+                .query(sql)
+                .all(partition, query.table, firstVariable, ...firstValues, afterRowId, batchSize, partition);
+            if (records.length === 0)
+                break;
+            for (const record of records) {
+                afterRowId = record.row_id;
+                // NULL payload: an index candidate whose row vanished — it still
+                // advances the keyset cursor (LEFT JOIN contract) but yields no row.
+                if (record.payload === null)
+                    continue;
+                const stored = toStoredRow(record);
+                if (!matchesEffective(stored.scopes, query.scopeFilter))
+                    continue;
+                rows.push(stored);
+                if (rows.length >= query.limit)
+                    break;
+            }
+            if (records.length < batchSize)
+                break;
+        }
+        return rows;
+    }
+    async getClientRecord(partition, clientId) {
+        const record = this.db
+            .query('SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? AND client_id=?')
+            .get(partition, clientId);
+        if (record === null)
+            return undefined;
+        return {
+            clientId: record.client_id,
+            actorId: record.actor_id,
+            cursor: record.cursor,
+            updatedAtMs: record.updated_at_ms,
+            subscriptions: JSON.parse(record.subscriptions),
+        };
+    }
+    async putClientRecord(partition, record) {
+        this.db
+            .query('INSERT OR REPLACE INTO sync_clients(partition, client_id, actor_id, cursor, subscriptions, updated_at_ms) VALUES (?,?,?,?,?,?)')
+            .run(partition, record.clientId, record.actorId, record.cursor, JSON.stringify(record.subscriptions), record.updatedAtMs);
+    }
+    async listClientCursors(partition) {
+        const records = this.db
+            .query('SELECT client_id, cursor, updated_at_ms FROM sync_clients WHERE partition=?')
+            .all(partition);
+        return records.map((r) => ({
+            clientId: r.client_id,
+            cursor: r.cursor,
+            updatedAtMs: r.updated_at_ms,
+        }));
+    }
+    async listRowsReferencingBlob(partition, blobId) {
+        // Candidate rows via the by-blob index (§5.9.4/§5.9.5); each row's
+        // stored scopes come from sync_rows for the §3.4 authorization test.
+        const refs = this.db
+            .query('SELECT tbl, row_id FROM sync_blob_refs WHERE partition=? AND blob_id=?')
+            .all(partition, blobId);
+        const out = [];
+        for (const ref of refs) {
+            const compiled = this.#tables?.get(ref.tbl);
+            if (compiled === undefined)
+                continue; // table no longer in the schema
+            const row = this.db
+                .query(selectRowScopesSql(compiled, 'sqlite'))
+                .get(partition, ref.row_id);
+            if (row === null)
+                continue;
+            out.push({
+                table: ref.tbl,
+                rowId: ref.row_id,
+                scopes: JSON.parse(row.scopes),
+            });
+        }
+        return out;
+    }
+    async listReferencedBlobIds(partition) {
+        const rows = this.db
+            .query('SELECT DISTINCT blob_id FROM sync_blob_refs WHERE partition=?')
+            .all(partition);
+        return rows.map((r) => r.blob_id);
+    }
+    // -- admin/console read surface (TODO §2.5) --------------------------------
+    async listClientRecords(partition) {
+        const records = this.db
+            .query('SELECT client_id, actor_id, cursor, subscriptions, updated_at_ms FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC')
+            .all(partition);
+        return records.map((record) => ({
+            clientId: record.client_id,
+            actorId: record.actor_id,
+            cursor: record.cursor,
+            updatedAtMs: record.updated_at_ms,
+            subscriptions: JSON.parse(record.subscriptions),
+        }));
+    }
+    async listCommitMetadata(partition, query) {
+        // Newest-first window, resumed by `afterSeq` (exclusive lower bound).
+        const rows = query.table
+            ? this.db
+                .query(`SELECT c.commit_seq, c.client_id, c.client_commit_id, c.actor_id, c.created_at_ms
+             FROM sync_commits c
+             WHERE c.partition=? AND c.commit_seq>?
+               AND EXISTS (SELECT 1 FROM sync_changes ch
+                 WHERE ch.partition=c.partition AND ch.commit_seq=c.commit_seq AND ch.tbl=?)
+             ORDER BY c.commit_seq DESC LIMIT ?`)
+                .all(partition, query.afterSeq, query.table, query.limit)
+            : this.db
+                .query(`SELECT commit_seq, client_id, client_commit_id, actor_id, created_at_ms
+             FROM sync_commits
+             WHERE partition=? AND commit_seq>?
+             ORDER BY commit_seq DESC LIMIT ?`)
+                .all(partition, query.afterSeq, query.limit);
+        return rows.map((row) => {
+            const changes = this.db
+                .query('SELECT tbl, count(*) AS n FROM sync_changes WHERE partition=? AND commit_seq=? GROUP BY tbl')
+                .all(partition, row.commit_seq);
+            return {
+                commitSeq: row.commit_seq,
+                clientId: row.client_id,
+                clientCommitId: row.client_commit_id,
+                actorId: row.actor_id,
+                createdAtMs: row.created_at_ms,
+                changeCount: changes.reduce((sum, c) => sum + c.n, 0),
+                tables: changes.map((c) => c.tbl),
+            };
+        });
+    }
+    async scopeActivity(partition, query) {
+        // Candidate commits via the change-scope index (§3.1) — never a scan.
+        const rows = this.db
+            .query(`SELECT DISTINCT commit_seq, tbl FROM sync_change_scopes
+         WHERE partition=? AND var=? AND value=?
+         ORDER BY commit_seq DESC LIMIT ?`)
+            .all(partition, query.variable, query.value, query.limit);
+        const out = [];
+        for (const row of rows) {
+            const meta = this.db
+                .query('SELECT actor_id, created_at_ms FROM sync_commits WHERE partition=? AND commit_seq=?')
+                .get(partition, row.commit_seq);
+            if (meta === null)
+                continue;
+            const count = this.db
+                .query('SELECT count(*) AS n FROM sync_changes WHERE partition=? AND commit_seq=? AND tbl=?')
+                .get(partition, row.commit_seq, row.tbl);
+            out.push({
+                commitSeq: row.commit_seq,
+                table: row.tbl,
+                createdAtMs: meta.created_at_ms,
+                actorId: meta.actor_id,
+                changeCount: count?.n ?? 0,
+            });
+        }
+        return out;
+    }
+    async getRowScopes(partition, table, rowId) {
+        const record = this.db
+            .query(selectRowScopesSql(this.table(table), 'sqlite'))
+            .get(partition, rowId);
+        if (record === null)
+            return undefined;
+        return {
+            serverVersion: record.server_version,
+            scopes: JSON.parse(record.scopes),
+        };
+    }
+}