@g9tm/syncdata 1.2.2 → 1.3.0

package/dist/sqlite/apply.js

@@ -14,11 +14,12 @@ import { G9ApplyReport, G9ERevocationReason, REASON_TABLE_NOT_EMPTY } from '../c
 import { G9EPullPartKind } from '../protocol/dtos.js';
 import { G9ERowBatchOp, G9RowBatchFormatError } from '../protocol/rowBatch/format.js';
 import { openRowBatch } from '../protocol/rowBatch/reader.js';
-import { asNumber, asString, G9StatementBatch } from './connection.js';
+import { asNumber, asString, G9StatementBatch, statementsOf } from './connection.js';
 import { findByServerName, keyValue, meta, nowMs, persistedJson, readPersistedDefinitions, readSubscriptions } from './internals.js';
-import { disposeReleasedRow, REVOKED_REJECTION_CODE } from './revocation.js';
+import { disposeReleasedRow, disposeReleasedRows, REVOKED_REJECTION_CODE } from './revocation.js';
 import { bit, EPHEMERAL_SLOT, quote } from './sql.js';
 import { frameValueToSql } from './values.js';
+import { applySnapshotFile, attachedSchema, attachFiles, deleteFiles, fileSnapshotTimes, readStagedFiles, takeStagedFiles, verifyStagedFile, } from './snapshotFiles.js';
 const NEXT_SEQ = '(SELECT seq FROM g9_clock WHERE id = 1) + 1';
 const BUMP_IF_CHANGED = 'UPDATE g9_clock SET seq = seq + 1 WHERE id = 1 AND changes() > 0';
 const BUMP = 'UPDATE g9_clock SET seq = seq + 1 WHERE id = 1';
@@ -35,6 +36,8 @@ export class ApplyContext {
         this.droppedTriggers = [];
         this.rows = 0;
         this.hasRevoked = false;
+        /** The database holds a trigger the library did not create: bulk disposals would not be exact (row by row then). */
+        this.hasForeignTriggers = false;
         this._dirtyTables = new Set();
         this._tableTime = new Map();
         this._tableNames = new Map();
@@ -58,6 +61,17 @@ export class ApplyContext {
         const keys = this.report.changedKeys[table];
         return !keys || keys.length < this._maxChangedKeys;
     }
+    get maxChangedKeys() {
+        return this._maxChangedKeys;
+    }
+    /** Counts rows applied in bulk (a snapshot file's table) and reports progress once for them. */
+    advance(rows) {
+        if (rows <= 0)
+            return;
+        this.rows += rows;
+        if (this._progress)
+            this._progress(this.rows);
+    }
     addChanged(table, key) {
         var _a;
         const keys = (_a = this.report.changedKeys[table]) !== null && _a !== void 0 ? _a : (this.report.changedKeys[table] = []);
@@ -135,8 +149,9 @@ export async function beginApply(store, connection, options) {
         context.backfillColumns.set(backfill.tableId, backfill.columns);
     const diagnostics = context.report.diagnostics;
     diagnostics.transactions = 1;
+    const foreign = await foreignTriggerNames(store, connection);
+    context.hasForeignTriggers = foreign.length > 0;
     if (store.options.skipTriggersDuringApply) {
-        const foreign = await foreignTriggerNames(store, connection);
         if (foreign.length > 0) {
             for (const name of foreign.slice(0, 10))
                 diagnostics.foreignTriggers.push(name);
@@ -197,12 +212,25 @@ export async function foreignTriggerNames(store, connection) {
     }
     return foreign;
 }
-/** Rows changed after a sequence, table by table (`g9_rows_feed (tid, local_seq)` serves each read). */
+/**
+ * Rows changed after a sequence, table by table (`g9_rows_feed (tid, local_seq)` serves each part) — as ONE statement:
+ * a query per table cost a bridge call per table on every apply (LES-0055).
+ */
 export async function readRowsChangedAfter(store, connection, sequence, add) {
-    for (const table of store.byLocal.values()) {
-        for (const r of await connection.query('SELECT pk FROM g9_rows WHERE tid = $t AND local_seq > $c', { $t: table.tid, $c: sequence }))
-            add(table.tid, asString(r[0]));
-    }
+    const tids = Array.from(store.byLocal.values()).map((table) => table.tid);
+    // SQLite allows 500 terms in a compound SELECT: one statement per 400 tables, all in one driver call.
+    const statements = [];
+    for (let i = 0; i < tids.length; i += 400) {
+        const sql = tids
+            .slice(i, i + 400)
+            .map((tid) => `SELECT ${tid} AS tid, pk FROM g9_rows WHERE tid = ${tid} AND local_seq > $c`)
+            .join(' UNION ALL ');
+        for (const statement of statementsOf(sql, { $c: sequence }))
+            statements.push(statement);
+    }
+    for (const result of await connection.runStatements(statements))
+        for (const r of result.rows)
+            add(asNumber(r[0]), asString(r[1]));
 }
 /** Rows each table receives from snapshot parts of a plan, by server table id. */
 export function snapshotRowsByTable(plan) {
@@ -241,13 +269,25 @@ async function deferIndexes(store, context, snapshotRows) {
         store.log('debug', 20111, `G9SyncData loads ${rows} row(s) into ${table.model.localName}: ${indexes.length} index(es) rebuilt after the load`);
     }
 }
+/**
+ * What `downloadDurability` reads of a connection, read ONCE per connection: the store's connections set their journal
+ * mode when they open and nobody else touches their `synchronous`, so asking again for every staged chunk only cost two
+ * bridge calls per chunk (LES-0056).
+ */
+const durabilityOf = new WeakMap();
 /** `synchronous=NORMAL` while a transaction writes only server-derived data (WAL only); returns the restore action. */
 export async function downloadDurability(store, connection) {
     if (!store.options.synchronousNormalForDownloads)
         return null;
-    if (asString(await connection.scalar('PRAGMA journal_mode')).toLowerCase() !== 'wal')
+    let known = durabilityOf.get(connection);
+    if (!known) {
+        const wal = asString(await connection.scalar('PRAGMA journal_mode')).toLowerCase() === 'wal';
+        known = { wal, synchronous: wal ? asNumber(await connection.scalar('PRAGMA synchronous')) : 0 };
+        durabilityOf.set(connection, known);
+    }
+    if (!known.wal)
         return null;
-    const previous = asNumber(await connection.scalar('PRAGMA synchronous'));
+    const previous = known.synchronous;
     if (previous <= 1)
         return null;
     await connection.exec('PRAGMA synchronous = NORMAL');
@@ -287,12 +327,14 @@ export async function applyFrame(store, context, frame, subscriptionKey) {
                 while (reader.readRow())
                     await applyTombstone(context, table, reader.key, reader.version);
                 break;
-            case G9ERowBatchOp.MoveOut:
-                while (reader.readRow()) {
-                    if (slot >= 0)
-                        await moveOut(store, context, table, reader.key, subscriptionKey, slot, ephemeral, policy);
-                }
+            case G9ERowBatchOp.MoveOut: {
+                const keys = [];
+                while (reader.readRow())
+                    keys.push(reader.key);
+                if (slot >= 0)
+                    await moveOutMany(store, context, table, keys, subscriptionKey, slot, ephemeral, policy);
                 break;
+            }
         }
         context.addTableTime(table.model.localName, context.rows - rowsBefore, Date.now() - started);
     }
@@ -446,6 +488,34 @@ export async function moveOut(store, context, table, key, subscriptionKey, slot,
     if (removed)
         context.addChanged(table.model.localName, key);
 }
+/**
+ * A run of move-outs of one table (a MoveOut section of a frame, or a reconciliation's move-out keys), with the database
+ * state the row-by-row {@link moveOut} leaves. Row by row a released row costs ~8 driver calls (flush, read metadata,
+ * commands, the app row, the writes, the clock): ~34 ms per row on the Android emulator, minutes for a large office
+ * change. Here every membership update is queued and flushed once, then the released rows are disposed in bulk by
+ * disposeReleasedRows — read in three queries, decided in the same order, written in batches with each row's explicit
+ * local_seq (the clock's value the row-by-row path gives it), the clock moved once by the number of rows. Exact for the
+ * same reasons as an unsubscribe (ADR-0071 addendum) plus one: the keys of a run are distinct, so no row's membership
+ * update can change another row's decision. Ephemeral subscriptions, a Keep policy, duplicate keys or a foreign trigger
+ * take the row-by-row path.
+ */
+export async function moveOutMany(store, context, table, keys, subscriptionKey, slot, ephemeral, policy) {
+    if (keys.length === 0)
+        return;
+    const distinct = new Set(keys).size === keys.length;
+    if (keys.length === 1 || ephemeral || policy !== G9EMoveOutPolicy.Delete || !distinct || context.hasForeignTriggers) {
+        for (const key of keys)
+            await moveOut(store, context, table, key, subscriptionKey, slot, ephemeral, policy);
+        return;
+    }
+    const notBit = bit(slot).not();
+    for (const key of keys)
+        await context.queue('UPDATE g9_rows SET subs = subs & $n WHERE tid = $t AND pk = $pk', { $n: notBit, $t: table.tid, $pk: key });
+    context.report.moveOuts += keys.length;
+    // The disposal reads the rows: run what is queued (this run's membership updates and everything the pull queued before).
+    await context.flush();
+    await disposeReleasedRows(store, context.connection, 'SELECT $tid AS tid, j.value AS pk FROM json_each($keys) AS j ORDER BY j.key', { $tid: table.tid, $keys: JSON.stringify(keys) }, G9ERevocationReason.LeftScope, subscriptionKey, context.report, (_tid, key) => context.addChanged(table.model.localName, key));
+}
 /** `AddMembership`. */
 export async function addMembership(context, table, key, slot, ephemeralKey) {
     context.batch.add('UPDATE g9_rows SET subs = subs | $b WHERE tid = $t AND pk = $pk AND deleted = 0', { $b: bit(slot), $t: table.tid, $pk: key });
@@ -474,110 +544,145 @@ export async function applyStagedPull(store, runId, rowsApplied, slices, readSta
             if (!staged)
                 throw new G9SyncException(G9SyncErrorCodes.LocalStoreError, 'The staged pull no longer exists.');
             const sequences = (await connection.query('SELECT seq FROM g9_stage WHERE run_id = $r AND payload IS NOT NULL ORDER BY seq', { $r: runId })).map((r) => asNumber(r[0]));
-            // Slicing only pays for itself when someone is listening for the slices (ADR-0053); a LARGE snapshot stays in one
-            // transaction with its indexes deferred (ADR-0056/0058).
-            const sliceRows = slices === null ? 0 : store.options.applySliceRows;
-            const snapshotRows = snapshotRowsByTable(staged.plan);
-            let largeSnapshot = false;
-            if (store.options.rebuildIndexesAfterRows > 0)
-                snapshotRows.forEach((rows) => (largeSnapshot = largeSnapshot || rows >= store.options.rebuildIndexesAfterRows));
-            const progressive = sliceRows > 0 && sequences.length > 1 && !largeSnapshot;
-            // Read ONCE, before any slice commits (ADR-0046 + ADR-0053).
-            const protectedRows = progressive ? await readProtectedRows(store, connection, staged.stageSequence) : null;
-            const skipTriggers = staged.plan.totalRows >= store.options.skipTriggersMinRows && !progressive;
-            const total = new G9ApplyReport();
-            let appliedTotal = 0;
-            let index = 0;
-            const started = Date.now();
-            let sliceBudget = sliceRows;
-            // do/while: a pull with no payload chunks still writes its cursors and clears its stage, exactly once.
-            do {
-                if (signal && signal.aborted)
-                    throw cancellation();
-                const slice = [];
-                let final = true;
-                for (; index < sequences.length; index++) {
-                    slice.push(sequences[index]);
-                    if (!progressive)
-                        continue;
-                    if ((await stagedRowCount(connection, runId, slice)) >= sliceBudget && index + 1 < sequences.length) {
-                        index++;
-                        final = false;
-                        break;
-                    }
-                }
-                if (store.options.applySliceRowsMax > sliceBudget)
-                    sliceBudget = Math.min(store.options.applySliceRowsMax, sliceBudget * 2);
-                let context = null;
-                const syncedUpdates = [];
-                await connection.transaction('immediate', async () => {
-                    var _a;
-                    context = await beginApply(store, connection, {
-                        progress: rowsApplied,
-                        backfills: staged.backfills,
-                        snapshotRows: progressive ? null : snapshotRows,
-                        changedAfterSequence: staged.stageSequence,
-                        protectedRows,
-                        skipTriggers,
-                        signal,
-                    });
-                    for (const sequence of slice) {
-                        if (signal && signal.aborted)
-                            throw cancellation();
-                        const rows = await connection.query('SELECT payload, subscription_key FROM g9_stage WHERE run_id = $r AND seq = $s', { $r: runId, $s: sequence });
-                        const row = rows[0];
-                        if (!row)
-                            continue;
-                        const payload = row[0];
-                        if (!(payload instanceof ArrayBuffer))
+            // Snapshot files (ADR-0072) apply where their part sits in the plan: at the sequence of their first segment. Checked and
+            // ATTACHed before the transaction (SQLite cannot attach inside one), detached whatever happens.
+            const files = await readStagedFiles(connection, runId);
+            for (const file of files)
+                await verifyStagedFile(store, file);
+            const units = sequences.map((sequence) => ({ sequence, file: -1 }));
+            files.forEach((file, i) => units.push({ sequence: file.firstSequence, file: i }));
+            units.sort((a, b) => a.sequence - b.sequence || a.file - b.file);
+            const detach = await attachFiles(connection, files);
+            try {
+                // Slicing only pays for itself when someone is listening for the slices (ADR-0053); a LARGE snapshot stays in one
+                // transaction with its indexes deferred (ADR-0056/0058).
+                const sliceRows = slices === null ? 0 : store.options.applySliceRows;
+                const snapshotRows = snapshotRowsByTable(staged.plan);
+                let largeSnapshot = files.length > 0;
+                if (store.options.rebuildIndexesAfterRows > 0)
+                    snapshotRows.forEach((rows) => (largeSnapshot = largeSnapshot || rows >= store.options.rebuildIndexesAfterRows));
+                const progressive = sliceRows > 0 && units.length > 1 && !largeSnapshot;
+                // Read ONCE, before any slice commits (ADR-0046 + ADR-0053).
+                const protectedRows = progressive ? await readProtectedRows(store, connection, staged.stageSequence) : null;
+                const skipTriggers = staged.plan.totalRows >= store.options.skipTriggersMinRows && !progressive;
+                const total = new G9ApplyReport();
+                let appliedTotal = 0;
+                let index = 0;
+                const started = Date.now();
+                let sliceBudget = sliceRows;
+                // do/while: a pull with no payload chunks still writes its cursors and clears its stage, exactly once.
+                do {
+                    if (signal && signal.aborted)
+                        throw cancellation();
+                    const slice = [];
+                    let final = true;
+                    for (; index < units.length; index++) {
+                        slice.push(units[index]);
+                        if (!progressive)
                             continue;
-                        await applyFrame(store, context, new Uint8Array(payload), row[1] === null ? null : asString(row[1]));
+                        if ((await stagedRowCount(connection, runId, slice.map((unit) => unit.sequence))) >= sliceBudget && index + 1 < units.length) {
+                            index++;
+                            final = false;
+                            break;
+                        }
                     }
-                    if (final) {
-                        // Cursors move only here, and only when still the one the plan started from (ADR-0060; IS, a first pull plans from NULL).
-                        for (const cursor of staged.plan.cursors) {
-                            const update = 'UPDATE g9_subscriptions SET cursor = $c, definition_hash = $d, auth_hash = $a, state = 0, error_code = NULL, last_success_utc = $now WHERE key = $k';
-                            const params = { $c: cursor.cursor, $d: cursor.definitionHash, $a: cursor.authHash, $now: nowMs(), $k: cursor.key };
-                            if (Object.prototype.hasOwnProperty.call(staged.plannedFromCursors, cursor.key)) {
-                                await context.flush();
-                                await connection.exec(update + ' AND cursor IS $from', { ...params, $from: (_a = staged.plannedFromCursors[cursor.key]) !== null && _a !== void 0 ? _a : null });
+                    if (store.options.applySliceRowsMax > sliceBudget)
+                        sliceBudget = Math.min(store.options.applySliceRowsMax, sliceBudget * 2);
+                    let context = null;
+                    let doneFiles = [];
+                    const syncedUpdates = [];
+                    await connection.transaction('immediate', async () => {
+                        var _a;
+                        context = await beginApply(store, connection, {
+                            progress: rowsApplied,
+                            backfills: staged.backfills,
+                            snapshotRows: progressive ? null : snapshotRows,
+                            changedAfterSequence: staged.stageSequence,
+                            protectedRows,
+                            skipTriggers,
+                            signal,
+                        });
+                        for (const unit of slice) {
+                            if (signal && signal.aborted)
+                                throw cancellation();
+                            if (unit.file >= 0) {
+                                await applySnapshotFile(store, context, attachedSchema(unit.file), files[unit.file]);
+                                continue;
                             }
-                            else {
-                                await context.flush();
-                                await connection.exec(update, params);
+                            const sequence = unit.sequence;
+                            const rows = await connection.query('SELECT payload, subscription_key FROM g9_stage WHERE run_id = $r AND seq = $s', { $r: runId, $s: sequence });
+                            const row = rows[0];
+                            if (!row)
+                                continue;
+                            const payload = row[0];
+                            if (!(payload instanceof ArrayBuffer))
+                                continue;
+                            await applyFrame(store, context, new Uint8Array(payload), row[1] === null ? null : asString(row[1]));
+                        }
+                        if (final) {
+                            // Cursors move only here, and only when still the one the plan started from (ADR-0060; IS, a first pull plans from NULL).
+                            // A subscription that came as a file counts its next safety reconciliation from the file's snapshot (ADR-0072).
+                            const reconciledAt = fileSnapshotTimes(staged.plan);
+                            for (const cursor of staged.plan.cursors) {
+                                const at = reconciledAt[cursor.key];
+                                const update = 'UPDATE g9_subscriptions SET cursor = $c, definition_hash = $d, auth_hash = $a, state = 0, error_code = NULL, last_success_utc = $now' +
+                                    (at === undefined ? '' : ', last_reconcile_utc = $rec') +
+                                    ' WHERE key = $k';
+                                const params = { $c: cursor.cursor, $d: cursor.definitionHash, $a: cursor.authHash, $now: nowMs(), $k: cursor.key };
+                                if (at !== undefined)
+                                    params['$rec'] = at;
+                                if (Object.prototype.hasOwnProperty.call(staged.plannedFromCursors, cursor.key)) {
+                                    await context.flush();
+                                    await connection.exec(update + ' AND cursor IS $from', { ...params, $from: (_a = staged.plannedFromCursors[cursor.key]) !== null && _a !== void 0 ? _a : null });
+                                }
+                                else {
+                                    await context.flush();
+                                    await connection.exec(update, params);
+                                }
+                            }
+                            for (const backfill of staged.backfills) {
+                                const table = store.byServerId.get(backfill.tableId);
+                                if (!table)
+                                    continue;
+                                const synced = table.syncedColumns.slice();
+                                for (const column of backfill.columns)
+                                    if (!synced.some((s) => s.toLowerCase() === column.toLowerCase()))
+                                        synced.push(column);
+                                await connection.exec('UPDATE g9_tables SET synced_columns = $c WHERE tid = $t', { $c: persistedJson(synced), $t: table.tid });
+                                syncedUpdates.push({ table, synced });
                             }
                         }
-                        for (const backfill of staged.backfills) {
-                            const table = store.byServerId.get(backfill.tableId);
-                            if (!table)
-                                continue;
-                            const synced = table.syncedColumns.slice();
-                            for (const column of backfill.columns)
-                                if (!synced.some((s) => s.toLowerCase() === column.toLowerCase()))
-                                    synced.push(column);
-                            await connection.exec('UPDATE g9_tables SET synced_columns = $c WHERE tid = $t', { $c: persistedJson(synced), $t: table.tid });
-                            syncedUpdates.push({ table, synced });
+                        await endApply(context);
+                        const consumed = slice.filter((unit) => unit.file < 0).map((unit) => unit.sequence);
+                        if (consumed.length > 0)
+                            await connection.exec('DELETE FROM g9_stage WHERE run_id = $r AND seq IN (' + consumed.join(',') + ')', { $r: runId });
+                        if (final) {
+                            // Everything left of the stage - markers, file checkpoints, file records - goes with the last slice.
+                            doneFiles = await takeStagedFiles(connection, '$r', { $r: runId });
+                            await connection.exec('DELETE FROM g9_stage WHERE run_id = $r; DELETE FROM g9_stage_runs WHERE run_id = $r', { $r: runId });
                         }
+                    });
+                    if (doneFiles.length > 0) {
+                        await detach();
+                        await deleteFiles(store, doneFiles);
                     }
-                    await endApply(context);
-                    if (slice.length > 0)
-                        await connection.exec('DELETE FROM g9_stage WHERE run_id = $r AND seq IN (' + slice.join(',') + ')', { $r: runId });
-                    if (final)
-                        await connection.exec('DELETE FROM g9_stage_runs WHERE run_id = $r', { $r: runId });
-                });
-                const applied = context;
-                for (const { table, synced } of syncedUpdates)
-                    table.syncedColumns = synced;
-                appliedTotal += applied.rows;
-                total.merge(applied.report);
-                rowsApplied === null || rowsApplied === void 0 ? void 0 : rowsApplied(appliedTotal);
-                slices === null || slices === void 0 ? void 0 : slices({ report: applied.report, rowsApplied: applied.rows, rowsAppliedTotal: appliedTotal, rowsPlanned: staged.plan.totalRows, isFinal: final });
-            } while (index < sequences.length);
-            total.diagnostics.elapsedMs = Date.now() - started;
-            if (total.diagnostics.transactions > 0 && appliedTotal > 0)
-                store.log('information', 20112, `G9SyncData applied ${appliedTotal} row(s): ${total.diagnostics.toString()}`);
-            await maintainAfterApply(store, connection, appliedTotal);
-            return total;
+                    const applied = context;
+                    for (const { table, synced } of syncedUpdates)
+                        table.syncedColumns = synced;
+                    appliedTotal += applied.rows;
+                    total.merge(applied.report);
+                    rowsApplied === null || rowsApplied === void 0 ? void 0 : rowsApplied(appliedTotal);
+                    slices === null || slices === void 0 ? void 0 : slices({ report: applied.report, rowsApplied: applied.rows, rowsAppliedTotal: appliedTotal, rowsPlanned: staged.plan.totalRows, isFinal: final });
+                } while (index < units.length);
+                total.diagnostics.elapsedMs = Date.now() - started;
+                if (total.diagnostics.transactions > 0 && appliedTotal > 0)
+                    store.log('information', 20112, `G9SyncData applied ${appliedTotal} row(s): ${total.diagnostics.toString()}`);
+                await maintainAfterApply(store, connection, appliedTotal);
+                return total;
+            }
+            finally {
+                await detach();
+            }
         }
         finally {
             if (restore)
@@ -637,8 +742,7 @@ export async function applyReconcile(store, subscriptionKey, table, fetchedFrame
                         await addMembership(context, local, key, subscription.slot, subscription.ephemeral ? subscriptionKey : null);
                         context.report.membershipAdded++;
                     }
-                    for (const key of moveOutKeys)
-                        await moveOut(store, context, local, key, subscriptionKey, subscription.slot, subscription.ephemeral, policy);
+                    await moveOutMany(store, context, local, moveOutKeys, subscriptionKey, subscription.slot, subscription.ephemeral, policy);
                 }
                 await endApply(context);
                 return context.report;