@quereus/sync 0.6.3 → 0.8.0

package/dist/src/sync/change-applicator.js

@@ -0,0 +1,243 @@
+/**
+ * Change application logic.
+ *
+ * Handles the 3-phase change application pattern:
+ *   1. resolveChange — CRDT conflict resolution (no writes)
+ *   2. applyToStore callback — write data to store
+ *   3. commitChangeMetadata — persist CRDT metadata
+ */
+import { compareHLC } from '../clock/hlc.js';
+import { siteIdEquals } from '../clock/site.js';
+import { persistHLCState, toError } from './sync-context.js';
+/**
+ * Apply change sets from a remote peer.
+ *
+ * Three-phase process:
+ *   1. Resolve all changes (no writes)
+ *   2. Apply data to store via callback
+ *   3. Commit CRDT metadata
+ */
+export async function applyChanges(ctx, changes) {
+    let applied = 0;
+    let skipped = 0;
+    let conflicts = 0;
+    const dataChangesToApply = [];
+    const schemaChangesToApply = [];
+    const appliedChanges = [];
+    const resolvedDataChanges = [];
+    const pendingSchemaMigrations = [];
+    // PHASE 1: Resolve all changes (no writes yet)
+    for (const changeSet of changes) {
+        ctx.hlcManager.receive(changeSet.hlc);
+        // Process schema migrations first (DDL before DML)
+        for (const migration of changeSet.schemaMigrations) {
+            const schemaVersion = migration.schemaVersion ??
+                (await ctx.schemaMigrations.getCurrentVersion(migration.schema, migration.table)) + 1;
+            const existingMigration = await ctx.schemaMigrations.getMigration(migration.schema, migration.table, schemaVersion);
+            if (existingMigration) {
+                if (compareHLC(migration.hlc, existingMigration.hlc) <= 0) {
+                    skipped++;
+                    continue;
+                }
+            }
+            schemaChangesToApply.push({
+                type: migration.type,
+                schema: migration.schema,
+                table: migration.table,
+                ddl: migration.ddl,
+            });
+            pendingSchemaMigrations.push({ migration, schemaVersion });
+            applied++;
+        }
+        // Resolve data changes
+        for (const change of changeSet.changes) {
+            const resolved = await resolveChange(ctx, change);
+            if (resolved.outcome === 'applied') {
+                applied++;
+                appliedChanges.push({ change, siteId: changeSet.siteId });
+                resolvedDataChanges.push(resolved);
+                if (resolved.dataChange) {
+                    dataChangesToApply.push(resolved.dataChange);
+                }
+            }
+            else if (resolved.outcome === 'skipped') {
+                skipped++;
+            }
+            else if (resolved.outcome === 'conflict') {
+                conflicts++;
+            }
+        }
+    }
+    // PHASE 2: Apply data and schema changes to the store via callback
+    if (ctx.applyToStore && (dataChangesToApply.length > 0 || schemaChangesToApply.length > 0)) {
+        try {
+            await ctx.applyToStore(dataChangesToApply, schemaChangesToApply, { remote: true });
+        }
+        catch (error) {
+            // Emit error state so UI can react. CRDT metadata is NOT committed,
+            // allowing the same changes to be re-resolved on the next sync attempt.
+            ctx.syncEvents.emitSyncStateChange({
+                status: 'error',
+                error: toError(error),
+            });
+            throw error;
+        }
+    }
+    // PHASE 3: Commit CRDT metadata
+    await commitChangeMetadata(ctx, resolvedDataChanges);
+    // Commit schema migration metadata
+    for (const { migration, schemaVersion } of pendingSchemaMigrations) {
+        await ctx.schemaMigrations.recordMigration(migration.schema, migration.table, {
+            type: migration.type,
+            ddl: migration.ddl,
+            hlc: migration.hlc,
+            schemaVersion,
+        });
+    }
+    // Emit remote change events
+    if (appliedChanges.length > 0) {
+        const changesBySite = new Map();
+        for (const { change, siteId } of appliedChanges) {
+            const siteKey = Array.from(siteId).join(',');
+            const siteChanges = changesBySite.get(siteKey);
+            if (siteChanges) {
+                siteChanges.push(change);
+            }
+            else {
+                changesBySite.set(siteKey, [change]);
+            }
+        }
+        const appliedAt = ctx.hlcManager.now();
+        for (const [siteKey, siteChanges] of changesBySite) {
+            const siteIdBytes = new Uint8Array(siteKey.split(',').map(Number));
+            ctx.syncEvents.emitRemoteChange({
+                siteId: siteIdBytes,
+                transactionId: crypto.randomUUID(),
+                changes: siteChanges,
+                appliedAt,
+            });
+        }
+    }
+    await persistHLCState(ctx);
+    return {
+        applied,
+        skipped,
+        conflicts,
+        transactions: changes.length,
+    };
+}
+/**
+ * Resolve CRDT conflicts for a single change WITHOUT writing metadata.
+ *
+ * Phase 1 of the 3-phase apply pattern.
+ */
+export async function resolveChange(ctx, change) {
+    // Skip changes that originated from ourselves (echo prevention)
+    if (siteIdEquals(change.hlc.siteId, ctx.getSiteId())) {
+        return { outcome: 'skipped', change };
+    }
+    if (change.type === 'delete') {
+        const existingTombstone = await ctx.tombstones.getTombstone(change.schema, change.table, change.pk);
+        if (existingTombstone && compareHLC(change.hlc, existingTombstone.hlc) <= 0) {
+            return { outcome: 'skipped', change };
+        }
+        return {
+            outcome: 'applied',
+            change,
+            dataChange: {
+                type: 'delete',
+                schema: change.schema,
+                table: change.table,
+                pk: change.pk,
+            },
+        };
+    }
+    else {
+        // Column change
+        const shouldApply = await ctx.columnVersions.shouldApplyWrite(change.schema, change.table, change.pk, change.column, change.hlc);
+        if (!shouldApply) {
+            const localVersion = await ctx.columnVersions.getColumnVersion(change.schema, change.table, change.pk, change.column);
+            if (localVersion) {
+                const conflictEvent = {
+                    table: change.table,
+                    pk: change.pk,
+                    column: change.column,
+                    localValue: localVersion.value,
+                    remoteValue: change.value,
+                    winner: 'local',
+                    winningHLC: localVersion.hlc,
+                };
+                ctx.syncEvents.emitConflictResolved(conflictEvent);
+            }
+            return { outcome: 'conflict', change };
+        }
+        // Check for tombstone blocking
+        const isBlocked = await ctx.tombstones.isDeletedAndBlocking(change.schema, change.table, change.pk, change.hlc, ctx.config.allowResurrection);
+        if (isBlocked) {
+            return { outcome: 'skipped', change };
+        }
+        const oldColumnVersion = await ctx.columnVersions.getColumnVersion(change.schema, change.table, change.pk, change.column) ?? undefined;
+        // Emit conflict event when remote overwrites an existing local version
+        if (oldColumnVersion) {
+            const conflictEvent = {
+                table: change.table,
+                pk: change.pk,
+                column: change.column,
+                localValue: oldColumnVersion.value,
+                remoteValue: change.value,
+                winner: 'remote',
+                winningHLC: change.hlc,
+            };
+            ctx.syncEvents.emitConflictResolved(conflictEvent);
+        }
+        return {
+            outcome: 'applied',
+            change,
+            oldColumnVersion,
+            dataChange: {
+                type: 'update',
+                schema: change.schema,
+                table: change.table,
+                pk: change.pk,
+                columns: { [change.column]: change.value },
+            },
+        };
+    }
+}
+/**
+ * Commit CRDT metadata for resolved changes.
+ *
+ * Phase 3 of the 3-phase apply pattern: called AFTER data is written to store.
+ */
+export async function commitChangeMetadata(ctx, resolvedChanges) {
+    if (resolvedChanges.length === 0)
+        return;
+    const batch = ctx.kv.batch();
+    for (const resolved of resolvedChanges) {
+        if (resolved.outcome !== 'applied')
+            continue;
+        const change = resolved.change;
+        if (change.type === 'delete') {
+            ctx.tombstones.setTombstoneBatch(batch, change.schema, change.table, change.pk, change.hlc);
+            ctx.changeLog.recordDeletionBatch(batch, change.hlc, change.schema, change.table, change.pk);
+        }
+        else {
+            // Delete old change log entry if exists
+            if (resolved.oldColumnVersion) {
+                ctx.changeLog.deleteEntryBatch(batch, resolved.oldColumnVersion.hlc, 'column', change.schema, change.table, change.pk, change.column);
+            }
+            ctx.columnVersions.setColumnVersionBatch(batch, change.schema, change.table, change.pk, change.column, { hlc: change.hlc, value: change.value });
+            ctx.changeLog.recordColumnChangeBatch(batch, change.hlc, change.schema, change.table, change.pk, change.column);
+        }
+    }
+    await batch.write();
+    // Handle column version deletions for delete operations (requires async iteration)
+    for (const resolved of resolvedChanges) {
+        if (resolved.outcome !== 'applied')
+            continue;
+        if (resolved.change.type === 'delete') {
+            await ctx.columnVersions.deleteRowVersions(resolved.change.schema, resolved.change.table, resolved.change.pk);
+        }
+    }
+}
+//# sourceMappingURL=change-applicator.js.map

package/dist/src/sync/change-applicator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"change-applicator.js","sourceRoot":"","sources":["../../../src/sync/change-applicator.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAe,MAAM,kBAAkB,CAAC;AAY7D,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAc7D;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,GAAgB,EAChB,OAAoB;IAEpB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,kBAAkB,GAAwB,EAAE,CAAC;IACnD,MAAM,oBAAoB,GAA0B,EAAE,CAAC;IACvD,MAAM,cAAc,GAA8C,EAAE,CAAC;IACrE,MAAM,mBAAmB,GAAqB,EAAE,CAAC;IACjD,MAAM,uBAAuB,GAGxB,EAAE,CAAC;IAER,+CAA+C;IAC/C,KAAK,MAAM,SAAS,IAAI,OAAO,EAAE,CAAC;QACjC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAEtC,mDAAmD;QACnD,KAAK,MAAM,SAAS,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;YACpD,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa;gBAC5C,CAAC,MAAM,GAAG,CAAC,gBAAgB,CAAC,iBAAiB,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC;YAEvF,MAAM,iBAAiB,GAAG,MAAM,GAAG,CAAC,gBAAgB,CAAC,YAAY,CAChE,SAAS,CAAC,MAAM,EAChB,SAAS,CAAC,KAAK,EACf,aAAa,CACb,CAAC;YAEF,IAAI,iBAAiB,EAAE,CAAC;gBACvB,IAAI,UAAU,CAAC,SAAS,CAAC,GAAG,EAAE,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC3D,OAAO,EAAE,CAAC;oBACV,SAAS;gBACV,CAAC;YACF,CAAC;YAED,oBAAoB,CAAC,IAAI,CAAC;gBACzB,IAAI,EAAE,SAAS,CAAC,IAAI;gBACpB,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,GAAG,EAAE,SAAS,CAAC,GAAG;aAClB,CAAC,CAAC;YACH,uBAAuB,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,aAAa,EAAE,CAAC,CAAC;YAC3D,OAAO,EAAE,CAAC;QACX,CAAC;QAED,uBAAuB;QACvB,KAAK,MAAM,MAAM,IAAI,SAAS,CAAC,OAAO,EAAE,CAAC;YACxC,MAAM,QAAQ,GAAG,MAAM,aAAa,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAClD,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBACpC,OAAO,EAAE,CAAC;gBACV,cAAc,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;gBAC1D,mBAAmB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACnC,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;oBACzB,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;gBAC9C,CAAC;YACF,CAAC;iBAAM,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC3C,OAAO,EAAE,CAAC;YACX,CAAC;iBAAM,IAAI,QAAQ,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;gBAC5C,SAAS,EAAE,CAAC;YACb,CAAC;QACF,CAAC;IACF,CAAC;IAED,mEAAmE;IACnE,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC;QAC5F,IAAI,CAAC;YACJ,MAAM,GAAG,CAAC,YAAY,CAAC,kBAAkB,EAAE,oBAAoB,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QACpF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,oEAAoE;YACpE,wEAAwE;YACxE,GAAG,CAAC,UAAU,CAAC,mBAAmB,CAAC;gBAClC,MAAM,EAAE,OAAO;gBACf,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC;aACrB,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACb,CAAC;IACF,CAAC;IAED,gCAAgC;IAChC,MAAM,oBAAoB,CAAC,GAAG,EAAE,mBAAmB,CAAC,CAAC;IAErD,mCAAmC;IACnC,KAAK,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,IAAI,uBAAuB,EAAE,CAAC;QACpE,MAAM,GAAG,CAAC,gBAAgB,CAAC,eAAe,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,KAAK,EAAE;YAC7E,IAAI,EAAE,SAAS,CAAC,IAAI;YACpB,GAAG,EAAE,SAAS,CAAC,GAAG;YAClB,GAAG,EAAE,SAAS,CAAC,GAAG;YAClB,aAAa;SACb,CAAC,CAAC;IACJ,CAAC;IAED,4BAA4B;IAC5B,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC/B,MAAM,aAAa,GAAG,IAAI,GAAG,EAAoB,CAAC;QAClD,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,cAAc,EAAE,CAAC;YACjD,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC7C,MAAM,WAAW,GAAG,aAAa,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC/C,IAAI,WAAW,EAAE,CAAC;gBACjB,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1B,CAAC;iBAAM,CAAC;gBACP,aAAa,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;YACtC,CAAC;QACF,CAAC;QAED,MAAM,SAAS,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,CAAC;QACvC,KAAK,MAAM,CAAC,OAAO,EAAE,WAAW,CAAC,IAAI,aAAa,EAAE,CAAC;YACpD,MAAM,WAAW,GAAG,IAAI,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;YACnE,GAAG,CAAC,UAAU,CAAC,gBAAgB,CAAC;gBAC/B,MAAM,EAAE,WAAW;gBACnB,aAAa,EAAE,MAAM,CAAC,UAAU,EAAE;gBAClC,OAAO,EAAE,WAAW;gBACpB,SAAS;aACT,CAAC,CAAC;QACJ,CAAC;IACF,CAAC;IAED,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;IAE3B,OAAO;QACN,OAAO;QACP,OAAO;QACP,SAAS;QACT,YAAY,EAAE,OAAO,CAAC,MAAM;KAC5B,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,GAAgB,EAChB,MAAc;IAEd,gEAAgE;IAChE,IAAI,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,SAAS,EAAE,CAAC,EAAE,CAAC;QACtD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;IACvC,CAAC;IAED,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,iBAAiB,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,YAAY,CAC1D,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,EAAE,CACT,CAAC;QAEF,IAAI,iBAAiB,IAAI,UAAU,CAAC,MAAM,CAAC,GAAG,EAAE,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7E,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;QACvC,CAAC;QAED,OAAO;YACN,OAAO,EAAE,SAAS;YAClB,MAAM;YACN,UAAU,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,EAAE,EAAE,MAAM,CAAC,EAAE;aACb;SACD,CAAC;IACH,CAAC;SAAM,CAAC;QACP,gBAAgB;QAChB,MAAM,WAAW,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,gBAAgB,CAC5D,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,GAAG,CACV,CAAC;QAEF,IAAI,CAAC,WAAW,EAAE,CAAC;YAClB,MAAM,YAAY,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,gBAAgB,CAC7D,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,MAAM,CACb,CAAC;YAEF,IAAI,YAAY,EAAE,CAAC;gBAClB,MAAM,aAAa,GAAkB;oBACpC,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,EAAE,EAAE,MAAM,CAAC,EAAE;oBACb,MAAM,EAAE,MAAM,CAAC,MAAM;oBACrB,UAAU,EAAE,YAAY,CAAC,KAAK;oBAC9B,WAAW,EAAE,MAAM,CAAC,KAAK;oBACzB,MAAM,EAAE,OAAO;oBACf,UAAU,EAAE,YAAY,CAAC,GAAG;iBAC5B,CAAC;gBACF,GAAG,CAAC,UAAU,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;YACpD,CAAC;YAED,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;QACxC,CAAC;QAED,+BAA+B;QAC/B,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,oBAAoB,CAC1D,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,GAAG,EACV,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAC5B,CAAC;QAEF,IAAI,SAAS,EAAE,CAAC;YACf,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;QACvC,CAAC;QAED,MAAM,gBAAgB,GAAG,MAAM,GAAG,CAAC,cAAc,CAAC,gBAAgB,CACjE,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,MAAM,CACb,IAAI,SAAS,CAAC;QAEf,uEAAuE;QACvE,IAAI,gBAAgB,EAAE,CAAC;YACtB,MAAM,aAAa,GAAkB;gBACpC,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,UAAU,EAAE,gBAAgB,CAAC,KAAK;gBAClC,WAAW,EAAE,MAAM,CAAC,KAAK;gBACzB,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,MAAM,CAAC,GAAG;aACtB,CAAC;YACF,GAAG,CAAC,UAAU,CAAC,oBAAoB,CAAC,aAAa,CAAC,CAAC;QACpD,CAAC;QAED,OAAO;YACN,OAAO,EAAE,SAAS;YAClB,MAAM;YACN,gBAAgB;YAChB,UAAU,EAAE;gBACX,IAAI,EAAE,QAAQ;gBACd,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,KAAK,EAAE;aAC1C;SACD,CAAC;IACH,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,GAAgB,EAChB,eAAiC;IAEjC,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAEzC,MAAM,KAAK,GAAG,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAE7B,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;QACxC,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS;YAAE,SAAS;QAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAE/B,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC9B,GAAG,CAAC,UAAU,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;YAC5F,GAAG,CAAC,SAAS,CAAC,mBAAmB,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAC9F,CAAC;aAAM,CAAC;YACP,wCAAwC;YACxC,IAAI,QAAQ,CAAC,gBAAgB,EAAE,CAAC;gBAC/B,GAAG,CAAC,SAAS,CAAC,gBAAgB,CAC7B,KAAK,EACL,QAAQ,CAAC,gBAAgB,CAAC,GAAG,EAC7B,QAAQ,EACR,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,MAAM,CACb,CAAC;YACH,CAAC;YAED,GAAG,CAAC,cAAc,CAAC,qBAAqB,CACvC,KAAK,EACL,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,MAAM,EACb,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CACxC,CAAC;YAEF,GAAG,CAAC,SAAS,CAAC,uBAAuB,CACpC,KAAK,EACL,MAAM,CAAC,GAAG,EACV,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,KAAK,EACZ,MAAM,CAAC,EAAE,EACT,MAAM,CAAC,MAAM,CACb,CAAC;QACH,CAAC;IACF,CAAC;IAED,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IAEpB,mFAAmF;IACnF,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;QACxC,IAAI,QAAQ,CAAC,OAAO,KAAK,SAAS;YAAE,SAAS;QAC7C,IAAI,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvC,MAAM,GAAG,CAAC,cAAc,CAAC,iBAAiB,CACzC,QAAQ,CAAC,MAAM,CAAC,MAAM,EACtB,QAAQ,CAAC,MAAM,CAAC,KAAK,EACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,CAClB,CAAC;QACH,CAAC;IACF,CAAC;AACF,CAAC"}

package/dist/src/sync/snapshot-stream.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Streaming snapshot operations.
+ *
+ * Handles chunked snapshot generation, application, and checkpoint
+ * management for memory-efficient sync of large databases.
+ */
+import type { SnapshotCheckpoint } from './manager.js';
+import type { SnapshotChunk, SnapshotProgress } from './protocol.js';
+import type { SyncContext } from './sync-context.js';
+/**
+ * Stream a snapshot as chunks for memory-efficient transfer.
+ */
+export declare function getSnapshotStream(ctx: SyncContext, chunkSize?: number): AsyncIterable<SnapshotChunk>;
+/**
+ * Resume a snapshot transfer from a checkpoint.
+ */
+export declare function resumeSnapshotStream(ctx: SyncContext, checkpoint: SnapshotCheckpoint): AsyncIterable<SnapshotChunk>;
+/**
+ * Apply a streamed snapshot, processing chunks as they arrive.
+ */
+export declare function applySnapshotStream(ctx: SyncContext, chunks: AsyncIterable<SnapshotChunk>, onProgress?: (progress: SnapshotProgress) => void): Promise<void>;
+/**
+ * Retrieve a saved checkpoint for an in-progress snapshot.
+ */
+export declare function getSnapshotCheckpoint(ctx: SyncContext, snapshotId: string): Promise<SnapshotCheckpoint | undefined>;
+//# sourceMappingURL=snapshot-stream.d.ts.map

package/dist/src/sync/snapshot-stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"snapshot-stream.d.ts","sourceRoot":"","sources":["../../../src/sync/snapshot-stream.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAiBH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAC;AACvD,OAAO,KAAK,EACX,aAAa,EACb,gBAAgB,EAShB,MAAM,eAAe,CAAC;AACvB,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAiKrD;;GAEG;AACH,wBAAuB,iBAAiB,CACvC,GAAG,EAAE,WAAW,EAChB,SAAS,GAAE,MAAoC,GAC7C,aAAa,CAAC,aAAa,CAAC,CAO9B;AAED;;GAEG;AACH,wBAAuB,oBAAoB,CAC1C,GAAG,EAAE,WAAW,EAChB,UAAU,EAAE,kBAAkB,GAC5B,aAAa,CAAC,aAAa,CAAC,CAS9B;AAMD;;GAEG;AACH,wBAAsB,mBAAmB,CACxC,GAAG,EAAE,WAAW,EAChB,MAAM,EAAE,aAAa,CAAC,aAAa,CAAC,EACpC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,GAC/C,OAAO,CAAC,IAAI,CAAC,CA4Mf;AAMD;;GAEG;AACH,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,WAAW,EAChB,UAAU,EAAE,MAAM,GAChB,OAAO,CAAC,kBAAkB,GAAG,SAAS,CAAC,CAiBzC"}

package/dist/src/sync/snapshot-stream.js

@@ -0,0 +1,375 @@
+/**
+ * Streaming snapshot operations.
+ *
+ * Handles chunked snapshot generation, application, and checkpoint
+ * management for memory-efficient sync of large databases.
+ */
+import { deserializeColumnVersion } from '../metadata/column-version.js';
+import { deserializeMigration } from '../metadata/schema-migration.js';
+import { buildAllColumnVersionsScanBounds, buildAllTombstonesScanBounds, buildAllSchemaMigrationsScanBounds, buildAllChangeLogScanBounds, buildTableColumnVersionScanBounds, parseColumnVersionKey, parseSchemaMigrationKey, encodePK, } from '../metadata/keys.js';
+import { persistHLCState } from './sync-context.js';
+/** Default chunk size for streaming snapshots. */
+const DEFAULT_SNAPSHOT_CHUNK_SIZE = 1000;
+/** Key prefix for snapshot checkpoints. */
+const CHECKPOINT_PREFIX = 'sc:';
+/**
+ * Shared generator that streams snapshot chunks.
+ *
+ * Both `getSnapshotStream` and `resumeSnapshotStream` delegate here,
+ * differing only in initial parameters (skip set, identity source, entry count).
+ */
+async function* streamSnapshotChunks(ctx, opts) {
+    const { snapshotId, siteId, hlc, chunkSize, completedTables, initialEntryCount } = opts;
+    const completedSet = completedTables ?? new Set();
+    // Count tables and migrations for header
+    const tableKeys = new Set();
+    const cvBounds = buildAllColumnVersionsScanBounds();
+    for await (const entry of ctx.kv.iterate(cvBounds)) {
+        const parsed = parseColumnVersionKey(entry.key);
+        if (parsed)
+            tableKeys.add(`${parsed.schema}.${parsed.table}`);
+    }
+    let migrationCount = 0;
+    const smBounds = buildAllSchemaMigrationsScanBounds();
+    for await (const _entry of ctx.kv.iterate(smBounds)) {
+        migrationCount++;
+    }
+    // Yield header
+    const header = {
+        type: 'header',
+        siteId,
+        hlc,
+        tableCount: tableKeys.size,
+        migrationCount,
+        snapshotId,
+    };
+    yield header;
+    // Stream each table, skipping completed ones
+    let totalEntries = initialEntryCount ?? 0;
+    for (const tableKey of tableKeys) {
+        if (completedSet.has(tableKey))
+            continue;
+        const [schema, table] = tableKey.split('.');
+        const tableCvBounds = buildTableColumnVersionScanBounds(schema, table);
+        // Yield table start (entry count filled in at table-end)
+        const tableStart = {
+            type: 'table-start',
+            schema,
+            table,
+            estimatedEntries: 0,
+        };
+        yield tableStart;
+        // Stream column versions in chunks (single pass per table)
+        let entries = [];
+        let entriesWritten = 0;
+        for await (const entry of ctx.kv.iterate(tableCvBounds)) {
+            const parsed = parseColumnVersionKey(entry.key);
+            if (!parsed)
+                continue;
+            const cv = deserializeColumnVersion(entry.value);
+            const versionKey = `${encodePK(parsed.pk)}:${parsed.column}`;
+            entries.push([versionKey, cv.hlc, cv.value]);
+            entriesWritten++;
+            if (entries.length >= chunkSize) {
+                const chunk = {
+                    type: 'column-versions',
+                    schema,
+                    table,
+                    entries,
+                };
+                yield chunk;
+                entries = [];
+            }
+        }
+        // Yield remaining entries
+        if (entries.length > 0) {
+            const chunk = {
+                type: 'column-versions',
+                schema,
+                table,
+                entries,
+            };
+            yield chunk;
+        }
+        // Yield table end
+        const tableEnd = {
+            type: 'table-end',
+            schema,
+            table,
+            entriesWritten,
+        };
+        yield tableEnd;
+        totalEntries += entriesWritten;
+    }
+    // Stream schema migrations
+    for await (const entry of ctx.kv.iterate(smBounds)) {
+        const parsed = parseSchemaMigrationKey(entry.key);
+        if (!parsed)
+            continue;
+        const migration = deserializeMigration(entry.value);
+        const migrationChunk = {
+            type: 'schema-migration',
+            migration: {
+                type: migration.type,
+                schema: parsed.schema,
+                table: parsed.table,
+                ddl: migration.ddl,
+                hlc: migration.hlc,
+                schemaVersion: migration.schemaVersion,
+            },
+        };
+        yield migrationChunk;
+    }
+    // Yield footer
+    const footer = {
+        type: 'footer',
+        snapshotId,
+        totalTables: tableKeys.size,
+        totalEntries,
+        totalMigrations: migrationCount,
+    };
+    yield footer;
+}
+/**
+ * Stream a snapshot as chunks for memory-efficient transfer.
+ */
+export async function* getSnapshotStream(ctx, chunkSize = DEFAULT_SNAPSHOT_CHUNK_SIZE) {
+    yield* streamSnapshotChunks(ctx, {
+        snapshotId: crypto.randomUUID(),
+        siteId: ctx.getSiteId(),
+        hlc: ctx.getCurrentHLC(),
+        chunkSize,
+    });
+}
+/**
+ * Resume a snapshot transfer from a checkpoint.
+ */
+export async function* resumeSnapshotStream(ctx, checkpoint) {
+    yield* streamSnapshotChunks(ctx, {
+        snapshotId: checkpoint.snapshotId,
+        siteId: checkpoint.siteId,
+        hlc: checkpoint.hlc,
+        chunkSize: DEFAULT_SNAPSHOT_CHUNK_SIZE,
+        completedTables: new Set(checkpoint.completedTables),
+        initialEntryCount: checkpoint.entriesProcessed,
+    });
+}
+// ============================================================================
+// Snapshot Application
+// ============================================================================
+/**
+ * Apply a streamed snapshot, processing chunks as they arrive.
+ */
+export async function applySnapshotStream(ctx, chunks, onProgress) {
+    let snapshotId;
+    let snapshotHLC;
+    let totalTables = 0;
+    let totalEntries = 0;
+    let tablesProcessed = 0;
+    let entriesProcessed = 0;
+    let currentTable;
+    const completedTables = [];
+    // Pending data to apply to store (batched for efficiency)
+    let pendingDataChanges = [];
+    let pendingSchemaChanges = [];
+    const DATA_FLUSH_SIZE = 100;
+    const flushDataToStore = async () => {
+        if (ctx.applyToStore && (pendingDataChanges.length > 0 || pendingSchemaChanges.length > 0)) {
+            await ctx.applyToStore(pendingDataChanges, pendingSchemaChanges, { remote: true });
+            pendingDataChanges = [];
+            pendingSchemaChanges = [];
+        }
+    };
+    // Clear existing metadata before applying
+    const clearBatch = ctx.kv.batch();
+    for await (const entry of ctx.kv.iterate(buildAllColumnVersionsScanBounds())) {
+        clearBatch.delete(entry.key);
+    }
+    for await (const entry of ctx.kv.iterate(buildAllTombstonesScanBounds())) {
+        clearBatch.delete(entry.key);
+    }
+    for await (const entry of ctx.kv.iterate(buildAllChangeLogScanBounds())) {
+        clearBatch.delete(entry.key);
+    }
+    await clearBatch.write();
+    // Process chunks
+    let batch = ctx.kv.batch();
+    let batchSize = 0;
+    const BATCH_FLUSH_SIZE = 1000;
+    let currentTableSchema;
+    let currentTableName;
+    const rowColumns = new Map();
+    for await (const chunk of chunks) {
+        switch (chunk.type) {
+            case 'header':
+                snapshotId = chunk.snapshotId;
+                snapshotHLC = chunk.hlc;
+                totalTables = chunk.tableCount;
+                break;
+            case 'table-start':
+                currentTable = `${chunk.schema}.${chunk.table}`;
+                currentTableSchema = chunk.schema;
+                currentTableName = chunk.table;
+                totalEntries += chunk.estimatedEntries;
+                rowColumns.clear();
+                break;
+            case 'column-versions':
+                for (const [versionKey, hlc, value] of chunk.entries) {
+                    const lastColon = versionKey.lastIndexOf(':');
+                    if (lastColon === -1)
+                        continue;
+                    const rowKey = versionKey.slice(0, lastColon);
+                    const column = versionKey.slice(lastColon + 1);
+                    const pk = JSON.parse(rowKey);
+                    // Track column for data application
+                    if (!rowColumns.has(rowKey)) {
+                        rowColumns.set(rowKey, {});
+                    }
+                    rowColumns.get(rowKey)[column] = value;
+                    // Write CRDT metadata
+                    ctx.columnVersions.setColumnVersionBatch(batch, chunk.schema, chunk.table, pk, column, { hlc, value });
+                    ctx.changeLog.recordColumnChangeBatch(batch, hlc, chunk.schema, chunk.table, pk, column);
+                    batchSize++;
+                    entriesProcessed++;
+                    if (batchSize >= BATCH_FLUSH_SIZE) {
+                        await batch.write();
+                        batch = ctx.kv.batch();
+                        batchSize = 0;
+                        // Save checkpoint
+                        if (snapshotId && snapshotHLC) {
+                            await saveSnapshotCheckpoint(ctx, {
+                                snapshotId,
+                                siteId: ctx.getSiteId(),
+                                hlc: snapshotHLC,
+                                lastTableIndex: tablesProcessed,
+                                lastEntryIndex: entriesProcessed,
+                                completedTables: [...completedTables],
+                                entriesProcessed,
+                                createdAt: Date.now(),
+                            });
+                        }
+                    }
+                }
+                if (onProgress && snapshotId) {
+                    onProgress({
+                        snapshotId,
+                        tablesProcessed,
+                        totalTables,
+                        entriesProcessed,
+                        totalEntries,
+                        currentTable,
+                    });
+                }
+                break;
+            case 'table-end':
+                // Flush accumulated rows to store
+                if (currentTableSchema && currentTableName) {
+                    for (const [rowKey, columns] of rowColumns) {
+                        const pk = JSON.parse(rowKey);
+                        pendingDataChanges.push({
+                            type: 'update',
+                            schema: currentTableSchema,
+                            table: currentTableName,
+                            pk,
+                            columns,
+                        });
+                        if (pendingDataChanges.length >= DATA_FLUSH_SIZE) {
+                            await flushDataToStore();
+                        }
+                    }
+                    rowColumns.clear();
+                }
+                tablesProcessed++;
+                if (currentTable) {
+                    completedTables.push(currentTable);
+                }
+                break;
+            case 'schema-migration': {
+                const migration = chunk.migration;
+                pendingSchemaChanges.push({
+                    type: migration.type,
+                    schema: migration.schema,
+                    table: migration.table,
+                    ddl: migration.ddl,
+                });
+                const schemaVersion = migration.schemaVersion ??
+                    (await ctx.schemaMigrations.getCurrentVersion(migration.schema, migration.table)) + 1;
+                await ctx.schemaMigrations.recordMigration(migration.schema, migration.table, {
+                    type: migration.type,
+                    ddl: migration.ddl,
+                    hlc: migration.hlc,
+                    schemaVersion,
+                });
+                break;
+            }
+            case 'footer':
+                // Flush remaining data to store
+                await flushDataToStore();
+                // Flush remaining metadata batch
+                if (batchSize > 0) {
+                    await batch.write();
+                }
+                // Update HLC
+                if (snapshotHLC) {
+                    ctx.hlcManager.receive(snapshotHLC);
+                    await persistHLCState(ctx);
+                }
+                // Clear checkpoint
+                if (snapshotId) {
+                    await clearSnapshotCheckpoint(ctx, snapshotId);
+                }
+                // Emit sync state change
+                if (snapshotHLC) {
+                    ctx.syncEvents.emitSyncStateChange({ status: 'synced', lastSyncHLC: snapshotHLC });
+                }
+                break;
+        }
+    }
+}
+// ============================================================================
+// Checkpoint Management
+// ============================================================================
+/**
+ * Retrieve a saved checkpoint for an in-progress snapshot.
+ */
+export async function getSnapshotCheckpoint(ctx, snapshotId) {
+    const key = new TextEncoder().encode(`${CHECKPOINT_PREFIX}${snapshotId}`);
+    const data = await ctx.kv.get(key);
+    if (!data)
+        return undefined;
+    const json = new TextDecoder().decode(data);
+    const obj = JSON.parse(json);
+    return {
+        ...obj,
+        hlc: {
+            wallTime: BigInt(obj.hlc.wallTime),
+            counter: obj.hlc.counter,
+            siteId: new Uint8Array(obj.hlc.siteId),
+        },
+        siteId: new Uint8Array(obj.siteId),
+    };
+}
+/**
+ * Save a checkpoint during a streaming snapshot apply.
+ */
+async function saveSnapshotCheckpoint(ctx, checkpoint) {
+    const key = new TextEncoder().encode(`${CHECKPOINT_PREFIX}${checkpoint.snapshotId}`);
+    const json = JSON.stringify({
+        ...checkpoint,
+        hlc: {
+            wallTime: checkpoint.hlc.wallTime.toString(),
+            counter: checkpoint.hlc.counter,
+            siteId: Array.from(checkpoint.hlc.siteId),
+        },
+        siteId: Array.from(checkpoint.siteId),
+    });
+    await ctx.kv.put(key, new TextEncoder().encode(json));
+}
+/**
+ * Clear checkpoint after a snapshot completes successfully.
+ */
+async function clearSnapshotCheckpoint(ctx, snapshotId) {
+    const key = new TextEncoder().encode(`${CHECKPOINT_PREFIX}${snapshotId}`);
+    await ctx.kv.delete(key);
+}
+//# sourceMappingURL=snapshot-stream.js.map