@syncular/server 0.0.6-95 → 0.1.0

package/dist/pull.js

@@ -1,38 +1,200 @@
-import { captureSyncException, countSyncMetric, distributionSyncMetric, encodeSnapshotRowFrames, encodeSnapshotRows, gzipBytes, gzipBytesToStream, randomId, SYNC_SNAPSHOT_CHUNK_COMPRESSION, SYNC_SNAPSHOT_CHUNK_ENCODING, sha256Hex, startSyncSpan, } from '@syncular/core';
-import { getServerBootstrapOrderFor, getServerHandlerOrThrow, } from './handlers/collection.js';
-import { EXTERNAL_CLIENT_ID } from './notify.js';
-import { insertSnapshotChunk, readSnapshotChunkRefByPageKey, } from './snapshot-chunks.js';
-import { createMemoryScopeCache, } from './subscriptions/cache.js';
-import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve.js';
+import { bytesToReadableStream, captureSyncException, concatByteChunks, countSyncMetric, createSnapshotManifest, distributionSyncMetric, encodeBinarySnapshotTable, gzipBytes, randomId, SYNC_SCOPED_SNAPSHOT_ARTIFACT_KIND_SQLITE_V1, SYNC_SNAPSHOT_CHUNK_COMPRESSION, SYNC_SNAPSHOT_CHUNK_ENCODING, sha256Hex, snapshotScopeDigestFromCacheKey, startSyncSpan, } from '@syncular/core';
+import { createWireSubscriptionIntegrity, SYNCULAR_COMMIT_GENESIS_ROOT, } from './commit-integrity';
+import { getServerBootstrapOrderFor, } from './handlers/collection';
+import { EXTERNAL_CLIENT_ID } from './notify';
+import { sortServerPullPlugins, } from './plugins/types';
+import { createScopedSnapshotArtifactScopeCacheKey, readBestScopedSnapshotArtifactRefForPageCapacity, } from './snapshot-artifacts';
+import { createSnapshotChunkScopeCacheKey, insertSnapshotChunk, readSnapshotChunkRefByPageKey, } from './snapshot-chunks';
+import { createMemoryScopeCache, } from './subscriptions/cache';
+import { resolveEffectiveScopesForSubscriptions } from './subscriptions/resolve';
 const defaultScopeCache = createMemoryScopeCache();
-function concatByteChunks(chunks) {
-    if (chunks.length === 1) {
-        return chunks[0] ?? new Uint8Array();
+const DEFAULT_MAX_BINARY_SNAPSHOT_BUNDLE_ROWS = 50_000;
+const DEFAULT_SNAPSHOT_CHUNK_GZIP_LEVEL = 1;
+const MAX_PULL_TRANSACTION_RETRIES = 2;
+const PULL_TRANSACTION_RETRY_DELAY_MS = 15;
+function toResponseChunkRef(ref) {
+    return {
+        id: ref.id,
+        byteLength: ref.byteLength,
+        sha256: ref.sha256,
+        encoding: ref.encoding,
+        compression: ref.compression,
+    };
+}
+async function createChunkedSnapshotManifest(args) {
+    return createSnapshotManifest({
+        version: 1,
+        table: args.table,
+        asOfCommitSeq: args.asOfCommitSeq,
+        scopeDigest: snapshotScopeDigestFromCacheKey(args.scopeKey),
+        rowCursor: args.rowCursor,
+        rowLimit: args.rowLimit,
+        nextRowCursor: args.nextRowCursor,
+        isFirstPage: args.isFirstPage,
+        isLastPage: args.isLastPage,
+        chunks: args.chunks.map(toResponseChunkRef),
+    });
+}
+function normalizeFeatureSet(features) {
+    return Array.from(new Set(features ?? [])).sort();
+}
+function resolveSnapshotArtifactSelection(request, clientSchemaVersion) {
+    if (!request)
+        return null;
+    if (!request.artifactKinds.includes(SYNC_SCOPED_SNAPSHOT_ARTIFACT_KIND_SQLITE_V1)) {
+        return null;
     }
-    let total = 0;
-    for (const chunk of chunks) {
-        total += chunk.length;
+    if (!request.compressions?.includes(SYNC_SNAPSHOT_CHUNK_COMPRESSION)) {
+        return null;
     }
-    const merged = new Uint8Array(total);
-    let offset = 0;
-    for (const chunk of chunks) {
-        merged.set(chunk, offset);
-        offset += chunk.length;
+    return {
+        artifactKind: SYNC_SCOPED_SNAPSHOT_ARTIFACT_KIND_SQLITE_V1,
+        compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+        schemaVersion: String(clientSchemaVersion),
+        featureSet: normalizeFeatureSet(request.featureSet),
+    };
+}
+function resolveSnapshotBinaryColumns(handler, schemaVersion) {
+    const versioned = handler.snapshotBinaryColumnsForVersion?.(schemaVersion);
+    return versioned === undefined
+        ? handler.snapshotBinaryColumns
+        : (versioned ?? undefined);
+}
+function resolveSnapshotBinaryEncoder(handler, schemaVersion) {
+    const versioned = handler.snapshotBinaryEncoderForVersion?.(schemaVersion);
+    return versioned === undefined
+        ? handler.snapshotBinaryEncoder
+        : (versioned ?? undefined);
+}
+function createPullBootstrapTimings() {
+    return {
+        snapshotQueryMs: 0,
+        binaryEncodeMs: 0,
+        chunkCacheLookupMs: 0,
+        artifactCacheLookupMs: 0,
+        chunkGzipMs: 0,
+        chunkHashMs: 0,
+        chunkPersistMs: 0,
+    };
+}
+function resolveBinarySnapshotBundleRowLimit(args) {
+    const pageSize = Math.max(1, args.limitSnapshotRows);
+    const pagesRemaining = Math.max(1, args.pagesRemaining);
+    const maxBundlePages = Math.max(1, Math.ceil(DEFAULT_MAX_BINARY_SNAPSHOT_BUNDLE_ROWS / pageSize));
+    return pageSize * Math.min(pagesRemaining, maxBundlePages);
+}
+async function gzipByteChunks(chunks, gzipLevel) {
+    return gzipBytes(concatByteChunks(chunks), {
+        level: gzipLevel,
+    });
+}
+async function encodeCompressedSnapshotChunk(chunks, gzipLevel) {
+    const gzipStartedAt = Date.now();
+    const body = await gzipByteChunks(chunks, gzipLevel);
+    const gzipMs = Math.max(0, Date.now() - gzipStartedAt);
+    const hashStartedAt = Date.now();
+    const sha256 = await sha256Hex(body);
+    const hashMs = Math.max(0, Date.now() - hashStartedAt);
+    return { body, sha256, gzipMs, hashMs };
+}
+async function encodeCompressedSnapshotChunkToStream(chunks, gzipLevel) {
+    const encoded = await encodeCompressedSnapshotChunk(chunks, gzipLevel);
+    return {
+        stream: bytesToReadableStream(encoded.body),
+        byteLength: encoded.body.length,
+        sha256: encoded.sha256,
+        gzipMs: encoded.gzipMs,
+        hashMs: encoded.hashMs,
+    };
+}
+function encodeBinarySnapshotRows(table, rows, columns) {
+    const recordRows = rows.map((row) => toSnapshotRecordRow(table, row));
+    return encodeBinarySnapshotTable({
+        table,
+        columns: columns ?? inferBinarySnapshotColumns(recordRows),
+        rows: recordRows,
+    });
+}
+function toSnapshotRecordRow(table, row) {
+    if (row == null ||
+        typeof row !== 'object' ||
+        Array.isArray(row) ||
+        row instanceof Uint8Array ||
+        row instanceof ArrayBuffer) {
+        throw new Error(`Cannot encode binary snapshot for table ${table}: snapshot rows must be objects`);
     }
-    return merged;
+    return row;
 }
-/**
- * Generate a stable cache key for snapshot chunks.
- */
-async function scopesToCacheKey(scopes) {
-    const sorted = Object.entries(scopes)
-        .sort(([a], [b]) => a.localeCompare(b))
-        .map(([k, v]) => {
-        const arr = Array.isArray(v) ? [...v].sort() : [v];
-        return `${k}:${arr.join(',')}`;
-    })
-        .join('|');
-    return await sha256Hex(sorted);
+function inferBinarySnapshotColumns(rows) {
+    const columns = [];
+    const columnsByName = new Map();
+    for (const row of rows) {
+        for (const name in row) {
+            if (!Object.hasOwn(row, name))
+                continue;
+            const value = row[name];
+            let column = columnsByName.get(name);
+            if (!column) {
+                column = { name, type: null, nullable: false, presentCount: 0 };
+                columnsByName.set(name, column);
+                columns.push(column);
+            }
+            column.presentCount += 1;
+            if (value == null) {
+                column.nullable = true;
+                continue;
+            }
+            column.type = mergeBinarySnapshotColumnTypes(column.type, inferBinarySnapshotColumnType(value));
+        }
+    }
+    return columns.map((column) => ({
+        name: column.name,
+        type: column.type ?? 'json',
+        ...(column.nullable || column.presentCount < rows.length
+            ? { nullable: true }
+            : {}),
+    }));
+}
+function inferBinarySnapshotColumnType(value) {
+    if (typeof value === 'string')
+        return 'string';
+    if (typeof value === 'boolean')
+        return 'boolean';
+    if (typeof value === 'bigint')
+        return 'integer';
+    if (typeof value === 'number') {
+        return Number.isSafeInteger(value) ? 'integer' : 'float';
+    }
+    if (value instanceof Uint8Array || value instanceof ArrayBuffer) {
+        return 'bytes';
+    }
+    return 'json';
+}
+function mergeBinarySnapshotColumnTypes(current, next) {
+    if (!current || current === next)
+        return next;
+    if ((current === 'integer' && next === 'float') ||
+        (current === 'float' && next === 'integer')) {
+        return 'float';
+    }
+    return 'json';
+}
+async function runWithConcurrency(items, concurrency, worker) {
+    if (items.length === 0)
+        return;
+    const workerCount = Math.max(1, Math.min(concurrency, items.length));
+    let nextIndex = 0;
+    async function runWorker() {
+        while (nextIndex < items.length) {
+            const index = nextIndex;
+            nextIndex += 1;
+            const item = items[index];
+            if (item === undefined)
+                continue;
+            await worker(item);
+        }
+    }
+    await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
 }
 /**
  * Sanitize a numeric limit parameter with bounds checking.
@@ -45,6 +207,22 @@ function sanitizeLimit(value, defaultValue, min, max) {
         return defaultValue;
     return Math.max(min, Math.min(max, value));
 }
+function sanitizeGzipLevel(value) {
+    if (value === undefined || value === null || !Number.isFinite(value)) {
+        return DEFAULT_SNAPSHOT_CHUNK_GZIP_LEVEL;
+    }
+    return Math.max(0, Math.min(9, Math.trunc(value)));
+}
+function isSerializablePullError(error) {
+    const withCode = error;
+    return (withCode.code === '40001' ||
+        error.message.toLowerCase().includes('could not serialize access'));
+}
+async function delay(ms) {
+    await new Promise((resolve) => {
+        setTimeout(resolve, ms);
+    });
+}
 /**
  * Merge all scope values into a flat ScopeValues for cursor tracking.
  */
@@ -68,6 +246,45 @@ function mergeScopes(subscriptions) {
     }
     return merged;
 }
+function assertPullChangeIdentityUnchanged(source, before, after) {
+    if (before.table !== after.table) {
+        throw new Error(`${source} cannot change change.table (${before.table} -> ${after.table})`);
+    }
+    if (before.row_id !== after.row_id) {
+        throw new Error(`${source} cannot change change.row_id (${before.row_id} -> ${after.row_id})`);
+    }
+    if (before.op !== after.op) {
+        throw new Error(`${source} cannot change change.op (${before.op} -> ${after.op})`);
+    }
+}
+function projectPullChangeForClientSchema(args) {
+    const projected = args.handler.projectChangeForVersion
+        ? args.handler.projectChangeForVersion(args.change, args.schemaVersion)
+        : args.change;
+    assertPullChangeIdentityUnchanged('Server table handler projectChangeForVersion', args.change, projected);
+    return projected;
+}
+async function transformPullChanges(args) {
+    let changes = [...args.changes];
+    for (const plugin of args.plugins) {
+        if (!plugin.transformPullChanges)
+            continue;
+        const nextChanges = await plugin.transformPullChanges({
+            ctx: args.ctx,
+            tableHandler: args.tableHandler,
+            subscription: args.subscription,
+            changes,
+        });
+        if (nextChanges.length !== changes.length) {
+            throw new Error(`Server pull plugin "${plugin.name}" cannot change pull change count (${changes.length} -> ${nextChanges.length})`);
+        }
+        for (let i = 0; i < changes.length; i += 1) {
+            assertPullChangeIdentityUnchanged(`Server pull plugin "${plugin.name}"`, changes[i], nextChanges[i]);
+        }
+        changes = [...nextChanges];
+    }
+    return changes;
+}
 function summarizePullResponse(response) {
     const subscriptions = response.subscriptions ?? [];
     let activeSubscriptionCount = 0;
@@ -126,29 +343,49 @@ function recordPullMetrics(args) {
     });
     distributionSyncMetric('sync.server.pull.snapshot_pages', stats.snapshotPageCount, { attributes });
 }
-/**
- * Read synthetic commits created by notifyExternalDataChange() after a given cursor.
- * Returns commit_seq and affected tables for each external change commit.
- */
-async function readExternalDataChanges(trx, dialect, args) {
+async function readLatestExternalCommitByTable(trx, args) {
+    const tableNames = Array.from(new Set(args.tables.filter((table) => typeof table === 'string')));
+    const latestByTable = new Map();
+    if (tableNames.length === 0) {
+        return latestByTable;
+    }
     const executor = trx;
     const rows = await executor
-        .selectFrom('sync_commits')
-        .select(['commit_seq', 'affected_tables'])
-        .where('partition_id', '=', args.partitionId)
-        .where('client_id', '=', EXTERNAL_CLIENT_ID)
-        .where('commit_seq', '>', args.afterCursor)
-        .orderBy('commit_seq', 'asc')
+        .selectFrom('sync_table_commits as tc')
+        .innerJoin('sync_commits as cm', (join) => join
+        .onRef('cm.commit_seq', '=', 'tc.commit_seq')
+        .onRef('cm.partition_id', '=', 'tc.partition_id'))
+        .select(['tc.table as table'])
+        .select((eb) => eb.fn.max('tc.commit_seq').as('latest_commit_seq'))
+        .where('tc.partition_id', '=', args.partitionId)
+        .where('cm.client_id', '=', EXTERNAL_CLIENT_ID)
+        .where('cm.change_count', '=', 0)
+        .where('tc.commit_seq', '>', args.afterCursor)
+        .where('tc.table', 'in', tableNames)
+        .groupBy('tc.table')
         .execute();
-    return rows.map((row) => ({
-        commitSeq: Number(row.commit_seq),
-        tables: dialect.dbToArray(row.affected_tables),
-    }));
+    for (const row of rows) {
+        const commitSeq = Number(row.latest_commit_seq ?? -1);
+        if (!Number.isFinite(commitSeq) || commitSeq < 0)
+            continue;
+        latestByTable.set(row.table, commitSeq);
+    }
+    return latestByTable;
 }
 export async function pull(args) {
     const { request, dialect } = args;
     const db = args.db;
+    const pullPlugins = sortServerPullPlugins(args.plugins);
     const partitionId = args.auth.partitionId ?? 'default';
+    const snapshotChunkGzipLevel = sanitizeGzipLevel(args.snapshotChunkGzipLevel);
+    const clientSchemaVersion = request.schemaVersion;
+    if (!Number.isInteger(clientSchemaVersion) || clientSchemaVersion < 1) {
+        throw new Error('Pull request schemaVersion must be a positive integer');
+    }
+    const snapshotChunkCacheSchemaVersion = args.snapshotChunkCacheSchemaVersion === null ||
+        args.snapshotChunkCacheSchemaVersion === undefined
+        ? clientSchemaVersion
+        : `${clientSchemaVersion}:${args.snapshotChunkCacheSchemaVersion}`;
     const requestedSubscriptionCount = Array.isArray(request.subscriptions)
         ? request.subscriptions.length
         : 0;
@@ -163,11 +400,13 @@ export async function pull(args) {
     }, async (span) => {
         try {
             // Validate and sanitize request limits
-            const limitCommits = sanitizeLimit(request.limitCommits, 50, 1, 500);
-            const limitSnapshotRows = sanitizeLimit(request.limitSnapshotRows, 1000, 1, 5000);
+            const limitCommits = sanitizeLimit(request.limitCommits, 1000, 1, 1000);
+            const limitSnapshotRows = sanitizeLimit(request.limitSnapshotRows, 1000, 1, 50000);
             const maxSnapshotPages = sanitizeLimit(request.maxSnapshotPages, 4, 1, 50);
             const dedupeRows = request.dedupeRows === true;
-            const pendingExternalChunkWrites = [];
+            const snapshotChunkEncoding = SYNC_SNAPSHOT_CHUNK_ENCODING;
+            const snapshotArtifactSelection = resolveSnapshotArtifactSelection(request.snapshotArtifacts, clientSchemaVersion);
+            const snapshotArtifactSchemaVersion = snapshotArtifactSelection?.schemaVersion ?? null;
             // Resolve effective scopes for each subscription
             const resolved = await resolveEffectiveScopesForSubscriptions({
                 db,
@@ -176,480 +415,781 @@ export async function pull(args) {
                 handlers: args.handlers,
                 scopeCache: args.scopeCache ?? defaultScopeCache,
             });
-            const result = await dialect.executeInTransaction(db, async (trx) => {
-                await dialect.setRepeatableRead(trx);
-                const maxCommitSeq = await dialect.readMaxCommitSeq(trx, {
-                    partitionId,
-                });
-                const minCommitSeq = await dialect.readMinCommitSeq(trx, {
-                    partitionId,
-                });
-                const subResponses = [];
-                const activeSubscriptions = [];
-                const nextCursors = [];
-                // Detect external data changes (synthetic commits from notifyExternalDataChange)
-                // Compute minimum cursor across all active subscriptions to scope the query.
-                let minSubCursor = Number.MAX_SAFE_INTEGER;
-                for (const sub of resolved) {
-                    if (sub.status === 'revoked' ||
-                        Object.keys(sub.scopes).length === 0)
-                        continue;
-                    const cursor = Math.max(-1, sub.cursor ?? -1);
-                    if (cursor >= 0 && cursor < minSubCursor) {
-                        minSubCursor = cursor;
-                    }
-                }
-                const externalDataChanges = minSubCursor < Number.MAX_SAFE_INTEGER && minSubCursor >= 0
-                    ? await readExternalDataChanges(trx, dialect, {
-                        partitionId,
-                        afterCursor: minSubCursor,
-                    })
-                    : [];
-                const maxExternalCommitByTable = new Map();
-                for (const change of externalDataChanges) {
-                    for (const table of change.tables) {
-                        const previous = maxExternalCommitByTable.get(table) ?? -1;
-                        if (change.commitSeq > previous) {
-                            maxExternalCommitByTable.set(table, change.commitSeq);
-                        }
-                    }
-                }
-                for (const sub of resolved) {
-                    const cursor = Math.max(-1, sub.cursor ?? -1);
-                    // Validate table handler exists (throws if not registered)
-                    getServerHandlerOrThrow(args.handlers, sub.table);
-                    if (sub.status === 'revoked' ||
-                        Object.keys(sub.scopes).length === 0) {
-                        subResponses.push({
-                            id: sub.id,
-                            status: 'revoked',
-                            scopes: {},
-                            bootstrap: false,
-                            nextCursor: cursor,
-                            commits: [],
+            for (let attemptIndex = 0; attemptIndex < MAX_PULL_TRANSACTION_RETRIES; attemptIndex += 1) {
+                const pendingExternalChunkWrites = [];
+                const bootstrapTimings = createPullBootstrapTimings();
+                try {
+                    const result = await dialect.executeInTransaction(db, async (trx) => {
+                        await dialect.setRepeatableRead(trx);
+                        const maxCommitSeq = await dialect.readMaxCommitSeq(trx, {
+                            partitionId,
                         });
-                        continue;
-                    }
-                    const effectiveScopes = sub.scopes;
-                    activeSubscriptions.push({ scopes: effectiveScopes });
-                    const latestExternalCommitForTable = maxExternalCommitByTable.get(sub.table);
-                    const needsBootstrap = sub.bootstrapState != null ||
-                        cursor < 0 ||
-                        cursor > maxCommitSeq ||
-                        (minCommitSeq > 0 && cursor < minCommitSeq - 1) ||
-                        (latestExternalCommitForTable !== undefined &&
-                            latestExternalCommitForTable > cursor);
-                    if (needsBootstrap) {
-                        const tables = getServerBootstrapOrderFor(args.handlers, sub.table).map((handler) => handler.table);
-                        const initState = {
-                            asOfCommitSeq: maxCommitSeq,
-                            tables,
-                            tableIndex: 0,
-                            rowCursor: null,
-                        };
-                        const requestedState = sub.bootstrapState ?? null;
-                        const state = requestedState &&
-                            typeof requestedState.asOfCommitSeq === 'number' &&
-                            Array.isArray(requestedState.tables) &&
-                            typeof requestedState.tableIndex === 'number'
-                            ? requestedState
-                            : initState;
-                        // If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
-                        const effectiveState = state.asOfCommitSeq < minCommitSeq - 1 ? initState : state;
-                        const tableName = effectiveState.tables[effectiveState.tableIndex];
-                        // No tables (or ran past the end): treat bootstrap as complete.
-                        if (!tableName) {
-                            subResponses.push({
-                                id: sub.id,
-                                status: 'active',
-                                scopes: effectiveScopes,
-                                bootstrap: true,
-                                bootstrapState: null,
-                                nextCursor: effectiveState.asOfCommitSeq,
-                                commits: [],
-                                snapshots: [],
-                            });
-                            nextCursors.push(effectiveState.asOfCommitSeq);
-                            continue;
+                        const minCommitSeq = await dialect.readMinCommitSeq(trx, {
+                            partitionId,
+                        });
+                        const subResponses = [];
+                        const activeSubscriptions = [];
+                        const nextCursors = [];
+                        // Detect external data changes (synthetic commits from notifyExternalDataChange)
+                        // Compute minimum cursor across all active subscriptions to scope the query.
+                        let minSubCursor = Number.MAX_SAFE_INTEGER;
+                        const activeTables = new Set();
+                        for (const sub of resolved) {
+                            if (sub.status === 'revoked' ||
+                                Object.keys(sub.scopes).length === 0)
+                                continue;
+                            activeTables.add(sub.table);
+                            const cursor = Math.max(-1, sub.cursor ?? -1);
+                            if (cursor >= 0 && cursor < minSubCursor) {
+                                minSubCursor = cursor;
+                            }
                         }
-                        const snapshots = [];
-                        let nextState = effectiveState;
-                        const cacheKey = `${partitionId}:${await scopesToCacheKey(effectiveScopes)}`;
-                        const flushSnapshotBundle = async (bundle) => {
-                            const nowIso = new Date().toISOString();
-                            const bundleRowLimit = Math.max(1, limitSnapshotRows * bundle.pageCount);
-                            const cached = await readSnapshotChunkRefByPageKey(trx, {
+                        const maxExternalCommitByTable = minSubCursor < Number.MAX_SAFE_INTEGER && minSubCursor >= 0
+                            ? await readLatestExternalCommitByTable(trx, {
                                 partitionId,
-                                scopeKey: cacheKey,
-                                scope: bundle.table,
-                                asOfCommitSeq: effectiveState.asOfCommitSeq,
-                                rowCursor: bundle.startCursor,
-                                rowLimit: bundleRowLimit,
-                                encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
-                                compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
-                                nowIso,
-                            });
-                            let chunkRef = cached;
-                            if (!chunkRef) {
-                                const rowFramePayload = concatByteChunks(bundle.rowFrameParts);
-                                const expiresAt = new Date(Date.now() + Math.max(1000, bundle.ttlMs)).toISOString();
-                                if (args.chunkStorage) {
-                                    const snapshot = {
-                                        table: bundle.table,
-                                        rows: [],
-                                        chunks: [],
-                                        isFirstPage: bundle.isFirstPage,
-                                        isLastPage: bundle.isLastPage,
+                                afterCursor: minSubCursor,
+                                tables: Array.from(activeTables),
+                            })
+                            : new Map();
+                        for (const sub of resolved) {
+                            const cursor = Math.max(-1, sub.cursor ?? -1);
+                            // Validate table handler exists (throws if not registered)
+                            if (!args.handlers.byTable.has(sub.table)) {
+                                throw new Error(`Unknown table: ${sub.table}`);
+                            }
+                            if (sub.status === 'revoked' ||
+                                Object.keys(sub.scopes).length === 0) {
+                                subResponses.push({
+                                    id: sub.id,
+                                    status: 'revoked',
+                                    scopes: {},
+                                    bootstrap: false,
+                                    nextCursor: cursor,
+                                    commits: [],
+                                });
+                                continue;
+                            }
+                            const effectiveScopes = sub.scopes;
+                            activeSubscriptions.push({ scopes: effectiveScopes });
+                            const latestExternalCommitForTable = maxExternalCommitByTable.get(sub.table);
+                            const needsBootstrap = sub.bootstrapState != null ||
+                                cursor < 0 ||
+                                cursor > maxCommitSeq ||
+                                (minCommitSeq > 0 && cursor < minCommitSeq - 1) ||
+                                (latestExternalCommitForTable !== undefined &&
+                                    latestExternalCommitForTable > cursor);
+                            if (needsBootstrap) {
+                                const tables = getServerBootstrapOrderFor(args.handlers, sub.table).map((handler) => handler.table);
+                                const initState = {
+                                    asOfCommitSeq: maxCommitSeq,
+                                    tables,
+                                    tableIndex: 0,
+                                    rowCursor: null,
+                                };
+                                const requestedState = sub.bootstrapState ?? null;
+                                const state = requestedState &&
+                                    typeof requestedState.asOfCommitSeq === 'number' &&
+                                    Array.isArray(requestedState.tables) &&
+                                    typeof requestedState.tableIndex === 'number'
+                                    ? requestedState
+                                    : initState;
+                                // If the bootstrap state's asOfCommitSeq is no longer catch-up-able, restart bootstrap.
+                                const effectiveState = state.asOfCommitSeq < minCommitSeq - 1
+                                    ? initState
+                                    : state;
+                                const tableName = effectiveState.tables[effectiveState.tableIndex];
+                                // No tables (or ran past the end): treat bootstrap as complete.
+                                if (!tableName) {
+                                    subResponses.push({
+                                        id: sub.id,
+                                        status: 'active',
+                                        scopes: effectiveScopes,
+                                        bootstrap: true,
+                                        bootstrapState: null,
+                                        nextCursor: effectiveState.asOfCommitSeq,
+                                        commits: [],
+                                        snapshots: [],
+                                    });
+                                    nextCursors.push(effectiveState.asOfCommitSeq);
+                                    continue;
+                                }
+                                const snapshots = [];
+                                let nextState = effectiveState;
+                                const cacheKey = await createSnapshotChunkScopeCacheKey({
+                                    partitionId,
+                                    scopes: effectiveScopes,
+                                    schemaVersion: snapshotChunkCacheSchemaVersion,
+                                    encoding: snapshotChunkEncoding,
+                                    compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+                                    gzipLevel: snapshotChunkGzipLevel,
+                                });
+                                const artifactScopeKey = snapshotArtifactSelection && snapshotArtifactSchemaVersion
+                                    ? await createScopedSnapshotArtifactScopeCacheKey({
+                                        partitionId,
+                                        subscriptionId: sub.id,
+                                        scopes: effectiveScopes,
+                                        schemaVersion: snapshotArtifactSchemaVersion,
+                                        artifactKind: snapshotArtifactSelection.artifactKind,
+                                        compression: snapshotArtifactSelection.compression,
+                                        features: snapshotArtifactSelection.featureSet,
+                                    })
+                                    : null;
+                                const createSnapshotBundle = (table, tableIndex, rowCursor, ttlMs, binaryColumns, binaryEncoder) => {
+                                    return {
+                                        table,
+                                        tableIndex,
+                                        startCursor: rowCursor,
+                                        nextRowCursor: null,
+                                        isFirstPage: rowCursor == null,
+                                        isLastPage: false,
+                                        pageCount: 0,
+                                        cacheRowLimit: null,
+                                        ttlMs,
+                                        binaryColumns,
+                                        binaryEncoder,
+                                        binaryRows: [],
                                     };
-                                    snapshots.push(snapshot);
-                                    pendingExternalChunkWrites.push({
-                                        snapshot,
-                                        cacheLookup: {
+                                };
+                                const snapshotBootstrapStateAfter = (args) => {
+                                    if (!args.isLastPage) {
+                                        return {
+                                            ...effectiveState,
+                                            tableIndex: args.tableIndex,
+                                            rowCursor: args.nextRowCursor,
+                                        };
+                                    }
+                                    if (args.tableIndex + 1 < effectiveState.tables.length) {
+                                        return {
+                                            ...effectiveState,
+                                            tableIndex: args.tableIndex + 1,
+                                            rowCursor: null,
+                                        };
+                                    }
+                                    return null;
+                                };
+                                const encodeSnapshotBundlePayload = (bundle) => {
+                                    const encodeStartedAt = Date.now();
+                                    const payload = bundle.binaryEncoder
+                                        ? bundle.binaryEncoder(bundle.binaryRows)
+                                        : encodeBinarySnapshotRows(bundle.table, bundle.binaryRows, bundle.binaryColumns);
+                                    bootstrapTimings.binaryEncodeMs += Math.max(0, Date.now() - encodeStartedAt);
+                                    return [payload];
+                                };
+                                const flushSnapshotBundle = async (bundle) => {
+                                    const nowIso = new Date().toISOString();
+                                    const bundleRowLimit = Math.max(1, bundle.cacheRowLimit ??
+                                        limitSnapshotRows * bundle.pageCount);
+                                    const cacheLookupStartedAt = Date.now();
+                                    const cached = await readSnapshotChunkRefByPageKey(trx, {
+                                        partitionId,
+                                        scopeKey: cacheKey,
+                                        scope: bundle.table,
+                                        asOfCommitSeq: effectiveState.asOfCommitSeq,
+                                        rowCursor: bundle.startCursor,
+                                        rowLimit: bundleRowLimit,
+                                        encoding: snapshotChunkEncoding,
+                                        compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+                                        nowIso,
+                                    });
+                                    bootstrapTimings.chunkCacheLookupMs += Math.max(0, Date.now() - cacheLookupStartedAt);
+                                    let chunkRef = cached;
+                                    if (!chunkRef) {
+                                        const expiresAt = new Date(Date.now() + Math.max(1000, bundle.ttlMs)).toISOString();
+                                        if (args.chunkStorage) {
+                                            const snapshot = {
+                                                table: bundle.table,
+                                                rows: [],
+                                                chunks: [],
+                                                isFirstPage: bundle.isFirstPage,
+                                                isLastPage: bundle.isLastPage,
+                                                bootstrapStateAfter: snapshotBootstrapStateAfter({
+                                                    tableIndex: bundle.tableIndex,
+                                                    nextRowCursor: bundle.nextRowCursor,
+                                                    isLastPage: bundle.isLastPage,
+                                                }),
+                                            };
+                                            snapshots.push(snapshot);
+                                            pendingExternalChunkWrites.push({
+                                                snapshot,
+                                                cacheLookup: {
+                                                    partitionId,
+                                                    scopeKey: cacheKey,
+                                                    scope: bundle.table,
+                                                    asOfCommitSeq: effectiveState.asOfCommitSeq,
+                                                    rowCursor: bundle.startCursor,
+                                                    rowLimit: bundleRowLimit,
+                                                    nextRowCursor: bundle.nextRowCursor,
+                                                    isLastPage: bundle.isLastPage,
+                                                },
+                                                payloadParts: encodeSnapshotBundlePayload(bundle),
+                                                expiresAt,
+                                            });
+                                            return;
+                                        }
+                                        const encodedChunk = await encodeCompressedSnapshotChunk(encodeSnapshotBundlePayload(bundle), snapshotChunkGzipLevel);
+                                        bootstrapTimings.chunkGzipMs += encodedChunk.gzipMs;
+                                        bootstrapTimings.chunkHashMs += encodedChunk.hashMs;
+                                        const chunkId = randomId();
+                                        const chunkPersistStartedAt = Date.now();
+                                        chunkRef = await insertSnapshotChunk(trx, {
+                                            chunkId,
                                             partitionId,
                                             scopeKey: cacheKey,
                                             scope: bundle.table,
                                             asOfCommitSeq: effectiveState.asOfCommitSeq,
                                             rowCursor: bundle.startCursor,
                                             rowLimit: bundleRowLimit,
-                                        },
-                                        rowFramePayload,
-                                        expiresAt,
+                                            nextRowCursor: bundle.nextRowCursor,
+                                            isLastPage: bundle.isLastPage,
+                                            encoding: snapshotChunkEncoding,
+                                            compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+                                            sha256: encodedChunk.sha256,
+                                            body: encodedChunk.body,
+                                            expiresAt,
+                                        });
+                                        bootstrapTimings.chunkPersistMs += Math.max(0, Date.now() - chunkPersistStartedAt);
+                                    }
+                                    const chunk = toResponseChunkRef(chunkRef);
+                                    snapshots.push({
+                                        table: bundle.table,
+                                        rows: [],
+                                        chunks: [chunk],
+                                        manifest: await createChunkedSnapshotManifest({
+                                            table: bundle.table,
+                                            asOfCommitSeq: effectiveState.asOfCommitSeq,
+                                            scopeKey: cacheKey,
+                                            rowCursor: bundle.startCursor,
+                                            rowLimit: bundleRowLimit,
+                                            nextRowCursor: bundle.nextRowCursor,
+                                            isFirstPage: bundle.isFirstPage,
+                                            isLastPage: bundle.isLastPage,
+                                            chunks: [chunk],
+                                        }),
+                                        isFirstPage: bundle.isFirstPage,
+                                        isLastPage: bundle.isLastPage,
+                                        bootstrapStateAfter: snapshotBootstrapStateAfter({
+                                            tableIndex: bundle.tableIndex,
+                                            nextRowCursor: bundle.nextRowCursor,
+                                            isLastPage: bundle.isLastPage,
+                                        }),
                                     });
-                                    return;
-                                }
-                                const sha256 = await sha256Hex(rowFramePayload);
-                                const compressedBody = await gzipBytes(rowFramePayload);
-                                const chunkId = randomId();
-                                chunkRef = await insertSnapshotChunk(trx, {
-                                    chunkId,
-                                    partitionId,
-                                    scopeKey: cacheKey,
-                                    scope: bundle.table,
-                                    asOfCommitSeq: effectiveState.asOfCommitSeq,
-                                    rowCursor: bundle.startCursor,
-                                    rowLimit: bundleRowLimit,
-                                    encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
-                                    compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
-                                    sha256,
-                                    body: compressedBody,
-                                    expiresAt,
-                                });
-                            }
-                            snapshots.push({
-                                table: bundle.table,
-                                rows: [],
-                                chunks: [chunkRef],
-                                isFirstPage: bundle.isFirstPage,
-                                isLastPage: bundle.isLastPage,
-                            });
-                        };
-                        let activeBundle = null;
-                        for (let pageIndex = 0; pageIndex < maxSnapshotPages; pageIndex++) {
-                            if (!nextState)
-                                break;
-                            const nextTableName = nextState.tables[nextState.tableIndex];
-                            if (!nextTableName) {
-                                if (activeBundle) {
+                                };
+                                let activeBundle = null;
+                                for (let pageIndex = 0; pageIndex < maxSnapshotPages; pageIndex++) {
+                                    if (!nextState)
+                                        break;
+                                    const nextTableName = nextState.tables[nextState.tableIndex];
+                                    if (!nextTableName) {
+                                        if (activeBundle) {
+                                            activeBundle.isLastPage = true;
+                                            await flushSnapshotBundle(activeBundle);
+                                            activeBundle = null;
+                                        }
+                                        nextState = null;
+                                        break;
+                                    }
+                                    const tableHandler = args.handlers.byTable.get(nextTableName);
+                                    if (!tableHandler) {
+                                        throw new Error(`Unknown table: ${nextTableName}`);
+                                    }
+                                    if (!activeBundle ||
+                                        activeBundle.table !== nextTableName) {
+                                        if (activeBundle) {
+                                            await flushSnapshotBundle(activeBundle);
+                                        }
+                                        activeBundle = createSnapshotBundle(nextTableName, nextState.tableIndex, nextState.rowCursor, tableHandler.snapshotChunkTtlMs ??
+                                            24 * 60 * 60 * 1000, resolveSnapshotBinaryColumns(tableHandler, clientSchemaVersion), resolveSnapshotBinaryEncoder(tableHandler, clientSchemaVersion));
+                                    }
+                                    if (artifactScopeKey && activeBundle.pageCount === 0) {
+                                        const pagesRemaining = Math.max(1, maxSnapshotPages - pageIndex);
+                                        const artifactRowLimit = resolveBinarySnapshotBundleRowLimit({
+                                            limitSnapshotRows,
+                                            pagesRemaining,
+                                        });
+                                        const artifactLookupStartedAt = Date.now();
+                                        const artifact = await readBestScopedSnapshotArtifactRefForPageCapacity(trx, {
+                                            partitionId,
+                                            scopeKey: artifactScopeKey,
+                                            subscriptionId: sub.id,
+                                            table: nextTableName,
+                                            asOfCommitSeq: effectiveState.asOfCommitSeq,
+                                            rowCursor: nextState.rowCursor,
+                                            maxRowLimit: artifactRowLimit,
+                                            artifactKind: snapshotArtifactSelection.artifactKind,
+                                            schemaVersion: snapshotArtifactSchemaVersion,
+                                            compression: snapshotArtifactSelection.compression,
+                                        });
+                                        bootstrapTimings.artifactCacheLookupMs += Math.max(0, Date.now() - artifactLookupStartedAt);
+                                        if (artifact &&
+                                            (artifact.isLastPage ||
+                                                artifact.nextRowCursor !== null)) {
+                                            snapshots.push({
+                                                table: nextTableName,
+                                                rows: [],
+                                                artifacts: [artifact],
+                                                isFirstPage: artifact.isFirstPage,
+                                                isLastPage: artifact.isLastPage,
+                                                bootstrapStateAfter: snapshotBootstrapStateAfter({
+                                                    tableIndex: nextState.tableIndex,
+                                                    nextRowCursor: artifact.nextRowCursor,
+                                                    isLastPage: artifact.isLastPage,
+                                                }),
+                                            });
+                                            activeBundle = null;
+                                            const selectedArtifactRowLimit = artifact.manifest.rowLimit;
+                                            pageIndex +=
+                                                Math.max(1, Math.ceil(selectedArtifactRowLimit / limitSnapshotRows)) - 1;
+                                            nextState = snapshotBootstrapStateAfter({
+                                                tableIndex: nextState.tableIndex,
+                                                nextRowCursor: artifact.nextRowCursor,
+                                                isLastPage: artifact.isLastPage,
+                                            });
+                                            continue;
+                                        }
+                                    }
+                                    if (activeBundle.pageCount === 0) {
+                                        const pagesRemaining = Math.max(1, maxSnapshotPages - pageIndex);
+                                        const cachedRowLimit = resolveBinarySnapshotBundleRowLimit({
+                                            limitSnapshotRows,
+                                            pagesRemaining,
+                                        });
+                                        activeBundle.cacheRowLimit = cachedRowLimit;
+                                        const cacheLookupStartedAt = Date.now();
+                                        const cached = await readSnapshotChunkRefByPageKey(trx, {
+                                            partitionId,
+                                            scopeKey: cacheKey,
+                                            scope: nextTableName,
+                                            asOfCommitSeq: effectiveState.asOfCommitSeq,
+                                            rowCursor: nextState.rowCursor,
+                                            rowLimit: cachedRowLimit,
+                                            encoding: snapshotChunkEncoding,
+                                            compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+                                        });
+                                        bootstrapTimings.chunkCacheLookupMs += Math.max(0, Date.now() - cacheLookupStartedAt);
+                                        if (cached &&
+                                            (cached.isLastPage || cached.nextRowCursor !== null)) {
+                                            const chunk = toResponseChunkRef(cached);
+                                            snapshots.push({
+                                                table: nextTableName,
+                                                rows: [],
+                                                chunks: [chunk],
+                                                manifest: await createChunkedSnapshotManifest({
+                                                    table: nextTableName,
+                                                    asOfCommitSeq: effectiveState.asOfCommitSeq,
+                                                    scopeKey: cacheKey,
+                                                    rowCursor: nextState.rowCursor,
+                                                    rowLimit: cachedRowLimit,
+                                                    nextRowCursor: cached.nextRowCursor,
+                                                    isFirstPage: nextState.rowCursor == null,
+                                                    isLastPage: cached.isLastPage,
+                                                    chunks: [chunk],
+                                                }),
+                                                isFirstPage: nextState.rowCursor == null,
+                                                isLastPage: cached.isLastPage,
+                                                bootstrapStateAfter: snapshotBootstrapStateAfter({
+                                                    tableIndex: nextState.tableIndex,
+                                                    nextRowCursor: cached.nextRowCursor,
+                                                    isLastPage: cached.isLastPage,
+                                                }),
+                                            });
+                                            activeBundle = null;
+                                            pageIndex +=
+                                                Math.max(1, Math.ceil(cachedRowLimit / limitSnapshotRows)) - 1;
+                                            nextState = snapshotBootstrapStateAfter({
+                                                tableIndex: nextState.tableIndex,
+                                                nextRowCursor: cached.nextRowCursor,
+                                                isLastPage: cached.isLastPage,
+                                            });
+                                            continue;
+                                        }
+                                    }
+                                    const snapshotQueryStartedAt = Date.now();
+                                    const page = await tableHandler.snapshot({
+                                        db: trx,
+                                        actorId: args.auth.actorId,
+                                        auth: args.auth,
+                                        scopeValues: effectiveScopes,
+                                        cursor: nextState.rowCursor,
+                                        limit: limitSnapshotRows,
+                                        schemaVersion: clientSchemaVersion,
+                                    }, sub.params);
+                                    bootstrapTimings.snapshotQueryMs += Math.max(0, Date.now() - snapshotQueryStartedAt);
+                                    const pageRows = page.rows ?? [];
+                                    activeBundle.nextRowCursor = page.nextCursor;
+                                    activeBundle.binaryRows.push(...pageRows);
+                                    activeBundle.pageCount += 1;
+                                    if (page.nextCursor != null) {
+                                        const shouldFlushBinaryBundle = activeBundle.binaryRows.length >=
+                                            DEFAULT_MAX_BINARY_SNAPSHOT_BUNDLE_ROWS;
+                                        if (shouldFlushBinaryBundle) {
+                                            await flushSnapshotBundle(activeBundle);
+                                            activeBundle = null;
+                                        }
+                                        nextState = {
+                                            ...nextState,
+                                            rowCursor: page.nextCursor,
+                                        };
+                                        continue;
+                                    }
                                     activeBundle.isLastPage = true;
                                     await flushSnapshotBundle(activeBundle);
                                     activeBundle = null;
+                                    if (nextState.tableIndex + 1 < nextState.tables.length) {
+                                        nextState = {
+                                            ...nextState,
+                                            tableIndex: nextState.tableIndex + 1,
+                                            rowCursor: null,
+                                        };
+                                        continue;
+                                    }
+                                    nextState = null;
+                                    break;
                                 }
-                                nextState = null;
-                                break;
-                            }
-                            const tableHandler = getServerHandlerOrThrow(args.handlers, nextTableName);
-                            if (!activeBundle || activeBundle.table !== nextTableName) {
                                 if (activeBundle) {
                                     await flushSnapshotBundle(activeBundle);
                                 }
-                                const bundleHeader = encodeSnapshotRows([]);
-                                activeBundle = {
-                                    table: nextTableName,
-                                    startCursor: nextState.rowCursor,
-                                    isFirstPage: nextState.rowCursor == null,
-                                    isLastPage: false,
-                                    pageCount: 0,
-                                    ttlMs: tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000,
-                                    rowFrameParts: [bundleHeader],
-                                };
-                            }
-                            const page = await tableHandler.snapshot({
-                                db: trx,
-                                actorId: args.auth.actorId,
-                                auth: args.auth,
-                                scopeValues: effectiveScopes,
-                                cursor: nextState.rowCursor,
-                                limit: limitSnapshotRows,
-                            }, sub.params);
-                            const rowFrames = encodeSnapshotRowFrames(page.rows ?? []);
-                            activeBundle.rowFrameParts.push(rowFrames);
-                            activeBundle.pageCount += 1;
-                            if (page.nextCursor != null) {
-                                nextState = { ...nextState, rowCursor: page.nextCursor };
+                                subResponses.push({
+                                    id: sub.id,
+                                    status: 'active',
+                                    scopes: effectiveScopes,
+                                    bootstrap: true,
+                                    bootstrapState: nextState,
+                                    nextCursor: effectiveState.asOfCommitSeq,
+                                    commits: [],
+                                    snapshots,
+                                });
+                                nextCursors.push(effectiveState.asOfCommitSeq);
                                 continue;
                             }
-                            activeBundle.isLastPage = true;
-                            await flushSnapshotBundle(activeBundle);
-                            activeBundle = null;
-                            if (nextState.tableIndex + 1 < nextState.tables.length) {
-                                nextState = {
-                                    ...nextState,
-                                    tableIndex: nextState.tableIndex + 1,
-                                    rowCursor: null,
-                                };
-                                continue;
+                            // Incremental pull for this subscription. The dialect row query
+                            // carries the scanned commit-window max when matching rows exist,
+                            // so we only need a separate commit-window scan when the row query
+                            // returns no matches at all.
+                            const incrementalRows = [];
+                            let maxScannedCommitSeq = cursor;
+                            for await (const row of dialect.iterateIncrementalPullRows(trx, {
+                                partitionId,
+                                table: sub.table,
+                                scopes: effectiveScopes,
+                                cursor,
+                                limitCommits,
+                            })) {
+                                incrementalRows.push(row);
+                                maxScannedCommitSeq = Math.max(maxScannedCommitSeq, row.scanned_max_commit_seq ?? row.commit_seq);
                             }
-                            nextState = null;
-                            break;
-                        }
-                        if (activeBundle) {
-                            await flushSnapshotBundle(activeBundle);
-                        }
-                        subResponses.push({
-                            id: sub.id,
-                            status: 'active',
-                            scopes: effectiveScopes,
-                            bootstrap: true,
-                            bootstrapState: nextState,
-                            nextCursor: effectiveState.asOfCommitSeq,
-                            commits: [],
-                            snapshots,
-                        });
-                        nextCursors.push(effectiveState.asOfCommitSeq);
-                        continue;
-                    }
-                    // Incremental pull for this subscription
-                    // Read the commit window for this table up-front so the subscription cursor
-                    // can advance past commits that don't match the requested scopes.
-                    const scannedCommitSeqs = await dialect.readCommitSeqsForPull(trx, {
-                        partitionId,
-                        cursor,
-                        limitCommits,
-                        tables: [sub.table],
-                    });
-                    const maxScannedCommitSeq = scannedCommitSeqs.length > 0
-                        ? scannedCommitSeqs[scannedCommitSeqs.length - 1]
-                        : cursor;
-                    if (scannedCommitSeqs.length === 0) {
-                        subResponses.push({
-                            id: sub.id,
-                            status: 'active',
-                            scopes: effectiveScopes,
-                            bootstrap: false,
-                            nextCursor: cursor,
-                            commits: [],
-                        });
-                        nextCursors.push(cursor);
-                        continue;
-                    }
-                    let nextCursor = cursor;
-                    if (dedupeRows) {
-                        const latestByRowKey = new Map();
-                        for await (const r of dialect.iterateIncrementalPullRows(trx, {
-                            partitionId,
-                            table: sub.table,
-                            scopes: effectiveScopes,
-                            cursor,
-                            limitCommits,
-                        })) {
-                            nextCursor = Math.max(nextCursor, r.commit_seq);
-                            const rowKey = `${r.table}\u0000${r.row_id}`;
-                            const change = {
-                                table: r.table,
-                                row_id: r.row_id,
-                                op: r.op,
-                                row_json: r.row_json,
-                                row_version: r.row_version,
-                                scopes: r.scopes,
-                            };
-                            // Move row keys to insertion tail so Map iteration yields
-                            // "latest change wins" order without a full array sort.
-                            if (latestByRowKey.has(rowKey)) {
-                                latestByRowKey.delete(rowKey);
+                            if (incrementalRows.length === 0) {
+                                const scannedCommitSeqs = await dialect.readCommitSeqsForPull(trx, {
+                                    partitionId,
+                                    cursor,
+                                    limitCommits,
+                                    tables: [sub.table],
+                                });
+                                maxScannedCommitSeq =
+                                    scannedCommitSeqs.length > 0
+                                        ? scannedCommitSeqs[scannedCommitSeqs.length - 1]
+                                        : cursor;
+                                if (scannedCommitSeqs.length === 0) {
+                                    subResponses.push({
+                                        id: sub.id,
+                                        status: 'active',
+                                        scopes: effectiveScopes,
+                                        bootstrap: false,
+                                        nextCursor: cursor,
+                                        commits: [],
+                                    });
+                                    nextCursors.push(cursor);
+                                    continue;
+                                }
+                            }
+                            const tableHandler = args.handlers.byTable.get(sub.table);
+                            if (!tableHandler) {
+                                throw new Error(`Unknown table: ${sub.table}`);
                             }
-                            latestByRowKey.set(rowKey, {
+                            const incrementalItems = incrementalRows.map((r) => ({
                                 commitSeq: r.commit_seq,
                                 createdAt: r.created_at,
                                 actorId: r.actor_id,
-                                change,
+                                change: projectPullChangeForClientSchema({
+                                    handler: tableHandler,
+                                    schemaVersion: clientSchemaVersion,
+                                    change: {
+                                        table: r.table,
+                                        row_id: r.row_id,
+                                        op: r.op,
+                                        row_json: r.row_json,
+                                        row_version: r.row_version,
+                                        scopes: r.scopes,
+                                    },
+                                }),
+                            }));
+                            if (pullPlugins.length > 0 && incrementalItems.length > 0) {
+                                const transformedChanges = await transformPullChanges({
+                                    plugins: pullPlugins,
+                                    ctx: {
+                                        db: trx,
+                                        actorId: args.auth.actorId,
+                                        auth: args.auth,
+                                    },
+                                    tableHandler,
+                                    subscription: {
+                                        id: sub.id,
+                                        table: sub.table,
+                                        scopes: effectiveScopes,
+                                        params: sub.params,
+                                        cursor,
+                                        crdtStateVectors: sub.crdtStateVectors,
+                                    },
+                                    changes: incrementalItems.map((item) => item.change),
+                                });
+                                for (let i = 0; i < incrementalItems.length; i += 1) {
+                                    incrementalItems[i].change = transformedChanges[i];
+                                }
+                            }
+                            let nextCursor = cursor;
+                            if (dedupeRows) {
+                                const latestByRowKey = new Map();
+                                for (const item of incrementalItems) {
+                                    nextCursor = Math.max(nextCursor, item.commitSeq);
+                                    const rowKey = `${item.change.table}\u0000${item.change.row_id}`;
+                                    // Move row keys to insertion tail so Map iteration yields
+                                    // "latest change wins" order without a full array sort.
+                                    if (latestByRowKey.has(rowKey)) {
+                                        latestByRowKey.delete(rowKey);
+                                    }
+                                    latestByRowKey.set(rowKey, {
+                                        commitSeq: item.commitSeq,
+                                        createdAt: item.createdAt,
+                                        actorId: item.actorId,
+                                        change: item.change,
+                                    });
+                                }
+                                nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
+                                if (latestByRowKey.size === 0) {
+                                    subResponses.push({
+                                        id: sub.id,
+                                        status: 'active',
+                                        scopes: effectiveScopes,
+                                        bootstrap: false,
+                                        nextCursor,
+                                        commits: [],
+                                    });
+                                    nextCursors.push(nextCursor);
+                                    continue;
+                                }
+                                const commits = [];
+                                for (const item of latestByRowKey.values()) {
+                                    const lastCommit = commits[commits.length - 1];
+                                    if (!lastCommit ||
+                                        lastCommit.commitSeq !== item.commitSeq) {
+                                        commits.push({
+                                            commitSeq: item.commitSeq,
+                                            createdAt: item.createdAt,
+                                            actorId: item.actorId,
+                                            changes: [item.change],
+                                        });
+                                        continue;
+                                    }
+                                    lastCommit.changes.push(item.change);
+                                }
+                                const integrity = await createWireSubscriptionIntegrity({
+                                    partitionId,
+                                    subscriptionId: sub.id,
+                                    previousRoot: typeof sub.verifiedRoot === 'string'
+                                        ? sub.verifiedRoot
+                                        : SYNCULAR_COMMIT_GENESIS_ROOT,
+                                    commits,
+                                });
+                                subResponses.push({
+                                    id: sub.id,
+                                    status: 'active',
+                                    scopes: effectiveScopes,
+                                    bootstrap: false,
+                                    nextCursor,
+                                    ...(integrity ? { integrity } : {}),
+                                    commits,
+                                });
+                                nextCursors.push(nextCursor);
+                                continue;
+                            }
+                            const commits = [];
+                            for (const item of incrementalItems) {
+                                nextCursor = Math.max(nextCursor, item.commitSeq);
+                                const seq = item.commitSeq;
+                                let commit = commits[commits.length - 1];
+                                if (!commit || commit.commitSeq !== seq) {
+                                    commit = {
+                                        commitSeq: seq,
+                                        createdAt: item.createdAt,
+                                        actorId: item.actorId,
+                                        changes: [],
+                                    };
+                                    commits.push(commit);
+                                }
+                                commit.changes.push(item.change);
+                            }
+                            const integrity = await createWireSubscriptionIntegrity({
+                                partitionId,
+                                subscriptionId: sub.id,
+                                previousRoot: typeof sub.verifiedRoot === 'string'
+                                    ? sub.verifiedRoot
+                                    : SYNCULAR_COMMIT_GENESIS_ROOT,
+                                commits,
                             });
-                        }
-                        nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
-                        if (latestByRowKey.size === 0) {
+                            nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
+                            if (commits.length === 0) {
+                                subResponses.push({
+                                    id: sub.id,
+                                    status: 'active',
+                                    scopes: effectiveScopes,
+                                    bootstrap: false,
+                                    nextCursor,
+                                    commits: [],
+                                });
+                                nextCursors.push(nextCursor);
+                                continue;
+                            }
                             subResponses.push({
                                 id: sub.id,
                                 status: 'active',
                                 scopes: effectiveScopes,
                                 bootstrap: false,
                                 nextCursor,
-                                commits: [],
+                                ...(integrity ? { integrity } : {}),
+                                commits,
                             });
                             nextCursors.push(nextCursor);
-                            continue;
                         }
-                        const commits = [];
-                        for (const item of latestByRowKey.values()) {
-                            const lastCommit = commits[commits.length - 1];
-                            if (!lastCommit || lastCommit.commitSeq !== item.commitSeq) {
-                                commits.push({
-                                    commitSeq: item.commitSeq,
-                                    createdAt: item.createdAt,
-                                    actorId: item.actorId,
-                                    changes: [item.change],
-                                });
-                                continue;
-                            }
-                            lastCommit.changes.push(item.change);
-                        }
-                        subResponses.push({
-                            id: sub.id,
-                            status: 'active',
-                            scopes: effectiveScopes,
-                            bootstrap: false,
-                            nextCursor,
-                            commits,
-                        });
-                        nextCursors.push(nextCursor);
-                        continue;
-                    }
-                    const commitsBySeq = new Map();
-                    const commitSeqs = [];
-                    for await (const r of dialect.iterateIncrementalPullRows(trx, {
-                        partitionId,
-                        table: sub.table,
-                        scopes: effectiveScopes,
-                        cursor,
-                        limitCommits,
-                    })) {
-                        nextCursor = Math.max(nextCursor, r.commit_seq);
-                        const seq = r.commit_seq;
-                        let commit = commitsBySeq.get(seq);
-                        if (!commit) {
-                            commit = {
-                                commitSeq: seq,
-                                createdAt: r.created_at,
-                                actorId: r.actor_id,
-                                changes: [],
-                            };
-                            commitsBySeq.set(seq, commit);
-                            commitSeqs.push(seq);
-                        }
-                        const change = {
-                            table: r.table,
-                            row_id: r.row_id,
-                            op: r.op,
-                            row_json: r.row_json,
-                            row_version: r.row_version,
-                            scopes: r.scopes,
+                        const effectiveScopes = mergeScopes(activeSubscriptions);
+                        const clientCursor = nextCursors.length > 0
+                            ? Math.min(...nextCursors)
+                            : maxCommitSeq;
+                        return {
+                            response: {
+                                ok: true,
+                                subscriptions: subResponses,
+                            },
+                            effectiveScopes,
+                            clientCursor,
                         };
-                        commit.changes.push(change);
-                    }
-                    nextCursor = Math.max(nextCursor, maxScannedCommitSeq);
-                    if (commitSeqs.length === 0) {
-                        subResponses.push({
-                            id: sub.id,
-                            status: 'active',
-                            scopes: effectiveScopes,
-                            bootstrap: false,
-                            nextCursor,
-                            commits: [],
-                        });
-                        nextCursors.push(nextCursor);
-                        continue;
-                    }
-                    const commits = commitSeqs
-                        .map((seq) => commitsBySeq.get(seq))
-                        .filter((c) => !!c)
-                        .filter((c) => c.changes.length > 0);
-                    subResponses.push({
-                        id: sub.id,
-                        status: 'active',
-                        scopes: effectiveScopes,
-                        bootstrap: false,
-                        nextCursor,
-                        commits,
-                    });
-                    nextCursors.push(nextCursor);
-                }
-                const effectiveScopes = mergeScopes(activeSubscriptions);
-                const clientCursor = nextCursors.length > 0 ? Math.min(...nextCursors) : maxCommitSeq;
-                return {
-                    response: {
-                        ok: true,
-                        subscriptions: subResponses,
-                    },
-                    effectiveScopes,
-                    clientCursor,
-                };
-            });
-            const chunkStorage = args.chunkStorage;
-            if (chunkStorage && pendingExternalChunkWrites.length > 0) {
-                for (const pending of pendingExternalChunkWrites) {
-                    let chunkRef = await readSnapshotChunkRefByPageKey(db, {
-                        partitionId: pending.cacheLookup.partitionId,
-                        scopeKey: pending.cacheLookup.scopeKey,
-                        scope: pending.cacheLookup.scope,
-                        asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
-                        rowCursor: pending.cacheLookup.rowCursor,
-                        rowLimit: pending.cacheLookup.rowLimit,
-                        encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
-                        compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
                     });
-                    if (!chunkRef) {
-                        const sha256 = await sha256Hex(pending.rowFramePayload);
-                        if (chunkStorage.storeChunkStream) {
-                            const { stream: bodyStream, byteLength } = await gzipBytesToStream(pending.rowFramePayload);
-                            chunkRef = await chunkStorage.storeChunkStream({
+                    const chunkStorage = args.chunkStorage;
+                    if (chunkStorage && pendingExternalChunkWrites.length > 0) {
+                        await runWithConcurrency(pendingExternalChunkWrites, 4, async (pending) => {
+                            const cacheLookupStartedAt = Date.now();
+                            let chunkRef = await readSnapshotChunkRefByPageKey(db, {
                                 partitionId: pending.cacheLookup.partitionId,
                                 scopeKey: pending.cacheLookup.scopeKey,
                                 scope: pending.cacheLookup.scope,
                                 asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
                                 rowCursor: pending.cacheLookup.rowCursor,
                                 rowLimit: pending.cacheLookup.rowLimit,
-                                encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
+                                encoding: snapshotChunkEncoding,
                                 compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
-                                sha256,
-                                byteLength,
-                                bodyStream,
-                                expiresAt: pending.expiresAt,
                             });
-                        }
-                        else {
-                            const compressedBody = await gzipBytes(pending.rowFramePayload);
-                            chunkRef = await chunkStorage.storeChunk({
-                                partitionId: pending.cacheLookup.partitionId,
-                                scopeKey: pending.cacheLookup.scopeKey,
-                                scope: pending.cacheLookup.scope,
-                                asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
-                                rowCursor: pending.cacheLookup.rowCursor,
-                                rowLimit: pending.cacheLookup.rowLimit,
-                                encoding: SYNC_SNAPSHOT_CHUNK_ENCODING,
-                                compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
-                                sha256,
-                                body: compressedBody,
-                                expiresAt: pending.expiresAt,
-                            });
-                        }
+                            bootstrapTimings.chunkCacheLookupMs += Math.max(0, Date.now() - cacheLookupStartedAt);
+                            if (!chunkRef) {
+                                if (chunkStorage.storeChunkStream) {
+                                    const { stream: bodyStream, byteLength, sha256, gzipMs, hashMs, } = await encodeCompressedSnapshotChunkToStream(pending.payloadParts, snapshotChunkGzipLevel);
+                                    bootstrapTimings.chunkGzipMs += gzipMs;
+                                    bootstrapTimings.chunkHashMs += hashMs;
+                                    const chunkPersistStartedAt = Date.now();
+                                    chunkRef = await chunkStorage.storeChunkStream({
+                                        partitionId: pending.cacheLookup.partitionId,
+                                        scopeKey: pending.cacheLookup.scopeKey,
+                                        scope: pending.cacheLookup.scope,
+                                        asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
+                                        rowCursor: pending.cacheLookup.rowCursor,
+                                        rowLimit: pending.cacheLookup.rowLimit,
+                                        nextRowCursor: pending.cacheLookup.nextRowCursor,
+                                        isLastPage: pending.cacheLookup.isLastPage,
+                                        encoding: snapshotChunkEncoding,
+                                        compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+                                        sha256,
+                                        byteLength,
+                                        bodyStream,
+                                        expiresAt: pending.expiresAt,
+                                    });
+                                    bootstrapTimings.chunkPersistMs += Math.max(0, Date.now() - chunkPersistStartedAt);
+                                }
+                                else {
+                                    const encodedChunk = await encodeCompressedSnapshotChunk(pending.payloadParts, snapshotChunkGzipLevel);
+                                    bootstrapTimings.chunkGzipMs += encodedChunk.gzipMs;
+                                    bootstrapTimings.chunkHashMs += encodedChunk.hashMs;
+                                    const chunkPersistStartedAt = Date.now();
+                                    chunkRef = await chunkStorage.storeChunk({
+                                        partitionId: pending.cacheLookup.partitionId,
+                                        scopeKey: pending.cacheLookup.scopeKey,
+                                        scope: pending.cacheLookup.scope,
+                                        asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
+                                        rowCursor: pending.cacheLookup.rowCursor,
+                                        rowLimit: pending.cacheLookup.rowLimit,
+                                        nextRowCursor: pending.cacheLookup.nextRowCursor,
+                                        isLastPage: pending.cacheLookup.isLastPage,
+                                        encoding: snapshotChunkEncoding,
+                                        compression: SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+                                        sha256: encodedChunk.sha256,
+                                        body: encodedChunk.body,
+                                        expiresAt: pending.expiresAt,
+                                    });
+                                    bootstrapTimings.chunkPersistMs += Math.max(0, Date.now() - chunkPersistStartedAt);
+                                }
+                            }
+                            const chunk = toResponseChunkRef(chunkRef);
+                            pending.snapshot.chunks = [chunk];
+                            pending.snapshot.manifest =
+                                await createChunkedSnapshotManifest({
+                                    table: pending.snapshot.table,
+                                    asOfCommitSeq: pending.cacheLookup.asOfCommitSeq,
+                                    scopeKey: pending.cacheLookup.scopeKey,
+                                    rowCursor: pending.cacheLookup.rowCursor,
+                                    rowLimit: pending.cacheLookup.rowLimit,
+                                    nextRowCursor: pending.cacheLookup.nextRowCursor,
+                                    isFirstPage: pending.snapshot.isFirstPage,
+                                    isLastPage: pending.cacheLookup.isLastPage,
+                                    chunks: [chunk],
+                                });
+                        });
                     }
-                    pending.snapshot.chunks = [chunkRef];
+                    const durationMs = Math.max(0, Date.now() - startedAtMs);
+                    const stats = summarizePullResponse(result.response);
+                    span.setAttribute('status', 'ok');
+                    span.setAttribute('duration_ms', durationMs);
+                    span.setAttribute('subscription_count', stats.subscriptionCount);
+                    span.setAttribute('commit_count', stats.commitCount);
+                    span.setAttribute('change_count', stats.changeCount);
+                    span.setAttribute('snapshot_page_count', stats.snapshotPageCount);
+                    span.setAttributes({
+                        bootstrap_snapshot_query_ms: bootstrapTimings.snapshotQueryMs,
+                        bootstrap_snapshot_binary_encode_ms: bootstrapTimings.binaryEncodeMs,
+                        bootstrap_chunk_cache_lookup_ms: bootstrapTimings.chunkCacheLookupMs,
+                        bootstrap_artifact_cache_lookup_ms: bootstrapTimings.artifactCacheLookupMs,
+                        bootstrap_chunk_gzip_ms: bootstrapTimings.chunkGzipMs,
+                        bootstrap_chunk_hash_ms: bootstrapTimings.chunkHashMs,
+                        bootstrap_chunk_persist_ms: bootstrapTimings.chunkPersistMs,
+                    });
+                    span.setStatus('ok');
+                    recordPullMetrics({
+                        status: 'ok',
+                        dedupeRows,
+                        durationMs,
+                        stats,
+                    });
+                    return {
+                        ...result,
+                        bootstrapTimings,
+                    };
+                }
+                catch (error) {
+                    if (error instanceof Error &&
+                        attemptIndex < MAX_PULL_TRANSACTION_RETRIES - 1 &&
+                        isSerializablePullError(error)) {
+                        await delay(PULL_TRANSACTION_RETRY_DELAY_MS * (attemptIndex + 1));
+                        continue;
+                    }
+                    throw error;
                 }
             }
-            const durationMs = Math.max(0, Date.now() - startedAtMs);
-            const stats = summarizePullResponse(result.response);
-            span.setAttribute('status', 'ok');
-            span.setAttribute('duration_ms', durationMs);
-            span.setAttribute('subscription_count', stats.subscriptionCount);
-            span.setAttribute('commit_count', stats.commitCount);
-            span.setAttribute('change_count', stats.changeCount);
-            span.setAttribute('snapshot_page_count', stats.snapshotPageCount);
-            span.setStatus('ok');
-            recordPullMetrics({
-                status: 'ok',
-                dedupeRows,
-                durationMs,
-                stats,
-            });
-            return result;
+            throw new Error('Pull transaction retry loop exhausted unexpectedly');
         }
         catch (error) {
             const durationMs = Math.max(0, Date.now() - startedAtMs);