@syncular/server 0.0.1-92 → 0.0.1-96

package/dist/snapshot-chunks/db-metadata.d.ts

@@ -26,7 +26,12 @@ export declare function createDbMetadataChunkStorage(options: DbMetadataSnapshot
     storeChunk: (metadata: Omit<SnapshotChunkMetadata, 'chunkId' | 'byteLength' | 'blobHash'> & {
         body: Uint8Array;
     }) => Promise<SyncSnapshotChunkRef>;
+    storeChunkStream: (metadata: Omit<SnapshotChunkMetadata, 'chunkId' | 'byteLength' | 'blobHash'> & {
+        bodyStream: ReadableStream<Uint8Array>;
+        byteLength: number;
+    }) => Promise<SyncSnapshotChunkRef>;
     readChunk: (chunkId: string) => Promise<Uint8Array | null>;
+    readChunkStream: (chunkId: string) => Promise<ReadableStream<Uint8Array> | null>;
     findChunk: (pageKey: SnapshotChunkPageKey) => Promise<SyncSnapshotChunkRef | null>;
     cleanupExpired: (beforeIso: string) => Promise<number>;
 };

package/dist/snapshot-chunks/db-metadata.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"db-metadata.d.ts","sourceRoot":"","sources":["../../src/snapshot-chunks/db-metadata.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAE3E,MAAM,WAAW,qCAAqC;IACpD,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACvB,4CAA4C;IAC5C,WAAW,EAAE,kBAAkB,CAAC;IAChC,oCAAoC;IACpC,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,qCAAqC,GAC7C;IACD,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,CACV,QAAQ,EAAE,IAAI,CACZ,qBAAqB,EACrB,SAAS,GAAG,YAAY,GAAG,UAAU,CACtC,GAAG;QACF,IAAI,EAAE,UAAU,CAAC;KAClB,KACE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACnC,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC3D,SAAS,EAAE,CACT,OAAO,EAAE,oBAAoB,KAC1B,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IAC1C,cAAc,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACxD,CAqNA"}
+{"version":3,"file":"db-metadata.d.ts","sourceRoot":"","sources":["../../src/snapshot-chunks/db-metadata.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/E,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAE3E,MAAM,WAAW,qCAAqC;IACpD,wBAAwB;IACxB,EAAE,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC;IACvB,4CAA4C;IAC5C,WAAW,EAAE,kBAAkB,CAAC;IAChC,oCAAoC;IACpC,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED;;;;GAIG;AACH,wBAAgB,4BAA4B,CAC1C,OAAO,EAAE,qCAAqC,GAC7C;IACD,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,CACV,QAAQ,EAAE,IAAI,CACZ,qBAAqB,EACrB,SAAS,GAAG,YAAY,GAAG,UAAU,CACtC,GAAG;QACF,IAAI,EAAE,UAAU,CAAC;KAClB,KACE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACnC,gBAAgB,EAAE,CAChB,QAAQ,EAAE,IAAI,CACZ,qBAAqB,EACrB,SAAS,GAAG,YAAY,GAAG,UAAU,CACtC,GAAG;QACF,UAAU,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACvC,UAAU,EAAE,MAAM,CAAC;KACpB,KACE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACnC,SAAS,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAC3D,eAAe,EAAE,CACf,OAAO,EAAE,MAAM,KACZ,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;IAChD,SAAS,EAAE,CACT,OAAO,EAAE,oBAAoB,KAC1B,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IAC1C,cAAc,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACxD,CA2WA"}

package/dist/snapshot-chunks/db-metadata.js

@@ -16,6 +16,40 @@ export function createDbMetadataChunkStorage(options) {
     function computeBlobHash(body) {
         return `sha256:${createHash('sha256').update(body).digest('hex')}`;
     }
+    function bytesToStream(bytes) {
+        return new ReadableStream({
+            start(controller) {
+                controller.enqueue(bytes);
+                controller.close();
+            },
+        });
+    }
+    async function streamToBytes(stream) {
+        const reader = stream.getReader();
+        try {
+            const chunks = [];
+            let total = 0;
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                if (!value)
+                    continue;
+                chunks.push(value);
+                total += value.length;
+            }
+            const out = new Uint8Array(total);
+            let offset = 0;
+            for (const chunk of chunks) {
+                out.set(chunk, offset);
+                offset += chunk.length;
+            }
+            return out;
+        }
+        finally {
+            reader.releaseLock();
+        }
+    }
     // Generate unique chunk ID
     function generateChunkId() {
         return `${chunkIdPrefix}${Date.now()}_${Math.random().toString(36).slice(2, 11)}`;
@@ -53,13 +87,73 @@ export function createDbMetadataChunkStorage(options) {
             compression: row.compression,
         };
     }
+    async function readBlobHash(chunkId) {
+        const row = await db
+            .selectFrom('sync_snapshot_chunks')
+            .select(['blob_hash'])
+            .where('chunk_id', '=', chunkId)
+            .executeTakeFirst();
+        return row?.blob_hash ?? null;
+    }
+    async function upsertChunkMetadata(metadata, args) {
+        const chunkId = generateChunkId();
+        const now = new Date().toISOString();
+        await db
+            .insertInto('sync_snapshot_chunks')
+            .values({
+            chunk_id: chunkId,
+            partition_id: metadata.partitionId,
+            scope_key: metadata.scopeKey,
+            scope: metadata.scope,
+            as_of_commit_seq: metadata.asOfCommitSeq,
+            row_cursor: metadata.rowCursor ?? '',
+            row_limit: metadata.rowLimit,
+            encoding: metadata.encoding,
+            compression: metadata.compression,
+            sha256: metadata.sha256,
+            byte_length: args.byteLength,
+            blob_hash: args.blobHash,
+            expires_at: metadata.expiresAt,
+            created_at: now,
+        })
+            .onConflict((oc) => oc
+            .columns([
+            'partition_id',
+            'scope_key',
+            'scope',
+            'as_of_commit_seq',
+            'row_cursor',
+            'row_limit',
+            'encoding',
+            'compression',
+        ])
+            .doUpdateSet({
+            expires_at: metadata.expiresAt,
+            blob_hash: args.blobHash,
+            sha256: metadata.sha256,
+            byte_length: args.byteLength,
+            row_cursor: metadata.rowCursor ?? '',
+        }))
+            .execute();
+    }
+    async function readChunkStreamById(chunkId) {
+        const blobHash = await readBlobHash(chunkId);
+        if (!blobHash)
+            return null;
+        if (blobAdapter.getStream) {
+            return blobAdapter.getStream(blobHash);
+        }
+        if (blobAdapter.get) {
+            const bytes = await blobAdapter.get(blobHash);
+            return bytes ? bytesToStream(bytes) : null;
+        }
+        throw new Error(`Blob adapter ${blobAdapter.name} does not support direct get() for snapshot chunks`);
+    }
     return {
         name: `db-metadata+${blobAdapter.name}`,
         async storeChunk(metadata) {
             const { body, ...metaWithoutBody } = metadata;
             const blobHash = computeBlobHash(body);
-            const chunkId = generateChunkId();
-            const now = new Date().toISOString();
             // Check if blob already exists (content-addressed dedup)
             const blobExists = await blobAdapter.exists(blobHash);
             if (!blobExists) {
@@ -67,48 +161,79 @@ export function createDbMetadataChunkStorage(options) {
                 if (blobAdapter.put) {
                     await blobAdapter.put(blobHash, body);
                 }
+                else if (blobAdapter.putStream) {
+                    await blobAdapter.putStream(blobHash, bytesToStream(body));
+                }
                 else {
                     throw new Error(`Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`);
                 }
             }
-            // Upsert metadata in database
-            await db
-                .insertInto('sync_snapshot_chunks')
-                .values({
-                chunk_id: chunkId,
-                partition_id: metaWithoutBody.partitionId,
-                scope_key: metaWithoutBody.scopeKey,
+            await upsertChunkMetadata(metaWithoutBody, {
+                blobHash,
+                byteLength: body.length,
+            });
+            const storedRef = await readStoredRef({
+                partitionId: metaWithoutBody.partitionId,
+                scopeKey: metaWithoutBody.scopeKey,
                 scope: metaWithoutBody.scope,
-                as_of_commit_seq: metaWithoutBody.asOfCommitSeq,
-                row_cursor: metaWithoutBody.rowCursor ?? '',
-                row_limit: metaWithoutBody.rowLimit,
+                asOfCommitSeq: metaWithoutBody.asOfCommitSeq,
+                rowCursor: metaWithoutBody.rowCursor,
+                rowLimit: metaWithoutBody.rowLimit,
                 encoding: metaWithoutBody.encoding,
                 compression: metaWithoutBody.compression,
-                sha256: metaWithoutBody.sha256,
-                byte_length: body.length,
-                blob_hash: blobHash,
-                expires_at: metaWithoutBody.expiresAt,
-                created_at: now,
-            })
-                .onConflict((oc) => oc
-                .columns([
-                'partition_id',
-                'scope_key',
-                'scope',
-                'as_of_commit_seq',
-                'row_cursor',
-                'row_limit',
-                'encoding',
-                'compression',
-            ])
-                .doUpdateSet({
-                expires_at: metaWithoutBody.expiresAt,
-                blob_hash: blobHash,
-                sha256: metaWithoutBody.sha256,
-                byte_length: body.length,
-                row_cursor: metaWithoutBody.rowCursor ?? '',
-            }))
-                .execute();
+                includeExpired: true,
+            });
+            if (!storedRef) {
+                throw new Error('Failed to read stored snapshot chunk reference');
+            }
+            return storedRef;
+        },
+        async storeChunkStream(metadata) {
+            const { bodyStream, byteLength, ...metaWithoutBody } = metadata;
+            const [uploadStream, hashStream] = bodyStream.tee();
+            const hasher = createHash('sha256');
+            let observedByteLength = 0;
+            const hashReader = hashStream.getReader();
+            try {
+                while (true) {
+                    const { done, value } = await hashReader.read();
+                    if (done)
+                        break;
+                    if (!value)
+                        continue;
+                    hasher.update(value);
+                    observedByteLength += value.length;
+                }
+            }
+            finally {
+                hashReader.releaseLock();
+            }
+            if (observedByteLength !== byteLength) {
+                throw new Error(`Snapshot chunk byte length mismatch: expected ${byteLength}, got ${observedByteLength}`);
+            }
+            const blobHash = `sha256:${hasher.digest('hex')}`;
+            const blobExists = await blobAdapter.exists(blobHash);
+            if (!blobExists) {
+                if (blobAdapter.putStream) {
+                    await blobAdapter.putStream(blobHash, uploadStream, {
+                        byteLength: observedByteLength,
+                    });
+                }
+                else if (blobAdapter.put) {
+                    const body = await streamToBytes(uploadStream);
+                    await blobAdapter.put(blobHash, body);
+                }
+                else {
+                    throw new Error(`Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`);
+                }
+            }
+            else {
+                await uploadStream.cancel();
+            }
+            await upsertChunkMetadata(metaWithoutBody, {
+                blobHash,
+                byteLength: observedByteLength,
+            });
             const storedRef = await readStoredRef({
                 partitionId: metaWithoutBody.partitionId,
                 scopeKey: metaWithoutBody.scopeKey,
@@ -126,19 +251,13 @@ export function createDbMetadataChunkStorage(options) {
             return storedRef;
         },
         async readChunk(chunkId) {
-            // Get metadata to find blob hash
-            const row = await db
-                .selectFrom('sync_snapshot_chunks')
-                .select(['blob_hash'])
-                .where('chunk_id', '=', chunkId)
-                .executeTakeFirst();
-            if (!row)
+            const stream = await readChunkStreamById(chunkId);
+            if (!stream)
                 return null;
-            // Read from blob adapter
-            if (blobAdapter.get) {
-                return blobAdapter.get(row.blob_hash);
-            }
-            throw new Error(`Blob adapter ${blobAdapter.name} does not support direct get() for snapshot chunks`);
+            return streamToBytes(stream);
+        },
+        async readChunkStream(chunkId) {
+            return readChunkStreamById(chunkId);
         },
         async findChunk(pageKey) {
             return readStoredRef(pageKey);

package/dist/snapshot-chunks/db-metadata.js.map

@@ -1 +1 @@
-{"version":3,"file":"db-metadata.js","sourceRoot":"","sources":["../../src/snapshot-chunks/db-metadata.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAezC;;;;GAIG;AACH,MAAM,UAAU,4BAA4B,CAC1C,OAA8C,EAgB9C;IACA,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,aAAa,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC;IAE9D,gDAAgD;IAChD,SAAS,eAAe,CAAC,IAAgB,EAAU;QACjD,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAAA,CACpE;IAED,2BAA2B;IAC3B,SAAS,eAAe,GAAW;QACjC,OAAO,GAAG,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAAA,CACnF;IAED,KAAK,UAAU,aAAa,CAAC,IAW5B,EAAwC;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,EAAE;aACjB,UAAU,CAAC,sBAAsB,CAAC;aAClC,MAAM,CAAC,CAAC,UAAU,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;aACxE,KAAK,CAAC,cAAc,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC;aAC5C,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC;aACtC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC;aAC/B,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC;aAClD,KAAK,CAAC,YAAY,EAAE,GAAG,EAAE,YAAY,CAAC;aACtC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC;aACtC,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC;aACrC,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc;YACpC,CAAC,CAAC,SAAS,CAAC,gBAAgB,EAAE;YAC9B,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAEnE,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,uCAAuC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC9D,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CACb,0CAA0C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CACpE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,QAAQ;YAChB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;YACxC,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,WAAW,EAAE,GAAG,CAAC,WAAW;SAC7B,CAAC;IAAA,CACH;IAED,OAAO;QACL,IAAI,EAAE,eAAe,WAAW,CAAC,IAAI,EAAE;QAEvC,KAAK,CAAC,UAAU,CACd,QAKC,EAC8B;YAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,eAAe,EAAE,GAAG,QAAQ,CAAC;YAC9C,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;YACvC,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAErC,yDAAyD;YACzD,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEtD,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,6BAA6B;gBAC7B,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;oBACpB,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBACxC,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CACb,gBAAgB,WAAW,CAAC,IAAI,oDAAoD,CACrF,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,8BAA8B;YAC9B,MAAM,EAAE;iBACL,UAAU,CAAC,sBAAsB,CAAC;iBAClC,MAAM,CAAC;gBACN,QAAQ,EAAE,OAAO;gBACjB,YAAY,EAAE,eAAe,CAAC,WAAW;gBACzC,SAAS,EAAE,eAAe,CAAC,QAAQ;gBACnC,KAAK,EAAE,eAAe,CAAC,KAAK;gBAC5B,gBAAgB,EAAE,eAAe,CAAC,aAAa;gBAC/C,UAAU,EAAE,eAAe,CAAC,SAAS,IAAI,EAAE;gBAC3C,SAAS,EAAE,eAAe,CAAC,QAAQ;gBACnC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,MAAM,EAAE,eAAe,CAAC,MAAM;gBAC9B,WAAW,EAAE,IAAI,CAAC,MAAM;gBACxB,SAAS,EAAE,QAAQ;gBACnB,UAAU,EAAE,eAAe,CAAC,SAAS;gBACrC,UAAU,EAAE,GAAG;aAChB,CAAC;iBACD,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CACjB,EAAE;iBACC,OAAO,CAAC;gBACP,cAAc;gBACd,WAAW;gBACX,OAAO;gBACP,kBAAkB;gBAClB,YAAY;gBACZ,WAAW;gBACX,UAAU;gBACV,aAAa;aACd,CAAC;iBACD,WAAW,CAAC;gBACX,UAAU,EAAE,eAAe,CAAC,SAAS;gBACrC,SAAS,EAAE,QAAQ;gBACnB,MAAM,EAAE,eAAe,CAAC,MAAM;gBAC9B,WAAW,EAAE,IAAI,CAAC,MAAM;gBACxB,UAAU,EAAE,eAAe,CAAC,SAAS,IAAI,EAAE;aAC5C,CAAC,CACL;iBACA,OAAO,EAAE,CAAC;YAEb,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC;gBACpC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,KAAK,EAAE,eAAe,CAAC,KAAK;gBAC5B,aAAa,EAAE,eAAe,CAAC,aAAa;gBAC5C,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACpE,CAAC;YAED,OAAO,SAAS,CAAC;QAAA,CAClB;QAED,KAAK,CAAC,SAAS,CAAC,OAAe,EAA8B;YAC3D,iCAAiC;YACjC,MAAM,GAAG,GAAG,MAAM,EAAE;iBACjB,UAAU,CAAC,sBAAsB,CAAC;iBAClC,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC;iBACrB,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,OAAO,CAAC;iBAC/B,gBAAgB,EAAE,CAAC;YAEtB,IAAI,CAAC,GAAG;gBAAE,OAAO,IAAI,CAAC;YAEtB,yBAAyB;YACzB,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;gBACpB,OAAO,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACxC,CAAC;YAED,MAAM,IAAI,KAAK,CACb,gBAAgB,WAAW,CAAC,IAAI,oDAAoD,CACrF,CAAC;QAAA,CACH;QAED,KAAK,CAAC,SAAS,CACb,OAA6B,EACS;YACtC,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;QAAA,CAC/B;QAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAmB;YACvD,sBAAsB;YACtB,MAAM,WAAW,GAAG,MAAM,EAAE;iBACzB,UAAU,CAAC,sBAAsB,CAAC;iBAClC,MAAM,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;iBACjC,KAAK,CAAC,YAAY,EAAE,IAAI,EAAE,SAAS,CAAC;iBACpC,OAAO,EAAE,CAAC;YAEb,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,CAAC,CAAC;YAEvC,yCAAyC;YACzC,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC9B,IAAI,CAAC;oBACH,MAAM,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;gBAAC,MAAM,CAAC;oBACP,iEAAiE;oBACjE,mDAAmD;oBACnD,OAAO,CAAC,IAAI,CACV,yBAAyB,GAAG,CAAC,SAAS,cAAc,GAAG,CAAC,QAAQ,oCAAoC,CACrG,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,gCAAgC;YAChC,MAAM,MAAM,GAAG,MAAM,EAAE;iBACpB,UAAU,CAAC,sBAAsB,CAAC;iBAClC,KAAK,CAAC,YAAY,EAAE,IAAI,EAAE,SAAS,CAAC;iBACpC,gBAAgB,EAAE,CAAC;YAEtB,OAAO,MAAM,CAAC,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC;QAAA,CAC3C;KACF,CAAC;AAAA,CACH"}
+{"version":3,"file":"db-metadata.js","sourceRoot":"","sources":["../../src/snapshot-chunks/db-metadata.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAezC;;;;GAIG;AACH,MAAM,UAAU,4BAA4B,CAC1C,OAA8C,EA4B9C;IACA,MAAM,EAAE,EAAE,EAAE,WAAW,EAAE,aAAa,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC;IAE9D,gDAAgD;IAChD,SAAS,eAAe,CAAC,IAAgB,EAAU;QACjD,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;IAAA,CACpE;IAED,SAAS,aAAa,CAAC,KAAiB,EAA8B;QACpE,OAAO,IAAI,cAAc,CAAa;YACpC,KAAK,CAAC,UAAU,EAAE;gBAChB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;gBAC1B,UAAU,CAAC,KAAK,EAAE,CAAC;YAAA,CACpB;SACF,CAAC,CAAC;IAAA,CACJ;IAED,KAAK,UAAU,aAAa,CAC1B,MAAkC,EACb;QACrB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,KAAK,GAAG,CAAC,CAAC;YAEd,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI;oBAAE,MAAM;gBAChB,IAAI,CAAC,KAAK;oBAAE,SAAS;gBACrB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;YACxB,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;YAClC,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBACvB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,WAAW,EAAE,CAAC;QACvB,CAAC;IAAA,CACF;IAED,2BAA2B;IAC3B,SAAS,eAAe,GAAW;QACjC,OAAO,GAAG,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAAA,CACnF;IAED,KAAK,UAAU,aAAa,CAAC,IAW5B,EAAwC;QACvC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,EAAE;aACjB,UAAU,CAAC,sBAAsB,CAAC;aAClC,MAAM,CAAC,CAAC,UAAU,EAAE,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;aACxE,KAAK,CAAC,cAAc,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC;aAC5C,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC;aACtC,KAAK,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC;aAC/B,KAAK,CAAC,kBAAkB,EAAE,GAAG,EAAE,IAAI,CAAC,aAAa,CAAC;aAClD,KAAK,CAAC,YAAY,EAAE,GAAG,EAAE,YAAY,CAAC;aACtC,KAAK,CAAC,WAAW,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC;aACtC,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC;aACrC,KAAK,CAAC,aAAa,EAAE,GAAG,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE/C,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,cAAc;YACpC,CAAC,CAAC,SAAS,CAAC,gBAAgB,EAAE;YAC9B,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC;QAEnE,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QAEtB,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,uCAAuC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC9D,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CACb,0CAA0C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CACpE,CAAC;QACJ,CAAC;QAED,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,QAAQ;YAChB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;YACxC,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,WAAW,EAAE,GAAG,CAAC,WAAW;SAC7B,CAAC;IAAA,CACH;IAED,KAAK,UAAU,YAAY,CAAC,OAAe,EAA0B;QACnE,MAAM,GAAG,GAAG,MAAM,EAAE;aACjB,UAAU,CAAC,sBAAsB,CAAC;aAClC,MAAM,CAAC,CAAC,WAAW,CAAC,CAAC;aACrB,KAAK,CAAC,UAAU,EAAE,GAAG,EAAE,OAAO,CAAC;aAC/B,gBAAgB,EAAE,CAAC;QACtB,OAAO,GAAG,EAAE,SAAS,IAAI,IAAI,CAAC;IAAA,CAC/B;IAED,KAAK,UAAU,mBAAmB,CAChC,QAGC,EACD,IAA8C,EAC/B;QACf,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;QAClC,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAErC,MAAM,EAAE;aACL,UAAU,CAAC,sBAAsB,CAAC;aAClC,MAAM,CAAC;YACN,QAAQ,EAAE,OAAO;YACjB,YAAY,EAAE,QAAQ,CAAC,WAAW;YAClC,SAAS,EAAE,QAAQ,CAAC,QAAQ;YAC5B,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,gBAAgB,EAAE,QAAQ,CAAC,aAAa;YACxC,UAAU,EAAE,QAAQ,CAAC,SAAS,IAAI,EAAE;YACpC,SAAS,EAAE,QAAQ,CAAC,QAAQ;YAC5B,QAAQ,EAAE,QAAQ,CAAC,QAAQ;YAC3B,WAAW,EAAE,QAAQ,CAAC,WAAW;YACjC,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,IAAI,CAAC,UAAU;YAC5B,SAAS,EAAE,IAAI,CAAC,QAAQ;YACxB,UAAU,EAAE,QAAQ,CAAC,SAAS;YAC9B,UAAU,EAAE,GAAG;SAChB,CAAC;aACD,UAAU,CAAC,CAAC,EAAE,EAAE,EAAE,CACjB,EAAE;aACC,OAAO,CAAC;YACP,cAAc;YACd,WAAW;YACX,OAAO;YACP,kBAAkB;YAClB,YAAY;YACZ,WAAW;YACX,UAAU;YACV,aAAa;SACd,CAAC;aACD,WAAW,CAAC;YACX,UAAU,EAAE,QAAQ,CAAC,SAAS;YAC9B,SAAS,EAAE,IAAI,CAAC,QAAQ;YACxB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,IAAI,CAAC,UAAU;YAC5B,UAAU,EAAE,QAAQ,CAAC,SAAS,IAAI,EAAE;SACrC,CAAC,CACL;aACA,OAAO,EAAE,CAAC;IAAA,CACd;IAED,KAAK,UAAU,mBAAmB,CAChC,OAAe,EAC6B;QAC5C,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,CAAC;QAC7C,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;YAC1B,OAAO,WAAW,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC9C,OAAO,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC7C,CAAC;QAED,MAAM,IAAI,KAAK,CACb,gBAAgB,WAAW,CAAC,IAAI,oDAAoD,CACrF,CAAC;IAAA,CACH;IAED,OAAO;QACL,IAAI,EAAE,eAAe,WAAW,CAAC,IAAI,EAAE;QAEvC,KAAK,CAAC,UAAU,CACd,QAKC,EAC8B;YAC/B,MAAM,EAAE,IAAI,EAAE,GAAG,eAAe,EAAE,GAAG,QAAQ,CAAC;YAC9C,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;YAEvC,yDAAyD;YACzD,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEtD,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,6BAA6B;gBAC7B,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;oBACpB,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBACxC,CAAC;qBAAM,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;oBACjC,MAAM,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC;gBAC7D,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CACb,gBAAgB,WAAW,CAAC,IAAI,oDAAoD,CACrF,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,MAAM,mBAAmB,CAAC,eAAe,EAAE;gBACzC,QAAQ;gBACR,UAAU,EAAE,IAAI,CAAC,MAAM;aACxB,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC;gBACpC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,KAAK,EAAE,eAAe,CAAC,KAAK;gBAC5B,aAAa,EAAE,eAAe,CAAC,aAAa;gBAC5C,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACpE,CAAC;YAED,OAAO,SAAS,CAAC;QAAA,CAClB;QAED,KAAK,CAAC,gBAAgB,CACpB,QAMC,EAC8B;YAC/B,MAAM,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,eAAe,EAAE,GAAG,QAAQ,CAAC;YAChE,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC;YAEpD,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;YACpC,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAC3B,MAAM,UAAU,GAAG,UAAU,CAAC,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;oBAChD,IAAI,IAAI;wBAAE,MAAM;oBAChB,IAAI,CAAC,KAAK;wBAAE,SAAS;oBACrB,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACrB,kBAAkB,IAAI,KAAK,CAAC,MAAM,CAAC;gBACrC,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,UAAU,CAAC,WAAW,EAAE,CAAC;YAC3B,CAAC;YAED,IAAI,kBAAkB,KAAK,UAAU,EAAE,CAAC;gBACtC,MAAM,IAAI,KAAK,CACb,iDAAiD,UAAU,SAAS,kBAAkB,EAAE,CACzF,CAAC;YACJ,CAAC;YAED,MAAM,QAAQ,GAAG,UAAU,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;YAElD,MAAM,UAAU,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACtD,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,IAAI,WAAW,CAAC,SAAS,EAAE,CAAC;oBAC1B,MAAM,WAAW,CAAC,SAAS,CAAC,QAAQ,EAAE,YAAY,EAAE;wBAClD,UAAU,EAAE,kBAAkB;qBAC/B,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,WAAW,CAAC,GAAG,EAAE,CAAC;oBAC3B,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,CAAC;oBAC/C,MAAM,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBACxC,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CACb,gBAAgB,WAAW,CAAC,IAAI,oDAAoD,CACrF,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,YAAY,CAAC,MAAM,EAAE,CAAC;YAC9B,CAAC;YAED,MAAM,mBAAmB,CAAC,eAAe,EAAE;gBACzC,QAAQ;gBACR,UAAU,EAAE,kBAAkB;aAC/B,CAAC,CAAC;YAEH,MAAM,SAAS,GAAG,MAAM,aAAa,CAAC;gBACpC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,KAAK,EAAE,eAAe,CAAC,KAAK;gBAC5B,aAAa,EAAE,eAAe,CAAC,aAAa;gBAC5C,SAAS,EAAE,eAAe,CAAC,SAAS;gBACpC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,QAAQ,EAAE,eAAe,CAAC,QAAQ;gBAClC,WAAW,EAAE,eAAe,CAAC,WAAW;gBACxC,cAAc,EAAE,IAAI;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;YACpE,CAAC;YAED,OAAO,SAAS,CAAC;QAAA,CAClB;QAED,KAAK,CAAC,SAAS,CAAC,OAAe,EAA8B;YAC3D,MAAM,MAAM,GAAG,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;YAClD,IAAI,CAAC,MAAM;gBAAE,OAAO,IAAI,CAAC;YACzB,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;QAAA,CAC9B;QAED,KAAK,CAAC,eAAe,CACnB,OAAe,EAC6B;YAC5C,OAAO,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAAA,CACrC;QAED,KAAK,CAAC,SAAS,CACb,OAA6B,EACS;YACtC,OAAO,aAAa,CAAC,OAAO,CAAC,CAAC;QAAA,CAC/B;QAED,KAAK,CAAC,cAAc,CAAC,SAAiB,EAAmB;YACvD,sBAAsB;YACtB,MAAM,WAAW,GAAG,MAAM,EAAE;iBACzB,UAAU,CAAC,sBAAsB,CAAC;iBAClC,MAAM,CAAC,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;iBACjC,KAAK,CAAC,YAAY,EAAE,IAAI,EAAE,SAAS,CAAC;iBACpC,OAAO,EAAE,CAAC;YAEb,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,CAAC,CAAC;YAEvC,yCAAyC;YACzC,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;gBAC9B,IAAI,CAAC;oBACH,MAAM,WAAW,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;gBAAC,MAAM,CAAC;oBACP,iEAAiE;oBACjE,mDAAmD;oBACnD,OAAO,CAAC,IAAI,CACV,yBAAyB,GAAG,CAAC,SAAS,cAAc,GAAG,CAAC,QAAQ,oCAAoC,CACrG,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,gCAAgC;YAChC,MAAM,MAAM,GAAG,MAAM,EAAE;iBACpB,UAAU,CAAC,sBAAsB,CAAC;iBAClC,KAAK,CAAC,YAAY,EAAE,IAAI,EAAE,SAAS,CAAC;iBACpC,gBAAgB,EAAE,CAAC;YAEtB,OAAO,MAAM,CAAC,MAAM,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC;QAAA,CAC3C;KACF,CAAC;AAAA,CACH"}

package/dist/snapshot-chunks/types.d.ts

@@ -49,10 +49,23 @@ export interface SnapshotChunkStorage {
     storeChunk(metadata: Omit<SnapshotChunkMetadata, 'chunkId' | 'byteLength' | 'blobHash'> & {
         body: Uint8Array;
     }): Promise<SyncSnapshotChunkRef>;
+    /**
+     * Store a chunk from a stream.
+     * Preferred for large payloads to avoid full buffering in memory.
+     */
+    storeChunkStream?(metadata: Omit<SnapshotChunkMetadata, 'chunkId' | 'byteLength' | 'blobHash'> & {
+        bodyStream: ReadableStream<Uint8Array>;
+        byteLength: number;
+    }): Promise<SyncSnapshotChunkRef>;
     /**
      * Read chunk body by chunk ID
      */
     readChunk(chunkId: string): Promise<Uint8Array | null>;
+    /**
+     * Read chunk body as a stream.
+     * Preferred for large payloads to avoid full buffering in memory.
+     */
+    readChunkStream?(chunkId: string): Promise<ReadableStream<Uint8Array> | null>;
     /**
      * Find existing chunk by page key
      */

package/dist/snapshot-chunks/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/snapshot-chunks/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,2BAA2B;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,UAAU,CACR,QAAQ,EAAE,IAAI,CACZ,qBAAqB,EACrB,SAAS,GAAG,YAAY,GAAG,UAAU,CACtC,GAAG;QACF,IAAI,EAAE,UAAU,CAAC;KAClB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAEvD;;OAEG;IACH,SAAS,CACP,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IAExC;;OAEG;IACH,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpD"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/snapshot-chunks/types.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAE3D;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,2BAA2B;IAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,UAAU,CACR,QAAQ,EAAE,IAAI,CACZ,qBAAqB,EACrB,SAAS,GAAG,YAAY,GAAG,UAAU,CACtC,GAAG;QACF,IAAI,EAAE,UAAU,CAAC;KAClB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;;OAGG;IACH,gBAAgB,CAAC,CACf,QAAQ,EAAE,IAAI,CACZ,qBAAqB,EACrB,SAAS,GAAG,YAAY,GAAG,UAAU,CACtC,GAAG;QACF,UAAU,EAAE,cAAc,CAAC,UAAU,CAAC,CAAC;QACvC,UAAU,EAAE,MAAM,CAAC;KACpB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEjC;;OAEG;IACH,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;IAEvD;;;OAGG;IACH,eAAe,CAAC,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;IAE9E;;OAEG;IACH,SAAS,CACP,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CAAC;IAExC;;OAEG;IACH,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACpD"}

package/dist/snapshot-chunks.d.ts

@@ -29,7 +29,7 @@ export interface SnapshotChunkRow {
     compression: 'gzip';
     sha256: string;
     byteLength: number;
-    body: Uint8Array;
+    body: Uint8Array | ReadableStream<Uint8Array>;
     expiresAt: string;
 }
 export declare function readSnapshotChunkRefByPageKey<DB extends SyncCoreDb>(db: Kysely<DB>, args: SnapshotChunkPageKey & {
@@ -53,6 +53,7 @@ export declare function readSnapshotChunk<DB extends SyncCoreDb>(db: Kysely<DB>,
     /** External chunk storage for reading from S3/R2/etc */
     chunkStorage?: {
         readChunk(chunkId: string): Promise<Uint8Array | null>;
+        readChunkStream?(chunkId: string): Promise<ReadableStream<Uint8Array> | null>;
     };
 }): Promise<SnapshotChunkRow | null>;
 export declare function deleteExpiredSnapshotChunks<DB extends SyncCoreDb>(db: Kysely<DB>, nowIso?: string): Promise<number>;

package/dist/snapshot-chunks.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-chunks.d.ts","sourceRoot":"","sources":["../src/snapshot-chunks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,EAAE,KAAK,MAAM,EAAO,MAAM,QAAQ,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAqBD,wBAAsB,6BAA6B,CAAC,EAAE,SAAS,UAAU,EACvE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,IAAI,EAAE,oBAAoB,GAAG;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/C,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CA+CtC;AAED,wBAAsB,mBAAmB,CAAC,EAAE,SAAS,UAAU,EAC7D,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,IAAI,EAAE;IACJ,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAyE/B;AAED,wBAAsB,iBAAiB,CAAC,EAAE,SAAS,UAAU,EAC3D,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;IACR,wDAAwD;IACxD,YAAY,CAAC,EAAE;QAAE,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAA;KAAE,CAAC;CAC3E,GACA,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAiFlC;AAED,wBAAsB,2BAA2B,CAAC,EAAE,SAAS,UAAU,EACrE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,MAAM,SAA2B,GAChC,OAAO,CAAC,MAAM,CAAC,CAOjB"}
+{"version":3,"file":"snapshot-chunks.d.ts","sourceRoot":"","sources":["../src/snapshot-chunks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC3D,OAAO,EAAE,KAAK,MAAM,EAAO,MAAM,QAAQ,CAAC;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAE3C,MAAM,WAAW,oBAAoB;IACnC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,UAAU,GAAG,cAAc,CAAC,UAAU,CAAC,CAAC;IAC9C,SAAS,EAAE,MAAM,CAAC;CACnB;AAqBD,wBAAsB,6BAA6B,CAAC,EAAE,SAAS,UAAU,EACvE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,IAAI,EAAE,oBAAoB,GAAG;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,GAC/C,OAAO,CAAC,oBAAoB,GAAG,IAAI,CAAC,CA+CtC;AAED,wBAAsB,mBAAmB,CAAC,EAAE,SAAS,UAAU,EAC7D,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,IAAI,EAAE;IACJ,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,UAAU,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB,GACA,OAAO,CAAC,oBAAoB,CAAC,CAyE/B;AAED,wBAAsB,iBAAiB,CAAC,EAAE,SAAS,UAAU,EAC3D,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,OAAO,EAAE,MAAM,EACf,OAAO,CAAC,EAAE;IACR,wDAAwD;IACxD,YAAY,CAAC,EAAE;QACb,SAAS,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC;QACvD,eAAe,CAAC,CACd,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,CAAC;KAC/C,CAAC;CACH,GACA,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAkGlC;AAED,wBAAsB,2BAA2B,CAAC,EAAE,SAAS,UAAU,EACrE,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,MAAM,SAA2B,GAChC,OAAO,CAAC,MAAM,CAAC,CAOjB"}

package/dist/snapshot-chunks.js

@@ -162,15 +162,35 @@ export async function readSnapshotChunk(db, chunkId, options) {
     // Read body from external storage if available, otherwise use inline body
     let body;
     if (options?.chunkStorage) {
-        const externalBody = await options.chunkStorage.readChunk(chunkId);
-        if (externalBody) {
-            body = externalBody;
-        }
-        else if (row.body) {
-            body = coerceChunkRow(row.body);
+        if (options.chunkStorage.readChunkStream) {
+            const externalBodyStream = await options.chunkStorage.readChunkStream(chunkId);
+            if (externalBodyStream) {
+                body = externalBodyStream;
+            }
+            else {
+                const externalBody = await options.chunkStorage.readChunk(chunkId);
+                if (externalBody) {
+                    body = externalBody;
+                }
+                else if (row.body) {
+                    body = coerceChunkRow(row.body);
+                }
+                else {
+                    throw new Error(`Snapshot chunk body missing for chunk ${chunkId}`);
+                }
+            }
         }
         else {
-            throw new Error(`Snapshot chunk body missing for chunk ${chunkId}`);
+            const externalBody = await options.chunkStorage.readChunk(chunkId);
+            if (externalBody) {
+                body = externalBody;
+            }
+            else if (row.body) {
+                body = coerceChunkRow(row.body);
+            }
+            else {
+                throw new Error(`Snapshot chunk body missing for chunk ${chunkId}`);
+            }
         }
     }
     else {

package/dist/snapshot-chunks.js.map

@@ -1 +1 @@
-{"version":3,"file":"snapshot-chunks.js","sourceRoot":"","sources":["../src/snapshot-chunks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAe,GAAG,EAAE,MAAM,QAAQ,CAAC;AA8B1C,SAAS,cAAc,CAAC,KAAc,EAAc;IAClD,wEAAwE;IACxE,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,OAAO,MAAM,KAAK,WAAW,IAAI,KAAK,YAAY,MAAM;QAAE,OAAO,KAAK,CAAC;IAC3E,IAAI,KAAK,YAAY,WAAW;QAAE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,EAAE,CAAC;QACtE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,IAAI,KAAK,CACb,wCAAwC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAChF,CAAC;AAAA,CACH;AAED,SAAS,eAAe,CAAC,KAAc,EAAU;IAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,KAAK,YAAY,IAAI;QAAE,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IACtD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AAAA,CACtB;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,EAAc,EACd,IAAgD,EACV;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACvD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IAE1C,MAAM,SAAS,GAAG,MAAM,GAAG,CAMzB;;WAEO,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;;uBAErB,IAAI,CAAC,WAAW;wBACf,IAAI,CAAC,QAAQ;oBACjB,IAAI,CAAC,KAAK;+BACC,IAAI,CAAC,aAAa;yBACxB,YAAY;wBACb,IAAI,CAAC,QAAQ;uBACd,IAAI,CAAC,QAAQ;0BACV,IAAI,CAAC,WAAW;yBACjB,MAAM;;GAE5B,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACd,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,uCAAuC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC9D,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CACb,0CAA0C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CACpE,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,QAAQ;QAChB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;QACxC,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,GAAG,CAAC,WAAW;KAC7B,CAAC;AAAA,CACH;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAc,EACd,IAaC,EAC8B;IAC/B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IAE1C,yDAAyD;IACzD,MAAM,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;IAEzC,MAAM,GAAG,CAAA;kBACO,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;;;;;;;;;;;;;;;;;;QAkB3C,IAAI,CAAC,OAAO;QACZ,IAAI,CAAC,WAAW;QAChB,IAAI,CAAC,QAAQ;QACb,IAAI,CAAC,KAAK;QACV,IAAI,CAAC,aAAa;QAClB,YAAY;QACZ,IAAI,CAAC,QAAQ;QACb,IAAI,CAAC,QAAQ;QACb,IAAI,CAAC,WAAW;QAChB,IAAI,CAAC,MAAM;QACX,IAAI,CAAC,IAAI,CAAC,MAAM;QAChB,QAAQ;QACR,IAAI,CAAC,IAAI;QACT,GAAG;QACH,IAAI,CAAC,SAAS;;;;;;;;;;;;;qBAaD,IAAI,CAAC,SAAS;oBACf,QAAQ;GACzB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAEd,MAAM,GAAG,GAAG,MAAM,6BAA6B,CAAC,EAAE,EAAE;QAClD,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,GAAG,CAAC;AAAA,CACZ;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAc,EACd,OAAe,EACf,OAGC,EACiC;IAClC,MAAM,SAAS,GAAG,MAAM,GAAG,CAezB;;;;;;;;;;;;;;;;WAgBO,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;uBACrB,OAAO;;GAE3B,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACd,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,uCAAuC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC9D,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CACb,0CAA0C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CACpE,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,IAAI,IAAgB,CAAC;IACrB,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QAC1B,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACnE,IAAI,YAAY,EAAE,CAAC;YACjB,IAAI,GAAG,YAAY,CAAC;QACtB,CAAC;aAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACpB,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAClC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,EAAE,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,QAAQ,EAAE,GAAG,CAAC,SAAS;QACvB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,CAAC;QAChD,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC;QACpC,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;QACxC,IAAI;QACJ,SAAS,EAAE,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC;KAC3C,CAAC;AAAA,CACH;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,EAAc,EACd,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAChB;IACjB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAA;kBACL,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;0BACzB,MAAM;GAC7B,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAEd,OAAO,MAAM,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC;AAAA,CACzC"}
+{"version":3,"file":"snapshot-chunks.js","sourceRoot":"","sources":["../src/snapshot-chunks.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAe,GAAG,EAAE,MAAM,QAAQ,CAAC;AA8B1C,SAAS,cAAc,CAAC,KAAc,EAAc;IAClD,wEAAwE;IACxE,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,KAAK,CAAC;IAC9C,IAAI,OAAO,MAAM,KAAK,WAAW,IAAI,KAAK,YAAY,MAAM;QAAE,OAAO,KAAK,CAAC;IAC3E,IAAI,KAAK,YAAY,WAAW;QAAE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/D,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,EAAE,CAAC;QACtE,OAAO,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,IAAI,KAAK,CACb,wCAAwC,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAChF,CAAC;AAAA,CACH;AAED,SAAS,eAAe,CAAC,KAAc,EAAU;IAC/C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,KAAK,YAAY,IAAI;QAAE,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IACtD,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;AAAA,CACtB;AAED,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,EAAc,EACd,IAAgD,EACV;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACvD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IAE1C,MAAM,SAAS,GAAG,MAAM,GAAG,CAMzB;;WAEO,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;;uBAErB,IAAI,CAAC,WAAW;wBACf,IAAI,CAAC,QAAQ;oBACjB,IAAI,CAAC,KAAK;+BACC,IAAI,CAAC,aAAa;yBACxB,YAAY;wBACb,IAAI,CAAC,QAAQ;uBACd,IAAI,CAAC,QAAQ;0BACV,IAAI,CAAC,WAAW;yBACjB,MAAM;;GAE5B,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACd,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,uCAAuC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC9D,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CACb,0CAA0C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CACpE,CAAC;IACJ,CAAC;IAED,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,QAAQ;QAChB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;QACxC,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,GAAG,CAAC,WAAW;KAC7B,CAAC;AAAA,CACH;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,EAAc,EACd,IAaC,EAC8B;IAC/B,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,IAAI,EAAE,CAAC;IAE1C,yDAAyD;IACzD,MAAM,QAAQ,GAAG,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;IAEzC,MAAM,GAAG,CAAA;kBACO,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;;;;;;;;;;;;;;;;;;QAkB3C,IAAI,CAAC,OAAO;QACZ,IAAI,CAAC,WAAW;QAChB,IAAI,CAAC,QAAQ;QACb,IAAI,CAAC,KAAK;QACV,IAAI,CAAC,aAAa;QAClB,YAAY;QACZ,IAAI,CAAC,QAAQ;QACb,IAAI,CAAC,QAAQ;QACb,IAAI,CAAC,WAAW;QAChB,IAAI,CAAC,MAAM;QACX,IAAI,CAAC,IAAI,CAAC,MAAM;QAChB,QAAQ;QACR,IAAI,CAAC,IAAI;QACT,GAAG;QACH,IAAI,CAAC,SAAS;;;;;;;;;;;;;qBAaD,IAAI,CAAC,SAAS;oBACf,QAAQ;GACzB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAEd,MAAM,GAAG,GAAG,MAAM,6BAA6B,CAAC,EAAE,EAAE;QAClD,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,WAAW,EAAE,IAAI,CAAC,WAAW;KAC9B,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,GAAG,CAAC;AAAA,CACZ;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,EAAc,EACd,OAAe,EACf,OAQC,EACiC;IAClC,MAAM,SAAS,GAAG,MAAM,GAAG,CAezB;;;;;;;;;;;;;;;;WAgBO,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;uBACrB,OAAO;;GAE3B,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACd,MAAM,GAAG,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE9B,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,uCAAuC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAC9D,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,WAAW,KAAK,MAAM,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CACb,0CAA0C,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CACpE,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,IAAI,IAA6C,CAAC;IAClD,IAAI,OAAO,EAAE,YAAY,EAAE,CAAC;QAC1B,IAAI,OAAO,CAAC,YAAY,CAAC,eAAe,EAAE,CAAC;YACzC,MAAM,kBAAkB,GACtB,MAAM,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;YACtD,IAAI,kBAAkB,EAAE,CAAC;gBACvB,IAAI,GAAG,kBAAkB,CAAC;YAC5B,CAAC;iBAAM,CAAC;gBACN,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;gBACnE,IAAI,YAAY,EAAE,CAAC;oBACjB,IAAI,GAAG,YAAY,CAAC;gBACtB,CAAC;qBAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;oBACpB,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC;qBAAM,CAAC;oBACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,EAAE,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACnE,IAAI,YAAY,EAAE,CAAC;gBACjB,IAAI,GAAG,YAAY,CAAC;YACtB,CAAC;iBAAM,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACpB,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,yCAAyC,OAAO,EAAE,CAAC,CAAC;YACtE,CAAC;QACH,CAAC;IACH,CAAC;SAAM,CAAC;QACN,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,OAAO;QACL,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,QAAQ,EAAE,GAAG,CAAC,SAAS;QACvB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,gBAAgB,IAAI,CAAC,CAAC;QAChD,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,CAAC;QACpC,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;QACxC,IAAI;QACJ,SAAS,EAAE,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC;KAC3C,CAAC;AAAA,CACH;AAED,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAC/C,EAAc,EACd,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAChB;IACjB,MAAM,GAAG,GAAG,MAAM,GAAG,CAAA;kBACL,GAAG,CAAC,KAAK,CAAC,sBAAsB,CAAC;0BACzB,MAAM;GAC7B,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAEd,OAAO,MAAM,CAAC,GAAG,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC;AAAA,CACzC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/server",
-  "version": "0.0.1-92",
+  "version": "0.0.1-96",
   "description": "Server-side sync engine with push/pull, pruning, and snapshot support",
   "license": "MIT",
   "author": "Benjamin Kniffler",

package/src/dialect/base.ts

@@ -5,12 +5,17 @@
  * database-specific sync dialect implementations.
  */
 
-import type { ScopeValues, StoredScopes, SyncOp } from '@syncular/core';
+import type { ScopeValues, StoredScopes } from '@syncular/core';
 import type { Kysely, RawBuilder, Transaction } from 'kysely';
 import { sql } from 'kysely';
 import type { SyncChangeRow, SyncCommitRow, SyncCoreDb } from '../schema';
 import { coerceIsoString, coerceNumber, parseScopes } from './helpers';
-import type { DbExecutor, ServerSyncDialect } from './types';
+import type {
+  DbExecutor,
+  IncrementalPullRow,
+  IncrementalPullRowsArgs,
+  ServerSyncDialect,
+} from './types';
 
 /**
  * Abstract base class for server sync dialects.
@@ -21,7 +26,7 @@ import type { DbExecutor, ServerSyncDialect } from './types';
  * provided via small abstract hook methods.
  *
  * Genuinely different methods (DDL, transaction control, scope filtering,
- * compaction, streaming) remain abstract for each dialect to implement.
+ * compaction) remain abstract for each dialect to implement.
  */
 export abstract class BaseServerSyncDialect implements ServerSyncDialect {
   abstract readonly name: string;
@@ -81,29 +86,10 @@ export abstract class BaseServerSyncDialect implements ServerSyncDialect {
     }
   ): Promise<SyncChangeRow[]>;
 
-  abstract readIncrementalPullRows<DB extends SyncCoreDb>(
+  protected abstract readIncrementalPullRowsBatch<DB extends SyncCoreDb>(
     db: DbExecutor<DB>,
-    args: {
-      table: string;
-      scopes: ScopeValues;
-      cursor: number;
-      limitCommits: number;
-      partitionId?: string;
-    }
-  ): Promise<
-    Array<{
-      commit_seq: number;
-      actor_id: string;
-      created_at: string;
-      change_id: number;
-      table: string;
-      row_id: string;
-      op: SyncOp;
-      row_json: unknown | null;
-      row_version: number | null;
-      scopes: StoredScopes;
-    }>
-  >;
+    args: Omit<IncrementalPullRowsArgs, 'batchSize'>
+  ): Promise<IncrementalPullRow[]>;
 
   abstract compactChanges<DB extends SyncCoreDb>(
     db: DbExecutor<DB>,
@@ -259,4 +245,48 @@ export abstract class BaseServerSyncDialect implements ServerSyncDialect {
         updated_at = ${now}
     `.execute(db);
   }
+
+  async *iterateIncrementalPullRows<DB extends SyncCoreDb>(
+    db: DbExecutor<DB>,
+    args: IncrementalPullRowsArgs
+  ): AsyncGenerator<IncrementalPullRow> {
+    const limitCommits = Math.max(1, Math.min(500, args.limitCommits));
+    const batchSize = Math.max(
+      1,
+      Math.min(limitCommits, args.batchSize ?? 100, 500)
+    );
+
+    let processedCommits = 0;
+    let cursor = args.cursor;
+
+    while (processedCommits < limitCommits) {
+      const remainingCommits = limitCommits - processedCommits;
+      const commitLimit = Math.min(batchSize, remainingCommits);
+      const rows = await this.readIncrementalPullRowsBatch(db, {
+        table: args.table,
+        scopes: args.scopes,
+        cursor,
+        limitCommits: commitLimit,
+        partitionId: args.partitionId,
+      });
+
+      if (rows.length === 0) break;
+
+      let maxCommitSeq = cursor;
+      const commitSeqs = new Set<number>();
+
+      for (const row of rows) {
+        maxCommitSeq = Math.max(maxCommitSeq, row.commit_seq);
+        commitSeqs.add(row.commit_seq);
+        yield row;
+      }
+
+      if (maxCommitSeq <= cursor) break;
+
+      processedCommits += commitSeqs.size;
+      cursor = maxCommitSeq;
+
+      if (commitSeqs.size < commitLimit) break;
+    }
+  }
 }