@syncular/server 0.0.1-91 → 0.0.1-95

package/src/dialect/types.ts

@@ -22,6 +22,28 @@ export type DbExecutor<DB extends SyncCoreDb = SyncCoreDb> =
  */
 export type ServerSyncDialectName = string;
 
+export interface IncrementalPullRowsArgs {
+  table: string;
+  scopes: ScopeValues;
+  cursor: number;
+  limitCommits: number;
+  partitionId?: string;
+  batchSize?: number;
+}
+
+export interface IncrementalPullRow {
+  commit_seq: number;
+  actor_id: string;
+  created_at: string;
+  change_id: number;
+  table: string;
+  row_id: string;
+  op: SyncOp;
+  row_json: unknown | null;
+  row_version: number | null;
+  scopes: StoredScopes;
+}
+
 export interface ServerSyncDialect {
   readonly name: ServerSyncDialectName;
 
@@ -88,62 +110,15 @@ export interface ServerSyncDialect {
   ): Promise<SyncChangeRow[]>;
 
   /**
-   * Optimized incremental pull for a subscription.
+   * Incremental pull iterator for a subscription.
    *
-   * Returns change rows joined with commit metadata and filtered by
+   * Yields change rows joined with commit metadata and filtered by
    * the subscription's table and scope values.
    */
-  readIncrementalPullRows<DB extends SyncCoreDb>(
-    db: DbExecutor<DB>,
-    args: {
-      table: string;
-      scopes: ScopeValues;
-      cursor: number;
-      limitCommits: number;
-      partitionId?: string;
-    }
-  ): Promise<
-    Array<{
-      commit_seq: number;
-      actor_id: string;
-      created_at: string;
-      change_id: number;
-      table: string;
-      row_id: string;
-      op: SyncOp;
-      row_json: unknown | null;
-      row_version: number | null;
-      scopes: StoredScopes;
-    }>
-  >;
-
-  /**
-   * Streaming incremental pull for large result sets.
-   *
-   * Yields changes one at a time instead of loading all into memory.
-   * Use this when expecting large numbers of changes.
-   */
-  streamIncrementalPullRows?<DB extends SyncCoreDb>(
+  iterateIncrementalPullRows<DB extends SyncCoreDb>(
     db: DbExecutor<DB>,
-    args: {
-      table: string;
-      scopes: ScopeValues;
-      cursor: number;
-      limitCommits: number;
-      partitionId?: string;
-    }
-  ): AsyncGenerator<{
-    commit_seq: number;
-    actor_id: string;
-    created_at: string;
-    change_id: number;
-    table: string;
-    row_id: string;
-    op: SyncOp;
-    row_json: unknown | null;
-    row_version: number | null;
-    scopes: StoredScopes;
-  }>;
+    args: IncrementalPullRowsArgs
+  ): AsyncGenerator<IncrementalPullRow>;
 
   /**
    * Optional compaction of the change log to reduce storage.

package/src/pull.ts

@@ -37,6 +37,20 @@ async function compressSnapshotNdjson(ndjson: string): Promise<Uint8Array> {
   return new Uint8Array(compressed);
 }
 
+function concatUint8Arrays(chunks: readonly Uint8Array[]): Uint8Array {
+  if (chunks.length === 0) return new Uint8Array();
+  if (chunks.length === 1) return chunks[0] ?? new Uint8Array();
+
+  const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0);
+  const out = new Uint8Array(total);
+  let offset = 0;
+  for (const chunk of chunks) {
+    out.set(chunk, offset);
+    offset += chunk.length;
+  }
+  return out;
+}
+
 export interface PullResult {
   response: SyncPullResponse;
   /**
@@ -346,120 +360,189 @@ export async function pull<DB extends SyncCoreDb>(args: {
 
               const snapshots: SyncSnapshot[] = [];
               let nextState: SyncBootstrapState | null = effectiveState;
+              const cacheKey = `${partitionId}:${scopesToCacheKey(effectiveScopes)}`;
+
+              interface SnapshotBundle {
+                table: string;
+                startCursor: string | null;
+                isFirstPage: boolean;
+                isLastPage: boolean;
+                pageCount: number;
+                ttlMs: number;
+                hash: ReturnType<typeof createHash>;
+                compressedParts: Uint8Array[];
+              }
 
-              for (
-                let pageIndex = 0;
-                pageIndex < maxSnapshotPages;
-                pageIndex++
-              ) {
-                if (!nextState) break;
-
-                const nextTableName = nextState.tables[nextState.tableIndex];
-                if (!nextTableName) {
-                  nextState = null;
-                  break;
-                }
-
-                const tableHandler = args.shapes.getOrThrow(nextTableName);
-                const isFirstPage = nextState.rowCursor == null;
-
-                const page = await tableHandler.snapshot(
-                  {
-                    db: trx,
-                    actorId: args.actorId,
-                    scopeValues: effectiveScopes,
-                    cursor: nextState.rowCursor,
-                    limit: limitSnapshotRows,
-                  },
-                  sub.params
-                );
-
-                const isLastPage = page.nextCursor == null;
-
-                // Always use NDJSON+gzip for bootstrap snapshots
-                const ttlMs =
-                  tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000; // 24h
+              const flushSnapshotBundle = async (
+                bundle: SnapshotBundle
+              ): Promise<void> => {
                 const nowIso = new Date().toISOString();
+                const bundleRowLimit = Math.max(
+                  1,
+                  limitSnapshotRows * bundle.pageCount
+                );
 
-                // Use scope hash for caching
-                const cacheKey = `${partitionId}:${scopesToCacheKey(effectiveScopes)}`;
                 const cached = await readSnapshotChunkRefByPageKey(trx, {
                   partitionId,
                   scopeKey: cacheKey,
-                  scope: nextTableName,
+                  scope: bundle.table,
                   asOfCommitSeq: effectiveState.asOfCommitSeq,
-                  rowCursor: nextState.rowCursor,
-                  rowLimit: limitSnapshotRows,
+                  rowCursor: bundle.startCursor,
+                  rowLimit: bundleRowLimit,
                   encoding: 'ndjson',
                   compression: 'gzip',
                   nowIso,
                 });
 
                 let chunkRef = cached;
-
                 if (!chunkRef) {
-                  const lines: string[] = [];
-                  for (const r of page.rows ?? []) {
-                    const s = JSON.stringify(r);
-                    lines.push(s === undefined ? 'null' : s);
-                  }
-                  const ndjson =
-                    lines.length > 0 ? `${lines.join('\n')}\n` : '';
-                  const gz = await compressSnapshotNdjson(ndjson);
-                  const sha256 = createHash('sha256')
-                    .update(ndjson)
-                    .digest('hex');
+                  const sha256 = bundle.hash.digest('hex');
                   const expiresAt = new Date(
-                    Date.now() + Math.max(1000, ttlMs)
+                    Date.now() + Math.max(1000, bundle.ttlMs)
                   ).toISOString();
+                  const byteLength = bundle.compressedParts.reduce(
+                    (sum, part) => sum + part.length,
+                    0
+                  );
 
-                  // Use external chunk storage if available, otherwise fall back to inline
                   if (args.chunkStorage) {
-                    chunkRef = await args.chunkStorage.storeChunk({
-                      partitionId,
-                      scopeKey: cacheKey,
-                      scope: nextTableName,
-                      asOfCommitSeq: effectiveState.asOfCommitSeq,
-                      rowCursor: nextState.rowCursor ?? null,
-                      rowLimit: limitSnapshotRows,
-                      encoding: 'ndjson',
-                      compression: 'gzip',
-                      sha256,
-                      body: gz,
-                      expiresAt,
-                    });
+                    if (args.chunkStorage.storeChunkStream) {
+                      chunkRef = await args.chunkStorage.storeChunkStream({
+                        partitionId,
+                        scopeKey: cacheKey,
+                        scope: bundle.table,
+                        asOfCommitSeq: effectiveState.asOfCommitSeq,
+                        rowCursor: bundle.startCursor,
+                        rowLimit: bundleRowLimit,
+                        encoding: 'ndjson',
+                        compression: 'gzip',
+                        sha256,
+                        byteLength,
+                        bodyStream: new ReadableStream<Uint8Array>({
+                          start(controller) {
+                            for (const part of bundle.compressedParts) {
+                              controller.enqueue(part);
+                            }
+                            controller.close();
+                          },
+                        }),
+                        expiresAt,
+                      });
+                    } else {
+                      const body = concatUint8Arrays(bundle.compressedParts);
+                      chunkRef = await args.chunkStorage.storeChunk({
+                        partitionId,
+                        scopeKey: cacheKey,
+                        scope: bundle.table,
+                        asOfCommitSeq: effectiveState.asOfCommitSeq,
+                        rowCursor: bundle.startCursor,
+                        rowLimit: bundleRowLimit,
+                        encoding: 'ndjson',
+                        compression: 'gzip',
+                        sha256,
+                        body,
+                        expiresAt,
+                      });
+                    }
                   } else {
+                    const body = concatUint8Arrays(bundle.compressedParts);
                     const chunkId = randomUUID();
                     chunkRef = await insertSnapshotChunk(trx, {
                       chunkId,
                       partitionId,
                       scopeKey: cacheKey,
-                      scope: nextTableName,
+                      scope: bundle.table,
                       asOfCommitSeq: effectiveState.asOfCommitSeq,
-                      rowCursor: nextState.rowCursor,
-                      rowLimit: limitSnapshotRows,
+                      rowCursor: bundle.startCursor,
+                      rowLimit: bundleRowLimit,
                       encoding: 'ndjson',
                       compression: 'gzip',
                       sha256,
-                      body: gz,
+                      body,
                       expiresAt,
                     });
                   }
                 }
 
                 snapshots.push({
-                  table: nextTableName,
+                  table: bundle.table,
                   rows: [],
                   chunks: [chunkRef],
-                  isFirstPage,
-                  isLastPage,
+                  isFirstPage: bundle.isFirstPage,
+                  isLastPage: bundle.isLastPage,
                 });
+              };
+
+              let activeBundle: SnapshotBundle | null = null;
+
+              for (
+                let pageIndex = 0;
+                pageIndex < maxSnapshotPages;
+                pageIndex++
+              ) {
+                if (!nextState) break;
+
+                const nextTableName = nextState.tables[nextState.tableIndex];
+                if (!nextTableName) {
+                  if (activeBundle) {
+                    activeBundle.isLastPage = true;
+                    await flushSnapshotBundle(activeBundle);
+                    activeBundle = null;
+                  }
+                  nextState = null;
+                  break;
+                }
+
+                const tableHandler = args.shapes.getOrThrow(nextTableName);
+                if (!activeBundle || activeBundle.table !== nextTableName) {
+                  if (activeBundle) {
+                    await flushSnapshotBundle(activeBundle);
+                  }
+                  activeBundle = {
+                    table: nextTableName,
+                    startCursor: nextState.rowCursor,
+                    isFirstPage: nextState.rowCursor == null,
+                    isLastPage: false,
+                    pageCount: 0,
+                    ttlMs:
+                      tableHandler.snapshotChunkTtlMs ?? 24 * 60 * 60 * 1000,
+                    hash: createHash('sha256'),
+                    compressedParts: [],
+                  };
+                }
+
+                const page = await tableHandler.snapshot(
+                  {
+                    db: trx,
+                    actorId: args.actorId,
+                    scopeValues: effectiveScopes,
+                    cursor: nextState.rowCursor,
+                    limit: limitSnapshotRows,
+                  },
+                  sub.params
+                );
+
+                const lines: string[] = [];
+                for (const r of page.rows ?? []) {
+                  const s = JSON.stringify(r);
+                  lines.push(s === undefined ? 'null' : s);
+                }
+                const ndjson = lines.length > 0 ? `${lines.join('\n')}\n` : '';
+                activeBundle.hash.update(ndjson);
+                activeBundle.compressedParts.push(
+                  await compressSnapshotNdjson(ndjson)
+                );
+                activeBundle.pageCount += 1;
 
                 if (page.nextCursor != null) {
                   nextState = { ...nextState, rowCursor: page.nextCursor };
                   continue;
                 }
 
+                activeBundle.isLastPage = true;
+                await flushSnapshotBundle(activeBundle);
+                activeBundle = null;
+
                 if (nextState.tableIndex + 1 < nextState.tables.length) {
                   nextState = {
                     ...nextState,
@@ -473,6 +556,10 @@ export async function pull<DB extends SyncCoreDb>(args: {
                 break;
               }
 
+              if (activeBundle) {
+                await flushSnapshotBundle(activeBundle);
+              }
+
               subResponses.push({
                 id: sub.id,
                 status: 'active',
@@ -501,17 +588,6 @@ export async function pull<DB extends SyncCoreDb>(args: {
                 ? scannedCommitSeqs[scannedCommitSeqs.length - 1]!
                 : cursor;
 
-            // Use streaming when available to reduce memory pressure for large pulls
-            const pullRowStream = dialect.streamIncrementalPullRows
-              ? dialect.streamIncrementalPullRows(trx, {
-                  partitionId,
-                  table: sub.shape,
-                  scopes: effectiveScopes,
-                  cursor,
-                  limitCommits,
-                })
-              : null;
-
             // Collect rows and compute nextCursor in a single pass
             const incrementalRows: Array<{
               commit_seq: number;
@@ -528,25 +604,15 @@ export async function pull<DB extends SyncCoreDb>(args: {
 
             let nextCursor = cursor;
 
-            if (pullRowStream) {
-              // Streaming path: process rows as they arrive
-              for await (const row of pullRowStream) {
-                incrementalRows.push(row);
-                nextCursor = Math.max(nextCursor, row.commit_seq);
-              }
-            } else {
-              // Non-streaming fallback: load all rows at once
-              const rows = await dialect.readIncrementalPullRows(trx, {
-                partitionId,
-                table: sub.shape,
-                scopes: effectiveScopes,
-                cursor,
-                limitCommits,
-              });
-              incrementalRows.push(...rows);
-              for (const r of incrementalRows) {
-                nextCursor = Math.max(nextCursor, r.commit_seq);
-              }
+            for await (const row of dialect.iterateIncrementalPullRows(trx, {
+              partitionId,
+              table: sub.shape,
+              scopes: effectiveScopes,
+              cursor,
+              limitCommits,
+            })) {
+              incrementalRows.push(row);
+              nextCursor = Math.max(nextCursor, row.commit_seq);
             }
 
             nextCursor = Math.max(nextCursor, maxScannedCommitSeq);