@syncular/server 0.0.6-95 → 0.1.0

package/src/snapshot-chunks/db-metadata.ts

@@ -7,9 +7,10 @@
 
 import {
   type BlobStorageAdapter,
+  bytesToReadableStream,
   randomId,
+  readAllBytesFromStream,
   SYNC_SNAPSHOT_CHUNK_COMPRESSION,
-  SYNC_SNAPSHOT_CHUNK_ENCODING,
   type SyncSnapshotChunkCompression,
   type SyncSnapshotChunkEncoding,
   type SyncSnapshotChunkRef,
@@ -19,6 +20,27 @@ import type { Kysely } from 'kysely';
 import type { SyncCoreDb } from '../schema';
 import type { SnapshotChunkMetadata, SnapshotChunkPageKey } from './types';
 
+export interface SnapshotChunkCleanupTuning {
+  /** Cleanup select/delete batch size. Default: 250 */
+  batchSize?: number;
+  /** Max concurrent blob deletes during cleanup. Default: 8 */
+  deleteConcurrency?: number;
+}
+
+export const SNAPSHOT_CHUNK_CLEANUP_TUNING_PRESETS = {
+  server: {
+    batchSize: 500,
+    deleteConcurrency: 16,
+  },
+  edge: {
+    batchSize: 100,
+    deleteConcurrency: 4,
+  },
+} as const satisfies Record<
+  'server' | 'edge',
+  Required<SnapshotChunkCleanupTuning>
+>;
+
 export interface DbMetadataSnapshotChunkStorageOptions {
   /** Database instance */
   db: Kysely<SyncCoreDb>;
@@ -26,6 +48,8 @@ export interface DbMetadataSnapshotChunkStorageOptions {
   blobAdapter: BlobStorageAdapter;
   /** Optional prefix for chunk IDs */
   chunkIdPrefix?: string;
+  /** Optional cleanup throughput tuning knobs. */
+  cleanupTuning?: SnapshotChunkCleanupTuning;
 }
 
 /**
@@ -63,7 +87,23 @@ export function createDbMetadataChunkStorage(
   ) => Promise<SyncSnapshotChunkRef | null>;
   cleanupExpired: (beforeIso: string) => Promise<number>;
 } {
-  const { db, blobAdapter, chunkIdPrefix = 'chunk_' } = options;
+  const { db, blobAdapter, chunkIdPrefix = 'chunk_', cleanupTuning } = options;
+
+  function positiveIntOrDefault(value: number | undefined, fallback: number) {
+    if (value === undefined) return fallback;
+    if (!Number.isFinite(value)) return fallback;
+    const normalized = Math.trunc(value);
+    return normalized > 0 ? normalized : fallback;
+  }
+
+  const CLEANUP_BATCH_SIZE = positiveIntOrDefault(
+    cleanupTuning?.batchSize,
+    250
+  );
+  const CLEANUP_DELETE_CONCURRENCY = positiveIntOrDefault(
+    cleanupTuning?.deleteConcurrency,
+    8
+  );
 
   // Generate deterministic blob hash from chunk identity metadata.
   async function computeBlobHash(metadata: {
@@ -77,59 +117,45 @@ export function createDbMetadataChunkStorage(
     return `sha256:${digest}`;
   }
 
-  function bytesToStream(bytes: Uint8Array): ReadableStream<Uint8Array> {
-    return new ReadableStream<Uint8Array>({
-      start(controller) {
-        controller.enqueue(bytes);
-        controller.close();
-      },
-    });
-  }
-
-  async function streamToBytes(
+  async function streamByteLength(
     stream: ReadableStream<Uint8Array>
-  ): Promise<Uint8Array> {
+  ): Promise<number> {
     const reader = stream.getReader();
     try {
-      const chunks: Uint8Array[] = [];
       let total = 0;
-
       while (true) {
         const { done, value } = await reader.read();
         if (done) break;
         if (!value) continue;
-        chunks.push(value);
         total += value.length;
       }
-
-      const out = new Uint8Array(total);
-      let offset = 0;
-      for (const chunk of chunks) {
-        out.set(chunk, offset);
-        offset += chunk.length;
-      }
-      return out;
+      return total;
     } finally {
       reader.releaseLock();
     }
   }
 
-  async function streamByteLength(
-    stream: ReadableStream<Uint8Array>
-  ): Promise<number> {
-    const reader = stream.getReader();
-    try {
-      let total = 0;
-      while (true) {
-        const { done, value } = await reader.read();
-        if (done) break;
-        if (!value) continue;
-        total += value.length;
+  async function runWithConcurrency<T>(
+    items: readonly T[],
+    concurrency: number,
+    worker: (item: T) => Promise<void>
+  ): Promise<void> {
+    if (items.length === 0) return;
+
+    const workerCount = Math.max(1, Math.min(concurrency, items.length));
+    let nextIndex = 0;
+
+    async function runWorker(): Promise<void> {
+      while (nextIndex < items.length) {
+        const index = nextIndex;
+        nextIndex += 1;
+        const item = items[index];
+        if (item === undefined) continue;
+        await worker(item);
       }
-      return total;
-    } finally {
-      reader.releaseLock();
     }
+
+    await Promise.all(Array.from({ length: workerCount }, () => runWorker()));
   }
 
   // Generate unique chunk ID
@@ -153,7 +179,15 @@ export function createDbMetadataChunkStorage(
     const rowCursorKey = args.rowCursor ?? '';
     const baseQuery = db
       .selectFrom('sync_snapshot_chunks')
-      .select(['chunk_id', 'sha256', 'byte_length', 'encoding', 'compression'])
+      .select([
+        'chunk_id',
+        'sha256',
+        'byte_length',
+        'next_row_cursor',
+        'is_last_page',
+        'encoding',
+        'compression',
+      ])
       .where('partition_id', '=', args.partitionId)
       .where('scope_key', '=', args.scopeKey)
       .where('scope', '=', args.scope)
@@ -169,7 +203,7 @@ export function createDbMetadataChunkStorage(
 
     if (!row) return null;
 
-    if (row.encoding !== SYNC_SNAPSHOT_CHUNK_ENCODING) {
+    if (row.encoding !== args.encoding) {
       throw new Error(
         `Unexpected snapshot chunk encoding: ${String(row.encoding)}`
       );
@@ -184,9 +218,12 @@ export function createDbMetadataChunkStorage(
       id: row.chunk_id,
       sha256: row.sha256,
       byteLength: Number(row.byte_length ?? 0),
-      encoding: row.encoding,
+      nextRowCursor:
+        row.next_row_cursor == null ? null : String(row.next_row_cursor),
+      isLastPage: Number(row.is_last_page ?? 0) === 1,
+      encoding: args.encoding,
       compression: row.compression,
-    };
+    } as SyncSnapshotChunkRef;
   }
 
   async function readBlobHash(chunkId: string): Promise<string | null> {
@@ -218,6 +255,8 @@ export function createDbMetadataChunkStorage(
         as_of_commit_seq: metadata.asOfCommitSeq,
         row_cursor: metadata.rowCursor ?? '',
         row_limit: metadata.rowLimit,
+        next_row_cursor: metadata.nextRowCursor ?? null,
+        is_last_page: metadata.isLastPage ? 1 : 0,
         encoding: metadata.encoding,
         compression: metadata.compression,
         sha256: metadata.sha256,
@@ -243,6 +282,8 @@ export function createDbMetadataChunkStorage(
             blob_hash: args.blobHash,
             sha256: metadata.sha256,
             byte_length: args.byteLength,
+            next_row_cursor: metadata.nextRowCursor ?? null,
+            is_last_page: metadata.isLastPage ? 1 : 0,
             row_cursor: metadata.rowCursor ?? '',
           })
       )
@@ -261,7 +302,7 @@ export function createDbMetadataChunkStorage(
 
     if (blobAdapter.get) {
       const bytes = await blobAdapter.get(blobHash);
-      return bytes ? bytesToStream(bytes) : null;
+      return bytes ? bytesToReadableStream(bytes) : null;
     }
 
     throw new Error(
@@ -295,7 +336,7 @@ export function createDbMetadataChunkStorage(
             contentLength: body.length,
           });
         } else if (blobAdapter.putStream) {
-          await blobAdapter.putStream(blobHash, bytesToStream(body), {
+          await blobAdapter.putStream(blobHash, bytesToReadableStream(body), {
             disableChecksum: true,
             byteLength: body.length,
             contentLength: body.length,
@@ -367,7 +408,7 @@ export function createDbMetadataChunkStorage(
           ]);
           observedByteLength = countedByteLength;
         } else if (blobAdapter.put) {
-          const body = await streamToBytes(bodyStream);
+          const body = await readAllBytesFromStream(bodyStream);
           await blobAdapter.put(blobHash, body);
           observedByteLength = body.length;
         } else {
@@ -428,7 +469,7 @@ export function createDbMetadataChunkStorage(
     async readChunk(chunkId: string): Promise<Uint8Array | null> {
       const stream = await readChunkStreamById(chunkId);
       if (!stream) return null;
-      return streamToBytes(stream);
+      return readAllBytesFromStream(stream);
     },
 
     async readChunkStream(
@@ -444,35 +485,53 @@ export function createDbMetadataChunkStorage(
     },
 
     async cleanupExpired(beforeIso: string): Promise<number> {
-      // Find expired chunks
-      const expiredRows = await db
-        .selectFrom('sync_snapshot_chunks')
-        .select(['chunk_id', 'blob_hash'])
-        .where('expires_at', '<=', beforeIso)
-        .execute();
-
-      if (expiredRows.length === 0) return 0;
-
-      // Delete from blob storage (best effort)
-      for (const row of expiredRows) {
-        try {
-          await blobAdapter.delete(row.blob_hash);
-        } catch {
-          // Ignore deletion errors - blob may be shared or already deleted
-          // Log for observability but don't fail the cleanup
-          console.warn(
-            `Failed to delete blob ${row.blob_hash} for chunk ${row.chunk_id}, may be already deleted or shared`
-          );
+      let deleted = 0;
+
+      while (true) {
+        const expiredRows = await db
+          .selectFrom('sync_snapshot_chunks')
+          .select(['chunk_id', 'blob_hash'])
+          .where('expires_at', '<=', beforeIso)
+          .orderBy('expires_at', 'asc')
+          .limit(CLEANUP_BATCH_SIZE)
+          .execute();
+
+        if (expiredRows.length === 0) {
+          break;
+        }
+
+        // Delete from blob storage (best effort).
+        await runWithConcurrency(
+          expiredRows,
+          CLEANUP_DELETE_CONCURRENCY,
+          async (row) => {
+            try {
+              await blobAdapter.delete(row.blob_hash);
+            } catch {
+              // Ignore deletion errors - blob may be shared or already deleted
+              // Log for observability but don't fail the cleanup
+              console.warn(
+                `Failed to delete blob ${row.blob_hash} for chunk ${row.chunk_id}, may be already deleted or shared`
+              );
+            }
+          }
+        );
+
+        const chunkIds = expiredRows.map((row) => row.chunk_id);
+        if (chunkIds.length > 0) {
+          const result = await db
+            .deleteFrom('sync_snapshot_chunks')
+            .where('chunk_id', 'in', chunkIds)
+            .executeTakeFirst();
+          deleted += Number(result.numDeletedRows ?? 0);
         }
-      }
 
-      // Delete metadata from database
-      const result = await db
-        .deleteFrom('sync_snapshot_chunks')
-        .where('expires_at', '<=', beforeIso)
-        .executeTakeFirst();
+        if (expiredRows.length < CLEANUP_BATCH_SIZE) {
+          break;
+        }
+      }
 
-      return Number(result.numDeletedRows ?? 0);
+      return deleted;
     },
   };
 }

package/src/snapshot-chunks/types.ts

@@ -36,6 +36,8 @@ export interface SnapshotChunkMetadata {
   asOfCommitSeq: number;
   rowCursor: string | null;
   rowLimit: number;
+  nextRowCursor?: string | null;
+  isLastPage?: boolean;
   encoding: SyncSnapshotChunkEncoding;
   compression: SyncSnapshotChunkCompression;
   sha256: string;