@syncular/server 0.0.1-92 → 0.0.1-95

package/src/snapshot-chunks/db-metadata.ts

@@ -37,7 +37,19 @@ export function createDbMetadataChunkStorage(
       body: Uint8Array;
     }
   ) => Promise<SyncSnapshotChunkRef>;
+  storeChunkStream: (
+    metadata: Omit<
+      SnapshotChunkMetadata,
+      'chunkId' | 'byteLength' | 'blobHash'
+    > & {
+      bodyStream: ReadableStream<Uint8Array>;
+      byteLength: number;
+    }
+  ) => Promise<SyncSnapshotChunkRef>;
   readChunk: (chunkId: string) => Promise<Uint8Array | null>;
+  readChunkStream: (
+    chunkId: string
+  ) => Promise<ReadableStream<Uint8Array> | null>;
   findChunk: (
     pageKey: SnapshotChunkPageKey
   ) => Promise<SyncSnapshotChunkRef | null>;
@@ -50,6 +62,43 @@ export function createDbMetadataChunkStorage(
     return `sha256:${createHash('sha256').update(body).digest('hex')}`;
   }
 
+  function bytesToStream(bytes: Uint8Array): ReadableStream<Uint8Array> {
+    return new ReadableStream<Uint8Array>({
+      start(controller) {
+        controller.enqueue(bytes);
+        controller.close();
+      },
+    });
+  }
+
+  async function streamToBytes(
+    stream: ReadableStream<Uint8Array>
+  ): Promise<Uint8Array> {
+    const reader = stream.getReader();
+    try {
+      const chunks: Uint8Array[] = [];
+      let total = 0;
+
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (!value) continue;
+        chunks.push(value);
+        total += value.length;
+      }
+
+      const out = new Uint8Array(total);
+      let offset = 0;
+      for (const chunk of chunks) {
+        out.set(chunk, offset);
+        offset += chunk.length;
+      }
+      return out;
+    } finally {
+      reader.releaseLock();
+    }
+  }
+
   // Generate unique chunk ID
   function generateChunkId(): string {
     return `${chunkIdPrefix}${Date.now()}_${Math.random().toString(36).slice(2, 11)}`;
@@ -107,6 +156,86 @@ export function createDbMetadataChunkStorage(
     };
   }
 
+  async function readBlobHash(chunkId: string): Promise<string | null> {
+    const row = await db
+      .selectFrom('sync_snapshot_chunks')
+      .select(['blob_hash'])
+      .where('chunk_id', '=', chunkId)
+      .executeTakeFirst();
+    return row?.blob_hash ?? null;
+  }
+
+  async function upsertChunkMetadata(
+    metadata: Omit<
+      SnapshotChunkMetadata,
+      'chunkId' | 'byteLength' | 'blobHash'
+    >,
+    args: { blobHash: string; byteLength: number }
+  ): Promise<void> {
+    const chunkId = generateChunkId();
+    const now = new Date().toISOString();
+
+    await db
+      .insertInto('sync_snapshot_chunks')
+      .values({
+        chunk_id: chunkId,
+        partition_id: metadata.partitionId,
+        scope_key: metadata.scopeKey,
+        scope: metadata.scope,
+        as_of_commit_seq: metadata.asOfCommitSeq,
+        row_cursor: metadata.rowCursor ?? '',
+        row_limit: metadata.rowLimit,
+        encoding: metadata.encoding,
+        compression: metadata.compression,
+        sha256: metadata.sha256,
+        byte_length: args.byteLength,
+        blob_hash: args.blobHash,
+        expires_at: metadata.expiresAt,
+        created_at: now,
+      })
+      .onConflict((oc) =>
+        oc
+          .columns([
+            'partition_id',
+            'scope_key',
+            'scope',
+            'as_of_commit_seq',
+            'row_cursor',
+            'row_limit',
+            'encoding',
+            'compression',
+          ])
+          .doUpdateSet({
+            expires_at: metadata.expiresAt,
+            blob_hash: args.blobHash,
+            sha256: metadata.sha256,
+            byte_length: args.byteLength,
+            row_cursor: metadata.rowCursor ?? '',
+          })
+      )
+      .execute();
+  }
+
+  async function readChunkStreamById(
+    chunkId: string
+  ): Promise<ReadableStream<Uint8Array> | null> {
+    const blobHash = await readBlobHash(chunkId);
+    if (!blobHash) return null;
+
+    if (blobAdapter.getStream) {
+      return blobAdapter.getStream(blobHash);
+    }
+
+    if (blobAdapter.get) {
+      const bytes = await blobAdapter.get(blobHash);
+      return bytes ? bytesToStream(bytes) : null;
+    }
+
+    throw new Error(
+      `Blob adapter ${blobAdapter.name} does not support direct get() for snapshot chunks`
+    );
+  }
+
   return {
     name: `db-metadata+${blobAdapter.name}`,
 
@@ -120,8 +249,6 @@ export function createDbMetadataChunkStorage(
     ): Promise<SyncSnapshotChunkRef> {
       const { body, ...metaWithoutBody } = metadata;
       const blobHash = computeBlobHash(body);
-      const chunkId = generateChunkId();
-      const now = new Date().toISOString();
 
       // Check if blob already exists (content-addressed dedup)
       const blobExists = await blobAdapter.exists(blobHash);
@@ -130,6 +257,8 @@ export function createDbMetadataChunkStorage(
         // Store body in blob adapter
         if (blobAdapter.put) {
           await blobAdapter.put(blobHash, body);
+        } else if (blobAdapter.putStream) {
+          await blobAdapter.putStream(blobHash, bytesToStream(body));
         } else {
           throw new Error(
             `Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`
@@ -137,46 +266,10 @@ export function createDbMetadataChunkStorage(
         }
       }
 
-      // Upsert metadata in database
-      await db
-        .insertInto('sync_snapshot_chunks')
-        .values({
-          chunk_id: chunkId,
-          partition_id: metaWithoutBody.partitionId,
-          scope_key: metaWithoutBody.scopeKey,
-          scope: metaWithoutBody.scope,
-          as_of_commit_seq: metaWithoutBody.asOfCommitSeq,
-          row_cursor: metaWithoutBody.rowCursor ?? '',
-          row_limit: metaWithoutBody.rowLimit,
-          encoding: metaWithoutBody.encoding,
-          compression: metaWithoutBody.compression,
-          sha256: metaWithoutBody.sha256,
-          byte_length: body.length,
-          blob_hash: blobHash,
-          expires_at: metaWithoutBody.expiresAt,
-          created_at: now,
-        })
-        .onConflict((oc) =>
-          oc
-            .columns([
-              'partition_id',
-              'scope_key',
-              'scope',
-              'as_of_commit_seq',
-              'row_cursor',
-              'row_limit',
-              'encoding',
-              'compression',
-            ])
-            .doUpdateSet({
-              expires_at: metaWithoutBody.expiresAt,
-              blob_hash: blobHash,
-              sha256: metaWithoutBody.sha256,
-              byte_length: body.length,
-              row_cursor: metaWithoutBody.rowCursor ?? '',
-            })
-        )
-        .execute();
+      await upsertChunkMetadata(metaWithoutBody, {
+        blobHash,
+        byteLength: body.length,
+      });
 
       const storedRef = await readStoredRef({
         partitionId: metaWithoutBody.partitionId,
@@ -197,24 +290,93 @@ export function createDbMetadataChunkStorage(
       return storedRef;
     },
 
-    async readChunk(chunkId: string): Promise<Uint8Array | null> {
-      // Get metadata to find blob hash
-      const row = await db
-        .selectFrom('sync_snapshot_chunks')
-        .select(['blob_hash'])
-        .where('chunk_id', '=', chunkId)
-        .executeTakeFirst();
+    async storeChunkStream(
+      metadata: Omit<
+        SnapshotChunkMetadata,
+        'chunkId' | 'byteLength' | 'blobHash'
+      > & {
+        bodyStream: ReadableStream<Uint8Array>;
+        byteLength: number;
+      }
+    ): Promise<SyncSnapshotChunkRef> {
+      const { bodyStream, byteLength, ...metaWithoutBody } = metadata;
+      const [uploadStream, hashStream] = bodyStream.tee();
 
-      if (!row) return null;
+      const hasher = createHash('sha256');
+      let observedByteLength = 0;
+      const hashReader = hashStream.getReader();
+      try {
+        while (true) {
+          const { done, value } = await hashReader.read();
+          if (done) break;
+          if (!value) continue;
+          hasher.update(value);
+          observedByteLength += value.length;
+        }
+      } finally {
+        hashReader.releaseLock();
+      }
 
-      // Read from blob adapter
-      if (blobAdapter.get) {
-        return blobAdapter.get(row.blob_hash);
+      if (observedByteLength !== byteLength) {
+        throw new Error(
+          `Snapshot chunk byte length mismatch: expected ${byteLength}, got ${observedByteLength}`
+        );
       }
 
-      throw new Error(
-        `Blob adapter ${blobAdapter.name} does not support direct get() for snapshot chunks`
-      );
+      const blobHash = `sha256:${hasher.digest('hex')}`;
+
+      const blobExists = await blobAdapter.exists(blobHash);
+      if (!blobExists) {
+        if (blobAdapter.putStream) {
+          await blobAdapter.putStream(blobHash, uploadStream, {
+            byteLength: observedByteLength,
+          });
+        } else if (blobAdapter.put) {
+          const body = await streamToBytes(uploadStream);
+          await blobAdapter.put(blobHash, body);
+        } else {
+          throw new Error(
+            `Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`
+          );
+        }
+      } else {
+        await uploadStream.cancel();
+      }
+
+      await upsertChunkMetadata(metaWithoutBody, {
+        blobHash,
+        byteLength: observedByteLength,
+      });
+
+      const storedRef = await readStoredRef({
+        partitionId: metaWithoutBody.partitionId,
+        scopeKey: metaWithoutBody.scopeKey,
+        scope: metaWithoutBody.scope,
+        asOfCommitSeq: metaWithoutBody.asOfCommitSeq,
+        rowCursor: metaWithoutBody.rowCursor,
+        rowLimit: metaWithoutBody.rowLimit,
+        encoding: metaWithoutBody.encoding,
+        compression: metaWithoutBody.compression,
+        includeExpired: true,
+      });
+
+      if (!storedRef) {
+        throw new Error('Failed to read stored snapshot chunk reference');
+      }
+
+      return storedRef;
+    },
+
+    async readChunk(chunkId: string): Promise<Uint8Array | null> {
+      const stream = await readChunkStreamById(chunkId);
+      if (!stream) return null;
+      return streamToBytes(stream);
+    },
+
+    async readChunkStream(
+      chunkId: string
+    ): Promise<ReadableStream<Uint8Array> | null> {
+      return readChunkStreamById(chunkId);
     },
 
     async findChunk(

package/src/snapshot-chunks/types.ts

@@ -60,11 +60,31 @@ export interface SnapshotChunkStorage {
     }
   ): Promise<SyncSnapshotChunkRef>;
 
+  /**
+   * Store a chunk from a stream.
+   * Preferred for large payloads to avoid full buffering in memory.
+   */
+  storeChunkStream?(
+    metadata: Omit<
+      SnapshotChunkMetadata,
+      'chunkId' | 'byteLength' | 'blobHash'
+    > & {
+      bodyStream: ReadableStream<Uint8Array>;
+      byteLength: number;
+    }
+  ): Promise<SyncSnapshotChunkRef>;
+
   /**
    * Read chunk body by chunk ID
    */
   readChunk(chunkId: string): Promise<Uint8Array | null>;
 
+  /**
+   * Read chunk body as a stream.
+   * Preferred for large payloads to avoid full buffering in memory.
+   */
+  readChunkStream?(chunkId: string): Promise<ReadableStream<Uint8Array> | null>;
+
   /**
    * Find existing chunk by page key
    */

package/src/snapshot-chunks.ts

@@ -32,7 +32,7 @@ export interface SnapshotChunkRow {
   compression: 'gzip';
   sha256: string;
   byteLength: number;
-  body: Uint8Array;
+  body: Uint8Array | ReadableStream<Uint8Array>;
   expiresAt: string;
 }
 
@@ -203,7 +203,12 @@ export async function readSnapshotChunk<DB extends SyncCoreDb>(
   chunkId: string,
   options?: {
     /** External chunk storage for reading from S3/R2/etc */
-    chunkStorage?: { readChunk(chunkId: string): Promise<Uint8Array | null> };
+    chunkStorage?: {
+      readChunk(chunkId: string): Promise<Uint8Array | null>;
+      readChunkStream?(
+        chunkId: string
+      ): Promise<ReadableStream<Uint8Array> | null>;
+    };
   }
 ): Promise<SnapshotChunkRow | null> {
   const rowResult = await sql<{
@@ -257,15 +262,32 @@ export async function readSnapshotChunk<DB extends SyncCoreDb>(
   }
 
   // Read body from external storage if available, otherwise use inline body
-  let body: Uint8Array;
+  let body: Uint8Array | ReadableStream<Uint8Array>;
   if (options?.chunkStorage) {
-    const externalBody = await options.chunkStorage.readChunk(chunkId);
-    if (externalBody) {
-      body = externalBody;
-    } else if (row.body) {
-      body = coerceChunkRow(row.body);
+    if (options.chunkStorage.readChunkStream) {
+      const externalBodyStream =
+        await options.chunkStorage.readChunkStream(chunkId);
+      if (externalBodyStream) {
+        body = externalBodyStream;
+      } else {
+        const externalBody = await options.chunkStorage.readChunk(chunkId);
+        if (externalBody) {
+          body = externalBody;
+        } else if (row.body) {
+          body = coerceChunkRow(row.body);
+        } else {
+          throw new Error(`Snapshot chunk body missing for chunk ${chunkId}`);
+        }
+      }
     } else {
-      throw new Error(`Snapshot chunk body missing for chunk ${chunkId}`);
+      const externalBody = await options.chunkStorage.readChunk(chunkId);
+      if (externalBody) {
+        body = externalBody;
+      } else if (row.body) {
+        body = coerceChunkRow(row.body);
+      } else {
+        throw new Error(`Snapshot chunk body missing for chunk ${chunkId}`);
+      }
     }
   } else {
     body = coerceChunkRow(row.body);