@syncular/server 0.0.1 → 0.0.2-126

package/src/snapshot-chunks/db-metadata.ts

@@ -6,7 +6,14 @@
  */
 
 import { createHash } from 'node:crypto';
-import type { BlobStorageAdapter, SyncSnapshotChunkRef } from '@syncular/core';
+import {
+  type BlobStorageAdapter,
+  SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+  SYNC_SNAPSHOT_CHUNK_ENCODING,
+  type SyncSnapshotChunkCompression,
+  type SyncSnapshotChunkEncoding,
+  type SyncSnapshotChunkRef,
+} from '@syncular/core';
 import type { Kysely } from 'kysely';
 import type { SyncCoreDb } from '../schema';
 import type { SnapshotChunkMetadata, SnapshotChunkPageKey } from './types';
@@ -37,7 +44,19 @@ export function createDbMetadataChunkStorage(
       body: Uint8Array;
     }
   ) => Promise<SyncSnapshotChunkRef>;
+  storeChunkStream: (
+    metadata: Omit<
+      SnapshotChunkMetadata,
+      'chunkId' | 'byteLength' | 'blobHash'
+    > & {
+      bodyStream: ReadableStream<Uint8Array>;
+      byteLength?: number;
+    }
+  ) => Promise<SyncSnapshotChunkRef>;
   readChunk: (chunkId: string) => Promise<Uint8Array | null>;
+  readChunkStream: (
+    chunkId: string
+  ) => Promise<ReadableStream<Uint8Array> | null>;
   findChunk: (
     pageKey: SnapshotChunkPageKey
   ) => Promise<SyncSnapshotChunkRef | null>;
@@ -45,9 +64,71 @@ export function createDbMetadataChunkStorage(
 } {
   const { db, blobAdapter, chunkIdPrefix = 'chunk_' } = options;
 
-  // Generate deterministic blob hash from content
-  function computeBlobHash(body: Uint8Array): string {
-    return `sha256:${createHash('sha256').update(body).digest('hex')}`;
+  // Generate deterministic blob hash from chunk identity metadata.
+  function computeBlobHash(metadata: {
+    encoding: SyncSnapshotChunkEncoding;
+    compression: SyncSnapshotChunkCompression;
+    sha256: string;
+  }): string {
+    const digest = createHash('sha256')
+      .update(`${metadata.encoding}:${metadata.compression}:${metadata.sha256}`)
+      .digest('hex');
+    return `sha256:${digest}`;
+  }
+
+  function bytesToStream(bytes: Uint8Array): ReadableStream<Uint8Array> {
+    return new ReadableStream<Uint8Array>({
+      start(controller) {
+        controller.enqueue(bytes);
+        controller.close();
+      },
+    });
+  }
+
+  async function streamToBytes(
+    stream: ReadableStream<Uint8Array>
+  ): Promise<Uint8Array> {
+    const reader = stream.getReader();
+    try {
+      const chunks: Uint8Array[] = [];
+      let total = 0;
+
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (!value) continue;
+        chunks.push(value);
+        total += value.length;
+      }
+
+      const out = new Uint8Array(total);
+      let offset = 0;
+      for (const chunk of chunks) {
+        out.set(chunk, offset);
+        offset += chunk.length;
+      }
+      return out;
+    } finally {
+      reader.releaseLock();
+    }
+  }
+
+  async function streamByteLength(
+    stream: ReadableStream<Uint8Array>
+  ): Promise<number> {
+    const reader = stream.getReader();
+    try {
+      let total = 0;
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        if (!value) continue;
+        total += value.length;
+      }
+      return total;
+    } finally {
+      reader.releaseLock();
+    }
   }
 
   // Generate unique chunk ID
@@ -55,6 +136,138 @@ export function createDbMetadataChunkStorage(
     return `${chunkIdPrefix}${Date.now()}_${Math.random().toString(36).slice(2, 11)}`;
   }
 
+  async function readStoredRef(args: {
+    partitionId: string;
+    scopeKey: string;
+    scope: string;
+    asOfCommitSeq: number;
+    rowCursor: string | null;
+    rowLimit: number;
+    encoding: SyncSnapshotChunkEncoding;
+    compression: SyncSnapshotChunkCompression;
+    nowIso?: string;
+    includeExpired?: boolean;
+  }): Promise<SyncSnapshotChunkRef | null> {
+    const nowIso = args.nowIso ?? new Date().toISOString();
+    const rowCursorKey = args.rowCursor ?? '';
+    const baseQuery = db
+      .selectFrom('sync_snapshot_chunks')
+      .select(['chunk_id', 'sha256', 'byte_length', 'encoding', 'compression'])
+      .where('partition_id', '=', args.partitionId)
+      .where('scope_key', '=', args.scopeKey)
+      .where('scope', '=', args.scope)
+      .where('as_of_commit_seq', '=', args.asOfCommitSeq)
+      .where('row_cursor', '=', rowCursorKey)
+      .where('row_limit', '=', args.rowLimit)
+      .where('encoding', '=', args.encoding)
+      .where('compression', '=', args.compression);
+
+    const row = await (args.includeExpired
+      ? baseQuery.executeTakeFirst()
+      : baseQuery.where('expires_at', '>', nowIso).executeTakeFirst());
+
+    if (!row) return null;
+
+    if (row.encoding !== SYNC_SNAPSHOT_CHUNK_ENCODING) {
+      throw new Error(
+        `Unexpected snapshot chunk encoding: ${String(row.encoding)}`
+      );
+    }
+    if (row.compression !== SYNC_SNAPSHOT_CHUNK_COMPRESSION) {
+      throw new Error(
+        `Unexpected snapshot chunk compression: ${String(row.compression)}`
+      );
+    }
+
+    return {
+      id: row.chunk_id,
+      sha256: row.sha256,
+      byteLength: Number(row.byte_length ?? 0),
+      encoding: row.encoding,
+      compression: row.compression,
+    };
+  }
+
+  async function readBlobHash(chunkId: string): Promise<string | null> {
+    const row = await db
+      .selectFrom('sync_snapshot_chunks')
+      .select(['blob_hash'])
+      .where('chunk_id', '=', chunkId)
+      .executeTakeFirst();
+    return row?.blob_hash ?? null;
+  }
+
+  async function upsertChunkMetadata(
+    metadata: Omit<
+      SnapshotChunkMetadata,
+      'chunkId' | 'byteLength' | 'blobHash'
+    >,
+    args: { blobHash: string; byteLength: number }
+  ): Promise<void> {
+    const chunkId = generateChunkId();
+    const now = new Date().toISOString();
+
+    await db
+      .insertInto('sync_snapshot_chunks')
+      .values({
+        chunk_id: chunkId,
+        partition_id: metadata.partitionId,
+        scope_key: metadata.scopeKey,
+        scope: metadata.scope,
+        as_of_commit_seq: metadata.asOfCommitSeq,
+        row_cursor: metadata.rowCursor ?? '',
+        row_limit: metadata.rowLimit,
+        encoding: metadata.encoding,
+        compression: metadata.compression,
+        sha256: metadata.sha256,
+        byte_length: args.byteLength,
+        blob_hash: args.blobHash,
+        expires_at: metadata.expiresAt,
+        created_at: now,
+      })
+      .onConflict((oc) =>
+        oc
+          .columns([
+            'partition_id',
+            'scope_key',
+            'scope',
+            'as_of_commit_seq',
+            'row_cursor',
+            'row_limit',
+            'encoding',
+            'compression',
+          ])
+          .doUpdateSet({
+            expires_at: metadata.expiresAt,
+            blob_hash: args.blobHash,
+            sha256: metadata.sha256,
+            byte_length: args.byteLength,
+            row_cursor: metadata.rowCursor ?? '',
+          })
+      )
+      .execute();
+  }
+
+  async function readChunkStreamById(
+    chunkId: string
+  ): Promise<ReadableStream<Uint8Array> | null> {
+    const blobHash = await readBlobHash(chunkId);
+    if (!blobHash) return null;
+
+    if (blobAdapter.getStream) {
+      return blobAdapter.getStream(blobHash);
+    }
+
+    if (blobAdapter.get) {
+      const bytes = await blobAdapter.get(blobHash);
+      return bytes ? bytesToStream(bytes) : null;
+    }
+
+    throw new Error(
+      `Blob adapter ${blobAdapter.name} does not support direct get() for snapshot chunks`
+    );
+  }
+
   return {
     name: `db-metadata+${blobAdapter.name}`,
 
@@ -67,9 +280,7 @@ export function createDbMetadataChunkStorage(
       }
     ): Promise<SyncSnapshotChunkRef> {
       const { body, ...metaWithoutBody } = metadata;
-      const blobHash = computeBlobHash(body);
-      const chunkId = generateChunkId();
-      const now = new Date().toISOString();
+      const blobHash = computeBlobHash(metaWithoutBody);
 
       // Check if blob already exists (content-addressed dedup)
       const blobExists = await blobAdapter.exists(blobHash);
@@ -77,7 +288,17 @@ export function createDbMetadataChunkStorage(
       if (!blobExists) {
         // Store body in blob adapter
         if (blobAdapter.put) {
-          await blobAdapter.put(blobHash, body);
+          await blobAdapter.put(blobHash, body, {
+            disableChecksum: true,
+            byteLength: body.length,
+            contentLength: body.length,
+          });
+        } else if (blobAdapter.putStream) {
+          await blobAdapter.putStream(blobHash, bytesToStream(body), {
+            disableChecksum: true,
+            byteLength: body.length,
+            contentLength: body.length,
+          });
         } else {
           throw new Error(
             `Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`
@@ -85,119 +306,140 @@ export function createDbMetadataChunkStorage(
         }
       }
 
-      // Upsert metadata in database
-      await db
-        .insertInto('sync_snapshot_chunks')
-        .values({
-          chunk_id: chunkId,
-          scope_key: metaWithoutBody.scopeKey,
-          scope: metaWithoutBody.scope,
-          as_of_commit_seq: metaWithoutBody.asOfCommitSeq,
-          row_cursor: metaWithoutBody.rowCursor ?? '',
-          row_limit: metaWithoutBody.rowLimit,
-          encoding: metaWithoutBody.encoding,
-          compression: metaWithoutBody.compression,
-          sha256: metaWithoutBody.sha256,
-          byte_length: body.length,
-          blob_hash: blobHash,
-          expires_at: metaWithoutBody.expiresAt,
-          created_at: now,
-        })
-        .onConflict((oc) =>
-          oc
-            .columns([
-              'scope_key',
-              'scope',
-              'as_of_commit_seq',
-              'row_cursor',
-              'row_limit',
-              'encoding',
-              'compression',
-            ])
-            .doUpdateSet({
-              expires_at: metaWithoutBody.expiresAt,
-              blob_hash: blobHash,
-              sha256: metaWithoutBody.sha256,
-              byte_length: body.length,
-              row_cursor: metaWithoutBody.rowCursor ?? '',
-            })
-        )
-        .execute();
-
-      return {
-        id: chunkId,
-        sha256: metaWithoutBody.sha256,
+      await upsertChunkMetadata(metaWithoutBody, {
+        blobHash,
         byteLength: body.length,
+      });
+
+      const storedRef = await readStoredRef({
+        partitionId: metaWithoutBody.partitionId,
+        scopeKey: metaWithoutBody.scopeKey,
+        scope: metaWithoutBody.scope,
+        asOfCommitSeq: metaWithoutBody.asOfCommitSeq,
+        rowCursor: metaWithoutBody.rowCursor,
+        rowLimit: metaWithoutBody.rowLimit,
         encoding: metaWithoutBody.encoding,
         compression: metaWithoutBody.compression,
-      };
-    },
+        includeExpired: true,
+      });
 
-    async readChunk(chunkId: string): Promise<Uint8Array | null> {
-      // Get metadata to find blob hash
-      const row = await db
-        .selectFrom('sync_snapshot_chunks')
-        .select(['blob_hash'])
-        .where('chunk_id', '=', chunkId)
-        .executeTakeFirst();
-
-      if (!row) return null;
-
-      // Read from blob adapter
-      if (blobAdapter.get) {
-        return blobAdapter.get(row.blob_hash);
+      if (!storedRef) {
+        throw new Error('Failed to read stored snapshot chunk reference');
       }
 
-      throw new Error(
-        `Blob adapter ${blobAdapter.name} does not support direct get() for snapshot chunks`
-      );
+      return storedRef;
     },
 
-    async findChunk(
-      pageKey: SnapshotChunkPageKey
-    ): Promise<SyncSnapshotChunkRef | null> {
-      const nowIso = new Date().toISOString();
-      const rowCursorKey = pageKey.rowCursor ?? '';
+    async storeChunkStream(
+      metadata: Omit<
+        SnapshotChunkMetadata,
+        'chunkId' | 'byteLength' | 'blobHash'
+      > & {
+        bodyStream: ReadableStream<Uint8Array>;
+        byteLength?: number;
+      }
+    ): Promise<SyncSnapshotChunkRef> {
+      const { bodyStream, byteLength, ...metaWithoutBody } = metadata;
+      const blobHash = computeBlobHash(metaWithoutBody);
 
-      const row = await db
-        .selectFrom('sync_snapshot_chunks')
-        .select([
-          'chunk_id',
-          'sha256',
-          'byte_length',
-          'encoding',
-          'compression',
-        ])
-        .where('scope_key', '=', pageKey.scopeKey)
-        .where('scope', '=', pageKey.scope)
-        .where('as_of_commit_seq', '=', pageKey.asOfCommitSeq)
-        .where('row_cursor', '=', rowCursorKey)
-        .where('row_limit', '=', pageKey.rowLimit)
-        .where('encoding', '=', pageKey.encoding)
-        .where('compression', '=', pageKey.compression)
-        .where('expires_at', '>', nowIso)
-        .executeTakeFirst();
+      const blobExists = await blobAdapter.exists(blobHash);
+      let observedByteLength: number;
 
-      if (!row) return null;
+      if (!blobExists) {
+        if (blobAdapter.putStream) {
+          const [uploadStream, countStream] = bodyStream.tee();
+          const uploadPromise =
+            typeof byteLength === 'number'
+              ? blobAdapter.putStream(blobHash, uploadStream, {
+                  disableChecksum: true,
+                  byteLength,
+                  contentLength: byteLength,
+                })
+              : blobAdapter.putStream(blobHash, uploadStream, {
+                  disableChecksum: true,
+                });
+          const countPromise = streamByteLength(countStream);
 
-      if (row.encoding !== 'ndjson') {
-        throw new Error(
-          `Unexpected snapshot chunk encoding: ${String(row.encoding)}`
-        );
+          const [, countedByteLength] = await Promise.all([
+            uploadPromise,
+            countPromise,
+          ]);
+          observedByteLength = countedByteLength;
+        } else if (blobAdapter.put) {
+          const body = await streamToBytes(bodyStream);
+          await blobAdapter.put(blobHash, body);
+          observedByteLength = body.length;
+        } else {
+          throw new Error(
+            `Blob adapter ${blobAdapter.name} does not support direct put() for snapshot chunks`
+          );
+        }
+      } else if (typeof byteLength === 'number') {
+        observedByteLength = byteLength;
+        await bodyStream.cancel();
+      } else if (blobAdapter.getMetadata) {
+        const metadata = await blobAdapter.getMetadata(blobHash);
+        if (!metadata) {
+          throw new Error(
+            `Blob metadata missing for existing chunk ${blobHash}`
+          );
+        }
+        observedByteLength = metadata.size;
+        await bodyStream.cancel();
+      } else {
+        observedByteLength = await streamByteLength(bodyStream);
       }
-      if (row.compression !== 'gzip') {
+
+      if (
+        typeof byteLength === 'number' &&
+        Number.isFinite(byteLength) &&
+        observedByteLength !== byteLength
+      ) {
         throw new Error(
-          `Unexpected snapshot chunk compression: ${String(row.compression)}`
+          `Snapshot chunk byte length mismatch: expected ${byteLength}, got ${observedByteLength}`
         );
       }
 
-      return {
-        id: row.chunk_id,
-        sha256: row.sha256,
-        byteLength: Number(row.byte_length ?? 0),
-        encoding: row.encoding,
-        compression: row.compression,
-      };
+      await upsertChunkMetadata(metaWithoutBody, {
+        blobHash,
+        byteLength: observedByteLength,
+      });
+
+      const storedRef = await readStoredRef({
+        partitionId: metaWithoutBody.partitionId,
+        scopeKey: metaWithoutBody.scopeKey,
+        scope: metaWithoutBody.scope,
+        asOfCommitSeq: metaWithoutBody.asOfCommitSeq,
+        rowCursor: metaWithoutBody.rowCursor,
+        rowLimit: metaWithoutBody.rowLimit,
+        encoding: metaWithoutBody.encoding,
+        compression: metaWithoutBody.compression,
+        includeExpired: true,
+      });
+
+      if (!storedRef) {
+        throw new Error('Failed to read stored snapshot chunk reference');
+      }
+
+      return storedRef;
+    },
+
+    async readChunk(chunkId: string): Promise<Uint8Array | null> {
+      const stream = await readChunkStreamById(chunkId);
+      if (!stream) return null;
+      return streamToBytes(stream);
+    },
+
+    async readChunkStream(
+      chunkId: string
+    ): Promise<ReadableStream<Uint8Array> | null> {
+      return readChunkStreamById(chunkId);
+    },
+
+    async findChunk(
+      pageKey: SnapshotChunkPageKey
+    ): Promise<SyncSnapshotChunkRef | null> {
+      return readStoredRef(pageKey);
     },
 
     async cleanupExpired(beforeIso: string): Promise<number> {

package/src/snapshot-chunks/index.ts

@@ -4,6 +4,5 @@
  * Separates chunk metadata (database) from body content (blob storage).
  */
 
-export * from './adapters/s3';
 export * from './db-metadata';
 export * from './types';

package/src/snapshot-chunks/types.ts

@@ -5,19 +5,24 @@
  * Enables flexible storage backends (database, S3, R2, etc.)
  */
 
-import type { SyncSnapshotChunkRef } from '@syncular/core';
+import type {
+  SyncSnapshotChunkCompression,
+  SyncSnapshotChunkEncoding,
+  SyncSnapshotChunkRef,
+} from '@syncular/core';
 
 /**
  * Page key for identifying a specific chunk
  */
 export interface SnapshotChunkPageKey {
+  partitionId: string;
   scopeKey: string;
   scope: string;
   asOfCommitSeq: number;
   rowCursor: string | null;
   rowLimit: number;
-  encoding: 'ndjson';
-  compression: 'gzip';
+  encoding: SyncSnapshotChunkEncoding;
+  compression: SyncSnapshotChunkCompression;
 }
 
 /**
@@ -25,13 +30,14 @@ export interface SnapshotChunkPageKey {
  */
 export interface SnapshotChunkMetadata {
   chunkId: string;
+  partitionId: string;
   scopeKey: string;
   scope: string;
   asOfCommitSeq: number;
   rowCursor: string | null;
   rowLimit: number;
-  encoding: 'ndjson';
-  compression: 'gzip';
+  encoding: SyncSnapshotChunkEncoding;
+  compression: SyncSnapshotChunkCompression;
   sha256: string;
   byteLength: number;
   blobHash: string; // Reference to blob storage
@@ -58,11 +64,31 @@ export interface SnapshotChunkStorage {
     }
   ): Promise<SyncSnapshotChunkRef>;
 
+  /**
+   * Store a chunk from a stream.
+   * Preferred for large payloads to avoid full buffering in memory.
+   */
+  storeChunkStream?(
+    metadata: Omit<
+      SnapshotChunkMetadata,
+      'chunkId' | 'byteLength' | 'blobHash'
+    > & {
+      bodyStream: ReadableStream<Uint8Array>;
+      byteLength?: number;
+    }
+  ): Promise<SyncSnapshotChunkRef>;
+
   /**
    * Read chunk body by chunk ID
    */
   readChunk(chunkId: string): Promise<Uint8Array | null>;
 
+  /**
+   * Read chunk body as a stream.
+   * Preferred for large payloads to avoid full buffering in memory.
+   */
+  readChunkStream?(chunkId: string): Promise<ReadableStream<Uint8Array> | null>;
+
   /**
    * Find existing chunk by page key
    */