@syncular/client 0.0.6-204 → 0.0.6-206

package/src/pull-engine.ts

@@ -13,6 +13,14 @@ import type {
   SyncSubscriptionRequest,
   SyncTransport,
 } from '@syncular/core';
+import {
+  bytesToReadableStream,
+  readAllBytesFromStream,
+  SYNC_SNAPSHOT_CHUNK_COMPRESSION,
+  SYNC_SNAPSHOT_CHUNK_ENCODING,
+  SYNC_SNAPSHOT_CHUNK_MAGIC,
+  sha256Hex,
+} from '@syncular/core';
 import { type Kysely, sql, type Transaction } from 'kysely';
 import {
   type ClientHandlerCollection,
@@ -30,9 +38,16 @@ import type { SyncClientDb, SyncSubscriptionStateTable } from './schema';
 const jsonCache = new WeakMap<object, string>();
 const jsonCacheStats = { hits: 0, misses: 0 };
 const SNAPSHOT_APPLY_BATCH_ROWS = 500;
-const SNAPSHOT_ROW_FRAME_MAGIC = new Uint8Array([0x53, 0x52, 0x46, 0x31]); // "SRF1"
 const FRAME_LENGTH_BYTES = 4;
 
+interface DigestStreamLike extends WritableStream<Uint8Array> {
+  readonly digest: Promise<ArrayBuffer>;
+}
+
+interface CryptoWithDigestStream extends Crypto {
+  DigestStream?: new (algorithm: 'SHA-256') => DigestStreamLike;
+}
+
 function serializeJsonCached(obj: object): string {
   if (obj === null || typeof obj !== 'object') {
     return JSON.stringify(obj);
@@ -51,34 +66,6 @@ function serializeJsonCached(obj: object): string {
   return serialized;
 }
 
-function isGzipBytes(bytes: Uint8Array): boolean {
-  return bytes.length >= 2 && bytes[0] === 0x1f && bytes[1] === 0x8b;
-}
-
-function bytesToReadableStream(bytes: Uint8Array): ReadableStream<Uint8Array> {
-  return new ReadableStream<Uint8Array>({
-    start(controller) {
-      controller.enqueue(bytes);
-      controller.close();
-    },
-  });
-}
-
-function concatBytes(chunks: readonly Uint8Array[]): Uint8Array {
-  if (chunks.length === 1) {
-    return chunks[0] ?? new Uint8Array();
-  }
-  let total = 0;
-  for (const chunk of chunks) total += chunk.length;
-  const out = new Uint8Array(total);
-  let offset = 0;
-  for (const chunk of chunks) {
-    out.set(chunk, offset);
-    offset += chunk.length;
-  }
-  return out;
-}
-
 function appendBytes(base: Uint8Array, next: Uint8Array): Uint8Array {
   if (base.length === 0) return next;
   if (next.length === 0) return base;
@@ -88,62 +75,64 @@ function appendBytes(base: Uint8Array, next: Uint8Array): Uint8Array {
   return out;
 }
 
-function toOwnedUint8Array(chunk: Uint8Array): Uint8Array<ArrayBuffer> {
-  const out = new ArrayBuffer(chunk.byteLength);
-  const bytes = new Uint8Array(out);
-  bytes.set(chunk);
-  return bytes;
+function toHex(bytes: Uint8Array): string {
+  return Array.from(bytes)
+    .map((byte) => byte.toString(16).padStart(2, '0'))
+    .join('');
 }
 
-async function maybeGunzipStream(
-  stream: ReadableStream<Uint8Array>
-): Promise<ReadableStream<Uint8Array>> {
-  const reader = stream.getReader();
-  const prefetched: Uint8Array[] = [];
-  let prefetchedBytes = 0;
-
-  while (prefetchedBytes < 2) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    if (!value || value.length === 0) continue;
-    prefetched.push(value);
-    prefetchedBytes += value.length;
-  }
-
-  const prefetchedCombined = concatBytes(prefetched);
-  const gzip = isGzipBytes(prefetchedCombined);
+function getDigestStreamConstructor():
+  | (new (
+      algorithm: 'SHA-256'
+    ) => DigestStreamLike)
+  | null {
+  if (typeof crypto === 'undefined') return null;
+  const cryptoWithDigestStream: CryptoWithDigestStream = crypto;
+  return cryptoWithDigestStream.DigestStream ?? null;
+}
 
-  const replayStream = new ReadableStream<Uint8Array<ArrayBuffer>>({
-    start(controller) {
-      if (prefetchedCombined.length > 0) {
-        controller.enqueue(toOwnedUint8Array(prefetchedCombined));
-      }
-    },
-    async pull(controller) {
-      const { done, value } = await reader.read();
-      if (done) {
-        controller.close();
-        reader.releaseLock();
-        return;
-      }
-      if (!value || value.length === 0) return;
-      controller.enqueue(toOwnedUint8Array(value));
-    },
-    async cancel(reason) {
-      await reader.cancel(reason);
-      reader.releaseLock();
-    },
-  });
+async function decodeSnapshotChunkStream(args: {
+  stream: ReadableStream<Uint8Array>;
+  compression: string;
+  encoding: string;
+}): Promise<ReadableStream<Uint8Array>> {
+  if (args.encoding !== SYNC_SNAPSHOT_CHUNK_ENCODING) {
+    throw new Error(`Unexpected snapshot chunk encoding: ${args.encoding}`);
+  }
+  if (args.compression !== SYNC_SNAPSHOT_CHUNK_COMPRESSION) {
+    throw new Error(
+      `Unexpected snapshot chunk compression: ${args.compression}`
+    );
+  }
+  if (typeof DecompressionStream === 'undefined') {
+    throw new Error(
+      'Snapshot chunk gzip decompression is not available in this runtime'
+    );
+  }
+  return args.stream.pipeThrough(
+    new DecompressionStream('gzip') as TransformStream<Uint8Array, Uint8Array>
+  );
+}
 
-  if (!gzip) return replayStream;
+async function computeSha256HexFromStream(
+  stream: ReadableStream<Uint8Array>,
+  sha256Override?: (bytes: Uint8Array) => Promise<string>
+): Promise<string> {
+  if (sha256Override) {
+    const bytes = await readAllBytesFromStream(stream);
+    return sha256Override(bytes);
+  }
 
-  if (typeof DecompressionStream !== 'undefined') {
-    return replayStream.pipeThrough(new DecompressionStream('gzip'));
+  const DigestStream = getDigestStreamConstructor();
+  if (DigestStream) {
+    const digester = new DigestStream('SHA-256');
+    await stream.pipeTo(digester);
+    const digest = await digester.digest;
+    return toHex(new Uint8Array(digest));
   }
 
-  throw new Error(
-    'Snapshot chunk appears gzip-compressed but gzip decompression is not available in this runtime'
-  );
+  const bytes = await readAllBytesFromStream(stream);
+  return sha256Hex(bytes);
 }
 
 async function* decodeSnapshotRowStreamBatches(
@@ -164,15 +153,15 @@ async function* decodeSnapshotRowStreamBatches(
       pending = appendBytes(pending, value);
 
       if (!headerValidated) {
-        if (pending.length < SNAPSHOT_ROW_FRAME_MAGIC.length) {
+        if (pending.length < SYNC_SNAPSHOT_CHUNK_MAGIC.length) {
           continue;
         }
-        for (let index = 0; index < SNAPSHOT_ROW_FRAME_MAGIC.length; index++) {
-          if (pending[index] !== SNAPSHOT_ROW_FRAME_MAGIC[index]) {
+        for (let index = 0; index < SYNC_SNAPSHOT_CHUNK_MAGIC.length; index++) {
+          if (pending[index] !== SYNC_SNAPSHOT_CHUNK_MAGIC[index]) {
             throw new Error('Unexpected snapshot chunk format');
           }
         }
-        pending = pending.subarray(SNAPSHOT_ROW_FRAME_MAGIC.length);
+        pending = pending.subarray(SYNC_SNAPSHOT_CHUNK_MAGIC.length);
         headerValidated = true;
       }
 
@@ -191,7 +180,11 @@ async function* decodeSnapshotRowStreamBatches(
           FRAME_LENGTH_BYTES,
           FRAME_LENGTH_BYTES + payloadLength
         );
-        rows.push(JSON.parse(decoder.decode(payload)));
+        const parsed = JSON.parse(decoder.decode(payload));
+        if (!Array.isArray(parsed)) {
+          throw new Error('Snapshot chunk frame payload must be a JSON array');
+        }
+        rows.push(...parsed);
         pending = pending.subarray(FRAME_LENGTH_BYTES + payloadLength);
 
         if (rows.length >= batchSize) {
@@ -232,7 +225,11 @@ async function* decodeSnapshotRowStreamBatches(
           FRAME_LENGTH_BYTES,
           FRAME_LENGTH_BYTES + nextLength
         );
-        rows.push(JSON.parse(decoder.decode(payload)));
+        const parsed = JSON.parse(decoder.decode(payload));
+        if (!Array.isArray(parsed)) {
+          throw new Error('Snapshot chunk frame payload must be a JSON array');
+        }
+        rows.push(...parsed);
         pending = pending.subarray(FRAME_LENGTH_BYTES + nextLength);
         if (rows.length >= batchSize) {
           yield rows;
@@ -252,33 +249,6 @@ async function* decodeSnapshotRowStreamBatches(
   }
 }
 
-async function computeSha256Hex(
-  bytes: Uint8Array,
-  sha256Override?: (bytes: Uint8Array) => Promise<string>
-): Promise<string> {
-  // Use injected implementation if provided (e.g. expo-crypto on React Native)
-  if (sha256Override) {
-    return sha256Override(bytes);
-  }
-
-  // Use crypto.subtle if available (browsers, modern Node/Bun)
-  if (typeof crypto !== 'undefined' && crypto.subtle) {
-    // Create a fresh ArrayBuffer to satisfy crypto.subtle's type requirements
-    const buffer = new ArrayBuffer(bytes.length);
-    new Uint8Array(buffer).set(bytes);
-    const hashBuffer = await crypto.subtle.digest('SHA-256', buffer);
-    const hashArray = new Uint8Array(hashBuffer);
-    return Array.from(hashArray, (b) => b.toString(16).padStart(2, '0')).join(
-      ''
-    );
-  }
-
-  throw new Error(
-    'No crypto implementation available for SHA-256. ' +
-      'Provide a sha256 function via options or ensure crypto.subtle is available.'
-  );
-}
-
 async function fetchSnapshotChunkStream(
   transport: SyncTransport,
   request: {
@@ -293,62 +263,32 @@ async function fetchSnapshotChunkStream(
   return bytesToReadableStream(bytes);
 }
 
-async function readAllBytesFromStream(
-  stream: ReadableStream<Uint8Array>
-): Promise<Uint8Array> {
-  const reader = stream.getReader();
-  const chunks: Uint8Array[] = [];
-  let totalLength = 0;
-
-  try {
-    while (true) {
-      // eslint-disable-next-line no-await-in-loop
-      const { done, value } = await reader.read();
-      if (done) break;
-      if (!value) continue;
-      chunks.push(value);
-      totalLength += value.byteLength;
-    }
-  } finally {
-    reader.releaseLock();
-  }
-
-  if (chunks.length === 0) {
-    return new Uint8Array();
-  }
-  if (chunks.length === 1) {
-    return chunks[0]!;
-  }
-
-  const bytes = new Uint8Array(totalLength);
-  let offset = 0;
-  for (const chunk of chunks) {
-    bytes.set(chunk, offset);
-    offset += chunk.byteLength;
-  }
-  return bytes;
-}
-
 async function materializeSnapshotChunkRows(
   transport: SyncTransport,
   request: {
     chunkId: string;
     scopeValues?: ScopeValues;
   },
-  expectedHash: string | undefined,
+  chunk: {
+    sha256: string;
+    compression: string;
+    encoding: string;
+  },
   sha256Override?: (bytes: Uint8Array) => Promise<string>
 ): Promise<unknown[]> {
   const rawStream = await fetchSnapshotChunkStream(transport, request);
-  const decodedStream = await maybeGunzipStream(rawStream);
+  const decodedStream = await decodeSnapshotChunkStream({
+    stream: rawStream,
+    compression: chunk.compression,
+    encoding: chunk.encoding,
+  });
   let streamForDecode = decodedStream;
   let chunkHashPromise: Promise<string> | null = null;
 
-  if (expectedHash) {
+  if (chunk.sha256) {
     const [hashStream, decodeStream] = decodedStream.tee();
     streamForDecode = decodeStream;
-    chunkHashPromise = readAllBytesFromStream(hashStream).then((bytes) =>
-      computeSha256Hex(bytes, sha256Override)
-    );
+    chunkHashPromise = computeSha256HexFromStream(hashStream, sha256Override);
   }
 
   const rows: unknown[] = [];
@@ -368,9 +308,9 @@ async function materializeSnapshotChunkRows(
   if (chunkHashPromise) {
     try {
       const actualHash = await chunkHashPromise;
-      if (!materializeError && actualHash !== expectedHash) {
+      if (!materializeError && actualHash !== chunk.sha256) {
         materializeError = new Error(
-          `Snapshot chunk integrity check failed: expected sha256 ${expectedHash}, got ${actualHash}`
+          `Snapshot chunk integrity check failed: expected sha256 ${chunk.sha256}, got ${actualHash}`
         );
       }
     } catch (hashError) {
@@ -416,7 +356,7 @@ async function materializeChunkedSnapshots(
             chunkId: chunk.id,
             scopeValues: sub.scopes,
           },
-          chunk.sha256,
+          chunk,
           sha256Override
         );
         rows.push(...chunkRows);
@@ -462,16 +402,18 @@ async function applyChunkedSnapshot<DB extends SyncClientDb>(
       chunkId: chunk.id,
       scopeValues,
     });
-    const decodedStream = await maybeGunzipStream(rawStream);
+    const decodedStream = await decodeSnapshotChunkStream({
+      stream: rawStream,
+      compression: chunk.compression,
+      encoding: chunk.encoding,
+    });
     let streamForDecode = decodedStream;
     let chunkHashPromise: Promise<string> | null = null;
 
     if (chunk.sha256) {
       const [hashStream, decodeStream] = decodedStream.tee();
       streamForDecode = decodeStream;
-      chunkHashPromise = readAllBytesFromStream(hashStream).then((bytes) =>
-        computeSha256Hex(bytes, sha256Override)
-      );
+      chunkHashPromise = computeSha256HexFromStream(hashStream, sha256Override);
     }
 
     const rowBatchIterator = decodeSnapshotRowStreamBatches(