@syncular/client 0.0.6-168 → 0.0.6-177

package/src/pull-engine.test.ts

@@ -149,6 +149,142 @@ describe('applyPullResponse chunk streaming', () => {
     expect(streamFetchCount).toBe(1);
   });
 
+  it('materializes chunked bootstrap snapshots for afterPull plugins via streaming transport', async () => {
+    const firstRows = Array.from({ length: 1200 }, (_, index) => ({
+      id: `${index + 1}`,
+      name: `Item ${index + 1}`,
+    }));
+    const secondRows = Array.from({ length: 1200 }, (_, index) => ({
+      id: `${index + 1201}`,
+      name: `Item ${index + 1201}`,
+    }));
+    const firstChunk = new Uint8Array(gzipSync(encodeSnapshotRows(firstRows)));
+    const secondChunk = new Uint8Array(
+      gzipSync(encodeSnapshotRows(secondRows))
+    );
+
+    let streamFetchCount = 0;
+    let activeStreamFetches = 0;
+    let maxConcurrentStreamFetches = 0;
+    const transport: SyncTransport = {
+      async sync() {
+        return {};
+      },
+      async fetchSnapshotChunk() {
+        throw new Error('fetchSnapshotChunk should not be used');
+      },
+      async fetchSnapshotChunkStream({ chunkId }) {
+        streamFetchCount += 1;
+        activeStreamFetches += 1;
+        maxConcurrentStreamFetches = Math.max(
+          maxConcurrentStreamFetches,
+          activeStreamFetches
+        );
+        await new Promise((resolve) => setTimeout(resolve, 5));
+        activeStreamFetches -= 1;
+
+        if (chunkId === 'chunk-1') {
+          return createStreamFromBytes(firstChunk, 193);
+        }
+        if (chunkId === 'chunk-2') {
+          return createStreamFromBytes(secondChunk, 211);
+        }
+        throw new Error(`Unexpected chunk id: ${chunkId}`);
+      },
+    };
+
+    const handlers: ClientHandlerCollection<TestDb> = [
+      createClientHandler({
+        table: 'items',
+        scopes: ['items:{id}'],
+      }),
+    ];
+
+    let pluginSawRows = 0;
+    const options = {
+      clientId: 'client-1',
+      subscriptions: [
+        {
+          id: 'items-sub',
+          table: 'items',
+          scopes: {},
+        },
+      ],
+      stateId: 'default',
+      plugins: [
+        {
+          name: 'after-pull-observer',
+          async afterPull(_ctx, { response }) {
+            pluginSawRows =
+              response.subscriptions[0]?.snapshots?.[0]?.rows.length ?? 0;
+            return response;
+          },
+        },
+      ],
+    };
+
+    const pullState = await buildPullRequest(db, options);
+
+    const response: SyncPullResponse = {
+      ok: true,
+      subscriptions: [
+        {
+          id: 'items-sub',
+          status: 'active',
+          scopes: {},
+          bootstrap: true,
+          bootstrapState: null,
+          nextCursor: 2,
+          commits: [],
+          snapshots: [
+            {
+              table: 'items',
+              rows: [],
+              chunks: [
+                {
+                  id: 'chunk-1',
+                  byteLength: firstChunk.length,
+                  sha256: '',
+                  encoding: 'json-row-frame-v1',
+                  compression: 'gzip',
+                },
+                {
+                  id: 'chunk-2',
+                  byteLength: secondChunk.length,
+                  sha256: '',
+                  encoding: 'json-row-frame-v1',
+                  compression: 'gzip',
+                },
+              ],
+              isFirstPage: true,
+              isLastPage: true,
+            },
+          ],
+        },
+      ],
+    };
+
+    await applyPullResponse(
+      db,
+      transport,
+      handlers,
+      options,
+      pullState,
+      response
+    );
+
+    const countResult = await sql<{ count: number }>`
+      select count(*) as count
+      from ${sql.table('items')}
+    `.execute(db);
+    expect(Number(countResult.rows[0]?.count ?? 0)).toBe(
+      firstRows.length + secondRows.length
+    );
+    expect(pluginSawRows).toBe(firstRows.length + secondRows.length);
+    expect(streamFetchCount).toBe(2);
+    expect(maxConcurrentStreamFetches).toBe(1);
+  });
+
   it('rolls back partial chunked bootstrap when a later chunk fails', async () => {
     const firstRows = Array.from({ length: 1500 }, (_, index) => ({
       id: `${index + 1}`,

package/src/pull-engine.ts

@@ -12,7 +12,6 @@ import type {
   SyncSubscriptionRequest,
   SyncTransport,
 } from '@syncular/core';
-import { decodeSnapshotRows } from '@syncular/core';
 import { type Kysely, sql, type Transaction } from 'kysely';
 import {
   type ClientHandlerCollection,
@@ -29,7 +28,6 @@ import type { SyncClientDb, SyncSubscriptionStateTable } from './schema';
 // of the same objects during pull operations
 const jsonCache = new WeakMap<object, string>();
 const jsonCacheStats = { hits: 0, misses: 0 };
-const SNAPSHOT_CHUNK_CONCURRENCY = 8;
 const SNAPSHOT_APPLY_BATCH_ROWS = 500;
 const SNAPSHOT_ROW_FRAME_MAGIC = new Uint8Array([0x53, 0x52, 0x46, 0x31]); // "SRF1"
 const FRAME_LENGTH_BYTES = 4;
@@ -96,24 +94,6 @@ function toOwnedUint8Array(chunk: Uint8Array): Uint8Array<ArrayBuffer> {
   return bytes;
 }
 
-async function streamToBytes(
-  stream: ReadableStream<Uint8Array>
-): Promise<Uint8Array> {
-  const reader = stream.getReader();
-  try {
-    const chunks: Uint8Array[] = [];
-    while (true) {
-      const { done, value } = await reader.read();
-      if (done) break;
-      if (!value || value.length === 0) continue;
-      chunks.push(value);
-    }
-    return concatBytes(chunks);
-  } finally {
-    reader.releaseLock();
-  }
-}
-
 async function maybeGunzipStream(
   stream: ReadableStream<Uint8Array>
 ): Promise<ReadableStream<Uint8Array>> {
@@ -165,14 +145,6 @@ async function maybeGunzipStream(
   );
 }
 
-async function maybeGunzip(bytes: Uint8Array): Promise<Uint8Array> {
-  if (!isGzipBytes(bytes)) return bytes;
-  const decompressedStream = await maybeGunzipStream(
-    bytesToReadableStream(bytes)
-  );
-  return streamToBytes(decompressedStream);
-}
-
 async function* decodeSnapshotRowStreamBatches(
   stream: ReadableStream<Uint8Array>,
   batchSize: number
@@ -356,29 +328,62 @@ async function readAllBytesFromStream(
   return bytes;
 }
 
-async function mapWithConcurrency<T, U>(
-  items: readonly T[],
-  concurrency: number,
-  mapper: (item: T, index: number) => Promise<U>
-): Promise<U[]> {
-  if (items.length === 0) return [];
-
-  const workerCount = Math.max(1, Math.min(concurrency, items.length));
-  const results = new Array<U>(items.length);
-  let nextIndex = 0;
-
-  async function worker(): Promise<void> {
-    while (nextIndex < items.length) {
-      const index = nextIndex;
-      nextIndex += 1;
-      const item = items[index];
-      if (item === undefined) continue;
-      results[index] = await mapper(item, index);
+async function materializeSnapshotChunkRows(
+  transport: SyncTransport,
+  request: {
+    chunkId: string;
+    scopeValues?: ScopeValues;
+  },
+  expectedHash: string | undefined,
+  sha256Override?: (bytes: Uint8Array) => Promise<string>
+): Promise<unknown[]> {
+  const rawStream = await fetchSnapshotChunkStream(transport, request);
+  const decodedStream = await maybeGunzipStream(rawStream);
+  let rowStream = decodedStream;
+  let chunkHashPromise: Promise<string> | null = null;
+
+  if (expectedHash) {
+    const [hashStream, streamForRows] = decodedStream.tee();
+    rowStream = streamForRows;
+    chunkHashPromise = readAllBytesFromStream(hashStream).then((bytes) =>
+      computeSha256Hex(bytes, sha256Override)
+    );
+  }
+
+  const rows: unknown[] = [];
+  let materializeError: unknown = null;
+
+  try {
+    for await (const batch of decodeSnapshotRowStreamBatches(
+      rowStream,
+      SNAPSHOT_APPLY_BATCH_ROWS
+    )) {
+      rows.push(...batch);
+    }
+  } catch (error) {
+    materializeError = error;
+  }
+
+  if (chunkHashPromise) {
+    try {
+      const actualHash = await chunkHashPromise;
+      if (!materializeError && actualHash !== expectedHash) {
+        materializeError = new Error(
+          `Snapshot chunk integrity check failed: expected sha256 ${expectedHash}, got ${actualHash}`
+        );
+      }
+    } catch (hashError) {
+      if (!materializeError) {
+        materializeError = hashError;
+      }
     }
   }
 
-  await Promise.all(Array.from({ length: workerCount }, () => worker()));
-  return results;
+  if (materializeError) {
+    throw materializeError;
+  }
+
+  return rows;
 }
 
 async function materializeChunkedSnapshots(
@@ -386,69 +391,48 @@ async function materializeChunkedSnapshots(
   response: SyncPullResponse,
   sha256Override?: (bytes: Uint8Array) => Promise<string>
 ): Promise<SyncPullResponse> {
-  const chunkCache = new Map<string, Promise<Uint8Array>>();
-
-  const subscriptions = await Promise.all(
-    response.subscriptions.map(async (sub) => {
-      if (!sub.bootstrap) return sub;
-      if (!sub.snapshots || sub.snapshots.length === 0) return sub;
-
-      const snapshots = await mapWithConcurrency(
-        sub.snapshots,
-        SNAPSHOT_CHUNK_CONCURRENCY,
-        async (snapshot) => {
-          const chunks = snapshot.chunks ?? [];
-          if (chunks.length === 0) {
-            return snapshot;
-          }
+  const subscriptions: SyncPullResponse['subscriptions'] = [];
 
-          const parsedRowsByChunk = await mapWithConcurrency(
-            chunks,
-            SNAPSHOT_CHUNK_CONCURRENCY,
-            async (chunk) => {
-              const promise =
-                chunkCache.get(chunk.id) ??
-                transport.fetchSnapshotChunk({
-                  chunkId: chunk.id,
-                  scopeValues: sub.scopes,
-                });
-              chunkCache.set(chunk.id, promise);
-
-              const raw = await promise;
-              const bytes = await maybeGunzip(raw);
-
-              // Verify chunk integrity using sha256 hash
-              if (chunk.sha256) {
-                const actualHash = await computeSha256Hex(
-                  bytes,
-                  sha256Override
-                );
-                if (actualHash !== chunk.sha256) {
-                  throw new Error(
-                    `Snapshot chunk integrity check failed: expected sha256 ${chunk.sha256}, got ${actualHash}`
-                  );
-                }
-              }
-
-              return decodeSnapshotRows(bytes);
-            }
-          );
+  for (const sub of response.subscriptions) {
+    if (!sub.bootstrap || !sub.snapshots || sub.snapshots.length === 0) {
+      subscriptions.push(sub);
+      continue;
+    }
 
-          const rows: unknown[] = [];
-          for (const parsedRows of parsedRowsByChunk) {
-            rows.push(...parsedRows);
-          }
+    const snapshots: SyncPullSubscriptionResponse['snapshots'] = [];
+    for (const snapshot of sub.snapshots) {
+      const chunks = snapshot.chunks ?? [];
+      if (chunks.length === 0) {
+        snapshots.push(snapshot);
+        continue;
+      }
 
-          return { ...snapshot, rows, chunks: undefined };
-        }
-      );
+      const rows: unknown[] = [];
+      for (const chunk of chunks) {
+        const chunkRows = await materializeSnapshotChunkRows(
+          transport,
+          {
+            chunkId: chunk.id,
+            scopeValues: sub.scopes,
+          },
+          chunk.sha256,
+          sha256Override
+        );
+        rows.push(...chunkRows);
+      }
 
-      return { ...sub, snapshots };
-    })
-  );
+      snapshots.push({
+        ...snapshot,
+        rows,
+        chunks: undefined,
+      });
+    }
 
-  // Clear chunk cache after processing to prevent memory accumulation
-  chunkCache.clear();
+    subscriptions.push({
+      ...sub,
+      snapshots,
+    });
+  }
 
   return { ...response, subscriptions };
 }

package/src/query-public.ts

@@ -0,0 +1,277 @@
+/**
+ * Public query exports used by packages that consume @syncular/client.
+ *
+ * This wrapper keeps query-builder tracking isolated per chain so branching a
+ * base Kysely builder does not leak joined tables into sibling branches.
+ */
+
+import type { Kysely } from 'kysely';
+import type { FingerprintCollector } from './query/FingerprintCollector';
+import {
+  computeRowFingerprint,
+  computeValueFingerprint,
+  hasKeyField,
+  type MutationTimestampSource,
+} from './query/fingerprint';
+import type { SyncClientDb } from './schema';
+
+export { FingerprintCollector } from './query/FingerprintCollector';
+export {
+  canFingerprint,
+  computeFingerprint,
+} from './query/fingerprint';
+
+export type FingerprintMode = 'auto' | 'value';
+
+type TrackedSelectFrom<DB> = Kysely<DB>['selectFrom'];
+type SelectFromArgs<DB> = Parameters<Kysely<DB>['selectFrom']>;
+type SelectFromResult<DB> = ReturnType<Kysely<DB>['selectFrom']>;
+
+type ExecutableQuery = {
+  execute: () => Promise<unknown>;
+  executeTakeFirst: () => Promise<unknown>;
+  executeTakeFirstOrThrow: () => Promise<unknown>;
+};
+
+const JOIN_METHODS = new Set([
+  'innerJoin',
+  'leftJoin',
+  'rightJoin',
+  'fullJoin',
+  'crossJoin',
+  'innerJoinLateral',
+  'leftJoinLateral',
+  'crossJoinLateral',
+]);
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+function isExecutableQuery(value: unknown): value is ExecutableQuery {
+  if (!isRecord(value)) return false;
+  return (
+    typeof Reflect.get(value, 'execute') === 'function' &&
+    typeof Reflect.get(value, 'executeTakeFirst') === 'function' &&
+    typeof Reflect.get(value, 'executeTakeFirstOrThrow') === 'function'
+  );
+}
+
+function extractTrackedTableNames(value: unknown): string[] {
+  if (typeof value === 'string') {
+    const normalized = value.trim();
+    if (normalized.length === 0) return [];
+
+    const aliasIndex = normalized.search(/\s+as\s+/i);
+    const withoutAlias =
+      aliasIndex >= 0 ? normalized.slice(0, aliasIndex) : normalized;
+    const firstToken = withoutAlias.split(/\s+/)[0] ?? '';
+    return firstToken.length > 0 ? [firstToken] : [];
+  }
+
+  if (Array.isArray(value)) {
+    return value.flatMap((entry) => extractTrackedTableNames(entry));
+  }
+
+  return [];
+}
+
+function addFingerprint(args: {
+  rows: unknown;
+  primaryTable: string | null;
+  trackedTables: ReadonlySet<string>;
+  collector: FingerprintCollector;
+  engine: MutationTimestampSource;
+  keyField: string;
+  fingerprintMode: FingerprintMode;
+}): void {
+  const {
+    rows,
+    primaryTable,
+    trackedTables,
+    collector,
+    engine,
+    keyField,
+    fingerprintMode,
+  } = args;
+
+  const fingerprintScope =
+    trackedTables.size > 0
+      ? Array.from(trackedTables).sort().join('+')
+      : (primaryTable ?? 'query');
+
+  if (
+    fingerprintMode === 'auto' &&
+    primaryTable &&
+    trackedTables.size === 1 &&
+    Array.isArray(rows) &&
+    hasKeyField(rows, keyField)
+  ) {
+    collector.add(computeRowFingerprint(rows, primaryTable, engine, keyField));
+    return;
+  }
+
+  collector.add(computeValueFingerprint(fingerprintScope, rows));
+}
+
+function addTrackedTablesToScopeCollector(
+  scopeCollector: Set<string>,
+  trackedTables: ReadonlySet<string>
+): void {
+  for (const trackedTable of trackedTables) {
+    scopeCollector.add(trackedTable);
+  }
+}
+
+function createExecuteProxy<B extends ExecutableQuery>(
+  builder: B,
+  primaryTable: string | null,
+  trackedTables: ReadonlySet<string>,
+  scopeCollector: Set<string>,
+  collector: FingerprintCollector,
+  engine: MutationTimestampSource,
+  keyField: string,
+  fingerprintMode: FingerprintMode
+): B {
+  return new Proxy(builder, {
+    get(target, prop, receiver) {
+      if (prop === 'execute') {
+        return async () => {
+          const rows = await target.execute();
+          addTrackedTablesToScopeCollector(scopeCollector, trackedTables);
+          addFingerprint({
+            rows,
+            primaryTable,
+            trackedTables,
+            collector,
+            engine,
+            keyField,
+            fingerprintMode,
+          });
+          return rows;
+        };
+      }
+
+      if (prop === 'executeTakeFirst') {
+        return async () => {
+          const row = await target.executeTakeFirst();
+          addTrackedTablesToScopeCollector(scopeCollector, trackedTables);
+          addFingerprint({
+            rows: row,
+            primaryTable,
+            trackedTables,
+            collector,
+            engine,
+            keyField,
+            fingerprintMode,
+          });
+          return row;
+        };
+      }
+
+      if (prop === 'executeTakeFirstOrThrow') {
+        return async () => {
+          const row = await target.executeTakeFirstOrThrow();
+          addTrackedTablesToScopeCollector(scopeCollector, trackedTables);
+          addFingerprint({
+            rows: row,
+            primaryTable,
+            trackedTables,
+            collector,
+            engine,
+            keyField,
+            fingerprintMode,
+          });
+          return row;
+        };
+      }
+
+      const value = Reflect.get(target, prop, receiver);
+      if (typeof value !== 'function') {
+        return value;
+      }
+
+      return (...args: unknown[]) => {
+        const nextTrackedTables = new Set(trackedTables);
+
+        if (
+          typeof prop === 'string' &&
+          JOIN_METHODS.has(prop) &&
+          args.length > 0
+        ) {
+          for (const tableName of extractTrackedTableNames(args[0])) {
+            nextTrackedTables.add(tableName);
+          }
+        }
+
+        const result = Reflect.apply(value, target, args);
+        if (!isExecutableQuery(result)) {
+          return result;
+        }
+
+        return createExecuteProxy(
+          result,
+          primaryTable,
+          nextTrackedTables,
+          scopeCollector,
+          collector,
+          engine,
+          keyField,
+          fingerprintMode
+        );
+      };
+    },
+  });
+}
+
+function createTrackedSelectFrom<DB extends SyncClientDb>(
+  db: Kysely<DB>,
+  scopeCollector: Set<string>,
+  fingerprintCollector: FingerprintCollector,
+  engine: MutationTimestampSource,
+  keyField = 'id',
+  fingerprintMode: FingerprintMode = 'auto'
+): TrackedSelectFrom<DB> {
+  const selectFrom = (...args: SelectFromArgs<DB>) => {
+    const trackedTables = new Set<string>(extractTrackedTableNames(args[0]));
+    const primaryTable = Array.from(trackedTables)[0] ?? null;
+    const builder = db.selectFrom(...args);
+
+    return createExecuteProxy(
+      builder,
+      primaryTable,
+      trackedTables,
+      scopeCollector,
+      fingerprintCollector,
+      engine,
+      keyField,
+      fingerprintMode
+    ) as SelectFromResult<DB>;
+  };
+
+  return selectFrom as TrackedSelectFrom<DB>;
+}
+
+export interface QueryContext<DB extends SyncClientDb = SyncClientDb> {
+  selectFrom: TrackedSelectFrom<DB>;
+}
+
+export function createQueryContext<DB extends SyncClientDb>(
+  db: Kysely<DB>,
+  scopeCollector: Set<string>,
+  fingerprintCollector: FingerprintCollector,
+  engine: MutationTimestampSource,
+  keyField = 'id',
+  fingerprintMode: FingerprintMode = 'auto'
+): QueryContext<DB> {
+  return {
+    selectFrom: createTrackedSelectFrom(
+      db,
+      scopeCollector,
+      fingerprintCollector,
+      engine,
+      keyField,
+      fingerprintMode
+    ),
+  };
+}

package/dist/query/QueryContext.d.ts

@@ -1,33 +0,0 @@
-/**
- * @syncular/client - Query Context
- *
- * Provides a query context with tracked selectFrom for scope tracking
- * and automatic fingerprint generation.
- */
-import type { Kysely } from 'kysely';
-import type { SyncClientDb } from '../schema';
-import type { FingerprintCollector } from './FingerprintCollector';
-import type { MutationTimestampSource } from './fingerprint';
-import { createTrackedSelectFrom } from './tracked-select';
-export type TrackedSelectFrom<DB> = ReturnType<typeof createTrackedSelectFrom<DB>>;
-/**
- * Query context provided to query functions.
- *
- * Only `selectFrom` is exposed to ensure proper scope tracking and fingerprinting.
- * If you need raw database access, use the db directly outside the query function.
- */
-export interface QueryContext<DB extends SyncClientDb = SyncClientDb> {
-    /**
-     * Wrapped selectFrom that:
-     * 1. Registers table as watched scope
-     * 2. Intercepts .execute() to auto-detect fingerprinting mode:
-     *    - Result has keyField (default: 'id')? -> row-level fingerprinting
-     *    - No keyField? -> value-based fingerprinting (for aggregates)
-     */
-    selectFrom: TrackedSelectFrom<DB>;
-}
-/**
- * Create a query context with tracked selectFrom.
- */
-export declare function createQueryContext<DB extends SyncClientDb>(db: Kysely<DB>, scopeCollector: Set<string>, fingerprintCollector: FingerprintCollector, engine: MutationTimestampSource, keyField?: string): QueryContext<DB>;
-//# sourceMappingURL=QueryContext.d.ts.map

package/dist/query/QueryContext.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"QueryContext.d.ts","sourceRoot":"","sources":["../../src/query/QueryContext.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AACrC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAC9C,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AACnE,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AAC7D,OAAO,EAAE,uBAAuB,EAAE,MAAM,kBAAkB,CAAC;AAE3D,MAAM,MAAM,iBAAiB,CAAC,EAAE,IAAI,UAAU,CAC5C,OAAO,uBAAuB,CAAC,EAAE,CAAC,CACnC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,WAAW,YAAY,CAAC,EAAE,SAAS,YAAY,GAAG,YAAY;IAClE;;;;;;OAMG;IACH,UAAU,EAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,EAAE,SAAS,YAAY,EACxD,EAAE,EAAE,MAAM,CAAC,EAAE,CAAC,EACd,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,EAC3B,oBAAoB,EAAE,oBAAoB,EAC1C,MAAM,EAAE,uBAAuB,EAC/B,QAAQ,SAAO,GACd,YAAY,CAAC,EAAE,CAAC,CAUlB"}