@syncular/client 0.0.6-232 → 0.0.6-239

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/client",
-  "version": "0.0.6-232",
+  "version": "0.0.6-239",
   "description": "Client-side sync engine with offline-first support, outbox, and conflict resolution",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -119,8 +119,8 @@
     "release": "bunx syncular-publish"
   },
   "dependencies": {
-    "@syncular/core": "0.0.6-232",
-    "@syncular/transport-http": "0.0.6-232"
+    "@syncular/core": "0.0.6-239",
+    "@syncular/transport-http": "0.0.6-239"
   },
   "peerDependencies": {
     "kysely": "*"

package/src/engine/SyncEngine.test.ts

@@ -924,6 +924,150 @@ describe('SyncEngine WS inline apply', () => {
     expect(state.error?.retryable).toBe(false);
   });
 
+  it('pulls newly eligible bootstrap phases in the same sync cycle', async () => {
+    const handlers: ClientHandlerCollection<TestDb> = [
+      {
+        table: 'tasks',
+        async applySnapshot() {},
+        async clearAll() {},
+        async applyChange() {},
+      },
+    ];
+
+    const syncRequests: string[][] = [];
+    const transport: SyncTransport = {
+      async sync(request) {
+        const ids = request.pull?.subscriptions.map((subscription) => {
+          return subscription.id;
+        }) ?? ['unexpected'];
+        syncRequests.push(ids);
+
+        if (syncRequests.length === 1) {
+          expect(ids).toEqual(['catalog-meta', 'catalog-codes']);
+          return {
+            pull: {
+              ok: true,
+              subscriptions: [
+                {
+                  id: 'catalog-meta',
+                  status: 'active',
+                  scopes: {},
+                  bootstrap: false,
+                  nextCursor: 1,
+                  commits: [],
+                  snapshots: [],
+                },
+                {
+                  id: 'catalog-codes',
+                  status: 'active',
+                  scopes: {},
+                  bootstrap: false,
+                  nextCursor: 1,
+                  commits: [],
+                  snapshots: [],
+                },
+              ],
+            },
+          };
+        }
+
+        expect(ids).toEqual([
+          'catalog-meta',
+          'catalog-codes',
+          'catalog-relations',
+        ]);
+        return {
+          pull: {
+            ok: true,
+            subscriptions: [
+              {
+                id: 'catalog-meta',
+                status: 'active',
+                scopes: {},
+                bootstrap: false,
+                nextCursor: 1,
+                commits: [],
+                snapshots: [],
+              },
+              {
+                id: 'catalog-codes',
+                status: 'active',
+                scopes: {},
+                bootstrap: false,
+                nextCursor: 1,
+                commits: [],
+                snapshots: [],
+              },
+              {
+                id: 'catalog-relations',
+                status: 'active',
+                scopes: {},
+                bootstrap: false,
+                nextCursor: 1,
+                commits: [],
+                snapshots: [],
+              },
+            ],
+          },
+        };
+      },
+      async fetchSnapshotChunk() {
+        return new Uint8Array();
+      },
+    };
+
+    const engine = new SyncEngine<TestDb>({
+      db,
+      transport,
+      handlers,
+      actorId: 'catalog-public',
+      clientId: 'client-bootstrap-phases',
+      subscriptions: [
+        {
+          id: 'catalog-meta',
+          table: 'tasks',
+          scopes: {},
+          bootstrapPhase: 0,
+        },
+        {
+          id: 'catalog-codes',
+          table: 'tasks',
+          scopes: {},
+          bootstrapPhase: 0,
+        },
+        {
+          id: 'catalog-relations',
+          table: 'tasks',
+          scopes: {},
+          bootstrapPhase: 1,
+        },
+      ],
+      stateId: 'default',
+      pollIntervalMs: 60_000,
+    });
+
+    await engine.start();
+    engine.stop();
+
+    expect(syncRequests).toEqual([
+      ['catalog-meta', 'catalog-codes'],
+      ['catalog-meta', 'catalog-codes', 'catalog-relations'],
+    ]);
+
+    const relationState = await db
+      .selectFrom('sync_subscription_state')
+      .select(['subscription_id', 'status', 'cursor'])
+      .where('state_id', '=', 'default')
+      .where('subscription_id', '=', 'catalog-relations')
+      .executeTakeFirst();
+
+    expect(relationState).toEqual({
+      subscription_id: 'catalog-relations',
+      status: 'active',
+      cursor: 1,
+    });
+  });
+
   it('skips outbox and conflict refresh after a read-only successful sync', async () => {
     const handlers: ClientHandlerCollection<TestDb> = [
       {

package/src/engine/SyncEngine.ts

@@ -2172,6 +2172,7 @@ export class SyncEngine<DB extends SyncClientDb = SyncClientDb> {
               limitCommits: this.config.limitCommits,
               limitSnapshotRows: this.config.limitSnapshotRows,
               maxSnapshotPages: this.config.maxSnapshotPages,
+              snapshotApplyYieldMs: this.config.snapshotApplyYieldMs,
               dedupeRows: this.config.dedupeRows,
               stateId: this.config.stateId,
               sha256: this.config.sha256,

package/src/engine/types.ts

@@ -371,6 +371,13 @@ export interface SyncEngineConfig<DB extends SyncClientDb = SyncClientDb> {
   limitSnapshotRows?: number;
   /** Bootstrap snapshot pages per pull */
   maxSnapshotPages?: number;
+  /**
+   * Yield delay between large bootstrap apply batches.
+   * - `0`: yield on the next macrotask
+   * - `false`: disable yielding
+   * - omitted: use transport/runtime defaults
+   */
+  snapshotApplyYieldMs?: number | false;
   /** Deduplicate rows in pull responses on the server */
   dedupeRows?: boolean;
   /** Optional state row id (multi-profile support) */

package/src/handlers/create-handler.ts

@@ -278,6 +278,7 @@ export function createClientHandler<
         )}
         on conflict (${sql.ref(primaryKey)}) ${onConflict}
       `.execute(ctx.trx);
+      await ctx.yieldToMainThread?.();
     }
   };
 
@@ -359,6 +360,7 @@ export function createClientHandler<
           delete from ${sql.table(table)}
           where ${sql.ref(primaryKey)} in ${sql`(${sql.join(batchIds.map((rowId) => sql.val(rowId)))})`}
         `.execute(ctx.trx);
+        await ctx.yieldToMainThread?.();
       }
     };
 
@@ -404,6 +406,7 @@ export function createClientHandler<
           )}
           on conflict (${sql.ref(primaryKey)}) ${onConflict}
         `.execute(ctx.trx);
+        await ctx.yieldToMainThread?.();
       }
     };
 

package/src/handlers/types.ts

@@ -16,6 +16,12 @@ import type { Transaction } from 'kysely';
 export interface ClientHandlerContext<DB> {
   /** Database transaction */
   trx: Transaction<DB>;
+  /**
+   * Yields control back to the runtime scheduler between large apply batches.
+   * Useful on React Native / Hermes where long bootstrap writes can otherwise
+   * monopolize the JS thread.
+   */
+  yieldToMainThread?: () => Promise<void>;
   /**
    * Commit metadata for server-delivered changes.
    * Undefined for local optimistic changes.

package/src/pull-engine.ts

@@ -639,6 +639,7 @@ async function applyChunkedSnapshot<DB extends SyncClientDb>(
   snapshot: SyncSnapshot,
   scopeValues: ScopeValues,
   sha256Override?: (bytes: Uint8Array) => Promise<string>,
+  yieldToMainThread?: () => Promise<void>,
   trace?: {
     stateId: string;
     subscriptionId: string;
@@ -647,7 +648,7 @@ async function applyChunkedSnapshot<DB extends SyncClientDb>(
 ): Promise<void> {
   const chunks = snapshot.chunks ?? [];
   if (chunks.length === 0) {
-    await handler.applySnapshot({ trx }, snapshot);
+    await handler.applySnapshot({ trx, yieldToMainThread }, snapshot);
     return;
   }
 
@@ -713,7 +714,7 @@ async function applyChunkedSnapshot<DB extends SyncClientDb>(
             // eslint-disable-next-line no-await-in-loop
             try {
               await handler.applySnapshot(
-                { trx },
+                { trx, yieldToMainThread },
                 {
                   ...snapshot,
                   rows: pendingBatch,
@@ -745,7 +746,7 @@ async function applyChunkedSnapshot<DB extends SyncClientDb>(
           // eslint-disable-next-line no-await-in-loop
           try {
             await handler.applySnapshot(
-              { trx },
+              { trx, yieldToMainThread },
               {
                 ...snapshot,
                 rows: pendingBatch,
@@ -1046,6 +1047,13 @@ export interface SyncPullOnceOptions {
   limitCommits?: number;
   limitSnapshotRows?: number;
   maxSnapshotPages?: number;
+  /**
+   * Yield delay between heavy bootstrap apply batches.
+   * - `0`: yield on the next macrotask
+   * - `false`: disable yielding
+   * - omitted: use transport/runtime defaults
+   */
+  snapshotApplyYieldMs?: number | false;
   dedupeRows?: boolean;
   stateId?: string;
   /**
@@ -1087,6 +1095,26 @@ function emitTrace(
   });
 }
 
+function resolveSnapshotApplyYieldToMainThread(
+  options: SyncPullOnceOptions,
+  capabilities?: SyncTransportCapabilities
+): (() => Promise<void>) | undefined {
+  const configuredDelay =
+    options.snapshotApplyYieldMs ??
+    capabilities?.preferredSnapshotApplyYieldMs ??
+    false;
+
+  if (configuredDelay === false) {
+    return undefined;
+  }
+
+  const delay = Math.max(0, Math.trunc(configuredDelay));
+  return () =>
+    new Promise<void>((resolve) => {
+      setTimeout(resolve, delay);
+    });
+}
+
 function countSubscriptionRows(
   subscription: SyncPullSubscriptionResponse
 ): number | undefined {
@@ -1247,10 +1275,12 @@ export async function applyIncrementalCommitChanges<DB extends SyncClientDb>(
     commitSeq?: number | null;
     actorId?: string | null;
     createdAt?: string | null;
+    yieldToMainThread?: () => Promise<void>;
   }
 ): Promise<void> {
   const ctx = {
     trx,
+    yieldToMainThread: args.yieldToMainThread,
     commitSeq: args.commitSeq ?? null,
     actorId: args.actorId ?? null,
     createdAt: args.createdAt ?? null,
@@ -1311,6 +1341,10 @@ export async function applyPullResponse<DB extends SyncClientDb>(
     rawResponse,
     transport.capabilities
   );
+  const yieldToMainThread = resolveSnapshotApplyYieldToMainThread(
+    options,
+    transport.capabilities
+  );
 
   let responseToApply = requiresMaterializedSnapshots
     ? await materializeChunkedSnapshots(
@@ -1365,6 +1399,7 @@ export async function applyPullResponse<DB extends SyncClientDb>(
             existingById,
             subsById,
             sub,
+            yieldToMainThread,
           });
         });
         emitTrace(options.onTrace, {
@@ -1409,6 +1444,7 @@ export async function applyPullResponse<DB extends SyncClientDb>(
             existingById,
             subsById,
             sub,
+            yieldToMainThread,
           });
         }
       });
@@ -1528,6 +1564,7 @@ async function applySubscriptionResponse<DB extends SyncClientDb>(args: {
   existingById: Map<string, SyncSubscriptionStateTable>;
   subsById: Map<string, SyncClientSubscription | undefined>;
   sub: SyncPullSubscriptionResponse;
+  yieldToMainThread?: () => Promise<void>;
 }): Promise<void> {
   const {
     trx,
@@ -1538,6 +1575,7 @@ async function applySubscriptionResponse<DB extends SyncClientDb>(args: {
     existingById,
     subsById,
     sub,
+    yieldToMainThread,
   } = args;
   const def = subsById.get(sub.id);
   const prev = existingById.get(sub.id);
@@ -1603,6 +1641,7 @@ async function applySubscriptionResponse<DB extends SyncClientDb>(args: {
           const scopes = parseScopeValuesJson(prev.scopes_json);
           await getClientHandlerOrThrow(handlers, prev.table).clearAll({
             trx,
+            yieldToMainThread,
             scopes,
           });
         } catch {
@@ -1644,6 +1683,7 @@ async function applySubscriptionResponse<DB extends SyncClientDb>(args: {
         if (clearScopes !== 'none') {
           await getClientHandlerOrThrow(handlers, prev.table).clearAll({
             trx,
+            yieldToMainThread,
             scopes: clearScopes ?? previousScopes,
           });
         }
@@ -1662,6 +1702,7 @@ async function applySubscriptionResponse<DB extends SyncClientDb>(args: {
           try {
             await handler.onSnapshotStart({
               trx,
+              yieldToMainThread,
               table: snapshot.table,
               scopes: sub.scopes,
             });
@@ -1687,6 +1728,7 @@ async function applySubscriptionResponse<DB extends SyncClientDb>(args: {
             snapshot,
             sub.scopes,
             options.sha256,
+            yieldToMainThread,
             {
               stateId,
               subscriptionId: sub.id,
@@ -1695,7 +1737,7 @@ async function applySubscriptionResponse<DB extends SyncClientDb>(args: {
           );
         } else {
           try {
-            await handler.applySnapshot({ trx }, snapshot);
+            await handler.applySnapshot({ trx, yieldToMainThread }, snapshot);
           } catch (error) {
             throw wrapSyncClientStageError(
               error,
@@ -1714,6 +1756,7 @@ async function applySubscriptionResponse<DB extends SyncClientDb>(args: {
           try {
             await handler.onSnapshotEnd({
               trx,
+              yieldToMainThread,
               table: snapshot.table,
               scopes: sub.scopes,
             });
@@ -1739,6 +1782,7 @@ async function applySubscriptionResponse<DB extends SyncClientDb>(args: {
             commitSeq: commit.commitSeq ?? null,
             actorId: commit.actorId ?? null,
             createdAt: commit.createdAt ?? null,
+            yieldToMainThread,
           });
         } catch (error) {
           throw wrapSyncClientStageError(

package/src/sync-loop.ts

@@ -459,6 +459,21 @@ function needsAnotherPull(
   return totalCommits >= limitCommits;
 }
 
+function hasNewlyEligibleBootstrapSubscriptions(
+  currentPullState: SyncPullRequestState,
+  nextPullState: SyncPullRequestState
+): boolean {
+  const currentIds = new Set(
+    (currentPullState.request.subscriptions ?? []).map((subscription) => {
+      return subscription.id;
+    })
+  );
+
+  return (nextPullState.request.subscriptions ?? []).some((subscription) => {
+    return !currentIds.has(subscription.id);
+  });
+}
+
 function mergePullResponse(
   targetBySubId: Map<string, SyncPullSubscriptionResponse>,
   res: SyncPullResponse
@@ -545,8 +560,14 @@ async function syncPullUntilSettled<DB extends SyncClientDb>(
     const res = await syncPullOnce(db, transport, handlers, options, pullState);
     mergePullResponse(aggregatedBySubId, res);
 
-    if (!needsAnotherPull(res, pullState.request.limitCommits)) break;
-    pullState = createFollowupPullState(pullState, res);
+    const nextPullState = createFollowupPullState(pullState, res);
+    if (
+      !needsAnotherPull(res, pullState.request.limitCommits) &&
+      !hasNewlyEligibleBootstrapSubscriptions(pullState, nextPullState)
+    ) {
+      break;
+    }
+    pullState = nextPullState;
   }
 
   return {
@@ -568,6 +589,7 @@ export interface SyncOnceOptions {
   limitCommits?: number;
   limitSnapshotRows?: number;
   maxSnapshotPages?: number;
+  snapshotApplyYieldMs?: number | false;
   dedupeRows?: boolean;
   stateId?: string;
   maxPushCommits?: number;
@@ -612,6 +634,7 @@ async function syncOnceCombined<DB extends SyncClientDb>(
     limitCommits: options.limitCommits,
     limitSnapshotRows: options.limitSnapshotRows,
     maxSnapshotPages: options.maxSnapshotPages,
+    snapshotApplyYieldMs: options.snapshotApplyYieldMs,
     dedupeRows: options.dedupeRows,
     stateId: options.stateId,
     sha256: options.sha256,
@@ -868,14 +891,18 @@ async function syncOnceCombined<DB extends SyncClientDb>(
     pullRounds = 1;
 
     // Continue pulling if more data
-    if (needsAnotherPull(pullResponse, pullState.request.limitCommits)) {
+    const nextPullState = createFollowupPullState(pullState, pullResponse);
+    if (
+      needsAnotherPull(pullResponse, pullState.request.limitCommits) ||
+      hasNewlyEligibleBootstrapSubscriptions(pullState, nextPullState)
+    ) {
       const aggregatedBySubId = new Map<string, SyncPullSubscriptionResponse>();
       mergePullResponse(aggregatedBySubId, pullResponse);
 
       const more = await syncPullUntilSettled(db, transport, handlers, {
         ...pullOpts,
         maxRounds: (options.maxPullRounds ?? 20) - 1,
-        initialPullState: createFollowupPullState(pullState, pullResponse),
+        initialPullState: nextPullState,
       });
       pullRounds += more.rounds;
       mergePullResponse(aggregatedBySubId, more.response);