@syncular/client 0.19.0 → 0.20.0

package/dist/worker-entry.js

@@ -16,7 +16,7 @@
 import { SyncClient } from './client.js';
 import { withClientDiagnosticsHost, } from './diagnostics.js';
 import { encryptionConfigFromKeyring } from './encryption.js';
-import { ClientSyncError } from './errors.js';
+import { ClientSyncError, STORAGE_BUSY_CODE } from './errors.js';
 import { httpBlobTransport, httpSegmentDownloader, httpSyncTransport, webSocketRealtimeConnector, } from './http.js';
 import { openPersistentWasmDatabase } from './wasm-database.js';
 import { WORKER_FAILED_CODE, } from './worker-protocol.js';
@@ -170,9 +170,9 @@ export function startSyncWorker(overrides = {}) {
         return Object.assign(gateOffline(inner), {
             ...(fetchUrl !== undefined
                 ? {
-                    fetchUrl: (url) => {
+                    fetchUrl: (url, onProgress) => {
                         assertOnline();
-                        return fetchUrl(url);
+                        return fetchUrl(url, onProgress);
                     },
                 }
                 : {}),
@@ -183,10 +183,33 @@ export function startSyncWorker(overrides = {}) {
             throw new ClientSyncError(WORKER_FAILED_CODE, 'the worker is already initialized');
         }
         autoSync = config.autoSync ?? true;
-        database =
-            overrides.openDatabase !== undefined
-                ? await overrides.openDatabase(config)
-                : await defaultOpenDatabase(config);
+        for (let attempt = 0;; attempt++) {
+            try {
+                database =
+                    overrides.openDatabase !== undefined
+                        ? await overrides.openDatabase(config)
+                        : await defaultOpenDatabase(config);
+                break;
+            }
+            catch (error) {
+                if (config.database.mode !== 'persistent' ||
+                    !(error instanceof ClientSyncError) ||
+                    error.code !== STORAGE_BUSY_CODE ||
+                    !error.retryable ||
+                    attempt === 6) {
+                    throw error;
+                }
+                // Retain the leader lease while the previous worker's OPFS handles
+                // drain. Only opening is retried; core initialization has not started.
+                const delayMs = Math.min(50 * 2 ** attempt, 1000);
+                if (overrides.waitForStorageRetry !== undefined) {
+                    await overrides.waitForStorageRetry(delayMs);
+                }
+                else {
+                    await new Promise((resolve) => setTimeout(resolve, delayMs));
+                }
+            }
+        }
         const transport = gateOffline(overrides.createTransport !== undefined
             ? overrides.createTransport(config)
             : httpSyncTransport(config.endpoints.syncUrl));
@@ -245,6 +268,10 @@ export function startSyncWorker(overrides = {}) {
                 post({ t: 'event', event: { kind: 'presence', scopeKey } });
             },
         });
+        started.onProgress((progress) => {
+            if (!closed)
+                post({ t: 'event', event: { kind: 'progress', progress } });
+        });
         started.onChange((batch) => {
             if (!closed)
                 post({ t: 'event', event: { kind: 'change', batch } });

package/dist/worker-host.d.ts

@@ -1,3 +1,4 @@
+import { type SyncProgress, type SyncProgressListener } from './progress.js';
 import type { PromiseMethods } from './client.js';
 /**
  * Main-thread side of the worker mode and the
@@ -179,6 +180,8 @@ export declare class SyncClientHandle implements PromiseMethods<WorkerApi> {
      * unsubscribe function.
      */
     onInvalidate(listener: InvalidationListener): () => void;
+    onProgress(listener: SyncProgressListener): () => void;
+    progressSnapshot(): SyncProgress | undefined;
     onChange(listener: ClientChangeListener): () => void;
     onDiagnostics(listener: ClientDiagnosticsListener): () => void;
     /**

package/dist/worker-host.js

@@ -1,3 +1,4 @@
+import { ProgressEmitter, } from './progress.js';
 import { registerDevtools } from './devtools.js';
 import { ClientDiagnosticsEmitter, withClientDiagnosticsHost, } from './diagnostics.js';
 import { ClientSyncError } from './errors.js';
@@ -68,6 +69,7 @@ export class SyncClientHandle {
     #core;
     #follower;
     #invalidation;
+    #progress = new ProgressEmitter();
     #changes;
     #presence;
     #diagnostics;
@@ -156,7 +158,10 @@ export class SyncClientHandle {
     }
     /** @internal — dispatch a worker/relayed event to handle-local listeners. */
     __dispatchEvent(event) {
-        if (event.kind === 'presence') {
+        if (event.kind === 'progress') {
+            this.#progress.emit(event.progress);
+        }
+        else if (event.kind === 'presence') {
             for (const listener of this.#presence) {
                 try {
                     listener(event.scopeKey);
@@ -189,6 +194,12 @@ export class SyncClientHandle {
     onInvalidate(listener) {
         return this.#invalidation.on(listener);
     }
+    onProgress(listener) {
+        return this.#progress.on(listener);
+    }
+    progressSnapshot() {
+        return this.#progress.snapshot();
+    }
     onChange(listener) {
         return this.#changes.on(listener);
     }
@@ -349,6 +360,7 @@ export class SyncClientHandle {
         if (this.#closed)
             return;
         this.#closed = true;
+        this.#progress.clear();
         this.#devtoolsUnregister();
         if (this.#follower !== undefined) {
             this.#follower.close();

package/dist/worker-protocol.d.ts

@@ -1,3 +1,4 @@
+import type { SyncProgress } from './progress.js';
 /**
  * The worker RPC protocol: the whole
  * client core runs in a Web Worker; the UI thread talks to it through
@@ -151,6 +152,9 @@ export interface WorkerErrorShape {
     readonly retryable: boolean;
 }
 export type SyncWorkerEvent = {
+    readonly kind: 'progress';
+    readonly progress: SyncProgress;
+} | {
     readonly kind: 'sync-needed';
     readonly reason: 'startup' | 'hello' | WakeReason;
 } | {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/client",
-  "version": "0.19.0",
+  "version": "0.20.0",
   "description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -89,9 +89,10 @@
   },
   "dependencies": {
     "@sqlite.org/sqlite-wasm": "^3.53.0-build1",
-    "@syncular/core": "0.19.0"
+    "@syncular/core": "0.20.0"
   },
   "devDependencies": {
-    "@syncular/server": "0.19.0"
+    "@syncular/server": "0.20.0",
+    "playwright": "1.63.0"
   }
 }

package/src/apply.ts

@@ -11,7 +11,7 @@ import {
   type ScopeMap,
   type SegmentRow,
 } from '@syncular/core';
-import type { ClientDatabase } from './database';
+import type { ClientDatabase, SqlValue } from './database';
 import type { EncryptionConfig } from './encryption';
 import { ClientSyncError } from './errors';
 import {
@@ -303,6 +303,7 @@ export function applySqliteSegment(
     readonly clearFirst: boolean;
     readonly effective: ScopeMap;
     readonly transaction?: ApplyTransaction;
+    readonly onProgress?: (rowsProcessed: number) => void;
   },
 ): number {
   const withImage = db.withSqliteImage?.bind(db);
@@ -381,23 +382,45 @@ export function applySqliteSegment(
       if (options.clearFirst) {
         deleteScopedRows(db, table, options.effective);
       }
-      db.exec(
-        `INSERT INTO ${quoteIdent(table.name)}
-           (${insertNames.join(', ')})
-         SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')}
-         FROM ${IMAGE_ALIAS}.${quoteIdent(table.name)}
-         WHERE true
-         ON CONFLICT (${quoteIdent(table.primaryKey)}) DO UPDATE SET ${updates}`,
-      );
-      const counted = db.query(
-        `SELECT count(*) AS n FROM ${IMAGE_ALIAS}.${quoteIdent(table.name)}`,
-      )[0];
-      const applied = Number(counted?.n ?? 0);
+      const source = `${IMAGE_ALIAS}.${quoteIdent(table.name)}`;
+      const primaryKey = quoteIdent(table.primaryKey);
+      let after: SqlValue | undefined;
+      let applied = 0;
+      for (;;) {
+        const boundary = db.query(
+          `SELECT ${primaryKey} AS boundary FROM ${source}
+           ${after === undefined ? '' : `WHERE ${primaryKey} > ?`}
+           ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`,
+          after === undefined ? [] : [after],
+        )[0]?.boundary;
+        const predicates: string[] = [];
+        const params: SqlValue[] = [];
+        if (after !== undefined) {
+          predicates.push(`${primaryKey} > ?`);
+          params.push(after);
+        }
+        if (boundary !== undefined) {
+          predicates.push(`${primaryKey} <= ?`);
+          params.push(boundary);
+        }
+        db.exec(
+          `INSERT INTO ${quoteIdent(table.name)} (${insertNames.join(', ')})
+           SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')}
+           FROM ${source} WHERE ${predicates.join(' AND ') || 'true'}
+           ON CONFLICT (${primaryKey}) DO UPDATE SET ${updates}`,
+          params,
+        );
+        applied += Number(db.query('SELECT changes() AS n')[0]?.n ?? 0);
+        if (boundary === undefined) break;
+        options.onProgress?.(applied);
+        after = boundary;
+      }
       if (applied !== descriptor.rowCount) {
         imageInvalid(
           `image holds ${applied} rows, descriptor says ${descriptor.rowCount}`,
         );
       }
+      options.onProgress?.(applied);
       return applied;
     });
   });
@@ -420,6 +443,7 @@ export async function applyRowsSegment(
     readonly clearFirst: boolean;
     readonly effective: ScopeMap;
     readonly transaction?: ApplyTransaction;
+    readonly onProgress?: (rowsProcessed: number) => void;
   },
   encryption?: EncryptionConfig,
 ): Promise<number> {
@@ -453,8 +477,10 @@ export async function applyRowsSegment(
       for (const row of rows) {
         upsertLocalRow(db, table, row.values, row.serverVersion);
         applied += 1;
+        if (applied % 1024 === 0) options.onProgress?.(applied);
       }
     });
   }
+  options.onProgress?.(applied);
   return applied;
 }

package/src/client.ts

@@ -1,3 +1,8 @@
+import {
+  ProgressEmitter,
+  type SyncProgress,
+  type SyncProgressListener,
+} from './progress';
 /**
  * SyncClient implements the client side of SPEC.md §§3–8.
  *
@@ -584,6 +589,8 @@ export class SyncClient {
   #realtimeGeneration = 0;
   #pendingRound: PendingRound | undefined;
   #needsPull = false;
+  readonly #progress = new ProgressEmitter();
+  #progressAttempt = 0;
   #syncing = false;
   /**
    * True from the synchronous entry of `sync()` until its serialized round
@@ -873,6 +880,7 @@ export class SyncClient {
     await this.#lease?.release();
     this.#lease = undefined;
     this.#started = false;
+    this.#progress.clear();
     this.#syncNeededListeners.clear();
     this.#syncIntentListeners.clear();
   }
@@ -1073,6 +1081,14 @@ export class SyncClient {
   }
 
   /** Subscribe to exact revisioned observer transactions (SPEC §7.5). */
+  onProgress(listener: SyncProgressListener): () => void {
+    return this.#progress.on(listener);
+  }
+
+  progressSnapshot(): SyncProgress | undefined {
+    return this.#progress.snapshot();
+  }
+
   onChange(listener: ClientChangeListener): () => void {
     return this.#changes.on(listener);
   }
@@ -2655,6 +2671,13 @@ export class SyncClient {
         (error: unknown) => {
           const completedAtMs = this.#now();
           const code = (error as { code?: unknown }).code;
+          this.#progress.update({
+            state: 'failed',
+            errorCode:
+              typeof code === 'string'
+                ? this.#diagnosticCode(code)
+                : 'client.unknown_failure',
+          });
           this.#lastRound = {
             status: 'failed',
             startedAtMs,
@@ -2709,7 +2732,15 @@ export class SyncClient {
   }
 
   async #runSync(): Promise<SyncSummary> {
+    this.#progress.emit({
+      attempt: ++this.#progressAttempt,
+      state: 'running',
+      phase: 'request',
+      bytesReceived: 0,
+      rowsProcessed: 0,
+    });
     if (this.#schemaFloor !== undefined) {
+      this.#progress.update({ state: 'complete' });
       return {
         ...emptySummary(0),
         bootstrapping: [],
@@ -2800,6 +2831,11 @@ export class SyncClient {
       // §4.8 E1: the push half may have drained commits that pinned rows of
       // a shrunk window unit — retry any deferred evictions now.
       this.#drainPendingEvictions();
+      this.#progress.update(
+        summary.failed.length > 0
+          ? { state: 'failed', errorCode: 'sync.scope_revoked' }
+          : { state: 'complete' },
+      );
       this.#retryDelayMs = 250;
       if (deferred > 0) {
         // §6.1 splitBatch remainder: more queued commits than this request
@@ -3368,6 +3404,19 @@ export class SyncClient {
               break;
             }
             const segment = decodeRowsSegment(frame.payload);
+            this.#progress.update({
+              phase: 'import',
+              subscriptionId: section.sub.id,
+              table: section.sub.table,
+              segmentId: undefined,
+              bytesReceived: frame.payload.byteLength,
+              bytesTotal: frame.payload.byteLength,
+              rowsProcessed: 0,
+              rowsTotal: segment.blocks.reduce(
+                (n, block) => n + block.length,
+                0,
+              ),
+            });
             await this.#applySegmentOrFail(
               section,
               summary,
@@ -3379,6 +3428,8 @@ export class SyncClient {
                   segment,
                   {
                     clearFirst,
+                    onProgress: (rowsProcessed) =>
+                      this.#progress.update({ rowsProcessed }),
                     effective,
                     transaction: (fn) =>
                       this.#applyBatch((batch) => {
@@ -3418,6 +3469,7 @@ export class SyncClient {
               );
             }
             const bytes = await this.#downloadSegment(frame, section.sub);
+            this.#progress.update({ phase: 'import' });
             if (frame.mediaType === 'sqlite') {
               // §5.3: images are whole-table — a paged descriptor is
               // invalid, and the image is always its table's first page.
@@ -3447,6 +3499,8 @@ export class SyncClient {
                     },
                     {
                       clearFirst,
+                      onProgress: (rowsProcessed) =>
+                        this.#progress.update({ rowsProcessed }),
                       effective,
                       transaction: (fn) =>
                         this.#applyBatch((batch) => {
@@ -3476,6 +3530,8 @@ export class SyncClient {
                     segment,
                     {
                       clearFirst,
+                      onProgress: (rowsProcessed) =>
+                        this.#progress.update({ rowsProcessed }),
                       effective,
                       transaction: (fn) =>
                         this.#applyBatch((batch) => {
@@ -3857,6 +3913,19 @@ export class SyncClient {
         'received SEGMENT_REF but no segment downloader is configured',
       );
     }
+    this.#progress.update({
+      phase: 'download',
+      subscriptionId: sub.id,
+      table: frame.table,
+      segmentId: frame.segmentId,
+      bytesReceived: 0,
+      bytesTotal: frame.byteLength,
+      rowsProcessed: 0,
+      rowsTotal: frame.rowCount,
+    });
+    const onProgress = (bytesReceived: number): void => {
+      this.#progress.update({ bytesReceived });
+    };
     let bytes: Uint8Array;
     if (frame.url !== undefined) {
       // §5.4: a url-carrying descriptor MUST be fetched from that URL —
@@ -3884,17 +3953,19 @@ export class SyncClient {
           true,
         );
       }
-      bytes = await fetchUrl(frame.url);
+      bytes = await fetchUrl(frame.url, onProgress);
     } else {
       bytes = await downloader({
         segmentId: frame.segmentId,
         table: frame.table,
         requestedScopesJson: canonicalScopeJson(sub.scopes),
+        onProgress,
       });
     }
     // §5.1: verify the content address before applying; on mismatch the
     // segment is discarded and the cursor/resume token stay unpersisted,
     // so the next pull re-delivers.
+    onProgress(bytes.byteLength);
     const hash = await sha256Hex(bytes);
     if (`sha256:${hash}` !== frame.segmentId) {
       throw new ClientSyncError(

package/src/http.ts

@@ -139,6 +139,40 @@ export function webSocketRemoteOperationConnector(
  * Resolution (which path a descriptor takes, expiry, no fall-through)
  * lives in the client core, not here.
  */
+async function readSegmentBody(
+  response: Response,
+  onProgress?: (bytesReceived: number) => void,
+): Promise<Uint8Array> {
+  const chunks: Uint8Array[] = [];
+  let received = 0;
+  let reported = 0;
+  const reader = response.body?.getReader();
+  if (reader !== undefined) {
+    try {
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        chunks.push(value);
+        received += value.byteLength;
+        if (received - reported >= 64 * 1024) {
+          onProgress?.(received);
+          reported = received;
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+  if (received !== reported) onProgress?.(received);
+  const bytes = new Uint8Array(received);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return bytes;
+}
+
 export function httpSegmentDownloader(
   segmentsBaseUrl: string,
   options?: HttpTransportOptions,
@@ -147,6 +181,7 @@ export function httpSegmentDownloader(
   const direct = async (request: {
     readonly segmentId: string;
     readonly requestedScopesJson: string;
+    readonly onProgress?: (bytesReceived: number) => void;
   }) => {
     const response = await doFetch(
       `${segmentsBaseUrl}/${encodeURIComponent(request.segmentId)}`,
@@ -158,9 +193,12 @@ export function httpSegmentDownloader(
       },
     );
     if (!response.ok) await throwHttpError(response);
-    return new Uint8Array(await response.arrayBuffer());
+    return readSegmentBody(response, request.onProgress);
   };
-  const fetchUrl = async (url: string) => {
+  const fetchUrl = async (
+    url: string,
+    onProgress?: (bytesReceived: number) => void,
+  ) => {
     // Deliberately headerless: the URL is the bearer grant (§5.4).
     const response = await doFetch(url);
     if (!response.ok) {
@@ -170,7 +208,7 @@ export function httpSegmentDownloader(
         true,
       );
     }
-    return new Uint8Array(await response.arrayBuffer());
+    return readSegmentBody(response, onProgress);
   };
   return Object.assign(direct, { fetchUrl });
 }

package/src/index.ts

@@ -40,3 +40,9 @@ export * from './transport';
 export * from './window';
 export * from './worker-host';
 export * from './worker-protocol';
+
+export {
+  ProgressEmitter,
+  type SyncProgress,
+  type SyncProgressListener,
+} from './progress';

package/src/progress.ts

@@ -0,0 +1,62 @@
+/** Ephemeral sync work, independent of committed change notifications (SPEC §7.6). */
+export interface SyncProgress {
+  readonly attempt: number;
+  readonly state: 'running' | 'complete' | 'failed';
+  readonly phase: 'request' | 'download' | 'import';
+  readonly subscriptionId?: string;
+  readonly table?: string;
+  readonly segmentId?: string | undefined;
+  readonly bytesReceived: number;
+  readonly bytesTotal?: number;
+  readonly rowsProcessed: number;
+  readonly rowsTotal?: number;
+  readonly errorCode?: string;
+}
+
+export type SyncProgressListener = (progress: SyncProgress) => void;
+
+/** Shared by the direct client and event-forwarding handles. */
+export class ProgressEmitter {
+  #snapshot: SyncProgress | undefined;
+  readonly #listeners = new Set<SyncProgressListener>();
+
+  clear(): void {
+    this.#listeners.clear();
+    this.#snapshot = undefined;
+  }
+
+  snapshot(): SyncProgress | undefined {
+    return this.#snapshot;
+  }
+
+  on(listener: SyncProgressListener): () => void {
+    this.#listeners.add(listener);
+    if (this.#snapshot !== undefined) {
+      try {
+        listener(this.#snapshot);
+      } catch {
+        /* Observers cannot interrupt sync. */
+      }
+    }
+    return () => {
+      this.#listeners.delete(listener);
+    };
+  }
+
+  emit(progress: SyncProgress): void {
+    const snapshot = Object.freeze({ ...progress });
+    this.#snapshot = snapshot;
+    for (const listener of [...this.#listeners]) {
+      try {
+        listener(snapshot);
+      } catch {
+        /* Observers cannot interrupt sync. */
+      }
+    }
+  }
+
+  update(update: Partial<SyncProgress>): void {
+    if (this.#snapshot?.state === 'running')
+      this.emit({ ...this.#snapshot, ...update });
+  }
+}

package/src/transport.ts

@@ -32,6 +32,7 @@ export interface SegmentFetchRequest {
   readonly table: string;
   /** Canonical JSON (§11.2) of the requested scope map (§5.5 header). */
   readonly requestedScopesJson: string;
+  readonly onProgress?: (bytesReceived: number) => void;
 }
 
 /**
@@ -44,7 +45,10 @@ export interface SegmentFetchRequest {
  */
 export interface SegmentDownloader {
   (request: SegmentFetchRequest): Promise<Uint8Array>;
-  readonly fetchUrl?: (url: string) => Promise<Uint8Array>;
+  readonly fetchUrl?: (
+    url: string,
+    onProgress?: (bytesReceived: number) => void,
+  ) => Promise<Uint8Array>;
 }
 
 export interface RealtimeHandlers {