@syncular/client 0.19.0 → 0.20.1

package/src/client.ts

@@ -1,3 +1,8 @@
+import {
+  ProgressEmitter,
+  type SyncProgress,
+  type SyncProgressListener,
+} from './progress';
 /**
  * SyncClient implements the client side of SPEC.md §§3–8.
  *
@@ -39,6 +44,7 @@ import {
   deleteLocalRow,
   deleteScopedRows,
   evictScopedRows,
+  yieldToHost,
   upsertLocalRow,
 } from './apply';
 import {
@@ -281,8 +287,8 @@ export interface SyncClientLimits {
   readonly maxSnapshotPages?: number;
   /**
    * §4.2 accept bitmask; defaults to inline + external rows (0b0011)
-   * plus sqlite images (bit 2) when the database backend implements
-   * `withSqliteImage` and a segment downloader is configured (§5.3).
+   * plus SQLite images and signed URLs when supported. Both image chunks
+   * and rows blocks commit before yielding to local reads (§7.7).
    */
   readonly accept?: number;
   /**
@@ -584,6 +590,8 @@ export class SyncClient {
   #realtimeGeneration = 0;
   #pendingRound: PendingRound | undefined;
   #needsPull = false;
+  readonly #progress = new ProgressEmitter();
+  #progressAttempt = 0;
   #syncing = false;
   /**
    * True from the synchronous entry of `sync()` until its serialized round
@@ -741,6 +749,7 @@ export class SyncClient {
     const startupWork =
       this.#schemaFloor === undefined &&
       (countOutbox(this.#db) > 0 ||
+        loadPendingEvictions(this.#db).length > 0 ||
         subscriptions.some((sub) => sub.status === 'active'));
     if (startupWork && this.#securityLifecycle === 'active') {
       this.#needsPull = true;
@@ -873,6 +882,7 @@ export class SyncClient {
     await this.#lease?.release();
     this.#lease = undefined;
     this.#started = false;
+    this.#progress.clear();
     this.#syncNeededListeners.clear();
     this.#syncIntentListeners.clear();
   }
@@ -1073,6 +1083,14 @@ export class SyncClient {
   }
 
   /** Subscribe to exact revisioned observer transactions (SPEC §7.5). */
+  onProgress(listener: SyncProgressListener): () => void {
+    return this.#progress.on(listener);
+  }
+
+  progressSnapshot(): SyncProgress | undefined {
+    return this.#progress.snapshot();
+  }
+
   onChange(listener: ClientChangeListener): () => void {
     return this.#changes.on(listener);
   }
@@ -1804,23 +1822,17 @@ export class SyncClient {
     socket.send(encodePresencePublish(scopeKey, doc));
   }
 
-  /**
-   * §4.2 accept mask: the configured override, or the rows baseline plus
-   * bit 2 when the backend can import sqlite images (§5.3) and a segment
-   * downloader exists (sqlite segments are never inline, §5.7), plus
-   * bit 3 when the downloader exposes a direct URL fetch (§5.4
-   * capability negotiation).
-   */
+  /** Advertise supported segment formats; all imports yield between commits. */
   #acceptMask(): number {
     const configured = this.#config.limits?.accept;
     if (configured !== undefined) return configured;
     const segments = this.#config.segments;
-    const sqliteCapable =
-      typeof this.#db.withSqliteImage === 'function' && segments !== undefined;
     const urlCapable = typeof segments?.fetchUrl === 'function';
     return (
       ACCEPT_ROWS_BASELINE |
-      (sqliteCapable ? ACCEPT_SQLITE : 0) |
+      (typeof this.#db.withSqliteImage === 'function' && segments !== undefined
+        ? ACCEPT_SQLITE
+        : 0) |
       (urlCapable ? ACCEPT_SIGNED_URLS : 0)
     );
   }
@@ -1893,8 +1905,8 @@ export class SyncClient {
    * Set the live window units for a base (§4.8): a value-sharded family of
    * subscriptions, one per unit. Computes the diff against the registry —
    * added units get fresh subscriptions (image-lane bootstrap on the next
-   * sync); removed units are unsubscribed and evicted, fused in one local
-   * transaction (E1–E4). Idempotent: calling with the same units is a
+   * sync); removed units are unsubscribed with their first eviction chunk
+   * in one local transaction (E1–E4). Idempotent: calling with the same units is a
    * no-op. Re-entry (a unit removed then re-added) cancels any deferred
    * eviction and fresh-bootstraps.
    *
@@ -1959,7 +1971,7 @@ export class SyncClient {
       // Shrink: units live but not wanted → unsubscribe fused with eviction.
       for (const { unit, subId } of live) {
         if (wanted.has(unit)) continue;
-        this.#evictUnit(baseKey, base, unit, subId);
+        await this.#evictUnit(baseKey, base, unit, subId);
         changed = true;
       }
     });
@@ -2002,8 +2014,8 @@ export class SyncClient {
   }
 
   /**
-   * §4.8 E1–E4: evict one departing unit, fused with its unsubscription in
-   * one transaction. Deletes the unit's rows EXCEPT those pinned by a
+   * §4.8 E1–E4: evict one departing unit in committed chunks, fusing
+   * its first chunk with unsubscription. Deletes the unit's rows EXCEPT those pinned by a
    * pending outbox commit (E1); if any pin remains, records a deferred
    * eviction retried on the next outbox drain. Discards the subscription's
    * cursor/resume/effective-echo (E3) and its version state with the rows
@@ -2011,24 +2023,26 @@ export class SyncClient {
    * with no local scope-column mapping, surfaces a configuration error and
    * evicts nothing (§4.8/§3.3).
    */
-  #evictUnit(
+  async #evictUnit(
     baseKey: string,
     base: WindowBase,
     unit: string,
     subId: string,
-  ): void {
+  ): Promise<void> {
     const table = this.#table(base.table);
     const sub = getSubscription(this.#db, subId);
     // The rows a unit holds live under its LAST effective scopes if it ever
     // synced; before first sync, the requested unit scopes are the match.
     const effective = sub?.effectiveScopes ?? unitScopes(base, unit);
     const pinned = this.#pinnedRowIds(base.table);
+    let remaining = false;
     this.#applyBatch((batch) => {
       this.#db.transaction(() => {
-        const deferred = evictScopedRows(this.#db, table, effective, pinned);
+        const result = evictScopedRows(this.#db, table, effective, pinned);
+        remaining = result.remaining;
         deleteWindowUnit(this.#db, baseKey, unit);
         deleteSubscription(this.#db, subId);
-        if (deferred) {
+        if (result.deferred) {
           savePendingEviction(this.#db, subId, base.table, effective);
         } else {
           deletePendingEviction(this.#db, subId);
@@ -2039,6 +2053,10 @@ export class SyncClient {
       this.#needsPull = true;
       batch.status();
     });
+    if (remaining) {
+      await yieldToHost();
+      await this.#drainPendingEvictions();
+    }
   }
 
   /**
@@ -2046,7 +2064,7 @@ export class SyncClient {
    * unit's rows are removed once no pending commit references them; a unit
    * that re-entered the window in the meantime has no pending record left.
    */
-  #drainPendingEvictions(): void {
+  async #drainPendingEvictions(): Promise<void> {
     const pending = loadPendingEvictions(this.#db);
     if (pending.length === 0) return;
     for (const entry of pending) {
@@ -2055,15 +2073,30 @@ export class SyncClient {
         deletePendingEviction(this.#db, entry.subId);
         continue;
       }
-      const pinned = this.#pinnedRowIds(entry.table);
-      this.#applyBatch((batch) => {
-        let deferred = false;
-        this.#db.transaction(() => {
-          deferred = evictScopedRows(this.#db, table, entry.effective, pinned);
-          if (!deferred) deletePendingEviction(this.#db, entry.subId);
+      let remaining: boolean;
+      do {
+        if (
+          !loadPendingEvictions(this.#db).some(
+            (item) => item.subId === entry.subId,
+          )
+        )
+          break;
+        // Local writes may add pins while the worker yields between chunks.
+        const pinned = this.#pinnedRowIds(entry.table);
+        remaining = false;
+        this.#applyBatch((batch) => {
+          const result = evictScopedRows(
+            this.#db,
+            table,
+            entry.effective,
+            pinned,
+          );
+          remaining = result.remaining;
+          if (!result.deferred) deletePendingEviction(this.#db, entry.subId);
+          batch.scopeMap(table, entry.effective);
         });
-        batch.scopeMap(table, entry.effective);
-      });
+        if (remaining) await yieldToHost();
+      } while (remaining);
     }
   }
 
@@ -2655,6 +2688,13 @@ export class SyncClient {
         (error: unknown) => {
           const completedAtMs = this.#now();
           const code = (error as { code?: unknown }).code;
+          this.#progress.update({
+            state: 'failed',
+            errorCode:
+              typeof code === 'string'
+                ? this.#diagnosticCode(code)
+                : 'client.unknown_failure',
+          });
           this.#lastRound = {
             status: 'failed',
             startedAtMs,
@@ -2709,7 +2749,15 @@ export class SyncClient {
   }
 
   async #runSync(): Promise<SyncSummary> {
+    this.#progress.emit({
+      attempt: ++this.#progressAttempt,
+      state: 'running',
+      phase: 'request',
+      bytesReceived: 0,
+      rowsProcessed: 0,
+    });
     if (this.#schemaFloor !== undefined) {
+      this.#progress.update({ state: 'complete' });
       return {
         ...emptySummary(0),
         bootstrapping: [],
@@ -2722,6 +2770,7 @@ export class SyncClient {
     // survive it — the reference server keeps no replay buffer (§8.2).
     this.#setSyncNeeded(false);
     try {
+      await this.#drainPendingEvictions();
       const logEpoch = getMeta(this.#db, LOG_EPOCH_META_KEY);
       // §5.9.7 B4: upload pending blobs BEFORE pushing rows that reference
       // them, so the server-side existence check (§6.6) passes.
@@ -2799,7 +2848,12 @@ export class SyncClient {
       );
       // §4.8 E1: the push half may have drained commits that pinned rows of
       // a shrunk window unit — retry any deferred evictions now.
-      this.#drainPendingEvictions();
+      await this.#drainPendingEvictions();
+      this.#progress.update(
+        summary.failed.length > 0
+          ? { state: 'failed', errorCode: 'sync.scope_revoked' }
+          : { state: 'complete' },
+      );
       this.#retryDelayMs = 250;
       if (deferred > 0) {
         // §6.1 splitBatch remainder: more queued commits than this request
@@ -2843,10 +2897,27 @@ export class SyncClient {
    * Pull repeatedly until quiescent: no commits delivered, no bootstrap
    * pages pending, no resets to recover (§4.5 "pull again" SHOULD).
    */
-  async syncUntilIdle(maxRounds = 20): Promise<SyncSummary> {
+  async syncUntilIdle(maxRounds?: number): Promise<SyncSummary> {
+    const budget = maxRounds ?? 20;
     let last: SyncSummary | undefined;
-    for (let round = 0; round < maxRounds; round++) {
+    for (let round = 0; round < budget; round++) {
+      const before = JSON.stringify(
+        loadSubscriptions(this.#db).map((sub) => [sub.id, sub.bootstrapState]),
+      );
       last = await this.sync();
+      if (
+        maxRounds === undefined &&
+        last.segmentRowsApplied > 0 &&
+        last.bootstrapping.length > 0 &&
+        before !==
+          JSON.stringify(
+            loadSubscriptions(this.#db).map((sub) => [
+              sub.id,
+              sub.bootstrapState,
+            ]),
+          )
+      )
+        round = -1;
       if (last.schemaFloor !== undefined) return last;
       if (
         last.commitsApplied === 0 &&
@@ -2861,7 +2932,7 @@ export class SyncClient {
     }
     throw new ClientSyncError(
       'sync.invalid_request',
-      `sync did not reach idle within ${maxRounds} rounds`,
+      'sync did not reach idle within the round budget',
     );
   }
 
@@ -3368,6 +3439,19 @@ export class SyncClient {
               break;
             }
             const segment = decodeRowsSegment(frame.payload);
+            this.#progress.update({
+              phase: 'import',
+              subscriptionId: section.sub.id,
+              table: section.sub.table,
+              segmentId: undefined,
+              bytesReceived: frame.payload.byteLength,
+              bytesTotal: frame.payload.byteLength,
+              rowsProcessed: 0,
+              rowsTotal: segment.blocks.reduce(
+                (n, block) => n + block.length,
+                0,
+              ),
+            });
             await this.#applySegmentOrFail(
               section,
               summary,
@@ -3379,6 +3463,9 @@ export class SyncClient {
                   segment,
                   {
                     clearFirst,
+                    isCurrent: () => this.#localResetEpoch === resetEpoch,
+                    onProgress: (rowsProcessed) =>
+                      this.#progress.update({ rowsProcessed }),
                     effective,
                     transaction: (fn) =>
                       this.#applyBatch((batch) => {
@@ -3389,7 +3476,9 @@ export class SyncClient {
                         ) {
                           batch.table(table.name);
                         }
-                        return fn();
+                        const result = fn();
+                        this.#replayOutbox();
+                        return result;
                       }),
                   },
                   this.#encryption,
@@ -3418,6 +3507,7 @@ export class SyncClient {
               );
             }
             const bytes = await this.#downloadSegment(frame, section.sub);
+            this.#progress.update({ phase: 'import' });
             if (frame.mediaType === 'sqlite') {
               // §5.3: images are whole-table — a paged descriptor is
               // invalid, and the image is always its table's first page.
@@ -3447,6 +3537,9 @@ export class SyncClient {
                     },
                     {
                       clearFirst,
+                      isCurrent: () => this.#localResetEpoch === resetEpoch,
+                      onProgress: (rowsProcessed) =>
+                        this.#progress.update({ rowsProcessed }),
                       effective,
                       transaction: (fn) =>
                         this.#applyBatch((batch) => {
@@ -3457,7 +3550,9 @@ export class SyncClient {
                           ) {
                             batch.table(table.name);
                           }
-                          return fn();
+                          const result = fn();
+                          this.#replayOutbox();
+                          return result;
                         }),
                     },
                   ),
@@ -3476,6 +3571,9 @@ export class SyncClient {
                     segment,
                     {
                       clearFirst,
+                      isCurrent: () => this.#localResetEpoch === resetEpoch,
+                      onProgress: (rowsProcessed) =>
+                        this.#progress.update({ rowsProcessed }),
                       effective,
                       transaction: (fn) =>
                         this.#applyBatch((batch) => {
@@ -3486,7 +3584,9 @@ export class SyncClient {
                           ) {
                             batch.table(table.name);
                           }
-                          return fn();
+                          const result = fn();
+                          this.#replayOutbox();
+                          return result;
                         }),
                     },
                     this.#encryption,
@@ -3857,6 +3957,19 @@ export class SyncClient {
         'received SEGMENT_REF but no segment downloader is configured',
       );
     }
+    this.#progress.update({
+      phase: 'download',
+      subscriptionId: sub.id,
+      table: frame.table,
+      segmentId: frame.segmentId,
+      bytesReceived: 0,
+      bytesTotal: frame.byteLength,
+      rowsProcessed: 0,
+      rowsTotal: frame.rowCount,
+    });
+    const onProgress = (bytesReceived: number): void => {
+      this.#progress.update({ bytesReceived });
+    };
     let bytes: Uint8Array;
     if (frame.url !== undefined) {
       // §5.4: a url-carrying descriptor MUST be fetched from that URL —
@@ -3884,17 +3997,19 @@ export class SyncClient {
           true,
         );
       }
-      bytes = await fetchUrl(frame.url);
+      bytes = await fetchUrl(frame.url, onProgress);
     } else {
       bytes = await downloader({
         segmentId: frame.segmentId,
         table: frame.table,
         requestedScopesJson: canonicalScopeJson(sub.scopes),
+        onProgress,
       });
     }
     // §5.1: verify the content address before applying; on mismatch the
     // segment is discarded and the cursor/resume token stay unpersisted,
     // so the next pull re-delivers.
+    onProgress(bytes.byteLength);
     const hash = await sha256Hex(bytes);
     if (`sha256:${hash}` !== frame.segmentId) {
       throw new ClientSyncError(

package/src/database.ts

@@ -22,11 +22,16 @@ export interface ClientDatabase {
   /**
    * Optional sqlite-image capability (SPEC §5.3): expose `bytes` as an
    * attached read-only database schema named `alias` for the duration of
-   * `fn`, then detach. Presence of this method is what makes the client
+   * `fn`, awaiting its result before detaching. No transaction spans its awaits.
+   * Presence of this method is what makes the client
    * advertise `accept` bit 2 (§4.2). MUST be called outside any open
    * transaction (SQLite cannot ATTACH inside one); `fn` may open its own.
    */
-  withSqliteImage?<T>(bytes: Uint8Array, alias: string, fn: () => T): T;
+  withSqliteImage?<T>(
+    bytes: Uint8Array,
+    alias: string,
+    fn: () => T | Promise<T>,
+  ): Promise<T>;
   close(): void;
 }
 

package/src/http.ts

@@ -139,6 +139,40 @@ export function webSocketRemoteOperationConnector(
  * Resolution (which path a descriptor takes, expiry, no fall-through)
  * lives in the client core, not here.
  */
+async function readSegmentBody(
+  response: Response,
+  onProgress?: (bytesReceived: number) => void,
+): Promise<Uint8Array> {
+  const chunks: Uint8Array[] = [];
+  let received = 0;
+  let reported = 0;
+  const reader = response.body?.getReader();
+  if (reader !== undefined) {
+    try {
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        chunks.push(value);
+        received += value.byteLength;
+        if (received - reported >= 64 * 1024) {
+          onProgress?.(received);
+          reported = received;
+        }
+      }
+    } finally {
+      reader.releaseLock();
+    }
+  }
+  if (received !== reported) onProgress?.(received);
+  const bytes = new Uint8Array(received);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  return bytes;
+}
+
 export function httpSegmentDownloader(
   segmentsBaseUrl: string,
   options?: HttpTransportOptions,
@@ -147,6 +181,7 @@ export function httpSegmentDownloader(
   const direct = async (request: {
     readonly segmentId: string;
     readonly requestedScopesJson: string;
+    readonly onProgress?: (bytesReceived: number) => void;
   }) => {
     const response = await doFetch(
       `${segmentsBaseUrl}/${encodeURIComponent(request.segmentId)}`,
@@ -158,9 +193,12 @@ export function httpSegmentDownloader(
       },
     );
     if (!response.ok) await throwHttpError(response);
-    return new Uint8Array(await response.arrayBuffer());
+    return readSegmentBody(response, request.onProgress);
   };
-  const fetchUrl = async (url: string) => {
+  const fetchUrl = async (
+    url: string,
+    onProgress?: (bytesReceived: number) => void,
+  ) => {
     // Deliberately headerless: the URL is the bearer grant (§5.4).
     const response = await doFetch(url);
     if (!response.ok) {
@@ -170,7 +208,7 @@ export function httpSegmentDownloader(
         true,
       );
     }
-    return new Uint8Array(await response.arrayBuffer());
+    return readSegmentBody(response, onProgress);
   };
   return Object.assign(direct, { fetchUrl });
 }

package/src/index.ts

@@ -40,3 +40,9 @@ export * from './transport';
 export * from './window';
 export * from './worker-host';
 export * from './worker-protocol';
+
+export {
+  ProgressEmitter,
+  type SyncProgress,
+  type SyncProgressListener,
+} from './progress';

package/src/node-database.ts

@@ -63,7 +63,11 @@ export class NodeClientDatabase implements ClientDatabase {
   }
 
   /** §5.3 image import through a private file attached for one callback. */
-  withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T): T {
+  async withSqliteImage<T>(
+    bytes: Uint8Array,
+    alias: string,
+    fn: () => T | Promise<T>,
+  ): Promise<T> {
     assertImageAlias(alias);
     const dir = mkdtempSync(join(tmpdir(), 'syncular-image-'));
     const path = join(dir, 'segment.db');
@@ -71,7 +75,7 @@ export class NodeClientDatabase implements ClientDatabase {
       writeFileSync(path, bytes);
       this.db.prepare(`ATTACH DATABASE ? AS ${alias}`).run(path);
       try {
-        return fn();
+        return await fn();
       } finally {
         this.db.prepare(`DETACH DATABASE ${alias}`).run();
       }

package/src/progress.ts

@@ -0,0 +1,62 @@
+/** Ephemeral sync work, independent of committed change notifications (SPEC §7.6). */
+export interface SyncProgress {
+  readonly attempt: number;
+  readonly state: 'running' | 'complete' | 'failed';
+  readonly phase: 'request' | 'download' | 'import';
+  readonly subscriptionId?: string;
+  readonly table?: string;
+  readonly segmentId?: string | undefined;
+  readonly bytesReceived: number;
+  readonly bytesTotal?: number;
+  readonly rowsProcessed: number;
+  readonly rowsTotal?: number;
+  readonly errorCode?: string;
+}
+
+export type SyncProgressListener = (progress: SyncProgress) => void;
+
+/** Shared by the direct client and event-forwarding handles. */
+export class ProgressEmitter {
+  #snapshot: SyncProgress | undefined;
+  readonly #listeners = new Set<SyncProgressListener>();
+
+  clear(): void {
+    this.#listeners.clear();
+    this.#snapshot = undefined;
+  }
+
+  snapshot(): SyncProgress | undefined {
+    return this.#snapshot;
+  }
+
+  on(listener: SyncProgressListener): () => void {
+    this.#listeners.add(listener);
+    if (this.#snapshot !== undefined) {
+      try {
+        listener(this.#snapshot);
+      } catch {
+        /* Observers cannot interrupt sync. */
+      }
+    }
+    return () => {
+      this.#listeners.delete(listener);
+    };
+  }
+
+  emit(progress: SyncProgress): void {
+    const snapshot = Object.freeze({ ...progress });
+    this.#snapshot = snapshot;
+    for (const listener of [...this.#listeners]) {
+      try {
+        listener(snapshot);
+      } catch {
+        /* Observers cannot interrupt sync. */
+      }
+    }
+  }
+
+  update(update: Partial<SyncProgress>): void {
+    if (this.#snapshot?.state === 'running')
+      this.emit({ ...this.#snapshot, ...update });
+  }
+}