@syncular/client 0.20.0 → 0.20.1

package/src/client.ts

@@ -44,6 +44,7 @@ import {
   deleteLocalRow,
   deleteScopedRows,
   evictScopedRows,
+  yieldToHost,
   upsertLocalRow,
 } from './apply';
 import {
@@ -286,8 +287,8 @@ export interface SyncClientLimits {
   readonly maxSnapshotPages?: number;
   /**
    * §4.2 accept bitmask; defaults to inline + external rows (0b0011)
-   * plus sqlite images (bit 2) when the database backend implements
-   * `withSqliteImage` and a segment downloader is configured (§5.3).
+   * plus SQLite images and signed URLs when supported. Both image chunks
+   * and rows blocks commit before yielding to local reads (§7.7).
    */
   readonly accept?: number;
   /**
@@ -748,6 +749,7 @@ export class SyncClient {
     const startupWork =
       this.#schemaFloor === undefined &&
       (countOutbox(this.#db) > 0 ||
+        loadPendingEvictions(this.#db).length > 0 ||
         subscriptions.some((sub) => sub.status === 'active'));
     if (startupWork && this.#securityLifecycle === 'active') {
       this.#needsPull = true;
@@ -1820,23 +1822,17 @@ export class SyncClient {
     socket.send(encodePresencePublish(scopeKey, doc));
   }
 
-  /**
-   * §4.2 accept mask: the configured override, or the rows baseline plus
-   * bit 2 when the backend can import sqlite images (§5.3) and a segment
-   * downloader exists (sqlite segments are never inline, §5.7), plus
-   * bit 3 when the downloader exposes a direct URL fetch (§5.4
-   * capability negotiation).
-   */
+  /** Advertise supported segment formats; all imports yield between commits. */
   #acceptMask(): number {
     const configured = this.#config.limits?.accept;
     if (configured !== undefined) return configured;
     const segments = this.#config.segments;
-    const sqliteCapable =
-      typeof this.#db.withSqliteImage === 'function' && segments !== undefined;
     const urlCapable = typeof segments?.fetchUrl === 'function';
     return (
       ACCEPT_ROWS_BASELINE |
-      (sqliteCapable ? ACCEPT_SQLITE : 0) |
+      (typeof this.#db.withSqliteImage === 'function' && segments !== undefined
+        ? ACCEPT_SQLITE
+        : 0) |
       (urlCapable ? ACCEPT_SIGNED_URLS : 0)
     );
   }
@@ -1909,8 +1905,8 @@ export class SyncClient {
    * Set the live window units for a base (§4.8): a value-sharded family of
    * subscriptions, one per unit. Computes the diff against the registry —
    * added units get fresh subscriptions (image-lane bootstrap on the next
-   * sync); removed units are unsubscribed and evicted, fused in one local
-   * transaction (E1–E4). Idempotent: calling with the same units is a
+   * sync); removed units are unsubscribed with their first eviction chunk
+   * in one local transaction (E1–E4). Idempotent: calling with the same units is a
    * no-op. Re-entry (a unit removed then re-added) cancels any deferred
    * eviction and fresh-bootstraps.
    *
@@ -1975,7 +1971,7 @@ export class SyncClient {
       // Shrink: units live but not wanted → unsubscribe fused with eviction.
       for (const { unit, subId } of live) {
         if (wanted.has(unit)) continue;
-        this.#evictUnit(baseKey, base, unit, subId);
+        await this.#evictUnit(baseKey, base, unit, subId);
         changed = true;
       }
     });
@@ -2018,8 +2014,8 @@ export class SyncClient {
   }
 
   /**
-   * §4.8 E1–E4: evict one departing unit, fused with its unsubscription in
-   * one transaction. Deletes the unit's rows EXCEPT those pinned by a
+   * §4.8 E1–E4: evict one departing unit in committed chunks, fusing
+   * its first chunk with unsubscription. Deletes the unit's rows EXCEPT those pinned by a
    * pending outbox commit (E1); if any pin remains, records a deferred
    * eviction retried on the next outbox drain. Discards the subscription's
    * cursor/resume/effective-echo (E3) and its version state with the rows
@@ -2027,24 +2023,26 @@ export class SyncClient {
    * with no local scope-column mapping, surfaces a configuration error and
    * evicts nothing (§4.8/§3.3).
    */
-  #evictUnit(
+  async #evictUnit(
     baseKey: string,
     base: WindowBase,
     unit: string,
     subId: string,
-  ): void {
+  ): Promise<void> {
     const table = this.#table(base.table);
     const sub = getSubscription(this.#db, subId);
     // The rows a unit holds live under its LAST effective scopes if it ever
     // synced; before first sync, the requested unit scopes are the match.
     const effective = sub?.effectiveScopes ?? unitScopes(base, unit);
     const pinned = this.#pinnedRowIds(base.table);
+    let remaining = false;
     this.#applyBatch((batch) => {
       this.#db.transaction(() => {
-        const deferred = evictScopedRows(this.#db, table, effective, pinned);
+        const result = evictScopedRows(this.#db, table, effective, pinned);
+        remaining = result.remaining;
         deleteWindowUnit(this.#db, baseKey, unit);
         deleteSubscription(this.#db, subId);
-        if (deferred) {
+        if (result.deferred) {
           savePendingEviction(this.#db, subId, base.table, effective);
         } else {
           deletePendingEviction(this.#db, subId);
@@ -2055,6 +2053,10 @@ export class SyncClient {
       this.#needsPull = true;
       batch.status();
     });
+    if (remaining) {
+      await yieldToHost();
+      await this.#drainPendingEvictions();
+    }
   }
 
   /**
@@ -2062,7 +2064,7 @@ export class SyncClient {
    * unit's rows are removed once no pending commit references them; a unit
    * that re-entered the window in the meantime has no pending record left.
    */
-  #drainPendingEvictions(): void {
+  async #drainPendingEvictions(): Promise<void> {
     const pending = loadPendingEvictions(this.#db);
     if (pending.length === 0) return;
     for (const entry of pending) {
@@ -2071,15 +2073,30 @@ export class SyncClient {
         deletePendingEviction(this.#db, entry.subId);
         continue;
       }
-      const pinned = this.#pinnedRowIds(entry.table);
-      this.#applyBatch((batch) => {
-        let deferred = false;
-        this.#db.transaction(() => {
-          deferred = evictScopedRows(this.#db, table, entry.effective, pinned);
-          if (!deferred) deletePendingEviction(this.#db, entry.subId);
+      let remaining: boolean;
+      do {
+        if (
+          !loadPendingEvictions(this.#db).some(
+            (item) => item.subId === entry.subId,
+          )
+        )
+          break;
+        // Local writes may add pins while the worker yields between chunks.
+        const pinned = this.#pinnedRowIds(entry.table);
+        remaining = false;
+        this.#applyBatch((batch) => {
+          const result = evictScopedRows(
+            this.#db,
+            table,
+            entry.effective,
+            pinned,
+          );
+          remaining = result.remaining;
+          if (!result.deferred) deletePendingEviction(this.#db, entry.subId);
+          batch.scopeMap(table, entry.effective);
         });
-        batch.scopeMap(table, entry.effective);
-      });
+        if (remaining) await yieldToHost();
+      } while (remaining);
     }
   }
 
@@ -2753,6 +2770,7 @@ export class SyncClient {
     // survive it — the reference server keeps no replay buffer (§8.2).
     this.#setSyncNeeded(false);
     try {
+      await this.#drainPendingEvictions();
       const logEpoch = getMeta(this.#db, LOG_EPOCH_META_KEY);
       // §5.9.7 B4: upload pending blobs BEFORE pushing rows that reference
       // them, so the server-side existence check (§6.6) passes.
@@ -2830,7 +2848,7 @@ export class SyncClient {
       );
       // §4.8 E1: the push half may have drained commits that pinned rows of
       // a shrunk window unit — retry any deferred evictions now.
-      this.#drainPendingEvictions();
+      await this.#drainPendingEvictions();
       this.#progress.update(
         summary.failed.length > 0
           ? { state: 'failed', errorCode: 'sync.scope_revoked' }
@@ -2879,10 +2897,27 @@ export class SyncClient {
    * Pull repeatedly until quiescent: no commits delivered, no bootstrap
    * pages pending, no resets to recover (§4.5 "pull again" SHOULD).
    */
-  async syncUntilIdle(maxRounds = 20): Promise<SyncSummary> {
+  async syncUntilIdle(maxRounds?: number): Promise<SyncSummary> {
+    const budget = maxRounds ?? 20;
     let last: SyncSummary | undefined;
-    for (let round = 0; round < maxRounds; round++) {
+    for (let round = 0; round < budget; round++) {
+      const before = JSON.stringify(
+        loadSubscriptions(this.#db).map((sub) => [sub.id, sub.bootstrapState]),
+      );
       last = await this.sync();
+      if (
+        maxRounds === undefined &&
+        last.segmentRowsApplied > 0 &&
+        last.bootstrapping.length > 0 &&
+        before !==
+          JSON.stringify(
+            loadSubscriptions(this.#db).map((sub) => [
+              sub.id,
+              sub.bootstrapState,
+            ]),
+          )
+      )
+        round = -1;
       if (last.schemaFloor !== undefined) return last;
       if (
         last.commitsApplied === 0 &&
@@ -2897,7 +2932,7 @@ export class SyncClient {
     }
     throw new ClientSyncError(
       'sync.invalid_request',
-      `sync did not reach idle within ${maxRounds} rounds`,
+      'sync did not reach idle within the round budget',
     );
   }
 
@@ -3428,6 +3463,7 @@ export class SyncClient {
                   segment,
                   {
                     clearFirst,
+                    isCurrent: () => this.#localResetEpoch === resetEpoch,
                     onProgress: (rowsProcessed) =>
                       this.#progress.update({ rowsProcessed }),
                     effective,
@@ -3440,7 +3476,9 @@ export class SyncClient {
                         ) {
                           batch.table(table.name);
                         }
-                        return fn();
+                        const result = fn();
+                        this.#replayOutbox();
+                        return result;
                       }),
                   },
                   this.#encryption,
@@ -3499,6 +3537,7 @@ export class SyncClient {
                     },
                     {
                       clearFirst,
+                      isCurrent: () => this.#localResetEpoch === resetEpoch,
                       onProgress: (rowsProcessed) =>
                         this.#progress.update({ rowsProcessed }),
                       effective,
@@ -3511,7 +3550,9 @@ export class SyncClient {
                           ) {
                             batch.table(table.name);
                           }
-                          return fn();
+                          const result = fn();
+                          this.#replayOutbox();
+                          return result;
                         }),
                     },
                   ),
@@ -3530,6 +3571,7 @@ export class SyncClient {
                     segment,
                     {
                       clearFirst,
+                      isCurrent: () => this.#localResetEpoch === resetEpoch,
                       onProgress: (rowsProcessed) =>
                         this.#progress.update({ rowsProcessed }),
                       effective,
@@ -3542,7 +3584,9 @@ export class SyncClient {
                           ) {
                             batch.table(table.name);
                           }
-                          return fn();
+                          const result = fn();
+                          this.#replayOutbox();
+                          return result;
                         }),
                     },
                     this.#encryption,

package/src/database.ts

@@ -22,11 +22,16 @@ export interface ClientDatabase {
   /**
    * Optional sqlite-image capability (SPEC §5.3): expose `bytes` as an
    * attached read-only database schema named `alias` for the duration of
-   * `fn`, then detach. Presence of this method is what makes the client
+   * `fn`, awaiting its result before detaching. No transaction spans its awaits.
+   * Presence of this method is what makes the client
    * advertise `accept` bit 2 (§4.2). MUST be called outside any open
    * transaction (SQLite cannot ATTACH inside one); `fn` may open its own.
    */
-  withSqliteImage?<T>(bytes: Uint8Array, alias: string, fn: () => T): T;
+  withSqliteImage?<T>(
+    bytes: Uint8Array,
+    alias: string,
+    fn: () => T | Promise<T>,
+  ): Promise<T>;
   close(): void;
 }
 

package/src/node-database.ts

@@ -63,7 +63,11 @@ export class NodeClientDatabase implements ClientDatabase {
   }
 
   /** §5.3 image import through a private file attached for one callback. */
-  withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T): T {
+  async withSqliteImage<T>(
+    bytes: Uint8Array,
+    alias: string,
+    fn: () => T | Promise<T>,
+  ): Promise<T> {
     assertImageAlias(alias);
     const dir = mkdtempSync(join(tmpdir(), 'syncular-image-'));
     const path = join(dir, 'segment.db');
@@ -71,7 +75,7 @@ export class NodeClientDatabase implements ClientDatabase {
       writeFileSync(path, bytes);
       this.db.prepare(`ATTACH DATABASE ? AS ${alias}`).run(path);
       try {
-        return fn();
+        return await fn();
       } finally {
         this.db.prepare(`DETACH DATABASE ${alias}`).run();
       }

package/src/reactive-store.ts

@@ -359,7 +359,8 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
   #running = false;
   #requested = false;
   #desiredRevision = 0n;
-  #claimReady: Promise<void> = Promise.resolve();
+  #claimError: Error | undefined;
+  #claimPending = false;
   #offStatus: (() => void) | undefined;
 
   constructor(
@@ -397,10 +398,25 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
             ),
           );
         }
-        this.#claimReady = Promise.all(claims).then(() => undefined);
-        // A render can lose its last subscriber before the read loop starts.
-        // The loop still observes the original rejection while it owns the claim.
-        void this.#claimReady.catch(() => undefined);
+        const generation = this.#generation;
+        this.#claimPending = claims.length > 0;
+        if (claims.length > 0)
+          void Promise.all(claims).then(
+            () => {
+              if (generation !== this.#generation || this.#listeners.size === 0)
+                return;
+              this.#claimPending = false;
+              if (!this.#running && this.#state.phase !== 'ready')
+                this.#requestRead();
+            },
+            (error: unknown) => {
+              if (generation !== this.#generation || this.#listeners.size === 0)
+                return;
+              this.#claimPending = false;
+              this.#claimError = errorOf(error);
+              this.#requestRead();
+            },
+          );
       }
       this.#requestRead();
     }
@@ -426,7 +442,8 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
     this.#running = false;
     this.#requested = false;
     this.#desiredRevision = 0n;
-    this.#claimReady = Promise.resolve();
+    this.#claimError = undefined;
+    this.#claimPending = false;
     this.#state = {
       rows: [],
       phase: 'loading',
@@ -520,9 +537,11 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
       do {
         this.#requested = false;
         if (this.store.availabilitySnapshot().state === 'blocked') break;
-        await this.#claimReady;
+
         if (generation !== this.#generation || this.#listeners.size === 0)
           return;
+        if (this.#claimError !== undefined) throw this.#claimError;
+        const claimWasPending = this.#claimPending;
         const snapshot = await this.store.client.querySnapshot<
           Record<string, SqlValue>
         >({
@@ -536,6 +555,7 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
         });
         if (generation !== this.#generation || this.#listeners.size === 0)
           return;
+        if (this.#claimError !== undefined) throw this.#claimError;
         const availability = this.store.availabilitySnapshot();
         if (availability.state === 'blocked') {
           this.#publish({
@@ -550,6 +570,12 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
           this.#requested = true;
           continue;
         }
+        if (
+          !snapshot.coverage.complete &&
+          claimWasPending &&
+          !this.#claimPending
+        )
+          this.#requested = true;
         const mappedRows =
           this.spec.mapRow === undefined
             ? (snapshot.rows as readonly Row[])
@@ -625,6 +651,8 @@ interface WindowClaimGroup {
   readonly base: WindowBase;
   readonly claims: Map<symbol, ReadonlySet<string>>;
   appliedKey: string;
+  appliedUnits: ReadonlySet<string>;
+  inFlightUnits?: ReadonlySet<string> | undefined;
   scheduled: boolean;
   running: boolean;
   requested: boolean;
@@ -889,6 +917,7 @@ export class ReactiveClientStore {
         base,
         claims: new Map(),
         appliedKey: '',
+        appliedUnits: new Set(),
         scheduled: false,
         running: false,
         requested: false,
@@ -897,6 +926,17 @@ export class ReactiveClientStore {
       this.#windowClaims.set(key, group);
     }
     group.claims.set(owner, new Set(units));
+    const alreadyHeld =
+      units.length > 0 &&
+      units.every(
+        (unit) =>
+          group.appliedUnits.has(unit) &&
+          (group.inFlightUnits === undefined || group.inFlightUnits.has(unit)),
+      );
+    if (alreadyHeld) {
+      this.#scheduleWindow(group);
+      return Promise.resolve();
+    }
     const result = new Promise<void>((resolve, reject) => {
       group?.waiters.push({ resolve, reject });
     });
@@ -932,13 +972,23 @@ export class ReactiveClientStore {
         ].sort();
         const key = canonicalValue(units);
         if (key !== group.appliedKey) {
-          await this.client.setWindow(group.base, units);
+          group.inFlightUnits = new Set(units);
+          try {
+            await this.client.setWindow(group.base, units);
+          } finally {
+            group.inFlightUnits = undefined;
+          }
           if (this.#windowClaims.get(baseKey) !== group) return;
           group.appliedKey = key;
+          group.appliedUnits = new Set(units);
         }
       }
       for (const waiter of group.waiters.splice(0)) waiter.resolve();
     } catch (error) {
+      // A failed window edit may already have committed its first eviction chunk.
+      // Reconcile through the core before acknowledging any later claim.
+      group.appliedKey = '';
+      group.appliedUnits = new Set();
       for (const waiter of group.waiters.splice(0)) waiter.reject(error);
     } finally {
       group.running = false;

package/src/schema.ts

@@ -374,6 +374,14 @@ function createFtsProjection(
     `CREATE VIRTUAL TABLE IF NOT EXISTS ${quoteIdent(index.name)} USING fts5(${quoteIdent(FTS_SOURCE_ID_COLUMN)} UNINDEXED, ${indexedColumns.join(', ')}, tokenize='${tokenizer}')`,
   );
 
+  const mapping = quoteIdent(`_syncular_fts_${index.name}`);
+  const mapped =
+    db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
+      `_syncular_fts_${index.name}`,
+    ]).length > 0;
+  db.exec(
+    `CREATE TABLE IF NOT EXISTS ${mapping} (id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)`,
+  );
   const sourceId = `CAST(${quoteIdent(table.primaryKey)} AS TEXT)`;
   const newSourceId = `CAST(new.${quoteIdent(table.primaryKey)} AS TEXT)`;
   const oldSourceId = `CAST(old.${quoteIdent(table.primaryKey)} AS TEXT)`;
@@ -386,9 +394,10 @@ function createFtsProjection(
     ...index.columns.map((column) => `new.${quoteIdent(column)}`),
   ].join(', ');
   const deleteFor = (value: string) =>
-    `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} = ${value}`;
+    `DELETE FROM ${quoteIdent(index.name)} WHERE rowid = (SELECT id FROM ${mapping} WHERE source_id = ${value}); DELETE FROM ${mapping} WHERE source_id = ${value}`;
   const deleteDisplaced = (select: string) =>
-    `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} IN (${select})`;
+    `DELETE FROM ${quoteIdent(index.name)} WHERE rowid IN (SELECT id FROM ${mapping} WHERE source_id IN (${select})); DELETE FROM ${mapping} WHERE source_id IN (${select})`;
+  const insertNew = `INSERT INTO ${mapping}(source_id) VALUES (${newSourceId}); INSERT INTO ${quoteIdent(index.name)} (rowid, ${projectionColumns}) VALUES ((SELECT id FROM ${mapping} WHERE source_id = ${newSourceId}), ${newValues})`;
 
   // A clean insert cannot already have a projection row because the source
   // primary key is unique. Keep clean inserts linear by moving replacement
@@ -441,13 +450,13 @@ function createFtsProjection(
     `CREATE TRIGGER ${quoteIdent(`${index.name}_bi`)} BEFORE INSERT ON ${quoteIdent(table.name)} WHEN ${insertGuardCondition} BEGIN ${insertGuardBody}; END`,
   );
   db.exec(
-    `CREATE TRIGGER ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`,
+    `CREATE TRIGGER ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN ${insertNew}; END`,
   );
   db.exec(
     `CREATE TRIGGER ${quoteIdent(`${index.name}_ad`)} AFTER DELETE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; END`,
   );
   db.exec(
-    `CREATE TRIGGER ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`,
+    `CREATE TRIGGER ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; ${insertNew}; END`,
   );
   // BEFORE UPDATE guard for `UPDATE OR REPLACE`: clear the projection of any
   // different-PK row about to be displaced through a secondary unique index by
@@ -468,6 +477,12 @@ function createFtsProjection(
       `INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) SELECT ${sourceId}, ${indexedColumns.join(', ')} FROM ${quoteIdent(table.name)}`,
     );
   }
+  if (!mapped || !existed) {
+    db.exec(`DELETE FROM ${mapping}`);
+    db.exec(
+      `INSERT INTO ${mapping}(id, source_id) SELECT rowid, ${quoteIdent(FTS_SOURCE_ID_COLUMN)} FROM ${quoteIdent(index.name)}`,
+    );
+  }
 }
 
 /**
@@ -602,6 +617,9 @@ export function dropAndRecreateSyncedTables(
        AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'`,
   );
   for (const row of virtualTables) {
+    db.exec(
+      `DROP TABLE IF EXISTS ${quoteIdent(`_syncular_fts_${String(row.name)}`)}`,
+    );
     db.exec(`DROP TABLE IF EXISTS ${quoteIdent(String(row.name))}`);
   }
   const existing = db.query(

package/src/wasm-database.ts

@@ -151,9 +151,13 @@ class WasmClientDatabase implements ClientDatabase {
    * §5.3 image import at near-file-copy speed: attach an empty in-memory
    * schema, then `sqlite3_deserialize` the image bytes into it (the
    * documented sqlite-wasm import path — no OPFS round-trip, no SQL-level
-   * row shuttling before the single INSERT…SELECT the caller runs).
+   * row shuttling before the chunked INSERT…SELECT statements the caller runs).
    */
-  withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T): T {
+  async withSqliteImage<T>(
+    bytes: Uint8Array,
+    alias: string,
+    fn: () => T | Promise<T>,
+  ): Promise<T> {
     assertImageAlias(alias);
     const pointer = this.#db.pointer;
     if (pointer === undefined) {
@@ -182,7 +186,7 @@ class WasmClientDatabase implements ClientDatabase {
           `sqlite3_deserialize failed with code ${rc} (§5.3)`,
         );
       }
-      return fn();
+      return await fn();
     } finally {
       this.exec(`DETACH ${alias}`);
     }