@syncular/client 0.19.0 → 0.20.1

package/src/reactive-store.ts

@@ -359,7 +359,8 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
   #running = false;
   #requested = false;
   #desiredRevision = 0n;
-  #claimReady: Promise<void> = Promise.resolve();
+  #claimError: Error | undefined;
+  #claimPending = false;
   #offStatus: (() => void) | undefined;
 
   constructor(
@@ -397,10 +398,25 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
             ),
           );
         }
-        this.#claimReady = Promise.all(claims).then(() => undefined);
-        // A render can lose its last subscriber before the read loop starts.
-        // The loop still observes the original rejection while it owns the claim.
-        void this.#claimReady.catch(() => undefined);
+        const generation = this.#generation;
+        this.#claimPending = claims.length > 0;
+        if (claims.length > 0)
+          void Promise.all(claims).then(
+            () => {
+              if (generation !== this.#generation || this.#listeners.size === 0)
+                return;
+              this.#claimPending = false;
+              if (!this.#running && this.#state.phase !== 'ready')
+                this.#requestRead();
+            },
+            (error: unknown) => {
+              if (generation !== this.#generation || this.#listeners.size === 0)
+                return;
+              this.#claimPending = false;
+              this.#claimError = errorOf(error);
+              this.#requestRead();
+            },
+          );
       }
       this.#requestRead();
     }
@@ -426,7 +442,8 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
     this.#running = false;
     this.#requested = false;
     this.#desiredRevision = 0n;
-    this.#claimReady = Promise.resolve();
+    this.#claimError = undefined;
+    this.#claimPending = false;
     this.#state = {
       rows: [],
       phase: 'loading',
@@ -520,9 +537,11 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
       do {
         this.#requested = false;
         if (this.store.availabilitySnapshot().state === 'blocked') break;
-        await this.#claimReady;
+
         if (generation !== this.#generation || this.#listeners.size === 0)
           return;
+        if (this.#claimError !== undefined) throw this.#claimError;
+        const claimWasPending = this.#claimPending;
         const snapshot = await this.store.client.querySnapshot<
           Record<string, SqlValue>
         >({
@@ -536,6 +555,7 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
         });
         if (generation !== this.#generation || this.#listeners.size === 0)
           return;
+        if (this.#claimError !== undefined) throw this.#claimError;
         const availability = this.store.availabilitySnapshot();
         if (availability.state === 'blocked') {
           this.#publish({
@@ -550,6 +570,12 @@ class QueryEntry<Row> implements ExternalStoreEntry<LiveQueryResult<Row>> {
           this.#requested = true;
           continue;
         }
+        if (
+          !snapshot.coverage.complete &&
+          claimWasPending &&
+          !this.#claimPending
+        )
+          this.#requested = true;
         const mappedRows =
           this.spec.mapRow === undefined
             ? (snapshot.rows as readonly Row[])
@@ -625,6 +651,8 @@ interface WindowClaimGroup {
   readonly base: WindowBase;
   readonly claims: Map<symbol, ReadonlySet<string>>;
   appliedKey: string;
+  appliedUnits: ReadonlySet<string>;
+  inFlightUnits?: ReadonlySet<string> | undefined;
   scheduled: boolean;
   running: boolean;
   requested: boolean;
@@ -889,6 +917,7 @@ export class ReactiveClientStore {
         base,
         claims: new Map(),
         appliedKey: '',
+        appliedUnits: new Set(),
         scheduled: false,
         running: false,
         requested: false,
@@ -897,6 +926,17 @@ export class ReactiveClientStore {
       this.#windowClaims.set(key, group);
     }
     group.claims.set(owner, new Set(units));
+    const alreadyHeld =
+      units.length > 0 &&
+      units.every(
+        (unit) =>
+          group.appliedUnits.has(unit) &&
+          (group.inFlightUnits === undefined || group.inFlightUnits.has(unit)),
+      );
+    if (alreadyHeld) {
+      this.#scheduleWindow(group);
+      return Promise.resolve();
+    }
     const result = new Promise<void>((resolve, reject) => {
       group?.waiters.push({ resolve, reject });
     });
@@ -932,13 +972,23 @@ export class ReactiveClientStore {
         ].sort();
         const key = canonicalValue(units);
         if (key !== group.appliedKey) {
-          await this.client.setWindow(group.base, units);
+          group.inFlightUnits = new Set(units);
+          try {
+            await this.client.setWindow(group.base, units);
+          } finally {
+            group.inFlightUnits = undefined;
+          }
           if (this.#windowClaims.get(baseKey) !== group) return;
           group.appliedKey = key;
+          group.appliedUnits = new Set(units);
         }
       }
       for (const waiter of group.waiters.splice(0)) waiter.resolve();
     } catch (error) {
+      // A failed window edit may already have committed its first eviction chunk.
+      // Reconcile through the core before acknowledging any later claim.
+      group.appliedKey = '';
+      group.appliedUnits = new Set();
       for (const waiter of group.waiters.splice(0)) waiter.reject(error);
     } finally {
       group.running = false;

package/src/schema.ts

@@ -374,6 +374,14 @@ function createFtsProjection(
     `CREATE VIRTUAL TABLE IF NOT EXISTS ${quoteIdent(index.name)} USING fts5(${quoteIdent(FTS_SOURCE_ID_COLUMN)} UNINDEXED, ${indexedColumns.join(', ')}, tokenize='${tokenizer}')`,
   );
 
+  const mapping = quoteIdent(`_syncular_fts_${index.name}`);
+  const mapped =
+    db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
+      `_syncular_fts_${index.name}`,
+    ]).length > 0;
+  db.exec(
+    `CREATE TABLE IF NOT EXISTS ${mapping} (id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)`,
+  );
   const sourceId = `CAST(${quoteIdent(table.primaryKey)} AS TEXT)`;
   const newSourceId = `CAST(new.${quoteIdent(table.primaryKey)} AS TEXT)`;
   const oldSourceId = `CAST(old.${quoteIdent(table.primaryKey)} AS TEXT)`;
@@ -386,9 +394,10 @@ function createFtsProjection(
     ...index.columns.map((column) => `new.${quoteIdent(column)}`),
   ].join(', ');
   const deleteFor = (value: string) =>
-    `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} = ${value}`;
+    `DELETE FROM ${quoteIdent(index.name)} WHERE rowid = (SELECT id FROM ${mapping} WHERE source_id = ${value}); DELETE FROM ${mapping} WHERE source_id = ${value}`;
   const deleteDisplaced = (select: string) =>
-    `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} IN (${select})`;
+    `DELETE FROM ${quoteIdent(index.name)} WHERE rowid IN (SELECT id FROM ${mapping} WHERE source_id IN (${select})); DELETE FROM ${mapping} WHERE source_id IN (${select})`;
+  const insertNew = `INSERT INTO ${mapping}(source_id) VALUES (${newSourceId}); INSERT INTO ${quoteIdent(index.name)} (rowid, ${projectionColumns}) VALUES ((SELECT id FROM ${mapping} WHERE source_id = ${newSourceId}), ${newValues})`;
 
   // A clean insert cannot already have a projection row because the source
   // primary key is unique. Keep clean inserts linear by moving replacement
@@ -441,13 +450,13 @@ function createFtsProjection(
     `CREATE TRIGGER ${quoteIdent(`${index.name}_bi`)} BEFORE INSERT ON ${quoteIdent(table.name)} WHEN ${insertGuardCondition} BEGIN ${insertGuardBody}; END`,
   );
   db.exec(
-    `CREATE TRIGGER ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`,
+    `CREATE TRIGGER ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN ${insertNew}; END`,
   );
   db.exec(
     `CREATE TRIGGER ${quoteIdent(`${index.name}_ad`)} AFTER DELETE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; END`,
   );
   db.exec(
-    `CREATE TRIGGER ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`,
+    `CREATE TRIGGER ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; ${insertNew}; END`,
   );
   // BEFORE UPDATE guard for `UPDATE OR REPLACE`: clear the projection of any
   // different-PK row about to be displaced through a secondary unique index by
@@ -468,6 +477,12 @@ function createFtsProjection(
       `INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) SELECT ${sourceId}, ${indexedColumns.join(', ')} FROM ${quoteIdent(table.name)}`,
     );
   }
+  if (!mapped || !existed) {
+    db.exec(`DELETE FROM ${mapping}`);
+    db.exec(
+      `INSERT INTO ${mapping}(id, source_id) SELECT rowid, ${quoteIdent(FTS_SOURCE_ID_COLUMN)} FROM ${quoteIdent(index.name)}`,
+    );
+  }
 }
 
 /**
@@ -602,6 +617,9 @@ export function dropAndRecreateSyncedTables(
        AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'`,
   );
   for (const row of virtualTables) {
+    db.exec(
+      `DROP TABLE IF EXISTS ${quoteIdent(`_syncular_fts_${String(row.name)}`)}`,
+    );
     db.exec(`DROP TABLE IF EXISTS ${quoteIdent(String(row.name))}`);
   }
   const existing = db.query(

package/src/transport.ts

@@ -32,6 +32,7 @@ export interface SegmentFetchRequest {
   readonly table: string;
   /** Canonical JSON (§11.2) of the requested scope map (§5.5 header). */
   readonly requestedScopesJson: string;
+  readonly onProgress?: (bytesReceived: number) => void;
 }
 
 /**
@@ -44,7 +45,10 @@ export interface SegmentFetchRequest {
  */
 export interface SegmentDownloader {
   (request: SegmentFetchRequest): Promise<Uint8Array>;
-  readonly fetchUrl?: (url: string) => Promise<Uint8Array>;
+  readonly fetchUrl?: (
+    url: string,
+    onProgress?: (bytesReceived: number) => void,
+  ) => Promise<Uint8Array>;
 }
 
 export interface RealtimeHandlers {

package/src/wasm-database.ts

@@ -53,6 +53,13 @@ interface SahPoolUtil {
   OpfsSAHPoolDb: new (filename: string) => Oo1Database;
 }
 
+interface SqliteIoMethods {
+  installMethod(
+    name: 'xCheckReservedLock',
+    callback: (file: number, output: number) => number,
+  ): unknown;
+}
+
 interface Sqlite3Static {
   oo1: {
     DB: new (filename: string, flags?: string) => Oo1Database;
@@ -60,6 +67,18 @@ interface Sqlite3Static {
   capi: {
     SQLITE_DESERIALIZE_FREEONCLOSE: number;
     SQLITE_DESERIALIZE_READONLY: number;
+    SQLITE_FCNTL_FILE_POINTER: number;
+    sqlite3_file_control(
+      db: number,
+      schema: string,
+      operation: number,
+      output: number,
+    ): number;
+    sqlite3_file: new (pointer: number) => {
+      readonly $pMethods: number;
+      dispose(): void;
+    };
+    sqlite3_io_methods: new (pointer: number) => SqliteIoMethods;
     sqlite3_deserialize(
       db: number,
       schema: string,
@@ -71,6 +90,13 @@ interface Sqlite3Static {
   };
   wasm: {
     allocFromTypedArray(bytes: Uint8Array): number;
+    peekPtr(pointer: number): number;
+    poke32(pointer: number, value: number): unknown;
+    pstack: {
+      readonly pointer: number;
+      allocPtr(): number;
+      restore(pointer: number): void;
+    };
   };
   installOpfsSAHPoolVfs(options: {
     name?: string;
@@ -125,9 +151,13 @@ class WasmClientDatabase implements ClientDatabase {
    * §5.3 image import at near-file-copy speed: attach an empty in-memory
    * schema, then `sqlite3_deserialize` the image bytes into it (the
    * documented sqlite-wasm import path — no OPFS round-trip, no SQL-level
-   * row shuttling before the single INSERT…SELECT the caller runs).
+   * row shuttling before the chunked INSERT…SELECT statements the caller runs).
    */
-  withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T): T {
+  async withSqliteImage<T>(
+    bytes: Uint8Array,
+    alias: string,
+    fn: () => T | Promise<T>,
+  ): Promise<T> {
     assertImageAlias(alias);
     const pointer = this.#db.pointer;
     if (pointer === undefined) {
@@ -156,7 +186,7 @@ class WasmClientDatabase implements ClientDatabase {
           `sqlite3_deserialize failed with code ${rc} (§5.3)`,
         );
       }
-      return fn();
+      return await fn();
     } finally {
       this.exec(`DETACH ${alias}`);
     }
@@ -217,6 +247,10 @@ function inWorkerContext(): boolean {
 /** One registered VFS per pool directory, reused across opens. */
 const sahPools = new Map<string, Promise<SahPoolUtil>>();
 
+// SAH pools share an I/O method table within the WASM runtime. Keep the
+// installed callback alive for that runtime, including across database closes.
+const sahIoMethods = new Map<number, SqliteIoMethods>();
+
 function opfsSahPoolError(error: unknown, directory: string): ClientSyncError {
   const detail = error instanceof Error ? error.message : String(error);
   const normalized = detail.toLowerCase();
@@ -306,5 +340,53 @@ export async function openPersistentWasmDatabase(
     sahPools.set(directory, pool);
   }
   const util = await pool;
-  return new WasmClientDatabase(new util.OpfsSAHPoolDb(`/${name}.db`), sqlite3);
+  const db = new util.OpfsSAHPoolDb(`/${name}.db`);
+  const { capi, wasm } = sqlite3;
+  const stack = wasm.pstack.pointer;
+  try {
+    // Do this before the first SQL statement, when SQLite checks for a hot
+    // journal. The SAH-pool callback in sqlite-wasm 3.53 reports a reserved
+    // lock unconditionally, which suppresses crash rollback. The pool already
+    // excludes other owners; report no competing writer, as the other OPFS
+    // VFS does. Keep DELETE/FULL and the existing database/journal files.
+    const output = wasm.pstack.allocPtr();
+    if (
+      db.pointer === undefined ||
+      capi.sqlite3_file_control(
+        db.pointer,
+        'main',
+        capi.SQLITE_FCNTL_FILE_POINTER,
+        output,
+      ) !== 0 ||
+      !wasm.peekPtr(output)
+    ) {
+      throw new ClientSyncError(
+        STORAGE_UNAVAILABLE_CODE,
+        'Could not configure OPFS crash recovery',
+      );
+    }
+    const file = new capi.sqlite3_file(wasm.peekPtr(output));
+    const methodsPointer = file.$pMethods;
+    file.dispose(); // Borrowed struct view; SQLite still owns the file.
+    if (!methodsPointer) {
+      throw new ClientSyncError(
+        STORAGE_UNAVAILABLE_CODE,
+        'Could not configure OPFS crash recovery',
+      );
+    }
+    if (!sahIoMethods.has(methodsPointer)) {
+      const methods = new capi.sqlite3_io_methods(methodsPointer);
+      methods.installMethod('xCheckReservedLock', (_file, reserved) => {
+        wasm.poke32(reserved, 0);
+        return 0;
+      });
+      sahIoMethods.set(methodsPointer, methods);
+    }
+    return new WasmClientDatabase(db, sqlite3);
+  } catch (error) {
+    db.close();
+    throw error;
+  } finally {
+    wasm.pstack.restore(stack);
+  }
 }

package/src/worker-entry.ts

@@ -20,7 +20,7 @@ import {
   withClientDiagnosticsHost,
 } from './diagnostics';
 import { encryptionConfigFromKeyring } from './encryption';
-import { ClientSyncError } from './errors';
+import { ClientSyncError, STORAGE_BUSY_CODE } from './errors';
 import {
   httpBlobTransport,
   httpSegmentDownloader,
@@ -51,6 +51,8 @@ export interface SyncWorkerOverrides {
   readonly openDatabase?: (
     config: WorkerInitConfig,
   ) => Promise<ClientDatabase> | ClientDatabase;
+  /** Retry deadline indirection for deterministic worker-host tests. */
+  readonly waitForStorageRetry?: (delayMs: number) => Promise<void>;
   readonly createTransport?: (config: WorkerInitConfig) => SyncTransport;
   readonly createSegments?: (
     config: WorkerInitConfig,
@@ -244,9 +246,12 @@ export function startSyncWorker(overrides: SyncWorkerOverrides = {}): void {
     return Object.assign(gateOffline(inner), {
       ...(fetchUrl !== undefined
         ? {
-            fetchUrl: (url: string) => {
+            fetchUrl: (
+              url: string,
+              onProgress?: (bytesReceived: number) => void,
+            ) => {
               assertOnline();
-              return fetchUrl(url);
+              return fetchUrl(url, onProgress);
             },
           }
         : {}),
@@ -262,10 +267,33 @@ export function startSyncWorker(overrides: SyncWorkerOverrides = {}): void {
     }
     autoSync = config.autoSync ?? true;
 
-    database =
-      overrides.openDatabase !== undefined
-        ? await overrides.openDatabase(config)
-        : await defaultOpenDatabase(config);
+    for (let attempt = 0; ; attempt++) {
+      try {
+        database =
+          overrides.openDatabase !== undefined
+            ? await overrides.openDatabase(config)
+            : await defaultOpenDatabase(config);
+        break;
+      } catch (error) {
+        if (
+          config.database.mode !== 'persistent' ||
+          !(error instanceof ClientSyncError) ||
+          error.code !== STORAGE_BUSY_CODE ||
+          !error.retryable ||
+          attempt === 6
+        ) {
+          throw error;
+        }
+        // Retain the leader lease while the previous worker's OPFS handles
+        // drain. Only opening is retried; core initialization has not started.
+        const delayMs = Math.min(50 * 2 ** attempt, 1000);
+        if (overrides.waitForStorageRetry !== undefined) {
+          await overrides.waitForStorageRetry(delayMs);
+        } else {
+          await new Promise<void>((resolve) => setTimeout(resolve, delayMs));
+        }
+      }
+    }
     const transport = gateOffline(
       overrides.createTransport !== undefined
         ? overrides.createTransport(config)
@@ -336,6 +364,9 @@ export function startSyncWorker(overrides: SyncWorkerOverrides = {}): void {
         post({ t: 'event', event: { kind: 'presence', scopeKey } });
       },
     });
+    started.onProgress((progress) => {
+      if (!closed) post({ t: 'event', event: { kind: 'progress', progress } });
+    });
     started.onChange((batch) => {
       if (!closed) post({ t: 'event', event: { kind: 'change', batch } });
     });

package/src/worker-host.ts

@@ -1,3 +1,8 @@
+import {
+  ProgressEmitter,
+  type SyncProgress,
+  type SyncProgressListener,
+} from './progress';
 import type { PromiseMethods } from './client';
 /**
  * Main-thread side of the worker mode and the
@@ -273,6 +278,7 @@ export class SyncClientHandle implements PromiseMethods<WorkerApi> {
   #core: LeaderCore | undefined;
   #follower: FollowerLink | undefined;
   readonly #invalidation: InvalidationEmitter;
+  readonly #progress = new ProgressEmitter();
   readonly #changes: ChangeEmitter;
   readonly #presence: Set<(scopeKey: string) => void>;
   readonly #diagnostics: ClientDiagnosticsEmitter;
@@ -376,7 +382,9 @@ export class SyncClientHandle implements PromiseMethods<WorkerApi> {
 
   /** @internal — dispatch a worker/relayed event to handle-local listeners. */
   __dispatchEvent(event: SyncWorkerEvent): void {
-    if (event.kind === 'presence') {
+    if (event.kind === 'progress') {
+      this.#progress.emit(event.progress);
+    } else if (event.kind === 'presence') {
       for (const listener of this.#presence) {
         try {
           listener(event.scopeKey);
@@ -409,6 +417,13 @@ export class SyncClientHandle implements PromiseMethods<WorkerApi> {
     return this.#invalidation.on(listener);
   }
 
+  onProgress(listener: SyncProgressListener): () => void {
+    return this.#progress.on(listener);
+  }
+  progressSnapshot(): SyncProgress | undefined {
+    return this.#progress.snapshot();
+  }
+
   onChange(listener: ClientChangeListener): () => void {
     return this.#changes.on(listener);
   }
@@ -646,6 +661,7 @@ export class SyncClientHandle implements PromiseMethods<WorkerApi> {
   async close(): Promise<void> {
     if (this.#closed) return;
     this.#closed = true;
+    this.#progress.clear();
     this.#devtoolsUnregister();
     if (this.#follower !== undefined) {
       this.#follower.close();

package/src/worker-protocol.ts

@@ -1,3 +1,4 @@
+import type { SyncProgress } from './progress';
 /**
  * The worker RPC protocol: the whole
  * client core runs in a Web Worker; the UI thread talks to it through
@@ -206,6 +207,7 @@ export interface WorkerErrorShape {
 }
 
 export type SyncWorkerEvent =
+  | { readonly kind: 'progress'; readonly progress: SyncProgress }
   | {
       readonly kind: 'sync-needed';
       readonly reason: 'startup' | 'hello' | WakeReason;