@syncular/client 0.19.0 → 0.20.1

package/dist/reactive-store.js

@@ -244,7 +244,8 @@ class QueryEntry {
     #running = false;
     #requested = false;
     #desiredRevision = 0n;
-    #claimReady = Promise.resolve();
+    #claimError;
+    #claimPending = false;
     #offStatus;
     constructor(store, spec, key, cache) {
         this.store = store;
@@ -270,10 +271,22 @@ class QueryEntry {
                 for (const coverage of this.spec.coverage ?? []) {
                     claims.push(this.store.setWindowClaim(this.#owner, coverage.base, coverage.units));
                 }
-                this.#claimReady = Promise.all(claims).then(() => undefined);
-                // A render can lose its last subscriber before the read loop starts.
-                // The loop still observes the original rejection while it owns the claim.
-                void this.#claimReady.catch(() => undefined);
+                const generation = this.#generation;
+                this.#claimPending = claims.length > 0;
+                if (claims.length > 0)
+                    void Promise.all(claims).then(() => {
+                        if (generation !== this.#generation || this.#listeners.size === 0)
+                            return;
+                        this.#claimPending = false;
+                        if (!this.#running && this.#state.phase !== 'ready')
+                            this.#requestRead();
+                    }, (error) => {
+                        if (generation !== this.#generation || this.#listeners.size === 0)
+                            return;
+                        this.#claimPending = false;
+                        this.#claimError = errorOf(error);
+                        this.#requestRead();
+                    });
             }
             this.#requestRead();
         }
@@ -300,7 +313,8 @@ class QueryEntry {
         this.#running = false;
         this.#requested = false;
         this.#desiredRevision = 0n;
-        this.#claimReady = Promise.resolve();
+        this.#claimError = undefined;
+        this.#claimPending = false;
         this.#state = {
             rows: [],
             phase: 'loading',
@@ -393,9 +407,11 @@ class QueryEntry {
                 this.#requested = false;
                 if (this.store.availabilitySnapshot().state === 'blocked')
                     break;
-                await this.#claimReady;
                 if (generation !== this.#generation || this.#listeners.size === 0)
                     return;
+                if (this.#claimError !== undefined)
+                    throw this.#claimError;
+                const claimWasPending = this.#claimPending;
                 const snapshot = await this.store.client.querySnapshot({
                     sql: this.spec.sql,
                     ...(this.spec.params !== undefined
@@ -407,6 +423,8 @@ class QueryEntry {
                 });
                 if (generation !== this.#generation || this.#listeners.size === 0)
                     return;
+                if (this.#claimError !== undefined)
+                    throw this.#claimError;
                 const availability = this.store.availabilitySnapshot();
                 if (availability.state === 'blocked') {
                     this.#publish({
@@ -421,6 +439,10 @@ class QueryEntry {
                     this.#requested = true;
                     continue;
                 }
+                if (!snapshot.coverage.complete &&
+                    claimWasPending &&
+                    !this.#claimPending)
+                    this.#requested = true;
                 const mappedRows = this.spec.mapRow === undefined
                     ? snapshot.rows
                     : snapshot.rows.map(this.spec.mapRow);
@@ -736,6 +758,7 @@ export class ReactiveClientStore {
                 base,
                 claims: new Map(),
                 appliedKey: '',
+                appliedUnits: new Set(),
                 scheduled: false,
                 running: false,
                 requested: false,
@@ -744,6 +767,13 @@ export class ReactiveClientStore {
             this.#windowClaims.set(key, group);
         }
         group.claims.set(owner, new Set(units));
+        const alreadyHeld = units.length > 0 &&
+            units.every((unit) => group.appliedUnits.has(unit) &&
+                (group.inFlightUnits === undefined || group.inFlightUnits.has(unit)));
+        if (alreadyHeld) {
+            this.#scheduleWindow(group);
+            return Promise.resolve();
+        }
         const result = new Promise((resolve, reject) => {
             group?.waiters.push({ resolve, reject });
         });
@@ -779,16 +809,27 @@ export class ReactiveClientStore {
                 ].sort();
                 const key = canonicalValue(units);
                 if (key !== group.appliedKey) {
-                    await this.client.setWindow(group.base, units);
+                    group.inFlightUnits = new Set(units);
+                    try {
+                        await this.client.setWindow(group.base, units);
+                    }
+                    finally {
+                        group.inFlightUnits = undefined;
+                    }
                     if (this.#windowClaims.get(baseKey) !== group)
                         return;
                     group.appliedKey = key;
+                    group.appliedUnits = new Set(units);
                 }
             }
             for (const waiter of group.waiters.splice(0))
                 waiter.resolve();
         }
         catch (error) {
+            // A failed window edit may already have committed its first eviction chunk.
+            // Reconcile through the core before acknowledging any later claim.
+            group.appliedKey = '';
+            group.appliedUnits = new Set();
             for (const waiter of group.waiters.splice(0))
                 waiter.reject(error);
         }

package/dist/schema.js

@@ -234,6 +234,11 @@ function createFtsProjection(db, table, index) {
     const indexedColumns = index.columns.map(quoteIdent);
     const tokenizer = index.tokenize.replaceAll("'", "''");
     db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${quoteIdent(index.name)} USING fts5(${quoteIdent(FTS_SOURCE_ID_COLUMN)} UNINDEXED, ${indexedColumns.join(', ')}, tokenize='${tokenizer}')`);
+    const mapping = quoteIdent(`_syncular_fts_${index.name}`);
+    const mapped = db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
+        `_syncular_fts_${index.name}`,
+    ]).length > 0;
+    db.exec(`CREATE TABLE IF NOT EXISTS ${mapping} (id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)`);
     const sourceId = `CAST(${quoteIdent(table.primaryKey)} AS TEXT)`;
     const newSourceId = `CAST(new.${quoteIdent(table.primaryKey)} AS TEXT)`;
     const oldSourceId = `CAST(old.${quoteIdent(table.primaryKey)} AS TEXT)`;
@@ -245,8 +250,9 @@ function createFtsProjection(db, table, index) {
         newSourceId,
         ...index.columns.map((column) => `new.${quoteIdent(column)}`),
     ].join(', ');
-    const deleteFor = (value) => `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} = ${value}`;
-    const deleteDisplaced = (select) => `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} IN (${select})`;
+    const deleteFor = (value) => `DELETE FROM ${quoteIdent(index.name)} WHERE rowid = (SELECT id FROM ${mapping} WHERE source_id = ${value}); DELETE FROM ${mapping} WHERE source_id = ${value}`;
+    const deleteDisplaced = (select) => `DELETE FROM ${quoteIdent(index.name)} WHERE rowid IN (SELECT id FROM ${mapping} WHERE source_id IN (${select})); DELETE FROM ${mapping} WHERE source_id IN (${select})`;
+    const insertNew = `INSERT INTO ${mapping}(source_id) VALUES (${newSourceId}); INSERT INTO ${quoteIdent(index.name)} (rowid, ${projectionColumns}) VALUES ((SELECT id FROM ${mapping} WHERE source_id = ${newSourceId}), ${newValues})`;
     // A clean insert cannot already have a projection row because the source
     // primary key is unique. Keep clean inserts linear by moving replacement
     // cleanup behind indexed source-table existence checks. SQLite does not
@@ -294,9 +300,9 @@ function createFtsProjection(db, table, index) {
         ...displacedByUnique.map(deleteDisplaced),
     ].join('; ');
     db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_bi`)} BEFORE INSERT ON ${quoteIdent(table.name)} WHEN ${insertGuardCondition} BEGIN ${insertGuardBody}; END`);
-    db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`);
+    db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN ${insertNew}; END`);
     db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_ad`)} AFTER DELETE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; END`);
-    db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`);
+    db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; ${insertNew}; END`);
     // BEFORE UPDATE guard for `UPDATE OR REPLACE`: clear the projection of any
     // different-PK row about to be displaced through a secondary unique index by
     // the new values. Only meaningful when the table has a unique index; with
@@ -311,6 +317,10 @@ function createFtsProjection(db, table, index) {
     if (!existed) {
         db.exec(`INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) SELECT ${sourceId}, ${indexedColumns.join(', ')} FROM ${quoteIdent(table.name)}`);
     }
+    if (!mapped || !existed) {
+        db.exec(`DELETE FROM ${mapping}`);
+        db.exec(`INSERT INTO ${mapping}(id, source_id) SELECT rowid, ${quoteIdent(FTS_SOURCE_ID_COLUMN)} FROM ${quoteIdent(index.name)}`);
+    }
 }
 /**
  * Create only the application-owned synced tables, indexes, and FTS
@@ -424,6 +434,7 @@ export function dropAndRecreateSyncedTables(db, schema) {
        AND sql LIKE 'CREATE VIRTUAL TABLE%'
        AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'`);
     for (const row of virtualTables) {
+        db.exec(`DROP TABLE IF EXISTS ${quoteIdent(`_syncular_fts_${String(row.name)}`)}`);
         db.exec(`DROP TABLE IF EXISTS ${quoteIdent(String(row.name))}`);
     }
     const existing = db.query(`SELECT name FROM sqlite_master WHERE type = 'table'

package/dist/transport.d.ts

@@ -22,6 +22,7 @@ export interface SegmentFetchRequest {
     readonly table: string;
     /** Canonical JSON (§11.2) of the requested scope map (§5.5 header). */
     readonly requestedScopesJson: string;
+    readonly onProgress?: (bytesReceived: number) => void;
 }
 /**
  * Fetch segment bytes from the direct endpoint (§5.5). `fetchUrl`, when
@@ -33,7 +34,7 @@ export interface SegmentFetchRequest {
  */
 export interface SegmentDownloader {
     (request: SegmentFetchRequest): Promise<Uint8Array>;
-    readonly fetchUrl?: (url: string) => Promise<Uint8Array>;
+    readonly fetchUrl?: (url: string, onProgress?: (bytesReceived: number) => void) => Promise<Uint8Array>;
 }
 export interface RealtimeHandlers {
     /** JSON control frame (§8.1): hello / sync / heartbeat / unknown. */

package/dist/wasm-database.js

@@ -65,9 +65,9 @@ class WasmClientDatabase {
      * §5.3 image import at near-file-copy speed: attach an empty in-memory
      * schema, then `sqlite3_deserialize` the image bytes into it (the
      * documented sqlite-wasm import path — no OPFS round-trip, no SQL-level
-     * row shuttling before the single INSERT…SELECT the caller runs).
+     * row shuttling before the chunked INSERT…SELECT statements the caller runs).
      */
-    withSqliteImage(bytes, alias, fn) {
+    async withSqliteImage(bytes, alias, fn) {
         assertImageAlias(alias);
         const pointer = this.#db.pointer;
         if (pointer === undefined) {
@@ -84,7 +84,7 @@ class WasmClientDatabase {
             if (rc !== 0) {
                 throw new ClientSyncError('sync.invalid_request', `sqlite3_deserialize failed with code ${rc} (§5.3)`);
             }
-            return fn();
+            return await fn();
         }
         finally {
             this.exec(`DETACH ${alias}`);
@@ -122,6 +122,9 @@ function inWorkerContext() {
 }
 /** One registered VFS per pool directory, reused across opens. */
 const sahPools = new Map();
+// SAH pools share an I/O method table within the WASM runtime. Keep the
+// installed callback alive for that runtime, including across database closes.
+const sahIoMethods = new Map();
 function opfsSahPoolError(error, directory) {
     const detail = error instanceof Error ? error.message : String(error);
     const normalized = detail.toLowerCase();
@@ -187,5 +190,42 @@ export async function openPersistentWasmDatabase(name, options) {
         sahPools.set(directory, pool);
     }
     const util = await pool;
-    return new WasmClientDatabase(new util.OpfsSAHPoolDb(`/${name}.db`), sqlite3);
+    const db = new util.OpfsSAHPoolDb(`/${name}.db`);
+    const { capi, wasm } = sqlite3;
+    const stack = wasm.pstack.pointer;
+    try {
+        // Do this before the first SQL statement, when SQLite checks for a hot
+        // journal. The SAH-pool callback in sqlite-wasm 3.53 reports a reserved
+        // lock unconditionally, which suppresses crash rollback. The pool already
+        // excludes other owners; report no competing writer, as the other OPFS
+        // VFS does. Keep DELETE/FULL and the existing database/journal files.
+        const output = wasm.pstack.allocPtr();
+        if (db.pointer === undefined ||
+            capi.sqlite3_file_control(db.pointer, 'main', capi.SQLITE_FCNTL_FILE_POINTER, output) !== 0 ||
+            !wasm.peekPtr(output)) {
+            throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, 'Could not configure OPFS crash recovery');
+        }
+        const file = new capi.sqlite3_file(wasm.peekPtr(output));
+        const methodsPointer = file.$pMethods;
+        file.dispose(); // Borrowed struct view; SQLite still owns the file.
+        if (!methodsPointer) {
+            throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, 'Could not configure OPFS crash recovery');
+        }
+        if (!sahIoMethods.has(methodsPointer)) {
+            const methods = new capi.sqlite3_io_methods(methodsPointer);
+            methods.installMethod('xCheckReservedLock', (_file, reserved) => {
+                wasm.poke32(reserved, 0);
+                return 0;
+            });
+            sahIoMethods.set(methodsPointer, methods);
+        }
+        return new WasmClientDatabase(db, sqlite3);
+    }
+    catch (error) {
+        db.close();
+        throw error;
+    }
+    finally {
+        wasm.pstack.restore(stack);
+    }
 }

package/dist/worker-entry.d.ts

@@ -4,6 +4,8 @@ import { type WorkerInitConfig } from './worker-protocol.js';
 export interface SyncWorkerOverrides {
     /** Database factory indirection: tests inject bun:sqlite here. */
     readonly openDatabase?: (config: WorkerInitConfig) => Promise<ClientDatabase> | ClientDatabase;
+    /** Retry deadline indirection for deterministic worker-host tests. */
+    readonly waitForStorageRetry?: (delayMs: number) => Promise<void>;
     readonly createTransport?: (config: WorkerInitConfig) => SyncTransport;
     readonly createSegments?: (config: WorkerInitConfig) => SegmentDownloader | undefined;
     readonly createRealtime?: (config: WorkerInitConfig, clientId: string) => RealtimeConnector | undefined;

package/dist/worker-entry.js

@@ -16,7 +16,7 @@
 import { SyncClient } from './client.js';
 import { withClientDiagnosticsHost, } from './diagnostics.js';
 import { encryptionConfigFromKeyring } from './encryption.js';
-import { ClientSyncError } from './errors.js';
+import { ClientSyncError, STORAGE_BUSY_CODE } from './errors.js';
 import { httpBlobTransport, httpSegmentDownloader, httpSyncTransport, webSocketRealtimeConnector, } from './http.js';
 import { openPersistentWasmDatabase } from './wasm-database.js';
 import { WORKER_FAILED_CODE, } from './worker-protocol.js';
@@ -170,9 +170,9 @@ export function startSyncWorker(overrides = {}) {
         return Object.assign(gateOffline(inner), {
             ...(fetchUrl !== undefined
                 ? {
-                    fetchUrl: (url) => {
+                    fetchUrl: (url, onProgress) => {
                         assertOnline();
-                        return fetchUrl(url);
+                        return fetchUrl(url, onProgress);
                     },
                 }
                 : {}),
@@ -183,10 +183,33 @@ export function startSyncWorker(overrides = {}) {
             throw new ClientSyncError(WORKER_FAILED_CODE, 'the worker is already initialized');
         }
         autoSync = config.autoSync ?? true;
-        database =
-            overrides.openDatabase !== undefined
-                ? await overrides.openDatabase(config)
-                : await defaultOpenDatabase(config);
+        for (let attempt = 0;; attempt++) {
+            try {
+                database =
+                    overrides.openDatabase !== undefined
+                        ? await overrides.openDatabase(config)
+                        : await defaultOpenDatabase(config);
+                break;
+            }
+            catch (error) {
+                if (config.database.mode !== 'persistent' ||
+                    !(error instanceof ClientSyncError) ||
+                    error.code !== STORAGE_BUSY_CODE ||
+                    !error.retryable ||
+                    attempt === 6) {
+                    throw error;
+                }
+                // Retain the leader lease while the previous worker's OPFS handles
+                // drain. Only opening is retried; core initialization has not started.
+                const delayMs = Math.min(50 * 2 ** attempt, 1000);
+                if (overrides.waitForStorageRetry !== undefined) {
+                    await overrides.waitForStorageRetry(delayMs);
+                }
+                else {
+                    await new Promise((resolve) => setTimeout(resolve, delayMs));
+                }
+            }
+        }
         const transport = gateOffline(overrides.createTransport !== undefined
             ? overrides.createTransport(config)
             : httpSyncTransport(config.endpoints.syncUrl));
@@ -245,6 +268,10 @@ export function startSyncWorker(overrides = {}) {
                 post({ t: 'event', event: { kind: 'presence', scopeKey } });
             },
         });
+        started.onProgress((progress) => {
+            if (!closed)
+                post({ t: 'event', event: { kind: 'progress', progress } });
+        });
         started.onChange((batch) => {
             if (!closed)
                 post({ t: 'event', event: { kind: 'change', batch } });

package/dist/worker-host.d.ts

@@ -1,3 +1,4 @@
+import { type SyncProgress, type SyncProgressListener } from './progress.js';
 import type { PromiseMethods } from './client.js';
 /**
  * Main-thread side of the worker mode and the
@@ -179,6 +180,8 @@ export declare class SyncClientHandle implements PromiseMethods<WorkerApi> {
      * unsubscribe function.
      */
     onInvalidate(listener: InvalidationListener): () => void;
+    onProgress(listener: SyncProgressListener): () => void;
+    progressSnapshot(): SyncProgress | undefined;
     onChange(listener: ClientChangeListener): () => void;
     onDiagnostics(listener: ClientDiagnosticsListener): () => void;
     /**

package/dist/worker-host.js

@@ -1,3 +1,4 @@
+import { ProgressEmitter, } from './progress.js';
 import { registerDevtools } from './devtools.js';
 import { ClientDiagnosticsEmitter, withClientDiagnosticsHost, } from './diagnostics.js';
 import { ClientSyncError } from './errors.js';
@@ -68,6 +69,7 @@ export class SyncClientHandle {
     #core;
     #follower;
     #invalidation;
+    #progress = new ProgressEmitter();
     #changes;
     #presence;
     #diagnostics;
@@ -156,7 +158,10 @@ export class SyncClientHandle {
     }
     /** @internal — dispatch a worker/relayed event to handle-local listeners. */
     __dispatchEvent(event) {
-        if (event.kind === 'presence') {
+        if (event.kind === 'progress') {
+            this.#progress.emit(event.progress);
+        }
+        else if (event.kind === 'presence') {
             for (const listener of this.#presence) {
                 try {
                     listener(event.scopeKey);
@@ -189,6 +194,12 @@ export class SyncClientHandle {
     onInvalidate(listener) {
         return this.#invalidation.on(listener);
     }
+    onProgress(listener) {
+        return this.#progress.on(listener);
+    }
+    progressSnapshot() {
+        return this.#progress.snapshot();
+    }
     onChange(listener) {
         return this.#changes.on(listener);
     }
@@ -349,6 +360,7 @@ export class SyncClientHandle {
         if (this.#closed)
             return;
         this.#closed = true;
+        this.#progress.clear();
         this.#devtoolsUnregister();
         if (this.#follower !== undefined) {
             this.#follower.close();

package/dist/worker-protocol.d.ts

@@ -1,3 +1,4 @@
+import type { SyncProgress } from './progress.js';
 /**
  * The worker RPC protocol: the whole
  * client core runs in a Web Worker; the UI thread talks to it through
@@ -151,6 +152,9 @@ export interface WorkerErrorShape {
     readonly retryable: boolean;
 }
 export type SyncWorkerEvent = {
+    readonly kind: 'progress';
+    readonly progress: SyncProgress;
+} | {
     readonly kind: 'sync-needed';
     readonly reason: 'startup' | 'hello' | WakeReason;
 } | {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/client",
-  "version": "0.19.0",
+  "version": "0.20.1",
   "description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -89,9 +89,10 @@
   },
   "dependencies": {
     "@sqlite.org/sqlite-wasm": "^3.53.0-build1",
-    "@syncular/core": "0.19.0"
+    "@syncular/core": "0.20.1"
   },
   "devDependencies": {
-    "@syncular/server": "0.19.0"
+    "@syncular/server": "0.20.1",
+    "playwright": "1.63.0"
   }
 }