@syncular/client 0.20.0 → 0.20.1

package/dist/reactive-store.js

@@ -244,7 +244,8 @@ class QueryEntry {
     #running = false;
     #requested = false;
     #desiredRevision = 0n;
-    #claimReady = Promise.resolve();
+    #claimError;
+    #claimPending = false;
     #offStatus;
     constructor(store, spec, key, cache) {
         this.store = store;
@@ -270,10 +271,22 @@ class QueryEntry {
                 for (const coverage of this.spec.coverage ?? []) {
                     claims.push(this.store.setWindowClaim(this.#owner, coverage.base, coverage.units));
                 }
-                this.#claimReady = Promise.all(claims).then(() => undefined);
-                // A render can lose its last subscriber before the read loop starts.
-                // The loop still observes the original rejection while it owns the claim.
-                void this.#claimReady.catch(() => undefined);
+                const generation = this.#generation;
+                this.#claimPending = claims.length > 0;
+                if (claims.length > 0)
+                    void Promise.all(claims).then(() => {
+                        if (generation !== this.#generation || this.#listeners.size === 0)
+                            return;
+                        this.#claimPending = false;
+                        if (!this.#running && this.#state.phase !== 'ready')
+                            this.#requestRead();
+                    }, (error) => {
+                        if (generation !== this.#generation || this.#listeners.size === 0)
+                            return;
+                        this.#claimPending = false;
+                        this.#claimError = errorOf(error);
+                        this.#requestRead();
+                    });
             }
             this.#requestRead();
         }
@@ -300,7 +313,8 @@ class QueryEntry {
         this.#running = false;
         this.#requested = false;
         this.#desiredRevision = 0n;
-        this.#claimReady = Promise.resolve();
+        this.#claimError = undefined;
+        this.#claimPending = false;
         this.#state = {
             rows: [],
             phase: 'loading',
@@ -393,9 +407,11 @@ class QueryEntry {
                 this.#requested = false;
                 if (this.store.availabilitySnapshot().state === 'blocked')
                     break;
-                await this.#claimReady;
                 if (generation !== this.#generation || this.#listeners.size === 0)
                     return;
+                if (this.#claimError !== undefined)
+                    throw this.#claimError;
+                const claimWasPending = this.#claimPending;
                 const snapshot = await this.store.client.querySnapshot({
                     sql: this.spec.sql,
                     ...(this.spec.params !== undefined
@@ -407,6 +423,8 @@ class QueryEntry {
                 });
                 if (generation !== this.#generation || this.#listeners.size === 0)
                     return;
+                if (this.#claimError !== undefined)
+                    throw this.#claimError;
                 const availability = this.store.availabilitySnapshot();
                 if (availability.state === 'blocked') {
                     this.#publish({
@@ -421,6 +439,10 @@ class QueryEntry {
                     this.#requested = true;
                     continue;
                 }
+                if (!snapshot.coverage.complete &&
+                    claimWasPending &&
+                    !this.#claimPending)
+                    this.#requested = true;
                 const mappedRows = this.spec.mapRow === undefined
                     ? snapshot.rows
                     : snapshot.rows.map(this.spec.mapRow);
@@ -736,6 +758,7 @@ export class ReactiveClientStore {
                 base,
                 claims: new Map(),
                 appliedKey: '',
+                appliedUnits: new Set(),
                 scheduled: false,
                 running: false,
                 requested: false,
@@ -744,6 +767,13 @@ export class ReactiveClientStore {
             this.#windowClaims.set(key, group);
         }
         group.claims.set(owner, new Set(units));
+        const alreadyHeld = units.length > 0 &&
+            units.every((unit) => group.appliedUnits.has(unit) &&
+                (group.inFlightUnits === undefined || group.inFlightUnits.has(unit)));
+        if (alreadyHeld) {
+            this.#scheduleWindow(group);
+            return Promise.resolve();
+        }
         const result = new Promise((resolve, reject) => {
             group?.waiters.push({ resolve, reject });
         });
@@ -779,16 +809,27 @@ export class ReactiveClientStore {
                 ].sort();
                 const key = canonicalValue(units);
                 if (key !== group.appliedKey) {
-                    await this.client.setWindow(group.base, units);
+                    group.inFlightUnits = new Set(units);
+                    try {
+                        await this.client.setWindow(group.base, units);
+                    }
+                    finally {
+                        group.inFlightUnits = undefined;
+                    }
                     if (this.#windowClaims.get(baseKey) !== group)
                         return;
                     group.appliedKey = key;
+                    group.appliedUnits = new Set(units);
                 }
             }
             for (const waiter of group.waiters.splice(0))
                 waiter.resolve();
         }
         catch (error) {
+            // A failed window edit may already have committed its first eviction chunk.
+            // Reconcile through the core before acknowledging any later claim.
+            group.appliedKey = '';
+            group.appliedUnits = new Set();
             for (const waiter of group.waiters.splice(0))
                 waiter.reject(error);
         }

package/dist/schema.js

@@ -234,6 +234,11 @@ function createFtsProjection(db, table, index) {
     const indexedColumns = index.columns.map(quoteIdent);
     const tokenizer = index.tokenize.replaceAll("'", "''");
     db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${quoteIdent(index.name)} USING fts5(${quoteIdent(FTS_SOURCE_ID_COLUMN)} UNINDEXED, ${indexedColumns.join(', ')}, tokenize='${tokenizer}')`);
+    const mapping = quoteIdent(`_syncular_fts_${index.name}`);
+    const mapped = db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
+        `_syncular_fts_${index.name}`,
+    ]).length > 0;
+    db.exec(`CREATE TABLE IF NOT EXISTS ${mapping} (id INTEGER PRIMARY KEY, source_id TEXT NOT NULL UNIQUE)`);
     const sourceId = `CAST(${quoteIdent(table.primaryKey)} AS TEXT)`;
     const newSourceId = `CAST(new.${quoteIdent(table.primaryKey)} AS TEXT)`;
     const oldSourceId = `CAST(old.${quoteIdent(table.primaryKey)} AS TEXT)`;
@@ -245,8 +250,9 @@ function createFtsProjection(db, table, index) {
         newSourceId,
         ...index.columns.map((column) => `new.${quoteIdent(column)}`),
     ].join(', ');
-    const deleteFor = (value) => `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} = ${value}`;
-    const deleteDisplaced = (select) => `DELETE FROM ${quoteIdent(index.name)} WHERE ${quoteIdent(FTS_SOURCE_ID_COLUMN)} IN (${select})`;
+    const deleteFor = (value) => `DELETE FROM ${quoteIdent(index.name)} WHERE rowid = (SELECT id FROM ${mapping} WHERE source_id = ${value}); DELETE FROM ${mapping} WHERE source_id = ${value}`;
+    const deleteDisplaced = (select) => `DELETE FROM ${quoteIdent(index.name)} WHERE rowid IN (SELECT id FROM ${mapping} WHERE source_id IN (${select})); DELETE FROM ${mapping} WHERE source_id IN (${select})`;
+    const insertNew = `INSERT INTO ${mapping}(source_id) VALUES (${newSourceId}); INSERT INTO ${quoteIdent(index.name)} (rowid, ${projectionColumns}) VALUES ((SELECT id FROM ${mapping} WHERE source_id = ${newSourceId}), ${newValues})`;
     // A clean insert cannot already have a projection row because the source
     // primary key is unique. Keep clean inserts linear by moving replacement
     // cleanup behind indexed source-table existence checks. SQLite does not
@@ -294,9 +300,9 @@ function createFtsProjection(db, table, index) {
         ...displacedByUnique.map(deleteDisplaced),
     ].join('; ');
     db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_bi`)} BEFORE INSERT ON ${quoteIdent(table.name)} WHEN ${insertGuardCondition} BEGIN ${insertGuardBody}; END`);
-    db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`);
+    db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_ai`)} AFTER INSERT ON ${quoteIdent(table.name)} BEGIN ${insertNew}; END`);
     db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_ad`)} AFTER DELETE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; END`);
-    db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) VALUES (${newValues}); END`);
+    db.exec(`CREATE TRIGGER ${quoteIdent(`${index.name}_au`)} AFTER UPDATE ON ${quoteIdent(table.name)} BEGIN ${deleteFor(oldSourceId)}; ${deleteFor(newSourceId)}; ${insertNew}; END`);
     // BEFORE UPDATE guard for `UPDATE OR REPLACE`: clear the projection of any
     // different-PK row about to be displaced through a secondary unique index by
     // the new values. Only meaningful when the table has a unique index; with
@@ -311,6 +317,10 @@ function createFtsProjection(db, table, index) {
     if (!existed) {
         db.exec(`INSERT INTO ${quoteIdent(index.name)} (${projectionColumns}) SELECT ${sourceId}, ${indexedColumns.join(', ')} FROM ${quoteIdent(table.name)}`);
     }
+    if (!mapped || !existed) {
+        db.exec(`DELETE FROM ${mapping}`);
+        db.exec(`INSERT INTO ${mapping}(id, source_id) SELECT rowid, ${quoteIdent(FTS_SOURCE_ID_COLUMN)} FROM ${quoteIdent(index.name)}`);
+    }
 }
 /**
  * Create only the application-owned synced tables, indexes, and FTS
@@ -424,6 +434,7 @@ export function dropAndRecreateSyncedTables(db, schema) {
        AND sql LIKE 'CREATE VIRTUAL TABLE%'
        AND name NOT LIKE '${RESERVED_TABLE_PREFIX}%'`);
     for (const row of virtualTables) {
+        db.exec(`DROP TABLE IF EXISTS ${quoteIdent(`_syncular_fts_${String(row.name)}`)}`);
         db.exec(`DROP TABLE IF EXISTS ${quoteIdent(String(row.name))}`);
     }
     const existing = db.query(`SELECT name FROM sqlite_master WHERE type = 'table'

package/dist/wasm-database.js

@@ -65,9 +65,9 @@ class WasmClientDatabase {
      * §5.3 image import at near-file-copy speed: attach an empty in-memory
      * schema, then `sqlite3_deserialize` the image bytes into it (the
      * documented sqlite-wasm import path — no OPFS round-trip, no SQL-level
-     * row shuttling before the single INSERT…SELECT the caller runs).
+     * row shuttling before the chunked INSERT…SELECT statements the caller runs).
      */
-    withSqliteImage(bytes, alias, fn) {
+    async withSqliteImage(bytes, alias, fn) {
         assertImageAlias(alias);
         const pointer = this.#db.pointer;
         if (pointer === undefined) {
@@ -84,7 +84,7 @@ class WasmClientDatabase {
             if (rc !== 0) {
                 throw new ClientSyncError('sync.invalid_request', `sqlite3_deserialize failed with code ${rc} (§5.3)`);
             }
-            return fn();
+            return await fn();
         }
         finally {
             this.exec(`DETACH ${alias}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@syncular/client",
-  "version": "0.20.0",
+  "version": "0.20.1",
   "description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
   "license": "Apache-2.0",
   "author": "Benjamin Kniffler",
@@ -89,10 +89,10 @@
   },
   "dependencies": {
     "@sqlite.org/sqlite-wasm": "^3.53.0-build1",
-    "@syncular/core": "0.20.0"
+    "@syncular/core": "0.20.1"
   },
   "devDependencies": {
-    "@syncular/server": "0.20.0",
+    "@syncular/server": "0.20.1",
     "playwright": "1.63.0"
   }
 }

package/src/apply.ts

@@ -215,8 +215,8 @@ export function deleteScopedRows(
  * scopes (same local-scope-column rule and fail-closed clause as
  * {@link deleteScopedRows}) EXCEPT rows whose primary key is in
  * `pinnedRowIds` (E1 — pinned by a still-pending outbox commit). Returns
- * `true` iff any pinned row was left behind, so the caller knows to defer
- * the rest of the eviction until the outbox drains. Also removes the
+ * whether unpinned work remains and whether any scoped rows remain. Each
+ * call removes at most 1,024 rows; the caller commits before yielding. Also removes the
  * evicted rows' `server_version` with them (E2 — no residual version
  * cache), which is automatic since the version column is per-row.
  */
@@ -225,9 +225,9 @@ export function evictScopedRows(
   table: CompiledClientTable,
   effective: ScopeMap,
   pinnedRowIds: ReadonlySet<string>,
-): boolean {
+): { remaining: boolean; deferred: boolean } {
   const entries = Object.entries(effective);
-  if (entries.length === 0) return false;
+  if (entries.length === 0) return { remaining: false, deferred: false };
   const clauses: string[] = [];
   const params: string[] = [];
   for (const [variable, values] of entries) {
@@ -238,7 +238,7 @@ export function evictScopedRows(
         `table ${JSON.stringify(table.name)} has no local scope-column mapping for ${JSON.stringify(variable)} (§4.8/§3.3 fail-closed)`,
       );
     }
-    if (values.length === 0) return false;
+    if (values.length === 0) return { remaining: false, deferred: false };
     clauses.push(
       `${quoteIdent(column)} IN (${values.map(() => '?').join(', ')})`,
     );
@@ -251,21 +251,25 @@ export function evictScopedRows(
     pinnedClause = ` AND ${pk} NOT IN (${ids.map(() => '?').join(', ')})`;
     params.push(...ids);
   }
+  const target = quoteIdent(table.name);
   db.exec(
-    `DELETE FROM ${quoteIdent(table.name)} WHERE ${clauses.join(' AND ')}${pinnedClause}`,
+    `DELETE FROM ${target} WHERE ${pk} IN (
+      SELECT ${pk} FROM ${target} WHERE ${clauses.join(' AND ')}${pinnedClause} LIMIT 1024
+    )`,
     params,
   );
-  if (pinnedRowIds.size === 0) return false;
-  // A pin still matters only if a pinned row actually falls inside this
-  // unit's effective scopes; check by re-selecting the survivors.
-  const survivors = db.query(
-    `SELECT ${pk} AS pk FROM ${quoteIdent(table.name)} WHERE ${clauses.join(' AND ')}`,
-    params.slice(0, params.length - pinnedRowIds.size),
-  );
-  for (const row of survivors) {
-    if (pinnedRowIds.has(String(row.pk))) return true;
-  }
-  return false;
+  const remaining =
+    db.query(
+      `SELECT 1 FROM ${target} WHERE ${clauses.join(' AND ')}${pinnedClause} LIMIT 1`,
+      params,
+    ).length > 0;
+  const deferred =
+    remaining ||
+    db.query(
+      `SELECT 1 FROM ${target} WHERE ${clauses.join(' AND ')} LIMIT 1`,
+      params.slice(0, params.length - pinnedRowIds.size),
+    ).length > 0;
+  return { remaining, deferred };
 }
 
 /** Descriptor fields a sqlite image is validated against (§5.3). */
@@ -286,14 +290,14 @@ function imageInvalid(detail: string): never {
 }
 
 /**
- * Apply a §5.3 sqlite-image segment in ONE local transaction: validate
+ * Apply a §5.3 sqlite image in committed chunks of at most 1,024 rows: validate
  * the in-file metadata against the descriptor, validate the data table's
  * column names/order against the generated schema, run the §5.6
- * first-page clear when fresh, then copy every row with a single
- * one primary-key upsert — `_syncular_version` lands in
+ * first-page clear when fresh, then bulk-copy each chunk through a
+ * primary-key upsert — `_syncular_version` lands in
  * `_sync_version` exactly like a rows segment's per-row `serverVersion`.
  */
-export function applySqliteSegment(
+export async function applySqliteSegment(
   db: ClientDatabase,
   schema: CompiledClientSchema,
   table: CompiledClientTable,
@@ -304,8 +308,9 @@ export function applySqliteSegment(
     readonly effective: ScopeMap;
     readonly transaction?: ApplyTransaction;
     readonly onProgress?: (rowsProcessed: number) => void;
+    readonly isCurrent?: () => boolean;
   },
-): number {
+): Promise<number> {
   const withImage = db.withSqliteImage?.bind(db);
   if (withImage === undefined) {
     throw new ClientSyncError(
@@ -316,7 +321,7 @@ export function applySqliteSegment(
   if (descriptor.table !== table.name) {
     imageInvalid(`descriptor table ${JSON.stringify(descriptor.table)}`);
   }
-  return withImage(bytes, IMAGE_ALIAS, () => {
+  return withImage(bytes, IMAGE_ALIAS, async () => {
     // 1. Metadata vs descriptor (§5.3 rule 2). A file that is not a
     //    SQLite database or lacks the metadata table fails right here.
     let meta: ReturnType<ClientDatabase['query']>;
@@ -371,38 +376,44 @@ export function applySqliteSegment(
       );
     }
 
-    // 3. One transaction: fresh-bootstrap clear, then primary-key upsert.
+    const primaryKeys = info.filter((column) => Number(column.pk) > 0);
+    if (primaryKeys.length !== 1 || primaryKeys[0]?.name !== table.primaryKey)
+      imageInvalid('image primary key does not match generated schema');
+    const source = `${IMAGE_ALIAS}.${quoteIdent(table.name)}`;
+    const count = Number(db.query(`SELECT count(*) AS n FROM ${source}`)[0]?.n);
+    if (count !== descriptor.rowCount)
+      imageInvalid('image row count does not match descriptor');
     const names = table.columns.map((column) => quoteIdent(column.name));
     const insertNames = [...names, quoteIdent(SYNC_VERSION_COLUMN)];
+    const primaryKey = quoteIdent(table.primaryKey);
     const updates = insertNames
-      .filter((name) => name !== quoteIdent(table.primaryKey))
+      .filter((name) => name !== primaryKey)
       .map((name) => `${name}=excluded.${name}`)
       .join(', ');
-    return (options.transaction ?? ((fn) => db.transaction(fn)))(() => {
-      if (options.clearFirst) {
-        deleteScopedRows(db, table, options.effective);
+    let after: SqlValue | undefined;
+    let applied = 0;
+    for (;;) {
+      if (options.isCurrent?.() === false) return applied;
+      const boundary = db.query(
+        `SELECT ${primaryKey} AS boundary FROM ${source}
+         ${after === undefined ? '' : `WHERE ${primaryKey} > ?`}
+         ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`,
+        after === undefined ? [] : [after],
+      )[0]?.boundary;
+      if (boundary === null) imageInvalid('image primary key must not be null');
+      const predicates: string[] = [];
+      const params: SqlValue[] = [];
+      if (after !== undefined) {
+        predicates.push(`${primaryKey} > ?`);
+        params.push(after);
       }
-      const source = `${IMAGE_ALIAS}.${quoteIdent(table.name)}`;
-      const primaryKey = quoteIdent(table.primaryKey);
-      let after: SqlValue | undefined;
-      let applied = 0;
-      for (;;) {
-        const boundary = db.query(
-          `SELECT ${primaryKey} AS boundary FROM ${source}
-           ${after === undefined ? '' : `WHERE ${primaryKey} > ?`}
-           ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`,
-          after === undefined ? [] : [after],
-        )[0]?.boundary;
-        const predicates: string[] = [];
-        const params: SqlValue[] = [];
-        if (after !== undefined) {
-          predicates.push(`${primaryKey} > ?`);
-          params.push(after);
-        }
-        if (boundary !== undefined) {
-          predicates.push(`${primaryKey} <= ?`);
-          params.push(boundary);
-        }
+      if (boundary !== undefined) {
+        predicates.push(`${primaryKey} <= ?`);
+        params.push(boundary);
+      }
+      (options.transaction ?? ((fn) => db.transaction(fn)))(() => {
+        if (after === undefined && options.clearFirst)
+          deleteScopedRows(db, table, options.effective);
         db.exec(
           `INSERT INTO ${quoteIdent(table.name)} (${insertNames.join(', ')})
            SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')}
@@ -411,18 +422,15 @@ export function applySqliteSegment(
           params,
         );
         applied += Number(db.query('SELECT changes() AS n')[0]?.n ?? 0);
-        if (boundary === undefined) break;
-        options.onProgress?.(applied);
-        after = boundary;
-      }
-      if (applied !== descriptor.rowCount) {
-        imageInvalid(
-          `image holds ${applied} rows, descriptor says ${descriptor.rowCount}`,
-        );
-      }
+      });
       options.onProgress?.(applied);
-      return applied;
-    });
+      await yieldToHost();
+      if (boundary === undefined || applied === count) break;
+      after = boundary;
+    }
+    if (applied !== count)
+      imageInvalid('image applied row count does not match descriptor');
+    return applied;
   });
 }
 
@@ -444,6 +452,7 @@ export async function applyRowsSegment(
     readonly effective: ScopeMap;
     readonly transaction?: ApplyTransaction;
     readonly onProgress?: (rowsProcessed: number) => void;
+    readonly isCurrent?: () => boolean;
   },
   encryption?: EncryptionConfig,
 ): Promise<number> {
@@ -453,6 +462,7 @@ export async function applyRowsSegment(
   const blocks: readonly (readonly SegmentRow[])[] =
     segment.blocks.length > 0 ? segment.blocks : [[]];
   for (const block of blocks) {
+    if (options.isCurrent?.() === false) break;
     // §5.11: decrypt this block's rows before opening the sync transaction
     // (WebCrypto is async; the SQLite transaction is not). Decrypt failure
     // aborts before any write in this block.
@@ -467,6 +477,7 @@ export async function applyRowsSegment(
           : row.values;
       rows.push({ values, serverVersion: row.serverVersion });
     }
+    if (options.isCurrent?.() === false) break;
     const clearThisBlock = first && options.clearFirst;
     first = false;
     if (!clearThisBlock && rows.length === 0) continue;
@@ -480,7 +491,22 @@ export async function applyRowsSegment(
         if (applied % 1024 === 0) options.onProgress?.(applied);
       }
     });
+    options.onProgress?.(applied);
+    await yieldToHost();
   }
   options.onProgress?.(applied);
   return applied;
 }
+
+/** Let worker messages run only after the current SQLite transaction commits. */
+export function yieldToHost(): Promise<void> {
+  return new Promise((resolve) => {
+    const channel = new MessageChannel();
+    channel.port1.onmessage = () => {
+      channel.port1.close();
+      channel.port2.close();
+      resolve();
+    };
+    channel.port2.postMessage(null);
+  });
+}

package/src/bun-database.ts

@@ -69,7 +69,11 @@ export class BunClientDatabase implements ClientDatabase {
    * §5.3 image import: bun:sqlite attaches files, not buffers, so the
    * image lands in a private temp file for the duration of the ATTACH.
    */
-  withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T): T {
+  async withSqliteImage<T>(
+    bytes: Uint8Array,
+    alias: string,
+    fn: () => T | Promise<T>,
+  ): Promise<T> {
     assertImageAlias(alias);
     const dir = mkdtempSync(join(tmpdir(), 'syncular-image-'));
     const path = join(dir, 'segment.db');
@@ -77,7 +81,7 @@ export class BunClientDatabase implements ClientDatabase {
       writeFileSync(path, bytes);
       this.db.run(`ATTACH DATABASE ? AS ${alias}`, [path]);
       try {
-        return fn();
+        return await fn();
       } finally {
         this.db.run(`DETACH DATABASE ${alias}`);
       }