@syncular/client 0.78.1 → 0.78.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/apply.d.ts CHANGED
@@ -94,6 +94,10 @@ export declare function applySqliteSegment(db: ClientDatabase, schema: CompiledC
94
94
  * to the next (§5.3).
95
95
  */
96
96
  readonly chunkRows?: Map<string, number>;
97
+ /** Rows of this image an earlier attempt committed (§5.6). */
98
+ readonly appliedRows?: number | undefined;
99
+ /** Runs inside each chunk's transaction with the rows applied so far. */
100
+ readonly onChunk?: (appliedRows: number) => void;
97
101
  }): Promise<number>;
98
102
  /**
99
103
  * Apply a decoded rows segment: each block in one local transaction
package/dist/apply.js CHANGED
@@ -362,8 +362,9 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
362
362
  }
363
363
  return [boundary, predicates.join(' AND ') || 'true', params];
364
364
  };
365
- let [boundary, where, params] = range(undefined, 1024);
366
- let applied = 0;
365
+ const resumed = options.appliedRows ?? 0;
366
+ let [boundary, where, params] = range(resumed > 0 ? range(undefined, resumed)[0] : undefined, 1024);
367
+ let applied = resumed;
367
368
  // §5.3: a table's first chunk holds 1,024 rows; a later range image
368
369
  // starts at the size its previous range reached. A later chunk holds at least
369
370
  // 1,024 rows, at most twice the rows of the one before, at most 65,536,
@@ -371,7 +372,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
371
372
  // to its COMMIT; a replaying chunk holds one statement.
372
373
  for (let size = (!options.clearFirst && options.chunkRows?.get(table.name)) || 1024;;) {
373
374
  if (options.isCurrent?.() === false)
374
- return applied;
375
+ return applied - resumed;
375
376
  const started = now();
376
377
  let chunk = 0;
377
378
  const first = applied === 0;
@@ -399,7 +400,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
399
400
  const skipped = new Set();
400
401
  const landing = rows.filter((row) => lands(db, table, String(row[table.primaryKey]), Number(row._syncular_version), kept, skipped));
401
402
  for (;;) {
402
- db.exec(`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates} WHERE excluded.${version}>${quoteIdent(table.name)}.${version}`, params);
403
+ db.exec(`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} AS image WHERE ${where} AND NOT EXISTS(SELECT 1 FROM ${quoteIdent(table.name)} AS held WHERE held.${primaryKey}=image.${primaryKey} AND held.${version}>=image._syncular_version) ON CONFLICT(${primaryKey})DO UPDATE SET ${updates}`, params);
403
404
  const copied = Number(db.query(`SELECT count(*)AS n FROM ${source} WHERE ${where}`, params)[0]?.n);
404
405
  applied += copied;
405
406
  chunk += copied;
@@ -418,6 +419,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
418
419
  rowValueToJson(row[column.name]),
419
420
  ])), Number(row._syncular_version));
420
421
  }
422
+ options.onChunk?.(applied);
421
423
  return rebaseScope(scope, kept, skipped);
422
424
  });
423
425
  const duration = now() - started;
@@ -432,7 +434,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
432
434
  imageInvalid('image applied row count does not match descriptor');
433
435
  // A file replica's WAL keeps its high-water size until truncated.
434
436
  db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
435
- return applied;
437
+ return applied - resumed;
436
438
  });
437
439
  }
438
440
  /**
package/dist/blob.js CHANGED
@@ -8,7 +8,7 @@ export const BLOB_COMMIT_REFS = '_syncular_blob_commit_refs';
8
8
  */
9
9
  import { parseBlobRef, serializeBlobRef } from '@syncular/core';
10
10
  import { ClientSyncError } from './errors.js';
11
- import { quoteIdent } from './schema.js';
11
+ import { quoteIdent, tableExists } from './schema.js';
12
12
  /** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
13
13
  export async function computeBlobId(bytes) {
14
14
  const input = bytes.buffer instanceof ArrayBuffer
@@ -21,8 +21,7 @@ export async function computeBlobId(bytes) {
21
21
  return `sha256:${hex}`;
22
22
  }
23
23
  export function ensureBlobSchema(db) {
24
- const existing = db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name='_syncular_blobs'");
25
- if (existing.length > 0) {
24
+ if (tableExists(db, '_syncular_blobs')) {
26
25
  const columns = db
27
26
  .query('PRAGMA table_info(_syncular_blobs)')
28
27
  .map((row) => row.name)
package/dist/client.d.ts CHANGED
@@ -187,7 +187,8 @@ export interface SyncClientConfig {
187
187
  * `transport.timeout` (`causeKind` `heartbeat`).
188
188
  */
189
189
  readonly realtimeHeartbeatMs?: number;
190
- /** Timer seam for the heartbeat watchdog; defaults to `setTimeout`. */
190
+ /** Timer seam for the heartbeat watchdog and the `syncUntilIdle` round
191
+ * bound; defaults to `setTimeout`. */
191
192
  readonly schedule?: (callback: () => void, delayMs: number) => () => void;
192
193
  /** Stable per-device id (§1.5); defaults to a persisted random UUID. */
193
194
  readonly clientId?: string;
package/dist/client.js CHANGED
@@ -30,7 +30,7 @@ import { appendOutboxCommit, chainsOn, deleteOutboxCommit, encodeOutboxCommit, l
30
30
  import { activeFailureRecords, isStoredOperation, listCommitOutcomes, persistCommitOutcomeResolution, persistedCommitOutcome, pruneCommitOutcomes, commitOutcome as readCommitOutcome, recordCommitOutcome, scrubPurgedOutcomes, } from './outcomes.js';
31
31
  import { buildPreviousVersionAudit, capturePreviousVersion, clearPreviousVersionAudit, clearPreviousVersionRefusal, dropPreviousVersionContainer, loadLocalSchemaDescriptor, PREVIOUS_VERSION_CONTAINER_NAME, PREVIOUS_VERSION_MAX_LIMIT, readPreviousVersionContainer, readPreviousVersionRows, setLocalSchemaDescriptor, storedPreviousVersionAudit, storedPreviousVersionRefusal, withPreviousVersionContainer, writePreviousVersionAudit, writePreviousVersionRefusal, } from './previous-version.js';
32
32
  import { bytesToHex, compileClientSchema, hexToBytes, matchesLocalScopes, resetLocalReplica, ensureLocalBookkeepingSchema, ensureLocalSyncedSchema, fromSqlValue, coerceSqlRepresentation, localColumnType, jsonToRowValue, LOCAL_SCHEMA_VERSION_KEY, normalizeRecordKeys, OPTIMISTIC_VERSION, quoteIdent, recordToRowValues, rowValueToJson, SEALED_COLUMN, SEALED_KEY_COLUMN, SYNC_VERSION_COLUMN, stripSyncColumns, toSqlValue, } from './schema.js';
33
- import { bootstrapping, bumpLocalRevision, covers, deleteUnit, ECHO_DIGEST_KEY, getLocalRevision, getMeta, getUnit, loadUnits, META_TABLE, owning, pairScopes, pruneUnknownUnits, REGISTERED_DIGEST_KEY, REPLICA_CURSOR_KEY, replicaCursor, saveUnit, saveUnitScopeSchema, setMeta, unitOwners, } from './state.js';
33
+ import { bootstrapping, bumpLocalRevision, covers, deleteUnit, ECHO_DIGEST_KEY, getLocalRevision, getMeta, getUnit, loadUnits, META_TABLE, owning, pairScopes, pruneUnknownUnits, recordAppliedRows, REGISTERED_DIGEST_KEY, REPLICA_CURSOR_KEY, replicaCursor, saveUnit, saveUnitScopeSchema, setMeta, unitOwners, } from './state.js';
34
34
  import { deletePendingEviction, loadPendingEvictions, loadWindowUnits, savePendingEviction, windowBaseKey, windowRefs, WINDOWS_TABLE, windowUnitScopes, } from './window.js';
35
35
  /** Stable client-local error while protected operations are preflight-gated. */
36
36
  export const SECURITY_PREFLIGHT_REQUIRED_CODE = 'client.security_preflight_required';
@@ -71,8 +71,6 @@ async function sha256Hex(bytes) {
71
71
  }
72
72
  const invalidHostResponse = (message) => new ClientSyncError('client.invalid_host_response', message);
73
73
  const outcomeNotFound = () => new ClientSyncError('sync.outcome_not_found', 'no durable outcome exists for the commit');
74
- /** The position a unit presents in a pull (§5.6 applied-segment rule). */
75
- const position = (unit) => JSON.stringify([unit.id, unit.cursor, unit.bootstrapState]);
76
74
  function emptySummary(pushed) {
77
75
  return {
78
76
  pushed,
@@ -198,13 +196,10 @@ export class SyncClient {
198
196
  #pendingRound;
199
197
  #needsPull = false;
200
198
  #progress = new ProgressEmitter();
201
- /**
202
- * §5.6 applied-segment rule: unit position → the segments applied in
203
- * sections whose SUB_END did not persist.
204
- */
205
- #appliedSegments = new Map();
206
199
  /** §5.3: a table's next image chunk size, carried across its ranges. */
207
200
  #chunkRows = new Map();
201
+ /** §7.8: restarts the running drive's round bound when rows apply. */
202
+ #roundProgressed;
208
203
  #progressAttempt = 0;
209
204
  /**
210
205
  * True from the synchronous entry of `sync()` until its serialized round
@@ -3345,26 +3340,16 @@ export class SyncClient {
3345
3340
  deferredCommits: 0,
3346
3341
  schemaFloor: undefined,
3347
3342
  };
3348
- const result = (budgetExhausted) => ({
3349
- pushed: aggregate.pushed,
3350
- applied: aggregate.applied,
3351
- rejected: aggregate.rejected,
3352
- retryable: aggregate.retryable,
3353
- conflicts: aggregate.conflicts,
3354
- commitsApplied: aggregate.commitsApplied,
3355
- segmentRowsApplied: aggregate.segmentRowsApplied,
3356
- bootstrapping: aggregate.bootstrapping,
3357
- resets: aggregate.resets,
3358
- revoked: aggregate.revoked,
3359
- failed: aggregate.failed,
3360
- ...(aggregate.deferredCommits > 0
3361
- ? { deferredCommits: aggregate.deferredCommits }
3362
- : {}),
3363
- ...(aggregate.schemaFloor !== undefined
3364
- ? { schemaFloor: aggregate.schemaFloor }
3365
- : {}),
3366
- budgetExhausted,
3367
- });
3343
+ const result = (budgetExhausted) => {
3344
+ const { deferredCommits, schemaFloor, ...summary } = aggregate;
3345
+ return {
3346
+ ...summary,
3347
+ ...(deferredCommits > 0 ? { deferredCommits } : {}),
3348
+ ...(schemaFloor !== undefined ? { schemaFloor } : {}),
3349
+ budgetExhausted,
3350
+ };
3351
+ };
3352
+ const states = () => JSON.stringify(loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState]));
3368
3353
  const startedAtMs = this.#now();
3369
3354
  let completed = false;
3370
3355
  for (let round = 0; round < budget; round++) {
@@ -3372,57 +3357,69 @@ export class SyncClient {
3372
3357
  // like an exhausted round budget; the next drive resumes.
3373
3358
  if (completed && this.#now() - startedAtMs >= timeoutMs)
3374
3359
  return result(true);
3375
- const before = JSON.stringify(loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState]));
3376
- // Each round is bounded by `timeoutMs` on its own. One signal reaches
3377
- // every transport call of the round.
3360
+ const before = states();
3361
+ // Each round is bounded by `timeoutMs` from its start or its last
3362
+ // applied rows. One signal reaches every transport call of the round.
3378
3363
  const controller = new AbortController();
3379
- let timer;
3364
+ let cancel;
3380
3365
  const sync = this.#sync(controller.signal);
3381
3366
  const last = await (timeoutMs === Number.POSITIVE_INFINITY
3382
3367
  ? sync
3383
3368
  : Promise.race([
3384
3369
  sync,
3385
3370
  new Promise((_, reject) => {
3386
- timer = setTimeout(() => {
3387
- const error = new ClientSyncError('sync.idle_timeout', 'syncUntilIdle exceeded its deadline', true, { timeoutMs, rounds: round });
3388
- // The transport may never answer; release the round so the
3389
- // client can run another one after the caller sees the error.
3390
- controller.abort(error);
3391
- this.#abortPendingRound('syncUntilIdle deadline exceeded');
3392
- reject(error);
3393
- }, timeoutMs);
3371
+ this.#roundProgressed = () => {
3372
+ cancel?.();
3373
+ cancel = this.#schedule(() => {
3374
+ const error = new ClientSyncError('sync.idle_timeout', 'syncUntilIdle exceeded its deadline', true, { timeoutMs, rounds: round });
3375
+ // The transport may never answer; release the round so the
3376
+ // client can run another one after the caller sees the error.
3377
+ controller.abort(error);
3378
+ this.#abortPendingRound(error);
3379
+ reject(error);
3380
+ }, timeoutMs);
3381
+ };
3382
+ this.#roundProgressed();
3394
3383
  }),
3395
- ])).finally(() => clearTimeout(timer));
3384
+ ])).finally(() => {
3385
+ cancel?.();
3386
+ this.#roundProgressed = undefined;
3387
+ });
3396
3388
  completed = true;
3397
3389
  // Historical counters accumulate; current readiness (`bootstrapping`,
3398
3390
  // `deferredCommits`, `schemaFloor`) describes the latest round.
3399
- aggregate.pushed += last.pushed;
3400
- aggregate.applied.push(...last.applied);
3401
- aggregate.rejected.push(...last.rejected);
3402
- aggregate.retryable.push(...last.retryable);
3391
+ for (const key of [
3392
+ 'pushed',
3393
+ 'commitsApplied',
3394
+ 'segmentRowsApplied',
3395
+ ])
3396
+ aggregate[key] += last[key];
3397
+ for (const key of [
3398
+ 'applied',
3399
+ 'rejected',
3400
+ 'retryable',
3401
+ 'resets',
3402
+ 'revoked',
3403
+ 'failed',
3404
+ ])
3405
+ aggregate[key].push(...last[key]);
3403
3406
  aggregate.conflicts.push(...last.conflicts);
3404
- aggregate.commitsApplied += last.commitsApplied;
3405
- aggregate.segmentRowsApplied += last.segmentRowsApplied;
3406
3407
  aggregate.bootstrapping = [...last.bootstrapping];
3407
- aggregate.resets.push(...last.resets);
3408
- aggregate.revoked.push(...last.revoked);
3409
- aggregate.failed.push(...last.failed);
3410
3408
  aggregate.deferredCommits = last.deferredCommits ?? 0;
3411
3409
  aggregate.schemaFloor = last.schemaFloor;
3412
3410
  // A disabled transport or a schema floor ends the loop, not the budget.
3413
- if (!this.#transportEnabled || last.schemaFloor !== undefined)
3411
+ if (!this.#transportEnabled || aggregate.schemaFloor)
3414
3412
  return result(false);
3415
3413
  if (maxRounds === undefined &&
3416
3414
  last.segmentRowsApplied > 0 &&
3417
3415
  last.bootstrapping.length > 0 &&
3418
- before !==
3419
- JSON.stringify(loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState])))
3416
+ before !== states())
3420
3417
  round = -1;
3421
3418
  if (last.commitsApplied === 0 &&
3422
3419
  last.segmentRowsApplied === 0 &&
3423
3420
  last.bootstrapping.length === 0 &&
3424
3421
  last.resets.length === 0 &&
3425
- (last.deferredCommits ?? 0) === 0 &&
3422
+ !aggregate.deferredCommits &&
3426
3423
  !this.#needsPull) {
3427
3424
  return result(false);
3428
3425
  }
@@ -3918,7 +3915,7 @@ export class SyncClient {
3918
3915
  else if (frame.type === 'SEGMENT_REF' &&
3919
3916
  open !== undefined &&
3920
3917
  this.#config.segments !== undefined &&
3921
- !this.#appliedSegments.get(position(open))?.has(frame.segmentId)) {
3918
+ open.applied?.[frame.segmentId] !== frame.rowCount) {
3922
3919
  queued.push([frame, open.scopes]);
3923
3920
  }
3924
3921
  }
@@ -4022,15 +4019,17 @@ export class SyncClient {
4022
4019
  image.rowCursor !== undefined ||
4023
4020
  image.nextRowCursor !== undefined))
4024
4021
  throw invalidRequest('unadvertised or paged sqlite segment (§4.2, §5.3)');
4025
- const applied = this.#appliedSegments.get(position(section.unit)) ??
4026
- new Set();
4027
- if (frame.type === 'SEGMENT_REF' && applied.has(frame.segmentId)) {
4028
- // §5.6: applied in an earlier attempt at this position.
4029
- section.seeded.add(frame.table);
4030
- break;
4031
- }
4032
4022
  let bytes;
4023
+ let appliedRows;
4033
4024
  if (frame.type === 'SEGMENT_REF') {
4025
+ // §5.6: rows an earlier attempt at this position applied. A
4026
+ // partly applied image already ran its table's first-page clear.
4027
+ appliedRows = section.unit.applied?.[frame.segmentId];
4028
+ if (appliedRows !== undefined) {
4029
+ section.seeded.add(frame.table);
4030
+ if (appliedRows === frame.rowCount)
4031
+ break;
4032
+ }
4034
4033
  this.#progress.update({
4035
4034
  ...NO_PAYLOAD,
4036
4035
  phase: 'download',
@@ -4083,11 +4082,15 @@ export class SyncClient {
4083
4082
  ? image.rowCount > 0
4084
4083
  : segment.blocks.some((block) => block.length > 0), resetEpoch, image !== undefined);
4085
4084
  return image !== undefined
4086
- ? applySqliteSegment(this.#db, this.#schema, table, bytes, image, options)
4085
+ ? applySqliteSegment(this.#db, this.#schema, table, bytes, image, {
4086
+ ...options,
4087
+ appliedRows,
4088
+ onChunk: (rows) => recordAppliedRows(this.#db, unitId, image.segmentId, rows),
4089
+ })
4087
4090
  : applyRowsSegment(this.#db, this.#schema, table, segment, options, this.#encryption);
4088
4091
  });
4089
4092
  if (frame.type === 'SEGMENT_REF' && !section.skip)
4090
- this.#appliedSegments.set(position(section.unit), applied.add(frame.segmentId));
4093
+ recordAppliedRows(this.#db, unitId, frame.segmentId, frame.rowCount);
4091
4094
  break;
4092
4095
  }
4093
4096
  case 'SUB_END': {
@@ -4098,7 +4101,6 @@ export class SyncClient {
4098
4101
  !section.skip &&
4099
4102
  this.#localResetEpoch === resetEpoch) {
4100
4103
  this.#finishSection(section, frame, summary);
4101
- this.#appliedSegments.delete(position(section.unit));
4102
4104
  }
4103
4105
  section = undefined;
4104
4106
  // The counters stay: `complete` and `failed` report them.
@@ -4504,7 +4506,10 @@ export class SyncClient {
4504
4506
  effective,
4505
4507
  owners,
4506
4508
  isCurrent: () => this.#localResetEpoch === resetEpoch,
4507
- onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
4509
+ onProgress: (rowsProcessed) => {
4510
+ this.#progress.update({ rowsProcessed });
4511
+ this.#roundProgressed?.();
4512
+ },
4508
4513
  transaction: (fn) => {
4509
4514
  const started = performance.now();
4510
4515
  let written = started;
@@ -1,6 +1,6 @@
1
1
  import { ClientSyncError } from './errors.js';
2
2
  import { INVALID_HOST_RESPONSE_CODE } from './local-rebootstrap.js';
3
- import { fromSqlValue, jsonToRowValue, localColumnType, quoteIdent, rowValueToJson, } from './schema.js';
3
+ import { fromSqlValue, jsonToRowValue, localColumnType, quoteIdent, rowValueToJson, tableExists, } from './schema.js';
4
4
  import { getMeta, setMeta, META_TABLE } from './state.js';
5
5
  /** `_syncular_meta` key holding the persisted {@link LocalSchemaDescriptor}. */
6
6
  const LOCAL_SCHEMA_DESCRIPTOR_KEY = 'localSchemaDescriptor';
@@ -228,8 +228,7 @@ export function withPreviousVersionContainer(db, fn, removeFile = false) {
228
228
  }
229
229
  /** Read the container's own metadata record; `undefined` when absent/corrupt. */
230
230
  export function readPreviousVersionContainer(db) {
231
- const meta = db.query("SELECT 1 AS present FROM sqlite_master WHERE type='table' AND name=?", [CONTAINER_META_TABLE]);
232
- if (meta.length === 0)
231
+ if (!tableExists(db, CONTAINER_META_TABLE))
233
232
  return undefined;
234
233
  const raw = db.query(`SELECT record FROM ${quoteIdent(CONTAINER_META_TABLE)} WHERE id=1`)[0]?.record;
235
234
  if (raw === undefined)
package/dist/schema.d.ts CHANGED
@@ -110,6 +110,7 @@ export declare function matchesLocalScopes(table: CompiledClientTable, scopes: R
110
110
  */
111
111
  export declare function pkSqlValue(table: CompiledClientTable, rowId: string): SqlValue;
112
112
  export declare function quoteIdent(name: string): string;
113
+ export declare function tableExists(db: ClientDatabase, name: string): boolean;
113
114
  /**
114
115
  * §5.11: the app-side type of a column for local (plaintext) storage. For an
115
116
  * encrypted column this is `declaredType` — the local mirror stays plaintext,
package/dist/schema.js CHANGED
@@ -216,6 +216,11 @@ export function pkSqlValue(table, rowId) {
216
216
  export function quoteIdent(name) {
217
217
  return `"${name.replaceAll('"', '""')}"`;
218
218
  }
219
+ export function tableExists(db, name) {
220
+ return (db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
221
+ name,
222
+ ]).length > 0);
223
+ }
219
224
  /**
220
225
  * §5.11: the app-side type of a column for local (plaintext) storage. For an
221
226
  * encrypted column this is `declaredType` — the local mirror stays plaintext,
@@ -288,21 +293,22 @@ const FTS_SOURCE_ID_COLUMN = '_syncular_source_id';
288
293
  * Limitation: because a BEFORE trigger runs before SQLite resolves the
289
294
  * conflict, an `OR IGNORE` write whose row is dropped still executes these
290
295
  * displacement DELETEs, transiently removing the surviving row's projection
291
- * entry. Syncular's mirror writes never use `OR IGNORE`; see the guard-block
292
- * comment below for the full rationale.
296
+ * entry. The same holds for an upsert whose `DO UPDATE ... WHERE` skips the
297
+ * row, so Syncular's mirror writes never offer a row that will not replace
298
+ * (§5.3); see the guard-block comment below for the full rationale. An open
299
+ * rebuilds a projection whose mapping holds a different row count than its
300
+ * source table.
293
301
  */
294
302
  function createFtsProjection(db, table, index) {
295
303
  const projection = quoteIdent(index.name);
296
304
  const sourceTable = quoteIdent(table.name);
297
305
  const primaryKey = quoteIdent(table.primaryKey);
298
- const existed = db.query("SELECT 1 AS present FROM sqlite_master WHERE type='table' AND name=?", [index.name]).length > 0;
306
+ const existed = tableExists(db, index.name);
299
307
  const indexedColumns = index.columns.map(quoteIdent);
300
308
  const tokenizer = index.tokenize.replaceAll("'", "''");
301
309
  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${projection} USING fts5(${quoteIdent(FTS_SOURCE_ID_COLUMN)} UNINDEXED,${indexedColumns.join(', ')},tokenize='${tokenizer}')`);
302
310
  const mapping = quoteIdent(`_syncular_fts_${index.name}`);
303
- const mapped = db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
304
- `_syncular_fts_${index.name}`,
305
- ]).length > 0;
311
+ const mapped = tableExists(db, `_syncular_fts_${index.name}`);
306
312
  db.exec(`CREATE TABLE IF NOT EXISTS ${mapping}(id INTEGER PRIMARY KEY,source_id TEXT NOT NULL UNIQUE)`);
307
313
  const sourceId = `CAST(${primaryKey} AS TEXT)`;
308
314
  const newSourceId = `CAST(new.${primaryKey} AS TEXT)`;
@@ -332,16 +338,19 @@ function createFtsProjection(db, table, index) {
332
338
  // `UPDATE OR REPLACE` needs no BEFORE guard: the new pk equals the displaced
333
339
  // row's pk, so the AFTER UPDATE delete of the new source id already clears it.
334
340
  //
335
- // Known limitation — `OR IGNORE`: a BEFORE trigger fires before SQLite
336
- // resolves the conflict, and SQLite exposes no signal for the eventual
337
- // resolution. So an `INSERT OR IGNORE` / `UPDATE OR IGNORE` whose row is
338
- // dropped on conflict still runs these displacement DELETEs, removing the
339
- // SURVIVING row's projection entry (a false negative until that row is next
340
- // rewritten). Moving cleanup to AFTER triggers would dodge the no-op but
341
- // reintroduce the REPLACE-doesn't-fire-DELETE gap and a full-projection scan
342
- // per write, so the guards stay BEFORE. Syncular's own mirror writes use
343
- // `INSERT … ON CONFLICT (pk) DO UPDATE` (apply.ts), which never takes the
344
- // IGNORE path; only hand-written `OR IGNORE` against a mirror table is exposed.
341
+ // Known limitation: a BEFORE trigger fires before SQLite resolves the
342
+ // conflict, and SQLite exposes no signal for the eventual resolution. So an
343
+ // `INSERT OR IGNORE` / `UPDATE OR IGNORE` whose row is dropped on conflict,
344
+ // and an upsert whose `DO UPDATE ... WHERE` skips the row, still run these
345
+ // displacement DELETEs, removing the SURVIVING row's projection entry (a
346
+ // false negative until that row is next rewritten). Moving cleanup to AFTER
347
+ // triggers would dodge the no-op but reintroduce the REPLACE-doesn't-fire-
348
+ // DELETE gap and a full-projection scan per write, so the guards stay
349
+ // BEFORE. Syncular's own mirror writes use an unconditional
350
+ // `INSERT … ON CONFLICT (pk) DO UPDATE` (apply.ts); the image copy filters
351
+ // held rows at the same or a newer version out of its SELECT instead of a
352
+ // `DO UPDATE ... WHERE`. Only hand-written `OR IGNORE` against a mirror
353
+ // table is exposed.
345
354
  for (const suffix of ['bi', 'ai', 'ad', 'au', 'bu']) {
346
355
  db.exec(`DROP TRIGGER IF EXISTS ${quoteIdent(`${index.name}_${suffix}`)}`);
347
356
  }
@@ -380,10 +389,16 @@ function createFtsProjection(db, table, index) {
380
389
  const updateGuardBody = displacedByUnique.map(deleteDisplaced).join('; ');
381
390
  createTrigger('bu', 'BEFORE UPDATE', updateGuardBody, updateGuardCondition);
382
391
  }
383
- if (!existed) {
392
+ // Every source row has exactly one mapping row, so a count mismatch marks a
393
+ // projection an earlier release damaged; the open rebuilds it.
394
+ const stale = !existed ||
395
+ (mapped &&
396
+ Number(db.query(`SELECT (SELECT count(*) FROM ${mapping})=(SELECT count(*) FROM ${sourceTable}) AS whole`)[0]?.whole) !== 1);
397
+ if (stale) {
398
+ db.exec(`DELETE FROM ${projection}`);
384
399
  db.exec(`INSERT INTO ${projection}(${projectionColumns})SELECT ${sourceId},${indexedColumns.join(', ')} FROM ${sourceTable}`);
385
400
  }
386
- if (!mapped || !existed) {
401
+ if (!mapped || stale) {
387
402
  db.exec(`DELETE FROM ${mapping}`);
388
403
  db.exec(`INSERT INTO ${mapping}(id,source_id)SELECT rowid,${quoteIdent(FTS_SOURCE_ID_COLUMN)} FROM ${projection}`);
389
404
  }
@@ -438,7 +453,7 @@ export function ensureLocalBookkeepingSchema(db) {
438
453
  createTable(BLOB_COMMIT_REFS, 'commit_id TEXT NOT NULL,blob_id TEXT NOT NULL,PRIMARY KEY(commit_id,blob_id)');
439
454
  db.exec(`CREATE INDEX IF NOT EXISTS ${BLOB_COMMIT_REFS}_body ON ${BLOB_COMMIT_REFS}(blob_id)`);
440
455
  db.exec(`INSERT OR IGNORE INTO ${META_TABLE}(key,value)VALUES('localRevision','0')`);
441
- const outboxExisted = db.query(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='${OUTBOX_TABLE}'`).length > 0;
456
+ const outboxExisted = tableExists(db, OUTBOX_TABLE);
442
457
  createTable(OUTBOX_TABLE, 'seq INTEGER PRIMARY KEY AUTOINCREMENT,client_commit_id TEXT NOT NULL UNIQUE,created_at_ms INTEGER NOT NULL,operations TEXT NOT NULL');
443
458
  // §7.1: `outboxId` names the outbox sequence. A new outbox table starts
444
459
  // a new sequence, so it replaces the id in the same transaction.
@@ -463,6 +478,13 @@ export function ensureLocalBookkeepingSchema(db) {
463
478
  }
464
479
  db.exec(`CREATE INDEX IF NOT EXISTS ${OUTCOMES_TABLE}_resolution_seq ON ${OUTCOMES_TABLE}(resolution,seq)`);
465
480
  createTable(UNITS_TABLE, "id TEXT PRIMARY KEY,scopes TEXT NOT NULL,explicit INTEGER NOT NULL DEFAULT 0,status TEXT NOT NULL DEFAULT 'active',reason_code TEXT,cursor INTEGER DEFAULT -1,bootstrap_state TEXT,seeded TEXT NOT NULL DEFAULT '[]',echo TEXT");
481
+ // §5.6 applied-segment record, added after the table shipped.
482
+ try {
483
+ db.exec(`ALTER TABLE ${UNITS_TABLE} ADD COLUMN applied TEXT`);
484
+ }
485
+ catch {
486
+ // column already exists
487
+ }
466
488
  // §4.8 window registry: base + value → the unit it holds, the
467
489
  // completeness oracle and one of a unit's references (§4.1).
468
490
  createTable(WINDOWS_TABLE, 'base TEXT NOT NULL,unit TEXT NOT NULL,sub_id TEXT NOT NULL,PRIMARY KEY(base,unit)');
@@ -474,7 +496,7 @@ export function ensureLocalBookkeepingSchema(db) {
474
496
  // rows and pending evictions and registers its units fresh. Its
475
497
  // subscriptions, cursors, echoes, resume tokens and window entries go.
476
498
  const legacyEvictions = '_syncular_window_pending_evict';
477
- if (db.query(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='${legacyEvictions}'`).length > 0) {
499
+ if (tableExists(db, legacyEvictions)) {
478
500
  if (db.query(`SELECT 1 FROM ${legacyEvictions} WHERE NOT json_valid(effective_scopes) OR json_type(effective_scopes)<>'object' OR EXISTS(SELECT 1 FROM json_each(effective_scopes) WHERE json_type(value)<>'array' OR EXISTS(SELECT 1 FROM json_each(value) v WHERE v.type<>'text'))`).length > 0)
479
501
  throw new ClientSyncError('sync.local_corrupt', 'a version 3 pending eviction cannot be upgraded');
480
502
  db.exec(`INSERT OR REPLACE INTO ${PENDING_EVICTIONS} SELECT sub_id,tbl,effective_scopes FROM ${legacyEvictions}`);
package/dist/state.d.ts CHANGED
@@ -38,6 +38,9 @@ export interface UnitRecord {
38
38
  readonly seeded: readonly string[];
39
39
  /** The echo persisted at the last `SUB_END` (§4.4). */
40
40
  readonly echo?: readonly FencedRecord[] | undefined;
41
+ /** §5.6 applied-segment record: segment id → rows applied at the unit's
42
+ * current cursor and token. */
43
+ readonly applied?: Readonly<Record<string, number>> | undefined;
41
44
  }
42
45
  /** §4.1: a catching-up unit holding cursor `-1` or a resume token. */
43
46
  export declare function bootstrapping(unit: UnitRecord): boolean;
@@ -57,8 +60,12 @@ export declare function unitOwners(units: readonly UnitRecord[], table: Compiled
57
60
  /** Every unit in registration order (§4.1). */
58
61
  export declare function loadUnits(db: ClientDatabase): UnitRecord[];
59
62
  export declare function getUnit(db: ClientDatabase, id: string): UnitRecord | undefined;
60
- /** Insert or update in place: an update keeps the registration order. */
63
+ /** Insert or update in place: an update keeps the registration order. The
64
+ * §5.6 applied-segment record ends when the cursor or the token changes. */
61
65
  export declare function saveUnit(db: ClientDatabase, record: UnitRecord): void;
66
+ /** §5.6: `rows` rows of `segmentId` applied at the unit's position. Runs
67
+ * in the transaction that commits them. */
68
+ export declare function recordAppliedRows(db: ClientDatabase, id: string, segmentId: string, rows: number): void;
62
69
  export declare function deleteUnit(db: ClientDatabase, id: string): void;
63
70
  /**
64
71
  * §7.4.3 reset, §2.1 log reset and local rebootstrap: keep the unit
package/dist/state.js CHANGED
@@ -77,6 +77,7 @@ function rowToRecord(row) {
77
77
  const scopes = json(row.scopes);
78
78
  const seeded = json(row.seeded);
79
79
  const echo = row.echo === null ? undefined : json(row.echo);
80
+ const applied = row.applied === null ? undefined : json(row.applied);
80
81
  const { id, status, cursor } = row;
81
82
  if (typeof id !== 'string' ||
82
83
  !isMap(scopes) ||
@@ -84,6 +85,7 @@ function rowToRecord(row) {
84
85
  (status !== 'active' && status !== 'revoked' && status !== 'failed') ||
85
86
  (cursor !== null && !Number.isInteger(cursor)) ||
86
87
  !isStrings(seeded) ||
88
+ (applied !== undefined && !isMap(applied)) ||
87
89
  (echo !== undefined &&
88
90
  !(Array.isArray(echo) &&
89
91
  echo.every((record) => isMap(record) &&
@@ -109,6 +111,8 @@ function rowToRecord(row) {
109
111
  record.bootstrapState = bootstrapState;
110
112
  if (echo !== undefined)
111
113
  record.echo = echo;
114
+ if (applied !== undefined)
115
+ record.applied = applied;
112
116
  return record;
113
117
  }
114
118
  /** Every unit in registration order (§4.1). */
@@ -121,9 +125,10 @@ export function getUnit(db, id) {
121
125
  const row = db.query(`SELECT * FROM ${UNITS_TABLE} WHERE id=?`, [id])[0];
122
126
  return row === undefined ? undefined : rowToRecord(row);
123
127
  }
124
- /** Insert or update in place: an update keeps the registration order. */
128
+ /** Insert or update in place: an update keeps the registration order. The
129
+ * §5.6 applied-segment record ends when the cursor or the token changes. */
125
130
  export function saveUnit(db, record) {
126
- db.exec(`INSERT INTO ${UNITS_TABLE}(id,scopes,explicit,status,reason_code,cursor,bootstrap_state,seeded,echo)VALUES(?,?,?,?,?,?,?,?,?)ON CONFLICT(id)DO UPDATE SET explicit=excluded.explicit,status=excluded.status,reason_code=excluded.reason_code,cursor=excluded.cursor,bootstrap_state=excluded.bootstrap_state,seeded=excluded.seeded,echo=excluded.echo`, [
131
+ db.exec(`INSERT INTO ${UNITS_TABLE}(id,scopes,explicit,status,reason_code,cursor,bootstrap_state,seeded,echo)VALUES(?,?,?,?,?,?,?,?,?)ON CONFLICT(id)DO UPDATE SET explicit=excluded.explicit,status=excluded.status,reason_code=excluded.reason_code,cursor=excluded.cursor,bootstrap_state=excluded.bootstrap_state,seeded=excluded.seeded,echo=excluded.echo,applied=iif((cursor,bootstrap_state)IS(excluded.cursor,excluded.bootstrap_state),applied,NULL)`, [
127
132
  record.id,
128
133
  JSON.stringify(record.scopes),
129
134
  record.explicit ? 1 : 0,
@@ -135,6 +140,11 @@ export function saveUnit(db, record) {
135
140
  record.echo === undefined ? null : JSON.stringify(record.echo),
136
141
  ]);
137
142
  }
143
+ /** §5.6: `rows` rows of `segmentId` applied at the unit's position. Runs
144
+ * in the transaction that commits them. */
145
+ export function recordAppliedRows(db, id, segmentId, rows) {
146
+ db.exec(`UPDATE ${UNITS_TABLE} SET applied=json_patch(coalesce(applied,'{}'),?) WHERE id=?`, [JSON.stringify({ [segmentId]: rows }), id]);
147
+ }
138
148
  export function deleteUnit(db, id) {
139
149
  db.exec(`DELETE FROM ${UNITS_TABLE} WHERE id=?`, [id]);
140
150
  }
@@ -146,7 +156,7 @@ export function deleteUnit(db, id) {
146
156
  * kept unit is live. Caller owns the transaction.
147
157
  */
148
158
  export function resetUnits(db, keep = []) {
149
- db.exec(`UPDATE ${UNITS_TABLE} SET status='active',reason_code=NULL,cursor=-1,bootstrap_state=NULL,seeded='[]',echo=NULL WHERE id NOT IN(${keep.map(() => '?').join(',')})`, [...keep]);
159
+ db.exec(`UPDATE ${UNITS_TABLE} SET status='active',reason_code=NULL,cursor=-1,bootstrap_state=NULL,seeded='[]',echo=NULL,applied=NULL WHERE id NOT IN(${keep.map(() => '?').join(',')})`, [...keep]);
150
160
  db.exec(`DELETE FROM ${META_TABLE} WHERE key IN(?,?) OR key=? AND NOT EXISTS(SELECT 1 FROM ${UNITS_TABLE} WHERE cursor IS NULL)`, [ECHO_DIGEST_KEY, REGISTERED_DIGEST_KEY, REPLICA_CURSOR_KEY]);
151
161
  }
152
162
  /** §4.5: the replica cursor, `-1` while no unit is live. */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/client",
3
- "version": "0.78.1",
3
+ "version": "0.78.3",
4
4
  "description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -121,11 +121,11 @@
121
121
  },
122
122
  "dependencies": {
123
123
  "@sqlite.org/sqlite-wasm": "^3.53.0-build1",
124
- "@syncular/core": "0.78.1"
124
+ "@syncular/core": "0.78.3"
125
125
  },
126
126
  "devDependencies": {
127
127
  "@electric-sql/pglite": "^0.5.4",
128
- "@syncular/server": "0.78.1",
128
+ "@syncular/server": "0.78.3",
129
129
  "playwright": "1.63.0",
130
130
  "vite": "8.3.4"
131
131
  }
package/src/apply.ts CHANGED
@@ -526,6 +526,10 @@ export async function applySqliteSegment(
526
526
  * to the next (§5.3).
527
527
  */
528
528
  readonly chunkRows?: Map<string, number>;
529
+ /** Rows of this image an earlier attempt committed (§5.6). */
530
+ readonly appliedRows?: number | undefined;
531
+ /** Runs inside each chunk's transaction with the rows applied so far. */
532
+ readonly onChunk?: (appliedRows: number) => void;
529
533
  },
530
534
  ): Promise<number> {
531
535
  const now = options.now ?? (() => performance.now());
@@ -619,8 +623,12 @@ export async function applySqliteSegment(
619
623
  }
620
624
  return [boundary, predicates.join(' AND ') || 'true', params] as const;
621
625
  };
622
- let [boundary, where, params] = range(undefined, 1024);
623
- let applied = 0;
626
+ const resumed = options.appliedRows ?? 0;
627
+ let [boundary, where, params] = range(
628
+ resumed > 0 ? range(undefined, resumed)[0] : undefined,
629
+ 1024,
630
+ );
631
+ let applied = resumed;
624
632
  // §5.3: a table's first chunk holds 1,024 rows; a later range image
625
633
  // starts at the size its previous range reached. A later chunk holds at least
626
634
  // 1,024 rows, at most twice the rows of the one before, at most 65,536,
@@ -631,7 +639,7 @@ export async function applySqliteSegment(
631
639
  (!options.clearFirst && options.chunkRows?.get(table.name)) || 1024;
632
640
  ;
633
641
  ) {
634
- if (options.isCurrent?.() === false) return applied;
642
+ if (options.isCurrent?.() === false) return applied - resumed;
635
643
  const started = now();
636
644
  let chunk = 0;
637
645
  const first = applied === 0;
@@ -672,7 +680,7 @@ export async function applySqliteSegment(
672
680
  );
673
681
  for (;;) {
674
682
  db.exec(
675
- `INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates} WHERE excluded.${version}>${quoteIdent(table.name)}.${version}`,
683
+ `INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} AS image WHERE ${where} AND NOT EXISTS(SELECT 1 FROM ${quoteIdent(table.name)} AS held WHERE held.${primaryKey}=image.${primaryKey} AND held.${version}>=image._syncular_version) ON CONFLICT(${primaryKey})DO UPDATE SET ${updates}`,
676
684
  params,
677
685
  );
678
686
  const copied = Number(
@@ -711,6 +719,7 @@ export async function applySqliteSegment(
711
719
  Number(row._syncular_version),
712
720
  );
713
721
  }
722
+ options.onChunk?.(applied);
714
723
  return rebaseScope(scope, kept, skipped);
715
724
  });
716
725
  const duration = now() - started;
@@ -727,7 +736,7 @@ export async function applySqliteSegment(
727
736
  imageInvalid('image applied row count does not match descriptor');
728
737
  // A file replica's WAL keeps its high-water size until truncated.
729
738
  db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
730
- return applied;
739
+ return applied - resumed;
731
740
  });
732
741
  }
733
742
 
package/src/blob.ts CHANGED
@@ -10,7 +10,7 @@ import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
10
10
  import type { ClientDatabase } from './database';
11
11
  import { ClientSyncError } from './errors';
12
12
  import type { CompiledClientSchema } from './schema';
13
- import { quoteIdent } from './schema';
13
+ import { quoteIdent, tableExists } from './schema';
14
14
 
15
15
  /**
16
16
  * A blob download result (§5.9.5). The authorized endpoint either serves the
@@ -121,10 +121,7 @@ export async function computeBlobId(bytes: Uint8Array): Promise<string> {
121
121
  }
122
122
 
123
123
  export function ensureBlobSchema(db: ClientDatabase): void {
124
- const existing = db.query(
125
- "SELECT 1 FROM sqlite_master WHERE type='table' AND name='_syncular_blobs'",
126
- );
127
- if (existing.length > 0) {
124
+ if (tableExists(db, '_syncular_blobs')) {
128
125
  const columns = db
129
126
  .query('PRAGMA table_info(_syncular_blobs)')
130
127
  .map((row) => row.name)
package/src/client.ts CHANGED
@@ -258,6 +258,7 @@ import {
258
258
  owning,
259
259
  pairScopes,
260
260
  pruneUnknownUnits,
261
+ recordAppliedRows,
261
262
  REGISTERED_DIGEST_KEY,
262
263
  REPLICA_CURSOR_KEY,
263
264
  replicaCursor,
@@ -476,7 +477,8 @@ export interface SyncClientConfig {
476
477
  * `transport.timeout` (`causeKind` `heartbeat`).
477
478
  */
478
479
  readonly realtimeHeartbeatMs?: number;
479
- /** Timer seam for the heartbeat watchdog; defaults to `setTimeout`. */
480
+ /** Timer seam for the heartbeat watchdog and the `syncUntilIdle` round
481
+ * bound; defaults to `setTimeout`. */
480
482
  readonly schedule?: (callback: () => void, delayMs: number) => () => void;
481
483
  /** Stable per-device id (§1.5); defaults to a persisted random UUID. */
482
484
  readonly clientId?: string;
@@ -748,10 +750,6 @@ const outcomeNotFound = () =>
748
750
  'no durable outcome exists for the commit',
749
751
  );
750
752
 
751
- /** The position a unit presents in a pull (§5.6 applied-segment rule). */
752
- const position = (unit: UnitRecord) =>
753
- JSON.stringify([unit.id, unit.cursor, unit.bootstrapState]);
754
-
755
753
  interface OpenSection {
756
754
  readonly start: SubStartFrame;
757
755
  readonly unit: UnitRecord;
@@ -986,13 +984,10 @@ export class SyncClient {
986
984
  #pendingRound: PendingRound | undefined;
987
985
  #needsPull = false;
988
986
  readonly #progress = new ProgressEmitter();
989
- /**
990
- * §5.6 applied-segment rule: unit position → the segments applied in
991
- * sections whose SUB_END did not persist.
992
- */
993
- readonly #appliedSegments = new Map<string, Set<string>>();
994
987
  /** §5.3: a table's next image chunk size, carried across its ranges. */
995
988
  readonly #chunkRows = new Map<string, number>();
989
+ /** §7.8: restarts the running drive's round bound when rows apply. */
990
+ #roundProgressed: (() => void) | undefined;
996
991
  #progressAttempt = 0;
997
992
  /**
998
993
  * True from the synchronous entry of `sync()` until its serialized round
@@ -4806,26 +4801,19 @@ export class SyncClient {
4806
4801
  deferredCommits: 0,
4807
4802
  schemaFloor: undefined as SchemaFloor | undefined,
4808
4803
  };
4809
- const result = (budgetExhausted: boolean): SyncIdleResult => ({
4810
- pushed: aggregate.pushed,
4811
- applied: aggregate.applied,
4812
- rejected: aggregate.rejected,
4813
- retryable: aggregate.retryable,
4814
- conflicts: aggregate.conflicts,
4815
- commitsApplied: aggregate.commitsApplied,
4816
- segmentRowsApplied: aggregate.segmentRowsApplied,
4817
- bootstrapping: aggregate.bootstrapping,
4818
- resets: aggregate.resets,
4819
- revoked: aggregate.revoked,
4820
- failed: aggregate.failed,
4821
- ...(aggregate.deferredCommits > 0
4822
- ? { deferredCommits: aggregate.deferredCommits }
4823
- : {}),
4824
- ...(aggregate.schemaFloor !== undefined
4825
- ? { schemaFloor: aggregate.schemaFloor }
4826
- : {}),
4827
- budgetExhausted,
4828
- });
4804
+ const result = (budgetExhausted: boolean): SyncIdleResult => {
4805
+ const { deferredCommits, schemaFloor, ...summary } = aggregate;
4806
+ return {
4807
+ ...summary,
4808
+ ...(deferredCommits > 0 ? { deferredCommits } : {}),
4809
+ ...(schemaFloor !== undefined ? { schemaFloor } : {}),
4810
+ budgetExhausted,
4811
+ };
4812
+ };
4813
+ const states = () =>
4814
+ JSON.stringify(
4815
+ loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState]),
4816
+ );
4829
4817
  const startedAtMs = this.#now();
4830
4818
  let completed = false;
4831
4819
  for (let round = 0; round < budget; round++) {
@@ -4833,13 +4821,11 @@ export class SyncClient {
4833
4821
  // like an exhausted round budget; the next drive resumes.
4834
4822
  if (completed && this.#now() - startedAtMs >= timeoutMs)
4835
4823
  return result(true);
4836
- const before = JSON.stringify(
4837
- loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState]),
4838
- );
4839
- // Each round is bounded by `timeoutMs` on its own. One signal reaches
4840
- // every transport call of the round.
4824
+ const before = states();
4825
+ // Each round is bounded by `timeoutMs` from its start or its last
4826
+ // applied rows. One signal reaches every transport call of the round.
4841
4827
  const controller = new AbortController();
4842
- let timer: ReturnType<typeof setTimeout> | undefined;
4828
+ let cancel: (() => void) | undefined;
4843
4829
  const sync = this.#sync(controller.signal);
4844
4830
  const last = await (
4845
4831
  timeoutMs === Number.POSITIVE_INFINITY
@@ -4847,49 +4833,59 @@ export class SyncClient {
4847
4833
  : Promise.race([
4848
4834
  sync,
4849
4835
  new Promise<never>((_, reject) => {
4850
- timer = setTimeout(() => {
4851
- const error = new ClientSyncError(
4852
- 'sync.idle_timeout',
4853
- 'syncUntilIdle exceeded its deadline',
4854
- true,
4855
- { timeoutMs, rounds: round },
4856
- );
4857
- // The transport may never answer; release the round so the
4858
- // client can run another one after the caller sees the error.
4859
- controller.abort(error);
4860
- this.#abortPendingRound('syncUntilIdle deadline exceeded');
4861
- reject(error);
4862
- }, timeoutMs);
4836
+ this.#roundProgressed = () => {
4837
+ cancel?.();
4838
+ cancel = this.#schedule(() => {
4839
+ const error = new ClientSyncError(
4840
+ 'sync.idle_timeout',
4841
+ 'syncUntilIdle exceeded its deadline',
4842
+ true,
4843
+ { timeoutMs, rounds: round },
4844
+ );
4845
+ // The transport may never answer; release the round so the
4846
+ // client can run another one after the caller sees the error.
4847
+ controller.abort(error);
4848
+ this.#abortPendingRound(error);
4849
+ reject(error);
4850
+ }, timeoutMs);
4851
+ };
4852
+ this.#roundProgressed();
4863
4853
  }),
4864
4854
  ])
4865
- ).finally(() => clearTimeout(timer));
4855
+ ).finally(() => {
4856
+ cancel?.();
4857
+ this.#roundProgressed = undefined;
4858
+ });
4866
4859
  completed = true;
4867
4860
  // Historical counters accumulate; current readiness (`bootstrapping`,
4868
4861
  // `deferredCommits`, `schemaFloor`) describes the latest round.
4869
- aggregate.pushed += last.pushed;
4870
- aggregate.applied.push(...last.applied);
4871
- aggregate.rejected.push(...last.rejected);
4872
- aggregate.retryable.push(...last.retryable);
4862
+ for (const key of [
4863
+ 'pushed',
4864
+ 'commitsApplied',
4865
+ 'segmentRowsApplied',
4866
+ ] as const)
4867
+ aggregate[key] += last[key];
4868
+ for (const key of [
4869
+ 'applied',
4870
+ 'rejected',
4871
+ 'retryable',
4872
+ 'resets',
4873
+ 'revoked',
4874
+ 'failed',
4875
+ ] as const)
4876
+ aggregate[key].push(...last[key]);
4873
4877
  aggregate.conflicts.push(...last.conflicts);
4874
- aggregate.commitsApplied += last.commitsApplied;
4875
- aggregate.segmentRowsApplied += last.segmentRowsApplied;
4876
4878
  aggregate.bootstrapping = [...last.bootstrapping];
4877
- aggregate.resets.push(...last.resets);
4878
- aggregate.revoked.push(...last.revoked);
4879
- aggregate.failed.push(...last.failed);
4880
4879
  aggregate.deferredCommits = last.deferredCommits ?? 0;
4881
4880
  aggregate.schemaFloor = last.schemaFloor;
4882
4881
  // A disabled transport or a schema floor ends the loop, not the budget.
4883
- if (!this.#transportEnabled || last.schemaFloor !== undefined)
4882
+ if (!this.#transportEnabled || aggregate.schemaFloor)
4884
4883
  return result(false);
4885
4884
  if (
4886
4885
  maxRounds === undefined &&
4887
4886
  last.segmentRowsApplied > 0 &&
4888
4887
  last.bootstrapping.length > 0 &&
4889
- before !==
4890
- JSON.stringify(
4891
- loadUnits(this.#db).map((unit) => [unit.id, unit.bootstrapState]),
4892
- )
4888
+ before !== states()
4893
4889
  )
4894
4890
  round = -1;
4895
4891
  if (
@@ -4897,7 +4893,7 @@ export class SyncClient {
4897
4893
  last.segmentRowsApplied === 0 &&
4898
4894
  last.bootstrapping.length === 0 &&
4899
4895
  last.resets.length === 0 &&
4900
- (last.deferredCommits ?? 0) === 0 &&
4896
+ !aggregate.deferredCommits &&
4901
4897
  !this.#needsPull
4902
4898
  ) {
4903
4899
  return result(false);
@@ -5465,7 +5461,7 @@ export class SyncClient {
5465
5461
  frame.type === 'SEGMENT_REF' &&
5466
5462
  open !== undefined &&
5467
5463
  this.#config.segments !== undefined &&
5468
- !this.#appliedSegments.get(position(open))?.has(frame.segmentId)
5464
+ open.applied?.[frame.segmentId] !== frame.rowCount
5469
5465
  ) {
5470
5466
  queued.push([frame, open.scopes]);
5471
5467
  }
@@ -5597,16 +5593,16 @@ export class SyncClient {
5597
5593
  throw invalidRequest(
5598
5594
  'unadvertised or paged sqlite segment (§4.2, §5.3)',
5599
5595
  );
5600
- const applied =
5601
- this.#appliedSegments.get(position(section.unit)) ??
5602
- new Set<string>();
5603
- if (frame.type === 'SEGMENT_REF' && applied.has(frame.segmentId)) {
5604
- // §5.6: applied in an earlier attempt at this position.
5605
- section.seeded.add(frame.table);
5606
- break;
5607
- }
5608
5596
  let bytes: Uint8Array;
5597
+ let appliedRows: number | undefined;
5609
5598
  if (frame.type === 'SEGMENT_REF') {
5599
+ // §5.6: rows an earlier attempt at this position applied. A
5600
+ // partly applied image already ran its table's first-page clear.
5601
+ appliedRows = section.unit.applied?.[frame.segmentId];
5602
+ if (appliedRows !== undefined) {
5603
+ section.seeded.add(frame.table);
5604
+ if (appliedRows === frame.rowCount) break;
5605
+ }
5610
5606
  this.#progress.update({
5611
5607
  ...NO_PAYLOAD,
5612
5608
  phase: 'download',
@@ -5685,7 +5681,17 @@ export class SyncClient {
5685
5681
  table,
5686
5682
  bytes,
5687
5683
  image,
5688
- options,
5684
+ {
5685
+ ...options,
5686
+ appliedRows,
5687
+ onChunk: (rows) =>
5688
+ recordAppliedRows(
5689
+ this.#db,
5690
+ unitId,
5691
+ image.segmentId,
5692
+ rows,
5693
+ ),
5694
+ },
5689
5695
  )
5690
5696
  : applyRowsSegment(
5691
5697
  this.#db,
@@ -5698,9 +5704,11 @@ export class SyncClient {
5698
5704
  },
5699
5705
  );
5700
5706
  if (frame.type === 'SEGMENT_REF' && !section.skip)
5701
- this.#appliedSegments.set(
5702
- position(section.unit),
5703
- applied.add(frame.segmentId),
5707
+ recordAppliedRows(
5708
+ this.#db,
5709
+ unitId,
5710
+ frame.segmentId,
5711
+ frame.rowCount,
5704
5712
  );
5705
5713
  break;
5706
5714
  }
@@ -5714,7 +5722,6 @@ export class SyncClient {
5714
5722
  this.#localResetEpoch === resetEpoch
5715
5723
  ) {
5716
5724
  this.#finishSection(section, frame, summary);
5717
- this.#appliedSegments.delete(position(section.unit));
5718
5725
  }
5719
5726
  section = undefined;
5720
5727
  // The counters stay: `complete` and `failed` report them.
@@ -6207,7 +6214,10 @@ export class SyncClient {
6207
6214
  effective,
6208
6215
  owners,
6209
6216
  isCurrent: () => this.#localResetEpoch === resetEpoch,
6210
- onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
6217
+ onProgress: (rowsProcessed) => {
6218
+ this.#progress.update({ rowsProcessed });
6219
+ this.#roundProgressed?.();
6220
+ },
6211
6221
  transaction: (fn) => {
6212
6222
  const started = performance.now();
6213
6223
  let written = started;
@@ -41,6 +41,7 @@ import {
41
41
  localColumnType,
42
42
  quoteIdent,
43
43
  rowValueToJson,
44
+ tableExists,
44
45
  } from './schema';
45
46
  import { getMeta, setMeta, META_TABLE } from './state';
46
47
 
@@ -449,11 +450,7 @@ export function withPreviousVersionContainer<T>(
449
450
  export function readPreviousVersionContainer(
450
451
  db: ClientDatabase,
451
452
  ): PreviousVersionRecord | undefined {
452
- const meta = db.query(
453
- "SELECT 1 AS present FROM sqlite_master WHERE type='table' AND name=?",
454
- [CONTAINER_META_TABLE],
455
- );
456
- if (meta.length === 0) return undefined;
453
+ if (!tableExists(db, CONTAINER_META_TABLE)) return undefined;
457
454
  const raw = db.query(
458
455
  `SELECT record FROM ${quoteIdent(CONTAINER_META_TABLE)} WHERE id=1`,
459
456
  )[0]?.record;
package/src/schema.ts CHANGED
@@ -356,6 +356,14 @@ export function quoteIdent(name: string): string {
356
356
  return `"${name.replaceAll('"', '""')}"`;
357
357
  }
358
358
 
359
+ export function tableExists(db: ClientDatabase, name: string): boolean {
360
+ return (
361
+ db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
362
+ name,
363
+ ]).length > 0
364
+ );
365
+ }
366
+
359
367
  /**
360
368
  * §5.11: the app-side type of a column for local (plaintext) storage. For an
361
369
  * encrypted column this is `declaredType` — the local mirror stays plaintext,
@@ -445,8 +453,11 @@ const FTS_SOURCE_ID_COLUMN = '_syncular_source_id';
445
453
  * Limitation: because a BEFORE trigger runs before SQLite resolves the
446
454
  * conflict, an `OR IGNORE` write whose row is dropped still executes these
447
455
  * displacement DELETEs, transiently removing the surviving row's projection
448
- * entry. Syncular's mirror writes never use `OR IGNORE`; see the guard-block
449
- * comment below for the full rationale.
456
+ * entry. The same holds for an upsert whose `DO UPDATE ... WHERE` skips the
457
+ * row, so Syncular's mirror writes never offer a row that will not replace
458
+ * (§5.3); see the guard-block comment below for the full rationale. An open
459
+ * rebuilds a projection whose mapping holds a different row count than its
460
+ * source table.
450
461
  */
451
462
  function createFtsProjection(
452
463
  db: ClientDatabase,
@@ -456,11 +467,7 @@ function createFtsProjection(
456
467
  const projection = quoteIdent(index.name);
457
468
  const sourceTable = quoteIdent(table.name);
458
469
  const primaryKey = quoteIdent(table.primaryKey);
459
- const existed =
460
- db.query(
461
- "SELECT 1 AS present FROM sqlite_master WHERE type='table' AND name=?",
462
- [index.name],
463
- ).length > 0;
470
+ const existed = tableExists(db, index.name);
464
471
  const indexedColumns = index.columns.map(quoteIdent);
465
472
  const tokenizer = index.tokenize.replaceAll("'", "''");
466
473
  db.exec(
@@ -468,10 +475,7 @@ function createFtsProjection(
468
475
  );
469
476
 
470
477
  const mapping = quoteIdent(`_syncular_fts_${index.name}`);
471
- const mapped =
472
- db.query("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", [
473
- `_syncular_fts_${index.name}`,
474
- ]).length > 0;
478
+ const mapped = tableExists(db, `_syncular_fts_${index.name}`);
475
479
  db.exec(
476
480
  `CREATE TABLE IF NOT EXISTS ${mapping}(id INTEGER PRIMARY KEY,source_id TEXT NOT NULL UNIQUE)`,
477
481
  );
@@ -506,16 +510,19 @@ function createFtsProjection(
506
510
  // `UPDATE OR REPLACE` needs no BEFORE guard: the new pk equals the displaced
507
511
  // row's pk, so the AFTER UPDATE delete of the new source id already clears it.
508
512
  //
509
- // Known limitation — `OR IGNORE`: a BEFORE trigger fires before SQLite
510
- // resolves the conflict, and SQLite exposes no signal for the eventual
511
- // resolution. So an `INSERT OR IGNORE` / `UPDATE OR IGNORE` whose row is
512
- // dropped on conflict still runs these displacement DELETEs, removing the
513
- // SURVIVING row's projection entry (a false negative until that row is next
514
- // rewritten). Moving cleanup to AFTER triggers would dodge the no-op but
515
- // reintroduce the REPLACE-doesn't-fire-DELETE gap and a full-projection scan
516
- // per write, so the guards stay BEFORE. Syncular's own mirror writes use
517
- // `INSERT … ON CONFLICT (pk) DO UPDATE` (apply.ts), which never takes the
518
- // IGNORE path; only hand-written `OR IGNORE` against a mirror table is exposed.
513
+ // Known limitation: a BEFORE trigger fires before SQLite resolves the
514
+ // conflict, and SQLite exposes no signal for the eventual resolution. So an
515
+ // `INSERT OR IGNORE` / `UPDATE OR IGNORE` whose row is dropped on conflict,
516
+ // and an upsert whose `DO UPDATE ... WHERE` skips the row, still run these
517
+ // displacement DELETEs, removing the SURVIVING row's projection entry (a
518
+ // false negative until that row is next rewritten). Moving cleanup to AFTER
519
+ // triggers would dodge the no-op but reintroduce the REPLACE-doesn't-fire-
520
+ // DELETE gap and a full-projection scan per write, so the guards stay
521
+ // BEFORE. Syncular's own mirror writes use an unconditional
522
+ // `INSERT … ON CONFLICT (pk) DO UPDATE` (apply.ts); the image copy filters
523
+ // held rows at the same or a newer version out of its SELECT instead of a
524
+ // `DO UPDATE ... WHERE`. Only hand-written `OR IGNORE` against a mirror
525
+ // table is exposed.
519
526
  for (const suffix of ['bi', 'ai', 'ad', 'au', 'bu']) {
520
527
  db.exec(`DROP TRIGGER IF EXISTS ${quoteIdent(`${index.name}_${suffix}`)}`);
521
528
  }
@@ -568,12 +575,23 @@ function createFtsProjection(
568
575
  createTrigger('bu', 'BEFORE UPDATE', updateGuardBody, updateGuardCondition);
569
576
  }
570
577
 
571
- if (!existed) {
578
+ // Every source row has exactly one mapping row, so a count mismatch marks a
579
+ // projection an earlier release damaged; the open rebuilds it.
580
+ const stale =
581
+ !existed ||
582
+ (mapped &&
583
+ Number(
584
+ db.query(
585
+ `SELECT (SELECT count(*) FROM ${mapping})=(SELECT count(*) FROM ${sourceTable}) AS whole`,
586
+ )[0]?.whole,
587
+ ) !== 1);
588
+ if (stale) {
589
+ db.exec(`DELETE FROM ${projection}`);
572
590
  db.exec(
573
591
  `INSERT INTO ${projection}(${projectionColumns})SELECT ${sourceId},${indexedColumns.join(', ')} FROM ${sourceTable}`,
574
592
  );
575
593
  }
576
- if (!mapped || !existed) {
594
+ if (!mapped || stale) {
577
595
  db.exec(`DELETE FROM ${mapping}`);
578
596
  db.exec(
579
597
  `INSERT INTO ${mapping}(id,source_id)SELECT rowid,${quoteIdent(FTS_SOURCE_ID_COLUMN)} FROM ${projection}`,
@@ -649,10 +667,7 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
649
667
  db.exec(
650
668
  `INSERT OR IGNORE INTO ${META_TABLE}(key,value)VALUES('localRevision','0')`,
651
669
  );
652
- const outboxExisted =
653
- db.query(
654
- `SELECT 1 FROM sqlite_master WHERE type='table' AND name='${OUTBOX_TABLE}'`,
655
- ).length > 0;
670
+ const outboxExisted = tableExists(db, OUTBOX_TABLE);
656
671
  createTable(
657
672
  OUTBOX_TABLE,
658
673
  'seq INTEGER PRIMARY KEY AUTOINCREMENT,client_commit_id TEXT NOT NULL UNIQUE,created_at_ms INTEGER NOT NULL,operations TEXT NOT NULL',
@@ -697,6 +712,12 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
697
712
  UNITS_TABLE,
698
713
  "id TEXT PRIMARY KEY,scopes TEXT NOT NULL,explicit INTEGER NOT NULL DEFAULT 0,status TEXT NOT NULL DEFAULT 'active',reason_code TEXT,cursor INTEGER DEFAULT -1,bootstrap_state TEXT,seeded TEXT NOT NULL DEFAULT '[]',echo TEXT",
699
714
  );
715
+ // §5.6 applied-segment record, added after the table shipped.
716
+ try {
717
+ db.exec(`ALTER TABLE ${UNITS_TABLE} ADD COLUMN applied TEXT`);
718
+ } catch {
719
+ // column already exists
720
+ }
700
721
  // §4.8 window registry: base + value → the unit it holds, the
701
722
  // completeness oracle and one of a unit's references (§4.1).
702
723
  createTable(
@@ -716,11 +737,7 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
716
737
  // rows and pending evictions and registers its units fresh. Its
717
738
  // subscriptions, cursors, echoes, resume tokens and window entries go.
718
739
  const legacyEvictions = '_syncular_window_pending_evict';
719
- if (
720
- db.query(
721
- `SELECT 1 FROM sqlite_master WHERE type='table' AND name='${legacyEvictions}'`,
722
- ).length > 0
723
- ) {
740
+ if (tableExists(db, legacyEvictions)) {
724
741
  if (
725
742
  db.query(
726
743
  `SELECT 1 FROM ${legacyEvictions} WHERE NOT json_valid(effective_scopes) OR json_type(effective_scopes)<>'object' OR EXISTS(SELECT 1 FROM json_each(effective_scopes) WHERE json_type(value)<>'array' OR EXISTS(SELECT 1 FROM json_each(value) v WHERE v.type<>'text'))`,
package/src/state.ts CHANGED
@@ -45,6 +45,9 @@ export interface UnitRecord {
45
45
  readonly seeded: readonly string[];
46
46
  /** The echo persisted at the last `SUB_END` (§4.4). */
47
47
  readonly echo?: readonly FencedRecord[] | undefined;
48
+ /** §5.6 applied-segment record: segment id → rows applied at the unit's
49
+ * current cursor and token. */
50
+ readonly applied?: Readonly<Record<string, number>> | undefined;
48
51
  }
49
52
 
50
53
  /** §4.1: a catching-up unit holding cursor `-1` or a resume token. */
@@ -138,6 +141,7 @@ function rowToRecord(row: Record<string, unknown>): UnitRecord {
138
141
  const scopes = json(row.scopes);
139
142
  const seeded = json(row.seeded);
140
143
  const echo = row.echo === null ? undefined : json(row.echo);
144
+ const applied = row.applied === null ? undefined : json(row.applied);
141
145
  const { id, status, cursor } = row;
142
146
  if (
143
147
  typeof id !== 'string' ||
@@ -146,6 +150,7 @@ function rowToRecord(row: Record<string, unknown>): UnitRecord {
146
150
  (status !== 'active' && status !== 'revoked' && status !== 'failed') ||
147
151
  (cursor !== null && !Number.isInteger(cursor)) ||
148
152
  !isStrings(seeded) ||
153
+ (applied !== undefined && !isMap(applied)) ||
149
154
  (echo !== undefined &&
150
155
  !(
151
156
  Array.isArray(echo) &&
@@ -173,6 +178,7 @@ function rowToRecord(row: Record<string, unknown>): UnitRecord {
173
178
  const bootstrapState = optionalText(row.bootstrap_state);
174
179
  if (bootstrapState !== undefined) record.bootstrapState = bootstrapState;
175
180
  if (echo !== undefined) record.echo = echo as FencedRecord[];
181
+ if (applied !== undefined) record.applied = applied as Record<string, number>;
176
182
  return record;
177
183
  }
178
184
 
@@ -191,10 +197,11 @@ export function getUnit(
191
197
  return row === undefined ? undefined : rowToRecord(row);
192
198
  }
193
199
 
194
- /** Insert or update in place: an update keeps the registration order. */
200
+ /** Insert or update in place: an update keeps the registration order. The
201
+ * §5.6 applied-segment record ends when the cursor or the token changes. */
195
202
  export function saveUnit(db: ClientDatabase, record: UnitRecord): void {
196
203
  db.exec(
197
- `INSERT INTO ${UNITS_TABLE}(id,scopes,explicit,status,reason_code,cursor,bootstrap_state,seeded,echo)VALUES(?,?,?,?,?,?,?,?,?)ON CONFLICT(id)DO UPDATE SET explicit=excluded.explicit,status=excluded.status,reason_code=excluded.reason_code,cursor=excluded.cursor,bootstrap_state=excluded.bootstrap_state,seeded=excluded.seeded,echo=excluded.echo`,
204
+ `INSERT INTO ${UNITS_TABLE}(id,scopes,explicit,status,reason_code,cursor,bootstrap_state,seeded,echo)VALUES(?,?,?,?,?,?,?,?,?)ON CONFLICT(id)DO UPDATE SET explicit=excluded.explicit,status=excluded.status,reason_code=excluded.reason_code,cursor=excluded.cursor,bootstrap_state=excluded.bootstrap_state,seeded=excluded.seeded,echo=excluded.echo,applied=iif((cursor,bootstrap_state)IS(excluded.cursor,excluded.bootstrap_state),applied,NULL)`,
198
205
  [
199
206
  record.id,
200
207
  JSON.stringify(record.scopes),
@@ -209,6 +216,20 @@ export function saveUnit(db: ClientDatabase, record: UnitRecord): void {
209
216
  );
210
217
  }
211
218
 
219
+ /** §5.6: `rows` rows of `segmentId` applied at the unit's position. Runs
220
+ * in the transaction that commits them. */
221
+ export function recordAppliedRows(
222
+ db: ClientDatabase,
223
+ id: string,
224
+ segmentId: string,
225
+ rows: number,
226
+ ): void {
227
+ db.exec(
228
+ `UPDATE ${UNITS_TABLE} SET applied=json_patch(coalesce(applied,'{}'),?) WHERE id=?`,
229
+ [JSON.stringify({ [segmentId]: rows }), id],
230
+ );
231
+ }
232
+
212
233
  export function deleteUnit(db: ClientDatabase, id: string): void {
213
234
  db.exec(`DELETE FROM ${UNITS_TABLE} WHERE id=?`, [id]);
214
235
  }
@@ -225,7 +246,7 @@ export function resetUnits(
225
246
  keep: readonly string[] = [],
226
247
  ): void {
227
248
  db.exec(
228
- `UPDATE ${UNITS_TABLE} SET status='active',reason_code=NULL,cursor=-1,bootstrap_state=NULL,seeded='[]',echo=NULL WHERE id NOT IN(${keep.map(() => '?').join(',')})`,
249
+ `UPDATE ${UNITS_TABLE} SET status='active',reason_code=NULL,cursor=-1,bootstrap_state=NULL,seeded='[]',echo=NULL,applied=NULL WHERE id NOT IN(${keep.map(() => '?').join(',')})`,
229
250
  [...keep],
230
251
  );
231
252
  db.exec(