@syncular/client 0.16.1 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -543,6 +543,14 @@ Node 22.13 or newer. No SQLite package or native addon is required. Both
543
543
  adapters support synchronous `exec`, `query`, nested transactions, boolean
544
544
  bindings, `null`, `Uint8Array` BLOB values, and §5.3 SQLite-image attachment.
545
545
 
546
+ Persistent Bun and Node databases use SQLite WAL journaling with
547
+ `synchronous=FULL`. Each application or sync commit retains its own durable
548
+ transaction. SQLite keeps in-memory databases on its memory journal. Use a
549
+ local filesystem path; WAL uses adjacent `-wal` and `-shm` files. Close every
550
+ connection before copying the database file, or use SQLite's backup facilities
551
+ while it is open. Copying only the main file while writers are active can omit
552
+ committed WAL contents.
553
+
546
554
  Runtime-specific imports remain available:
547
555
 
548
556
  ```ts
package/dist/blob.d.ts CHANGED
@@ -115,12 +115,14 @@ export declare function schemaHasBlobs(schema: CompiledClientSchema): boolean;
115
115
  * to zero refs *and* have no pending upload (a pending upload pins its body,
116
116
  * B4). Called after every apply/purge that may add or remove references.
117
117
  *
118
+ * `blobId` narrows a fresh-download refresh to the inserted cache row.
118
119
  * `deleteOrphans` distinguishes the two B2 transitions: revocation purge
119
120
  * passes `true` (drop the now-unauthorized body); a benign apply passes
120
121
  * `false` (retain zero-ref bodies as LRU cache entries — the shipped
121
122
  * default). Bodies pinned by a pending upload are always retained.
122
123
  */
123
124
  export declare function reconcileBlobRefcounts(db: ClientDatabase, schema: CompiledClientSchema, options?: {
125
+ readonly blobId?: string;
124
126
  readonly deleteOrphans?: boolean;
125
127
  }): void;
126
128
  export type { BlobRef };
package/dist/blob.js CHANGED
@@ -11,10 +11,14 @@
11
11
  * outbox-adjacent uploads table (B4) and flushed before push.
12
12
  */
13
13
  import { parseBlobRef, serializeBlobRef } from '@syncular/core';
14
+ import { ClientSyncError } from './errors.js';
14
15
  import { quoteIdent } from './schema.js';
15
16
  /** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
16
17
  export async function computeBlobId(bytes) {
17
- const digest = await crypto.subtle.digest('SHA-256', bytes.slice().buffer);
18
+ const input = bytes.buffer instanceof ArrayBuffer
19
+ ? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
20
+ : bytes.slice();
21
+ const digest = await crypto.subtle.digest('SHA-256', input);
18
22
  const hex = [...new Uint8Array(digest)]
19
23
  .map((b) => b.toString(16).padStart(2, '0'))
20
24
  .join('');
@@ -42,6 +46,9 @@ export function ensureBlobSchema(db) {
42
46
  blob_id TEXT PRIMARY KEY,
43
47
  media_type TEXT,
44
48
  created_at_ms INTEGER NOT NULL)`);
49
+ db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
50
+ commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
51
+ db.exec('CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)');
45
52
  }
46
53
  /** Put bytes into the content-addressed cache (idempotent); touches LRU. */
47
54
  export function putCachedBlob(db, blobId, bytes, nowMs, mediaType) {
@@ -90,6 +97,7 @@ export function enforceBlobCacheCap(db, maxBytes) {
90
97
  const candidates = db.query(`SELECT blob_id, byte_length FROM _syncular_blobs
91
98
  WHERE refcount = 0
92
99
  AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
100
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
93
101
  ORDER BY last_used_ms ASC, created_at_ms ASC`);
94
102
  const evicted = [];
95
103
  db.transaction(() => {
@@ -111,13 +119,23 @@ export function recordPendingUpload(db, blobId, nowMs, mediaType) {
111
119
  }
112
120
  export function listPendingUploads(db) {
113
121
  return db
114
- .query('SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms')
115
- .map((row) => ({
116
- blobId: row.blob_id,
117
- ...(row.media_type !== null
118
- ? { mediaType: row.media_type }
119
- : {}),
120
- }));
122
+ .query(`SELECT blob_id, media_type FROM (
123
+ SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
124
+ UNION ALL
125
+ SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
126
+ FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
127
+ WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
128
+ ) ORDER BY created_at_ms, blob_id`)
129
+ .map((row) => {
130
+ if (typeof row.blob_id !== 'string' ||
131
+ (row.media_type !== null && typeof row.media_type !== 'string')) {
132
+ throw new ClientSyncError('sync.local_corrupt', 'Pending blob upload metadata is invalid');
133
+ }
134
+ return {
135
+ blobId: row.blob_id,
136
+ ...(row.media_type !== null ? { mediaType: row.media_type } : {}),
137
+ };
138
+ });
121
139
  }
122
140
  export function clearPendingUpload(db, blobId) {
123
141
  db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id = ?', [blobId]);
@@ -150,6 +168,7 @@ export function schemaHasBlobs(schema) {
150
168
  * to zero refs *and* have no pending upload (a pending upload pins its body,
151
169
  * B4). Called after every apply/purge that may add or remove references.
152
170
  *
171
+ * `blobId` narrows a fresh-download refresh to the inserted cache row.
153
172
  * `deleteOrphans` distinguishes the two B2 transitions: revocation purge
154
173
  * passes `true` (drop the now-unauthorized body); a benign apply passes
155
174
  * `false` (retain zero-ref bodies as LRU cache entries — the shipped
@@ -159,6 +178,25 @@ export function reconcileBlobRefcounts(db, schema, options) {
159
178
  const byTable = blobRefColumnsBySchema(schema);
160
179
  if (byTable.size === 0)
161
180
  return;
181
+ if (options?.blobId !== undefined) {
182
+ let count = 0;
183
+ for (const [tableName, columns] of byTable) {
184
+ for (const column of columns) {
185
+ const identifier = quoteIdent(column);
186
+ const row = db.query(`SELECT count(*) AS n FROM ${quoteIdent(tableName)}
187
+ WHERE CASE
188
+ WHEN typeof(${identifier}) = 'text' AND json_valid(${identifier})
189
+ THEN json_extract(${identifier}, '$.blobId')
190
+ END = ?`, [options.blobId])[0];
191
+ count += Number(row?.n ?? 0);
192
+ }
193
+ }
194
+ db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
195
+ count,
196
+ options.blobId,
197
+ ]);
198
+ return;
199
+ }
162
200
  // Count references to each blobId across every blob_ref column.
163
201
  const counts = new Map();
164
202
  for (const [tableName, columns] of byTable) {
@@ -194,7 +232,8 @@ export function reconcileBlobRefcounts(db, schema, options) {
194
232
  // pending upload.
195
233
  db.exec(`DELETE FROM _syncular_blobs
196
234
  WHERE refcount = 0
197
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)`);
235
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
236
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)`);
198
237
  }
199
238
  });
200
239
  }
@@ -20,6 +20,17 @@ export class BunClientDatabase {
20
20
  #tx = { depth: 0 };
21
21
  constructor(path = ':memory:') {
22
22
  this.db = new Database(path);
23
+ // Match native Rust persistence: append durable commits to the WAL
24
+ // instead of creating and syncing a rollback journal per transaction.
25
+ // SQLite retains its in-memory journal for :memory: databases.
26
+ try {
27
+ this.db.run('PRAGMA journal_mode = WAL');
28
+ this.db.run('PRAGMA synchronous = FULL');
29
+ }
30
+ catch (error) {
31
+ this.db.close();
32
+ throw error;
33
+ }
23
34
  }
24
35
  exec(sql, params = []) {
25
36
  this.db.query(sql).run(...coerceParams(params));
package/dist/client.js CHANGED
@@ -191,8 +191,39 @@ export class SyncClient {
191
191
  // Do not materialize new app indexes/FTS projections yet: on a version
192
192
  // bump they may reference columns that only exist after the reset.
193
193
  ensureLocalBookkeepingSchema(this.#db);
194
- if (this.#hasBlobs)
195
- ensureBlobSchema(this.#db);
194
+ this.#db.transaction(() => {
195
+ for (const trigger of ['insert', 'update', 'delete', 'cache'])
196
+ this.#db.exec(`DROP TRIGGER IF EXISTS _syncular_blob_commit_${trigger}`);
197
+ if (this.#hasBlobs) {
198
+ ensureBlobSchema(this.#db);
199
+ const columns = JSON.stringify([...this.#schema.tables.values()].flatMap((table) => table.columns
200
+ .filter((column) => column.type === 'blob_ref')
201
+ .map((column) => [table.name, column.name]))).replaceAll("'", "''");
202
+ const references = `SELECT o.client_commit_id, b.blob_id
203
+ FROM _syncular_outbox o JOIN json_each(o.operations) op
204
+ JOIN json_each(op.value, '$.values') v JOIN json_each('${columns}') c
205
+ JOIN _syncular_blobs b ON b.blob_id = CASE WHEN v.type = 'text' AND json_valid(v.value) THEN json_extract(v.value, '$.blobId') END
206
+ WHERE json_extract(op.value, '$.op') = 'upsert'
207
+ AND json_extract(op.value, '$.table') = json_extract(c.value, '$[0]')
208
+ AND v.key = json_extract(c.value, '$[1]')`;
209
+ this.#db
210
+ .exec(`CREATE TRIGGER _syncular_blob_commit_insert AFTER INSERT ON _syncular_outbox BEGIN
211
+ INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
212
+ this.#db
213
+ .exec(`CREATE TRIGGER _syncular_blob_commit_update AFTER UPDATE OF operations ON _syncular_outbox BEGIN
214
+ DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id;
215
+ INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
216
+ this.#db
217
+ .exec(`CREATE TRIGGER _syncular_blob_commit_delete BEFORE DELETE ON _syncular_outbox BEGIN
218
+ DELETE FROM _syncular_blob_uploads WHERE blob_id IN (SELECT blob_id FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id)
219
+ AND NOT EXISTS (SELECT 1 FROM _syncular_blob_commit_refs r WHERE r.blob_id = _syncular_blob_uploads.blob_id AND r.commit_id != OLD.client_commit_id);
220
+ DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id; END`);
221
+ this.#db
222
+ .exec(`CREATE TRIGGER _syncular_blob_commit_cache AFTER INSERT ON _syncular_blobs BEGIN
223
+ INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND b.blob_id = NEW.blob_id; END`);
224
+ this.#db.exec(`INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references}`);
225
+ }
226
+ });
196
227
  this.#db.transaction(() => {
197
228
  pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
198
229
  });
@@ -770,7 +801,7 @@ export class SyncClient {
770
801
  * Re-entrant calls share the outer batch so a nested apply never
771
802
  * double-emits (e.g. purge → blob reconcile → replay inside one round).
772
803
  */
773
- #applyBatch(fn, statusSnapshotOverride) {
804
+ #applyBatch(fn, statusSnapshotOverride, onRollback) {
774
805
  if (this.#batch !== undefined)
775
806
  return fn(this.#batch);
776
807
  const batch = new ChangeAccumulator();
@@ -796,6 +827,7 @@ export class SyncClient {
796
827
  }
797
828
  catch (error) {
798
829
  this.#batch = undefined;
830
+ onRollback?.(error);
799
831
  throw error;
800
832
  }
801
833
  if (revision !== undefined) {
@@ -853,7 +885,8 @@ export class SyncClient {
853
885
  * (via `mutate`) after this call so upload-before-push holds (§5.9.3).
854
886
  */
855
887
  uploadBlob(bytes, options) {
856
- return this.#runProtectedAsync(() => this.#uploadBlob(bytes, options));
888
+ const owned = bytes.slice();
889
+ return this.#runProtectedAsync(() => this.#uploadBlob(owned, options));
857
890
  }
858
891
  async #uploadBlob(bytes, options) {
859
892
  if (this.#config.blobs === undefined) {
@@ -929,6 +962,9 @@ export class SyncClient {
929
962
  throw new ClientSyncError('sync.invalid_request', `blob content address mismatch for ${blobId} (§5.9.5)`);
930
963
  }
931
964
  putCachedBlob(this.#db, blobId, bytes, this.#now());
965
+ // The referencing row can arrive before its body. Pin the new cache entry
966
+ // from current visible references before applying the size cap (§5.9.7 B1).
967
+ this.#reconcileBlobs(false, blobId);
932
968
  this.#enforceBlobCacheCap();
933
969
  const stored = getCachedBlob(this.#db, blobId);
934
970
  if (stored === undefined) {
@@ -953,11 +989,12 @@ export class SyncClient {
953
989
  return;
954
990
  for (const pending of listPendingUploads(this.#db)) {
955
991
  const cached = getCachedBlob(this.#db, pending.blobId);
956
- if (cached === undefined) {
957
- // The bytes are gone (never happens for a well-behaved client);
958
- // drop the upload so it does not wedge the queue.
959
- clearPendingUpload(this.#db, pending.blobId);
960
- continue;
992
+ if (cached === undefined ||
993
+ !(cached.bytes instanceof Uint8Array) ||
994
+ !Number.isSafeInteger(cached.byteLength) ||
995
+ cached.byteLength !== cached.bytes.byteLength ||
996
+ (await computeBlobId(cached.bytes)) !== pending.blobId) {
997
+ throw new ClientSyncError('sync.local_corrupt', 'Pending blob upload body is missing or corrupt');
961
998
  }
962
999
  await this.#uploadOne(transport, pending.blobId, cached.bytes, pending.mediaType);
963
1000
  clearPendingUpload(this.#db, pending.blobId);
@@ -2067,8 +2104,16 @@ export class SyncClient {
2067
2104
  }
2068
2105
  let openedSocket;
2069
2106
  const socket = await connector({
2070
- onText: (text) => this.#handleRealtimeText(text),
2071
- onBinary: (bytes) => this.#routeRealtimeBinary(bytes),
2107
+ onText: (text) => {
2108
+ if (generation !== this.#realtimeGeneration || !this.#started)
2109
+ return;
2110
+ this.#handleRealtimeText(text);
2111
+ },
2112
+ onBinary: (bytes) => {
2113
+ if (generation !== this.#realtimeGeneration || !this.#started)
2114
+ return;
2115
+ this.#routeRealtimeBinary(bytes);
2116
+ },
2072
2117
  onClose: () => {
2073
2118
  if (openedSocket === undefined || this.#socket !== openedSocket)
2074
2119
  return;
@@ -2311,12 +2356,14 @@ export class SyncClient {
2311
2356
  let section;
2312
2357
  let errorFrame;
2313
2358
  let deltaCursor = -1;
2314
- let responseOutboxCount;
2315
2359
  // Each durable observer transaction emits its own revisioned batch.
2316
2360
  // Async decrypt/download work happens outside SQLite transactions.
2317
2361
  this.#beginDiagnosticsDeferral();
2318
2362
  try {
2319
- for (const frame of message.frames.slice(1)) {
2363
+ for (let index = 1; index < message.frames.length; index += 1) {
2364
+ const frame = message.frames[index];
2365
+ if (frame === undefined)
2366
+ break;
2320
2367
  switch (frame.type) {
2321
2368
  case 'RESP_HEADER':
2322
2369
  break;
@@ -2329,13 +2376,41 @@ export class SyncClient {
2329
2376
  });
2330
2377
  break;
2331
2378
  case 'PUSH_RESULT': {
2332
- let outboxCount = responseOutboxCount ?? countOutbox(this.#db);
2379
+ const results = [frame];
2380
+ if (frame.status !== 'rejected') {
2381
+ while (index + 1 < message.frames.length) {
2382
+ const next = message.frames[index + 1];
2383
+ if (next?.type !== 'PUSH_RESULT' || next.status === 'rejected')
2384
+ break;
2385
+ results.push(next);
2386
+ index += 1;
2387
+ }
2388
+ }
2389
+ const conflictCount = this.#conflicts.length;
2390
+ const rejectionCount = this.#rejections.length;
2391
+ let outboxCount = countOutbox(this.#db);
2333
2392
  this.#applyBatch((batch) => {
2334
- const drained = this.#handlePushResult(frame, commitsById, summary, batch, rejectionDetailsByCommit.get(frame.clientCommitId), frame === lastFinalPushResult);
2335
- if (drained)
2336
- outboxCount -= 1;
2337
- }, () => this.#statusSnapshot(outboxCount));
2338
- responseOutboxCount = outboxCount;
2393
+ let drained = false;
2394
+ for (const result of results) {
2395
+ if (this.#handlePushResult(result, commitsById, summary, batch, rejectionDetailsByCommit.get(result.clientCommitId))) {
2396
+ outboxCount -= 1;
2397
+ drained = true;
2398
+ }
2399
+ }
2400
+ if (drained &&
2401
+ results.some((result) => result === lastFinalPushResult)) {
2402
+ pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
2403
+ }
2404
+ }, () => this.#statusSnapshot(outboxCount), (cause) => {
2405
+ this.#conflicts.length = conflictCount;
2406
+ this.#rejections.length = rejectionCount;
2407
+ const error = new ClientSyncError('client.outcome_persistence_failed', 'local commit outcome could not be persisted');
2408
+ error.cause = cause;
2409
+ throw error;
2410
+ });
2411
+ for (const conflict of this.#conflicts.slice(conflictCount)) {
2412
+ this.#config.onConflict?.(conflict);
2413
+ }
2339
2414
  break;
2340
2415
  }
2341
2416
  case 'PUSH_RESULT_DETAILS':
@@ -2501,7 +2576,7 @@ export class SyncClient {
2501
2576
  }
2502
2577
  return { ...summary, bootstrapping };
2503
2578
  }
2504
- #handlePushResult(frame, commitsById, summary, batch, rejectionDetails, pruneOutcomes) {
2579
+ #handlePushResult(frame, commitsById, summary, batch, rejectionDetails) {
2505
2580
  const commit = commitsById.get(frame.clientCommitId);
2506
2581
  if (commit === undefined)
2507
2582
  return false;
@@ -2525,9 +2600,6 @@ export class SyncClient {
2525
2600
  })),
2526
2601
  });
2527
2602
  deleteOutboxCommit(this.#db, frame.clientCommitId);
2528
- if (pruneOutcomes) {
2529
- pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
2530
- }
2531
2603
  batch.status();
2532
2604
  batch.outcomes();
2533
2605
  summary.applied.push(frame.clientCommitId);
@@ -2562,7 +2634,6 @@ export class SyncClient {
2562
2634
  outcomeResults.push({ status: 'conflict', conflict });
2563
2635
  batch.conflicts();
2564
2636
  summary.conflicts.push(conflict);
2565
- this.#config.onConflict?.(conflict);
2566
2637
  }
2567
2638
  else if (result.status === 'error') {
2568
2639
  const details = rejectionDetails?.get(result.opIndex);
@@ -2592,9 +2663,6 @@ export class SyncClient {
2592
2663
  results: outcomeResults,
2593
2664
  operations: commit.operations,
2594
2665
  });
2595
- if (pruneOutcomes) {
2596
- pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
2597
- }
2598
2666
  batch.outcomes();
2599
2667
  // §7.2: remove the rejected optimistic layer. Before-images restore
2600
2668
  // validator-rejected updates even when the server emitted no new COMMIT;
@@ -3044,10 +3112,13 @@ export class SyncClient {
3044
3112
  * columns. No-op unless the schema has blob columns. `deleteOrphans`
3045
3113
  * triggers the revocation-side body deletion (B2).
3046
3114
  */
3047
- #reconcileBlobs(deleteOrphans) {
3115
+ #reconcileBlobs(deleteOrphans, blobId) {
3048
3116
  if (!this.#hasBlobs)
3049
3117
  return;
3050
- reconcileBlobRefcounts(this.#db, this.#schema, { deleteOrphans });
3118
+ reconcileBlobRefcounts(this.#db, this.#schema, {
3119
+ deleteOrphans,
3120
+ ...(blobId === undefined ? {} : { blobId }),
3121
+ });
3051
3122
  }
3052
3123
  // -- helpers -----------------------------------------------------------------
3053
3124
  #table(name) {
@@ -32,6 +32,14 @@ export class NodeClientDatabase {
32
32
  #tx = { depth: 0 };
33
33
  constructor(path = ':memory:') {
34
34
  this.db = new DatabaseSync(path);
35
+ try {
36
+ this.db.exec('PRAGMA journal_mode = WAL');
37
+ this.db.exec('PRAGMA synchronous = FULL');
38
+ }
39
+ catch (error) {
40
+ this.db.close();
41
+ throw error;
42
+ }
35
43
  }
36
44
  exec(sql, params = []) {
37
45
  this.db.prepare(sql).run(...coerceParams(params));
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/client",
3
- "version": "0.16.1",
3
+ "version": "0.18.0",
4
4
  "description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -89,9 +89,9 @@
89
89
  },
90
90
  "dependencies": {
91
91
  "@sqlite.org/sqlite-wasm": "^3.53.0-build1",
92
- "@syncular/core": "0.16.1"
92
+ "@syncular/core": "0.18.0"
93
93
  },
94
94
  "devDependencies": {
95
- "@syncular/server": "0.16.1"
95
+ "@syncular/server": "0.18.0"
96
96
  }
97
97
  }
package/src/blob.ts CHANGED
@@ -12,6 +12,7 @@
12
12
  */
13
13
  import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
14
14
  import type { ClientDatabase } from './database';
15
+ import { ClientSyncError } from './errors';
15
16
  import type { CompiledClientSchema } from './schema';
16
17
  import { quoteIdent } from './schema';
17
18
 
@@ -93,9 +94,13 @@ export interface CachedBlob {
93
94
 
94
95
  /** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
95
96
  export async function computeBlobId(bytes: Uint8Array): Promise<string> {
97
+ const input =
98
+ bytes.buffer instanceof ArrayBuffer
99
+ ? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
100
+ : bytes.slice();
96
101
  const digest = await crypto.subtle.digest(
97
102
  'SHA-256',
98
- bytes.slice().buffer as ArrayBuffer,
103
+ input as Uint8Array<ArrayBuffer>,
99
104
  );
100
105
  const hex = [...new Uint8Array(digest)]
101
106
  .map((b) => b.toString(16).padStart(2, '0'))
@@ -126,6 +131,11 @@ export function ensureBlobSchema(db: ClientDatabase): void {
126
131
  blob_id TEXT PRIMARY KEY,
127
132
  media_type TEXT,
128
133
  created_at_ms INTEGER NOT NULL)`);
134
+ db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
135
+ commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
136
+ db.exec(
137
+ 'CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)',
138
+ );
129
139
  }
130
140
 
131
141
  /** Put bytes into the content-addressed cache (idempotent); touches LRU. */
@@ -197,6 +207,7 @@ export function enforceBlobCacheCap(
197
207
  `SELECT blob_id, byte_length FROM _syncular_blobs
198
208
  WHERE refcount = 0
199
209
  AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
210
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
200
211
  ORDER BY last_used_ms ASC, created_at_ms ASC`,
201
212
  );
202
213
  const evicted: string[] = [];
@@ -231,14 +242,29 @@ export function listPendingUploads(
231
242
  ): { blobId: string; mediaType?: string }[] {
232
243
  return db
233
244
  .query(
234
- 'SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms',
245
+ `SELECT blob_id, media_type FROM (
246
+ SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
247
+ UNION ALL
248
+ SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
249
+ FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
250
+ WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
251
+ ) ORDER BY created_at_ms, blob_id`,
235
252
  )
236
- .map((row) => ({
237
- blobId: row.blob_id as string,
238
- ...(row.media_type !== null
239
- ? { mediaType: row.media_type as string }
240
- : {}),
241
- }));
253
+ .map((row) => {
254
+ if (
255
+ typeof row.blob_id !== 'string' ||
256
+ (row.media_type !== null && typeof row.media_type !== 'string')
257
+ ) {
258
+ throw new ClientSyncError(
259
+ 'sync.local_corrupt',
260
+ 'Pending blob upload metadata is invalid',
261
+ );
262
+ }
263
+ return {
264
+ blobId: row.blob_id,
265
+ ...(row.media_type !== null ? { mediaType: row.media_type } : {}),
266
+ };
267
+ });
242
268
  }
243
269
 
244
270
  export function clearPendingUpload(db: ClientDatabase, blobId: string): void {
@@ -275,6 +301,7 @@ export function schemaHasBlobs(schema: CompiledClientSchema): boolean {
275
301
  * to zero refs *and* have no pending upload (a pending upload pins its body,
276
302
  * B4). Called after every apply/purge that may add or remove references.
277
303
  *
304
+ * `blobId` narrows a fresh-download refresh to the inserted cache row.
278
305
  * `deleteOrphans` distinguishes the two B2 transitions: revocation purge
279
306
  * passes `true` (drop the now-unauthorized body); a benign apply passes
280
307
  * `false` (retain zero-ref bodies as LRU cache entries — the shipped
@@ -283,10 +310,35 @@ export function schemaHasBlobs(schema: CompiledClientSchema): boolean {
283
310
  export function reconcileBlobRefcounts(
284
311
  db: ClientDatabase,
285
312
  schema: CompiledClientSchema,
286
- options?: { readonly deleteOrphans?: boolean },
313
+ options?: {
314
+ readonly blobId?: string;
315
+ readonly deleteOrphans?: boolean;
316
+ },
287
317
  ): void {
288
318
  const byTable = blobRefColumnsBySchema(schema);
289
319
  if (byTable.size === 0) return;
320
+ if (options?.blobId !== undefined) {
321
+ let count = 0;
322
+ for (const [tableName, columns] of byTable) {
323
+ for (const column of columns) {
324
+ const identifier = quoteIdent(column);
325
+ const row = db.query(
326
+ `SELECT count(*) AS n FROM ${quoteIdent(tableName)}
327
+ WHERE CASE
328
+ WHEN typeof(${identifier}) = 'text' AND json_valid(${identifier})
329
+ THEN json_extract(${identifier}, '$.blobId')
330
+ END = ?`,
331
+ [options.blobId],
332
+ )[0];
333
+ count += Number(row?.n ?? 0);
334
+ }
335
+ }
336
+ db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
337
+ count,
338
+ options.blobId,
339
+ ]);
340
+ return;
341
+ }
290
342
  // Count references to each blobId across every blob_ref column.
291
343
  const counts = new Map<string, number>();
292
344
  for (const [tableName, columns] of byTable) {
@@ -323,7 +375,8 @@ export function reconcileBlobRefcounts(
323
375
  db.exec(
324
376
  `DELETE FROM _syncular_blobs
325
377
  WHERE refcount = 0
326
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)`,
378
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
379
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)`,
327
380
  );
328
381
  }
329
382
  });
@@ -36,6 +36,16 @@ export class BunClientDatabase implements ClientDatabase {
36
36
 
37
37
  constructor(path = ':memory:') {
38
38
  this.db = new Database(path);
39
+ // Match native Rust persistence: append durable commits to the WAL
40
+ // instead of creating and syncing a rollback journal per transaction.
41
+ // SQLite retains its in-memory journal for :memory: databases.
42
+ try {
43
+ this.db.run('PRAGMA journal_mode = WAL');
44
+ this.db.run('PRAGMA synchronous = FULL');
45
+ } catch (error) {
46
+ this.db.close();
47
+ throw error;
48
+ }
39
49
  }
40
50
 
41
51
  exec(sql: string, params: readonly SqlValue[] = []): void {
package/src/client.ts CHANGED
@@ -688,7 +688,47 @@ export class SyncClient {
688
688
  // Do not materialize new app indexes/FTS projections yet: on a version
689
689
  // bump they may reference columns that only exist after the reset.
690
690
  ensureLocalBookkeepingSchema(this.#db);
691
- if (this.#hasBlobs) ensureBlobSchema(this.#db);
691
+ this.#db.transaction(() => {
692
+ for (const trigger of ['insert', 'update', 'delete', 'cache'])
693
+ this.#db.exec(
694
+ `DROP TRIGGER IF EXISTS _syncular_blob_commit_${trigger}`,
695
+ );
696
+ if (this.#hasBlobs) {
697
+ ensureBlobSchema(this.#db);
698
+ const columns = JSON.stringify(
699
+ [...this.#schema.tables.values()].flatMap((table) =>
700
+ table.columns
701
+ .filter((column) => column.type === 'blob_ref')
702
+ .map((column) => [table.name, column.name]),
703
+ ),
704
+ ).replaceAll("'", "''");
705
+ const references = `SELECT o.client_commit_id, b.blob_id
706
+ FROM _syncular_outbox o JOIN json_each(o.operations) op
707
+ JOIN json_each(op.value, '$.values') v JOIN json_each('${columns}') c
708
+ JOIN _syncular_blobs b ON b.blob_id = CASE WHEN v.type = 'text' AND json_valid(v.value) THEN json_extract(v.value, '$.blobId') END
709
+ WHERE json_extract(op.value, '$.op') = 'upsert'
710
+ AND json_extract(op.value, '$.table') = json_extract(c.value, '$[0]')
711
+ AND v.key = json_extract(c.value, '$[1]')`;
712
+ this.#db
713
+ .exec(`CREATE TRIGGER _syncular_blob_commit_insert AFTER INSERT ON _syncular_outbox BEGIN
714
+ INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
715
+ this.#db
716
+ .exec(`CREATE TRIGGER _syncular_blob_commit_update AFTER UPDATE OF operations ON _syncular_outbox BEGIN
717
+ DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id;
718
+ INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
719
+ this.#db
720
+ .exec(`CREATE TRIGGER _syncular_blob_commit_delete BEFORE DELETE ON _syncular_outbox BEGIN
721
+ DELETE FROM _syncular_blob_uploads WHERE blob_id IN (SELECT blob_id FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id)
722
+ AND NOT EXISTS (SELECT 1 FROM _syncular_blob_commit_refs r WHERE r.blob_id = _syncular_blob_uploads.blob_id AND r.commit_id != OLD.client_commit_id);
723
+ DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id; END`);
724
+ this.#db
725
+ .exec(`CREATE TRIGGER _syncular_blob_commit_cache AFTER INSERT ON _syncular_blobs BEGIN
726
+ INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND b.blob_id = NEW.blob_id; END`);
727
+ this.#db.exec(
728
+ `INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references}`,
729
+ );
730
+ }
731
+ });
692
732
  this.#db.transaction(() => {
693
733
  pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
694
734
  });
@@ -1360,6 +1400,7 @@ export class SyncClient {
1360
1400
  #applyBatch<T>(
1361
1401
  fn: (batch: ChangeAccumulator) => T,
1362
1402
  statusSnapshotOverride?: () => SyncStatusSnapshot,
1403
+ onRollback?: (error: unknown) => never,
1363
1404
  ): T {
1364
1405
  if (this.#batch !== undefined) return fn(this.#batch);
1365
1406
  const batch = new ChangeAccumulator();
@@ -1383,6 +1424,7 @@ export class SyncClient {
1383
1424
  });
1384
1425
  } catch (error) {
1385
1426
  this.#batch = undefined;
1427
+ onRollback?.(error);
1386
1428
  throw error;
1387
1429
  }
1388
1430
  if (revision !== undefined) {
@@ -1453,7 +1495,8 @@ export class SyncClient {
1453
1495
  bytes: Uint8Array,
1454
1496
  options?: { readonly mediaType?: string; readonly name?: string },
1455
1497
  ): Promise<BlobRef> {
1456
- return this.#runProtectedAsync(() => this.#uploadBlob(bytes, options));
1498
+ const owned = bytes.slice();
1499
+ return this.#runProtectedAsync(() => this.#uploadBlob(owned, options));
1457
1500
  }
1458
1501
 
1459
1502
  async #uploadBlob(
@@ -1552,6 +1595,9 @@ export class SyncClient {
1552
1595
  );
1553
1596
  }
1554
1597
  putCachedBlob(this.#db, blobId, bytes, this.#now());
1598
+ // The referencing row can arrive before its body. Pin the new cache entry
1599
+ // from current visible references before applying the size cap (§5.9.7 B1).
1600
+ this.#reconcileBlobs(false, blobId);
1555
1601
  this.#enforceBlobCacheCap();
1556
1602
  const stored = getCachedBlob(this.#db, blobId);
1557
1603
  if (stored === undefined) {
@@ -1580,11 +1626,17 @@ export class SyncClient {
1580
1626
  if (transport === undefined || !this.#hasBlobs) return;
1581
1627
  for (const pending of listPendingUploads(this.#db)) {
1582
1628
  const cached = getCachedBlob(this.#db, pending.blobId);
1583
- if (cached === undefined) {
1584
- // The bytes are gone (never happens for a well-behaved client);
1585
- // drop the upload so it does not wedge the queue.
1586
- clearPendingUpload(this.#db, pending.blobId);
1587
- continue;
1629
+ if (
1630
+ cached === undefined ||
1631
+ !(cached.bytes instanceof Uint8Array) ||
1632
+ !Number.isSafeInteger(cached.byteLength) ||
1633
+ cached.byteLength !== cached.bytes.byteLength ||
1634
+ (await computeBlobId(cached.bytes)) !== pending.blobId
1635
+ ) {
1636
+ throw new ClientSyncError(
1637
+ 'sync.local_corrupt',
1638
+ 'Pending blob upload body is missing or corrupt',
1639
+ );
1588
1640
  }
1589
1641
  await this.#uploadOne(
1590
1642
  transport,
@@ -2931,8 +2983,14 @@ export class SyncClient {
2931
2983
  }
2932
2984
  let openedSocket: RealtimeSocket | undefined;
2933
2985
  const socket = await connector({
2934
- onText: (text) => this.#handleRealtimeText(text),
2935
- onBinary: (bytes) => this.#routeRealtimeBinary(bytes),
2986
+ onText: (text) => {
2987
+ if (generation !== this.#realtimeGeneration || !this.#started) return;
2988
+ this.#handleRealtimeText(text);
2989
+ },
2990
+ onBinary: (bytes) => {
2991
+ if (generation !== this.#realtimeGeneration || !this.#started) return;
2992
+ this.#routeRealtimeBinary(bytes);
2993
+ },
2936
2994
  onClose: () => {
2937
2995
  if (openedSocket === undefined || this.#socket !== openedSocket) return;
2938
2996
  this.#socket = undefined;
@@ -3223,13 +3281,14 @@ export class SyncClient {
3223
3281
  let section: OpenSection | undefined;
3224
3282
  let errorFrame: ClientSyncError | undefined;
3225
3283
  let deltaCursor = -1;
3226
- let responseOutboxCount: number | undefined;
3227
3284
 
3228
3285
  // Each durable observer transaction emits its own revisioned batch.
3229
3286
  // Async decrypt/download work happens outside SQLite transactions.
3230
3287
  this.#beginDiagnosticsDeferral();
3231
3288
  try {
3232
- for (const frame of message.frames.slice(1)) {
3289
+ for (let index = 1; index < message.frames.length; index += 1) {
3290
+ const frame = message.frames[index];
3291
+ if (frame === undefined) break;
3233
3292
  switch (frame.type) {
3234
3293
  case 'RESP_HEADER':
3235
3294
  break;
@@ -3242,22 +3301,61 @@ export class SyncClient {
3242
3301
  });
3243
3302
  break;
3244
3303
  case 'PUSH_RESULT': {
3245
- let outboxCount = responseOutboxCount ?? countOutbox(this.#db);
3304
+ const results: PushResultFrame[] = [frame];
3305
+ if (frame.status !== 'rejected') {
3306
+ while (index + 1 < message.frames.length) {
3307
+ const next = message.frames[index + 1];
3308
+ if (next?.type !== 'PUSH_RESULT' || next.status === 'rejected')
3309
+ break;
3310
+ results.push(next);
3311
+ index += 1;
3312
+ }
3313
+ }
3314
+ const conflictCount = this.#conflicts.length;
3315
+ const rejectionCount = this.#rejections.length;
3316
+ let outboxCount = countOutbox(this.#db);
3246
3317
  this.#applyBatch(
3247
3318
  (batch) => {
3248
- const drained = this.#handlePushResult(
3249
- frame,
3250
- commitsById,
3251
- summary,
3252
- batch,
3253
- rejectionDetailsByCommit.get(frame.clientCommitId),
3254
- frame === lastFinalPushResult,
3255
- );
3256
- if (drained) outboxCount -= 1;
3319
+ let drained = false;
3320
+ for (const result of results) {
3321
+ if (
3322
+ this.#handlePushResult(
3323
+ result,
3324
+ commitsById,
3325
+ summary,
3326
+ batch,
3327
+ rejectionDetailsByCommit.get(result.clientCommitId),
3328
+ )
3329
+ ) {
3330
+ outboxCount -= 1;
3331
+ drained = true;
3332
+ }
3333
+ }
3334
+ if (
3335
+ drained &&
3336
+ results.some((result) => result === lastFinalPushResult)
3337
+ ) {
3338
+ pruneCommitOutcomes(
3339
+ this.#db,
3340
+ this.#outcomeRetentionMaxEntries,
3341
+ );
3342
+ }
3257
3343
  },
3258
3344
  () => this.#statusSnapshot(outboxCount),
3345
+ (cause) => {
3346
+ this.#conflicts.length = conflictCount;
3347
+ this.#rejections.length = rejectionCount;
3348
+ const error = new ClientSyncError(
3349
+ 'client.outcome_persistence_failed',
3350
+ 'local commit outcome could not be persisted',
3351
+ );
3352
+ error.cause = cause;
3353
+ throw error;
3354
+ },
3259
3355
  );
3260
- responseOutboxCount = outboxCount;
3356
+ for (const conflict of this.#conflicts.slice(conflictCount)) {
3357
+ this.#config.onConflict?.(conflict);
3358
+ }
3261
3359
  break;
3262
3360
  }
3263
3361
  case 'PUSH_RESULT_DETAILS':
@@ -3506,7 +3604,6 @@ export class SyncClient {
3506
3604
  summary: MutableSummary,
3507
3605
  batch: ChangeAccumulator,
3508
3606
  rejectionDetails: ReadonlyMap<number, RejectionDetails> | undefined,
3509
- pruneOutcomes: boolean,
3510
3607
  ): boolean {
3511
3608
  const commit = commitsById.get(frame.clientCommitId);
3512
3609
  if (commit === undefined) return false;
@@ -3529,9 +3626,6 @@ export class SyncClient {
3529
3626
  })),
3530
3627
  });
3531
3628
  deleteOutboxCommit(this.#db, frame.clientCommitId);
3532
- if (pruneOutcomes) {
3533
- pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
3534
- }
3535
3629
  batch.status();
3536
3630
  batch.outcomes();
3537
3631
  summary.applied.push(frame.clientCommitId);
@@ -3569,7 +3663,6 @@ export class SyncClient {
3569
3663
  outcomeResults.push({ status: 'conflict', conflict });
3570
3664
  batch.conflicts();
3571
3665
  summary.conflicts.push(conflict);
3572
- this.#config.onConflict?.(conflict);
3573
3666
  } else if (result.status === 'error') {
3574
3667
  const details = rejectionDetails?.get(result.opIndex);
3575
3668
  const rejection: RejectionRecord = {
@@ -3597,9 +3690,6 @@ export class SyncClient {
3597
3690
  results: outcomeResults,
3598
3691
  operations: commit.operations,
3599
3692
  });
3600
- if (pruneOutcomes) {
3601
- pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
3602
- }
3603
3693
  batch.outcomes();
3604
3694
  // §7.2: remove the rejected optimistic layer. Before-images restore
3605
3695
  // validator-rejected updates even when the server emitted no new COMMIT;
@@ -4180,9 +4270,12 @@ export class SyncClient {
4180
4270
  * columns. No-op unless the schema has blob columns. `deleteOrphans`
4181
4271
  * triggers the revocation-side body deletion (B2).
4182
4272
  */
4183
- #reconcileBlobs(deleteOrphans: boolean): void {
4273
+ #reconcileBlobs(deleteOrphans: boolean, blobId?: string): void {
4184
4274
  if (!this.#hasBlobs) return;
4185
- reconcileBlobRefcounts(this.#db, this.#schema, { deleteOrphans });
4275
+ reconcileBlobRefcounts(this.#db, this.#schema, {
4276
+ deleteOrphans,
4277
+ ...(blobId === undefined ? {} : { blobId }),
4278
+ });
4186
4279
  }
4187
4280
 
4188
4281
  // -- helpers -----------------------------------------------------------------
@@ -40,6 +40,13 @@ export class NodeClientDatabase implements ClientDatabase {
40
40
 
41
41
  constructor(path = ':memory:') {
42
42
  this.db = new DatabaseSync(path);
43
+ try {
44
+ this.db.exec('PRAGMA journal_mode = WAL');
45
+ this.db.exec('PRAGMA synchronous = FULL');
46
+ } catch (error) {
47
+ this.db.close();
48
+ throw error;
49
+ }
43
50
  }
44
51
 
45
52
  exec(sql: string, params: readonly SqlValue[] = []): void {