@syncular/client 0.18.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -375,7 +375,7 @@ lifecycle with an `EncryptionConfig`; Worker handles use the portable keyring.
375
375
  Within one local SQLite transaction the engine deletes exactly the matching
376
376
  synced rows, lets generated FTS triggers remove their projections, drops every
377
377
  whole pending commit with a matching operation, restores/replays unrelated
378
- optimistic state, reconciles blob references, persists the `purgeId`, and emits
378
+ optimistic state, removes unreferenced cached blob bodies, persists the `purgeId`, and emits
379
379
  one revisioned change batch. A retry with the same canonical plan returns
380
380
  `alreadyApplied: true`; reusing an id with different selectors fails closed.
381
381
  Only bounded, non-empty, code-like values on plaintext string schema columns
@@ -484,9 +484,8 @@ and are cached locally. **Blob bytes live as `BLOB` columns in the client's own
484
484
  SQLite database** (a `_syncular_blobs` cache table), not in a separate OPFS
485
485
  directory or IndexedDB store. This is the pinned decision (SPEC §5.9.7 B1):
486
486
 
487
- - **One storage system.** The bytes are transactional with the refcount rows
488
- that pin them — a refcount adjust and a body insert/delete commit atomically,
489
- so a crash never strands a body against a stale count.
487
+ - **One storage system.** The bytes, upload state, and pending-commit references
488
+ share the client database. SQLite commits each change atomically.
490
489
  - **Survives restarts for free.** The client DB already rides OPFS via the
491
490
  sahpool VFS in the browser (and a plain file under `rusqlite`, `bun:sqlite`,
492
491
  or `node:sqlite` on native runtimes), so there is no second persistence
@@ -495,25 +494,24 @@ directory or IndexedDB store. This is the pinned decision (SPEC §5.9.7 B1):
495
494
  - **SQLite handles multi-MB images fine.** A page-cached `BLOB` read is a memory
496
495
  copy, well within the image/document envelope this targets.
497
496
 
498
- ### Size cap + LRU eviction
497
+ ### Size cap
499
498
 
500
499
  Pass `blobCacheMaxBytes` to cap the on-device cache. When the sum of cached body
501
- sizes exceeds the cap, the client evicts **zero-ref, non-pinned** bodies in
502
- least-recently-used order until back under the cap:
500
+ sizes exceeds the cap, the client evicts bodies with no visible row or pending
501
+ commit by creation timestamp, then blob ID, until back under the cap:
503
502
 
504
503
  ```ts
505
504
  new SyncClient({ /* … */, blobCacheMaxBytes: 256 * 1024 * 1024 }); // 256 MiB
506
505
  ```
507
506
 
508
- - A body **referenced by a live row** (refcount > 0) is **never** evicted — it
507
+ - A body **referenced by a live row** is **never** evicted. It
509
508
  stays resolvable without a re-download.
510
- - A body **pinned by a pending upload** (not yet pushed) is never evicted — its
509
+ - A body **needed by a pending upload** (not yet pushed) is never evicted. Its
511
510
  bytes are the only copy until the commit drains.
512
- - Evicting a zero-ref body only costs a future re-download, never correctness:
511
+ - Evicting an unreferenced body only costs a future re-download, never correctness:
513
512
  any surviving `blob_ref` value re-enables the fetch (§5.9.7 B3). If every
514
513
  over-cap body is referenced or pinned, the cache stays over the cap
515
- (correctness beats the cap). A cache-hit read touches "recently used", so a
516
- hot image survives a trim. Absent `blobCacheMaxBytes` ⇒ retain until storage
514
+ (correctness beats the cap). Absent `blobCacheMaxBytes` ⇒ retain until storage
517
515
  pressure (the default).
518
516
 
519
517
  ### Very large media — the escape hatch
@@ -522,7 +520,7 @@ SQLite is **not** the store for gigabyte video: a single `BLOB` must fit the
522
520
  client's memory and the SQLite row-size envelope. For very large media, run the
523
521
  server with presigned downloads (`blobSignedUrls`) and hand the presigned URL
524
522
  straight to a media element instead of pulling bytes through the cache — the
525
- image-app default (refcounted `BLOB` cache) and the large-media path (presigned
523
+ image-app default (`BLOB` cache) and the large-media path (presigned
526
524
  URL, no byte cache) coexist per attachment.
527
525
 
528
526
  ## Node and Bun SQLite backend (`./sqlite`)
package/dist/blob.d.ts CHANGED
@@ -1,14 +1,9 @@
1
1
  /**
2
2
  * Client-side blob cache + transport (SPEC.md §5.9.7).
3
3
  *
4
- * Blob bytes are cached content-addressed by `blobId` and refcounted by the
5
- * local rows whose `blob_ref` columns reference them (B1). The cache is
6
- * derived from live-row references: after any apply/purge, refcounts are
7
- * reconciled from the current `blob_ref` column contents, and a body whose
8
- * only referencing rows were revocation-purged is deleted (B2, evicted ≠
9
- * revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
10
- * row value is the whole download key. Pending uploads are tracked in the
11
- * outbox-adjacent uploads table (B4) and flushed before push.
4
+ * Blob bytes are cached by `blobId`. Retention reads live references directly
5
+ * from `blob_ref` columns and keeps explicit pending-commit dependencies.
6
+ * A small upload row queues each staged body for transfer before push.
12
7
  */
13
8
  import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
14
9
  import type { ClientDatabase } from './database.js';
@@ -82,48 +77,26 @@ export interface CachedBlob {
82
77
  /** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
83
78
  export declare function computeBlobId(bytes: Uint8Array): Promise<string>;
84
79
  export declare function ensureBlobSchema(db: ClientDatabase): void;
85
- /** Put bytes into the content-addressed cache (idempotent); touches LRU. */
80
+ /** Put bytes into the content-addressed cache without rewriting an existing body. */
86
81
  export declare function putCachedBlob(db: ClientDatabase, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): void;
87
- export declare function getCachedBlob(db: ClientDatabase, blobId: string, nowMs?: number): CachedBlob | undefined;
82
+ export declare function getCachedBlob(db: ClientDatabase, blobId: string): CachedBlob | undefined;
88
83
  /**
89
- * §5.9.7 B1 size cap + LRU eviction. When the sum of cached body sizes exceeds
90
- * `maxBytes`, evict **zero-ref, non-pinned** bodies in least-recently-used
91
- * order until back under the cap. NEVER evicts a referenced body (refcount > 0
92
- * — it must stay resolvable without a re-download) nor a pending-upload-pinned
93
- * body (its bytes are the only copy until push, B4). If every over-cap body is
94
- * referenced or pinned, the cache stays over the cap (correctness beats the
95
- * cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
96
- * any surviving `blob_ref` value. Returns the evicted blobIds.
84
+ * §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
85
+ * evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
86
+ * until back under the cap. Returns the evicted blobIds.
97
87
  */
98
- export declare function enforceBlobCacheCap(db: ClientDatabase, maxBytes: number): string[];
99
- /** Record a pending upload (§5.9.7 B4); flushed before the next push. */
100
- export declare function recordPendingUpload(db: ClientDatabase, blobId: string, nowMs: number, mediaType?: string): void;
88
+ export declare function enforceBlobCacheCap(db: ClientDatabase, schema: CompiledClientSchema, maxBytes: number): string[];
101
89
  export declare function listPendingUploads(db: ClientDatabase): {
102
90
  blobId: string;
103
91
  mediaType?: string;
104
92
  }[];
93
+ export declare function recordPendingUpload(db: ClientDatabase, blobId: string, nowMs: number, mediaType?: string): void;
105
94
  export declare function clearPendingUpload(db: ClientDatabase, blobId: string): void;
106
95
  /**
107
- * All `blob_ref` column names per table (for refcount reconciliation).
108
- * Blank result ⇒ the schema has no attachments; callers skip reconciliation.
96
+ * All `blob_ref` column names per table.
109
97
  */
110
98
  export declare function blobRefColumnsBySchema(schema: CompiledClientSchema): Map<string, string[]>;
111
99
  export declare function schemaHasBlobs(schema: CompiledClientSchema): boolean;
112
- /**
113
- * §5.9.7 B1/B2: recompute cache refcounts from the current `blob_ref` column
114
- * contents across all synced tables, then delete cache bodies that dropped
115
- * to zero refs *and* have no pending upload (a pending upload pins its body,
116
- * B4). Called after every apply/purge that may add or remove references.
117
- *
118
- * `blobId` narrows a fresh-download refresh to the inserted cache row.
119
- * `deleteOrphans` distinguishes the two B2 transitions: revocation purge
120
- * passes `true` (drop the now-unauthorized body); a benign apply passes
121
- * `false` (retain zero-ref bodies as LRU cache entries — the shipped
122
- * default). Bodies pinned by a pending upload are always retained.
123
- */
124
- export declare function reconcileBlobRefcounts(db: ClientDatabase, schema: CompiledClientSchema, options?: {
125
- readonly blobId?: string;
126
- readonly deleteOrphans?: boolean;
127
- }): void;
100
+ export declare function deleteUnreferencedCachedBlobs(db: ClientDatabase, schema: CompiledClientSchema): void;
128
101
  export type { BlobRef };
129
102
  export { parseBlobRef, serializeBlobRef };
package/dist/blob.js CHANGED
@@ -1,14 +1,9 @@
1
1
  /**
2
2
  * Client-side blob cache + transport (SPEC.md §5.9.7).
3
3
  *
4
- * Blob bytes are cached content-addressed by `blobId` and refcounted by the
5
- * local rows whose `blob_ref` columns reference them (B1). The cache is
6
- * derived from live-row references: after any apply/purge, refcounts are
7
- * reconciled from the current `blob_ref` column contents, and a body whose
8
- * only referencing rows were revocation-purged is deleted (B2, evicted ≠
9
- * revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
10
- * row value is the whole download key. Pending uploads are tracked in the
11
- * outbox-adjacent uploads table (B4) and flushed before push.
4
+ * Blob bytes are cached by `blobId`. Retention reads live references directly
5
+ * from `blob_ref` columns and keeps explicit pending-commit dependencies.
6
+ * A small upload row queues each staged body for transfer before push.
12
7
  */
13
8
  import { parseBlobRef, serializeBlobRef } from '@syncular/core';
14
9
  import { ClientSyncError } from './errors.js';
@@ -25,51 +20,46 @@ export async function computeBlobId(bytes) {
25
20
  return `sha256:${hex}`;
26
21
  }
27
22
  export function ensureBlobSchema(db) {
28
- db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blobs(
29
- blob_id TEXT PRIMARY KEY,
30
- bytes BLOB NOT NULL,
31
- byte_length INTEGER NOT NULL,
32
- media_type TEXT,
33
- refcount INTEGER NOT NULL DEFAULT 0,
34
- created_at_ms INTEGER NOT NULL,
35
- last_used_ms INTEGER NOT NULL DEFAULT 0)`);
36
- // Migrate a cache created before the §5.9.7 B1 LRU column: additive,
37
- // idempotent (a duplicate-column error on an already-migrated DB is
38
- // swallowed). last_used_ms drives cap eviction (LRU of zero-ref bodies).
39
- try {
40
- db.exec('ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0');
23
+ const existing = db.query("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs'");
24
+ if (existing.length > 0) {
25
+ const columns = db
26
+ .query('PRAGMA table_info(_syncular_blobs)')
27
+ .map((row) => row.name)
28
+ .sort();
29
+ const expected = [
30
+ 'blob_id',
31
+ 'byte_length',
32
+ 'bytes',
33
+ 'created_at_ms',
34
+ 'media_type',
35
+ ].sort();
36
+ if (columns.length !== expected.length ||
37
+ columns.some((column, index) => column !== expected[index])) {
38
+ throw new ClientSyncError('sync.schema_mismatch', 'Local blob schema is incompatible');
39
+ }
41
40
  }
42
- catch {
43
- // column already exists — the CREATE above included it
41
+ else {
42
+ db.exec(`CREATE TABLE _syncular_blobs(
43
+ blob_id TEXT PRIMARY KEY,
44
+ bytes BLOB NOT NULL,
45
+ byte_length INTEGER NOT NULL,
46
+ media_type TEXT,
47
+ created_at_ms INTEGER NOT NULL)`);
44
48
  }
45
49
  db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
46
- blob_id TEXT PRIMARY KEY,
47
- media_type TEXT,
48
- created_at_ms INTEGER NOT NULL)`);
49
- db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
50
- commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
51
- db.exec('CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)');
50
+ blob_id TEXT PRIMARY KEY, media_type TEXT, created_at_ms INTEGER NOT NULL)`);
52
51
  }
53
- /** Put bytes into the content-addressed cache (idempotent); touches LRU. */
52
+ /** Put bytes into the content-addressed cache without rewriting an existing body. */
54
53
  export function putCachedBlob(db, blobId, bytes, nowMs, mediaType) {
55
54
  db.exec(`INSERT INTO _syncular_blobs(
56
- blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms)
57
- VALUES (?,?,?,?,0,?,?)
58
- ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms`, [blobId, bytes, bytes.length, mediaType ?? null, nowMs, nowMs]);
55
+ blob_id, bytes, byte_length, media_type, created_at_ms)
56
+ VALUES (?,?,?,?,?) ON CONFLICT(blob_id) DO NOTHING`, [blobId, bytes, bytes.length, mediaType ?? null, nowMs]);
59
57
  }
60
- export function getCachedBlob(db, blobId, nowMs) {
58
+ export function getCachedBlob(db, blobId) {
61
59
  const rows = db.query('SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?', [blobId]);
62
60
  const row = rows[0];
63
61
  if (row === undefined)
64
62
  return undefined;
65
- // §5.9.7 B1 LRU: a cache-hit read touches "recently used" so a hot image
66
- // survives a cap trim. Skipped when no clock is supplied (pure read).
67
- if (nowMs !== undefined) {
68
- db.exec('UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?', [
69
- nowMs,
70
- blobId,
71
- ]);
72
- }
73
63
  return {
74
64
  blobId,
75
65
  bytes: row.bytes,
@@ -78,27 +68,21 @@ export function getCachedBlob(db, blobId, nowMs) {
78
68
  };
79
69
  }
80
70
  /**
81
- * §5.9.7 B1 size cap + LRU eviction. When the sum of cached body sizes exceeds
82
- * `maxBytes`, evict **zero-ref, non-pinned** bodies in least-recently-used
83
- * order until back under the cap. NEVER evicts a referenced body (refcount > 0
84
- * — it must stay resolvable without a re-download) nor a pending-upload-pinned
85
- * body (its bytes are the only copy until push, B4). If every over-cap body is
86
- * referenced or pinned, the cache stays over the cap (correctness beats the
87
- * cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
88
- * any surviving `blob_ref` value. Returns the evicted blobIds.
71
+ * §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
72
+ * evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
73
+ * until back under the cap. Returns the evicted blobIds.
89
74
  */
90
- export function enforceBlobCacheCap(db, maxBytes) {
75
+ export function enforceBlobCacheCap(db, schema, maxBytes) {
91
76
  const totalRow = db.query('SELECT COALESCE(SUM(byte_length), 0) AS total FROM _syncular_blobs')[0];
92
77
  let total = Number(totalRow?.total ?? 0);
93
78
  if (total <= maxBytes)
94
79
  return [];
95
- // Eviction candidates: zero-ref AND not pinned by a pending upload, oldest
96
- // (LRU) first, then oldest created as a stable tiebreak.
80
+ // Oldest eligible body first, with blob ID as a stable tie.
97
81
  const candidates = db.query(`SELECT blob_id, byte_length FROM _syncular_blobs
98
- WHERE refcount = 0
99
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
82
+ WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
100
83
  AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
101
- ORDER BY last_used_ms ASC, created_at_ms ASC`);
84
+ AND blob_id NOT IN (${visibleBlobIdsSql(schema)})
85
+ ORDER BY created_at_ms ASC, blob_id ASC`);
102
86
  const evicted = [];
103
87
  db.transaction(() => {
104
88
  for (const row of candidates) {
@@ -112,11 +96,6 @@ export function enforceBlobCacheCap(db, maxBytes) {
112
96
  });
113
97
  return evicted;
114
98
  }
115
- /** Record a pending upload (§5.9.7 B4); flushed before the next push. */
116
- export function recordPendingUpload(db, blobId, nowMs, mediaType) {
117
- db.exec(`INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms)
118
- VALUES (?,?,?)`, [blobId, mediaType ?? null, nowMs]);
119
- }
120
99
  export function listPendingUploads(db) {
121
100
  return db
122
101
  .query(`SELECT blob_id, media_type FROM (
@@ -137,12 +116,15 @@ export function listPendingUploads(db) {
137
116
  };
138
117
  });
139
118
  }
119
+ export function recordPendingUpload(db, blobId, nowMs, mediaType) {
120
+ db.exec(`INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms)
121
+ VALUES (?,?,?)`, [blobId, mediaType ?? null, nowMs]);
122
+ }
140
123
  export function clearPendingUpload(db, blobId) {
141
124
  db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id = ?', [blobId]);
142
125
  }
143
126
  /**
144
- * All `blob_ref` column names per table (for refcount reconciliation).
145
- * Blank result ⇒ the schema has no attachments; callers skip reconciliation.
127
+ * All `blob_ref` column names per table.
146
128
  */
147
129
  export function blobRefColumnsBySchema(schema) {
148
130
  const out = new Map();
@@ -162,79 +144,26 @@ export function schemaHasBlobs(schema) {
162
144
  }
163
145
  return false;
164
146
  }
165
- /**
166
- * §5.9.7 B1/B2: recompute cache refcounts from the current `blob_ref` column
167
- * contents across all synced tables, then delete cache bodies that dropped
168
- * to zero refs *and* have no pending upload (a pending upload pins its body,
169
- * B4). Called after every apply/purge that may add or remove references.
170
- *
171
- * `blobId` narrows a fresh-download refresh to the inserted cache row.
172
- * `deleteOrphans` distinguishes the two B2 transitions: revocation purge
173
- * passes `true` (drop the now-unauthorized body); a benign apply passes
174
- * `false` (retain zero-ref bodies as LRU cache entries — the shipped
175
- * default). Bodies pinned by a pending upload are always retained.
176
- */
177
- export function reconcileBlobRefcounts(db, schema, options) {
178
- const byTable = blobRefColumnsBySchema(schema);
179
- if (byTable.size === 0)
180
- return;
181
- if (options?.blobId !== undefined) {
182
- let count = 0;
183
- for (const [tableName, columns] of byTable) {
184
- for (const column of columns) {
185
- const identifier = quoteIdent(column);
186
- const row = db.query(`SELECT count(*) AS n FROM ${quoteIdent(tableName)}
187
- WHERE CASE
188
- WHEN typeof(${identifier}) = 'text' AND json_valid(${identifier})
189
- THEN json_extract(${identifier}, '$.blobId')
190
- END = ?`, [options.blobId])[0];
191
- count += Number(row?.n ?? 0);
192
- }
193
- }
194
- db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
195
- count,
196
- options.blobId,
197
- ]);
198
- return;
199
- }
200
- // Count references to each blobId across every blob_ref column.
201
- const counts = new Map();
202
- for (const [tableName, columns] of byTable) {
147
+ function visibleBlobIdsSql(schema) {
148
+ const selects = [];
149
+ for (const [tableName, columns] of blobRefColumnsBySchema(schema)) {
203
150
  for (const column of columns) {
204
- const rows = db.query(`SELECT ${quoteIdent(column)} AS v FROM ${quoteIdent(tableName)}
205
- WHERE ${quoteIdent(column)} IS NOT NULL`);
206
- for (const row of rows) {
207
- const raw = row.v;
208
- if (typeof raw !== 'string')
209
- continue;
210
- let ref;
211
- try {
212
- ref = parseBlobRef(raw);
213
- }
214
- catch {
215
- continue;
216
- }
217
- counts.set(ref.blobId, (counts.get(ref.blobId) ?? 0) + 1);
218
- }
151
+ const identifier = quoteIdent(column);
152
+ selects.push(`SELECT json_extract(${identifier}, '$.blobId') AS blob_id
153
+ FROM ${quoteIdent(tableName)}
154
+ WHERE typeof(${identifier}) = 'text'
155
+ AND json_valid(${identifier})
156
+ AND json_type(${identifier}, '$.blobId') = 'text'`);
219
157
  }
220
158
  }
221
- db.transaction(() => {
222
- // Reset all refcounts, then apply the recomputed counts.
223
- db.exec('UPDATE _syncular_blobs SET refcount = 0');
224
- for (const [blobId, count] of counts) {
225
- db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
226
- count,
227
- blobId,
228
- ]);
229
- }
230
- if (options?.deleteOrphans === true) {
231
- // §5.9.7 B2 revocation side: delete zero-ref bodies not pinned by a
232
- // pending upload.
233
- db.exec(`DELETE FROM _syncular_blobs
234
- WHERE refcount = 0
235
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
236
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)`);
237
- }
238
- });
159
+ return selects.length === 0
160
+ ? 'SELECT NULL AS blob_id WHERE 0'
161
+ : selects.join(' UNION ');
162
+ }
163
+ export function deleteUnreferencedCachedBlobs(db, schema) {
164
+ db.exec(`DELETE FROM _syncular_blobs
165
+ WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
166
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
167
+ AND blob_id NOT IN (${visibleBlobIdsSql(schema)})`);
239
168
  }
240
169
  export { parseBlobRef, serializeBlobRef };
package/dist/client.d.ts CHANGED
@@ -110,9 +110,9 @@ export interface SyncClientConfig {
110
110
  readonly blobs?: BlobTransport;
111
111
  /**
112
112
  * §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
113
- * exceeds this, zero-ref, non-pinned bodies are evicted LRU-first after each
114
- * cache write (referenced/pinned bodies are never evicted — correctness
115
- * beats the cap). Absent ⇒ retain until storage pressure (the shipped
113
+ * exceeds this, unreferenced bodies without pending work are evicted in
114
+ * creation timestamp order after each cache write. Referenced bodies remain cached
115
+ * even when that exceeds the cap. Absent means retain until storage pressure (the shipped
116
116
  * default; a referenced body always stays resolvable without a re-download).
117
117
  */
118
118
  readonly blobCacheMaxBytes?: number;
package/dist/client.js CHANGED
@@ -9,7 +9,7 @@
9
9
  */
10
10
  import { canonicalScopeJson, decodeMessage, decodeRow, decodeRowsSegment, encodeMessage, encodePresencePublish, MessageStreamScanner, PROTOCOL_WIRE_VERSION, parseRealtimeServerEvent, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
11
11
  import { applyCommitFrame, applyRowsSegment, applySqliteSegment, deleteLocalRow, deleteScopedRows, evictScopedRows, upsertLocalRow, } from './apply.js';
12
- import { clearPendingUpload, computeBlobId, enforceBlobCacheCap, ensureBlobSchema, getCachedBlob, listPendingUploads, parseBlobRef, putCachedBlob, reconcileBlobRefcounts, recordPendingUpload, schemaHasBlobs, serializeBlobRef, } from './blob.js';
12
+ import { clearPendingUpload, computeBlobId, deleteUnreferencedCachedBlobs, enforceBlobCacheCap, ensureBlobSchema, getCachedBlob, listPendingUploads, parseBlobRef, putCachedBlob, recordPendingUpload, schemaHasBlobs, serializeBlobRef, } from './blob.js';
13
13
  import { registerDevtools } from './devtools.js';
14
14
  import { CLIENT_DIAGNOSTICS_VERSION, ClientDiagnosticsEmitter, MAX_DIAGNOSTIC_DOMAINS, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS, } from './diagnostics.js';
15
15
  import { ClientSyncError } from './errors.js';
@@ -191,39 +191,8 @@ export class SyncClient {
191
191
  // Do not materialize new app indexes/FTS projections yet: on a version
192
192
  // bump they may reference columns that only exist after the reset.
193
193
  ensureLocalBookkeepingSchema(this.#db);
194
- this.#db.transaction(() => {
195
- for (const trigger of ['insert', 'update', 'delete', 'cache'])
196
- this.#db.exec(`DROP TRIGGER IF EXISTS _syncular_blob_commit_${trigger}`);
197
- if (this.#hasBlobs) {
198
- ensureBlobSchema(this.#db);
199
- const columns = JSON.stringify([...this.#schema.tables.values()].flatMap((table) => table.columns
200
- .filter((column) => column.type === 'blob_ref')
201
- .map((column) => [table.name, column.name]))).replaceAll("'", "''");
202
- const references = `SELECT o.client_commit_id, b.blob_id
203
- FROM _syncular_outbox o JOIN json_each(o.operations) op
204
- JOIN json_each(op.value, '$.values') v JOIN json_each('${columns}') c
205
- JOIN _syncular_blobs b ON b.blob_id = CASE WHEN v.type = 'text' AND json_valid(v.value) THEN json_extract(v.value, '$.blobId') END
206
- WHERE json_extract(op.value, '$.op') = 'upsert'
207
- AND json_extract(op.value, '$.table') = json_extract(c.value, '$[0]')
208
- AND v.key = json_extract(c.value, '$[1]')`;
209
- this.#db
210
- .exec(`CREATE TRIGGER _syncular_blob_commit_insert AFTER INSERT ON _syncular_outbox BEGIN
211
- INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
212
- this.#db
213
- .exec(`CREATE TRIGGER _syncular_blob_commit_update AFTER UPDATE OF operations ON _syncular_outbox BEGIN
214
- DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id;
215
- INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
216
- this.#db
217
- .exec(`CREATE TRIGGER _syncular_blob_commit_delete BEFORE DELETE ON _syncular_outbox BEGIN
218
- DELETE FROM _syncular_blob_uploads WHERE blob_id IN (SELECT blob_id FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id)
219
- AND NOT EXISTS (SELECT 1 FROM _syncular_blob_commit_refs r WHERE r.blob_id = _syncular_blob_uploads.blob_id AND r.commit_id != OLD.client_commit_id);
220
- DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id; END`);
221
- this.#db
222
- .exec(`CREATE TRIGGER _syncular_blob_commit_cache AFTER INSERT ON _syncular_blobs BEGIN
223
- INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND b.blob_id = NEW.blob_id; END`);
224
- this.#db.exec(`INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references}`);
225
- }
226
- });
194
+ if (this.#hasBlobs)
195
+ ensureBlobSchema(this.#db);
227
196
  this.#db.transaction(() => {
228
197
  pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
229
198
  });
@@ -893,13 +862,11 @@ export class SyncClient {
893
862
  throw new ClientSyncError('sync.invalid_request', 'uploadBlob requires a blob transport (SyncClientConfig.blobs, §5.9)');
894
863
  }
895
864
  const blobId = await computeBlobId(bytes);
865
+ const nowMs = this.#now();
896
866
  this.#db.transaction(() => {
897
- putCachedBlob(this.#db, blobId, bytes, this.#now(), options?.mediaType);
898
- recordPendingUpload(this.#db, blobId, this.#now(), options?.mediaType);
867
+ putCachedBlob(this.#db, blobId, bytes, nowMs, options?.mediaType);
868
+ recordPendingUpload(this.#db, blobId, nowMs, options?.mediaType);
899
869
  });
900
- // §5.9.7 B1: a staged upload is pinned (recordPendingUpload), so the cap
901
- // trim below will never evict it — but a stage may push other zero-ref
902
- // bodies over the cap, so run the trim.
903
870
  this.#enforceBlobCacheCap();
904
871
  return {
905
872
  blobId,
@@ -927,7 +894,7 @@ export class SyncClient {
927
894
  const blobId = blobIdOrRef.startsWith('sha256:')
928
895
  ? blobIdOrRef
929
896
  : parseBlobRef(blobIdOrRef).blobId;
930
- const cached = getCachedBlob(this.#db, blobId, this.#now());
897
+ const cached = getCachedBlob(this.#db, blobId);
931
898
  if (cached !== undefined)
932
899
  return cached;
933
900
  const transport = this.#config.blobs;
@@ -962,9 +929,7 @@ export class SyncClient {
962
929
  throw new ClientSyncError('sync.invalid_request', `blob content address mismatch for ${blobId} (§5.9.5)`);
963
930
  }
964
931
  putCachedBlob(this.#db, blobId, bytes, this.#now());
965
- // The referencing row can arrive before its body. Pin the new cache entry
966
- // from current visible references before applying the size cap (§5.9.7 B1).
967
- this.#reconcileBlobs(false, blobId);
932
+ this.#db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
968
933
  this.#enforceBlobCacheCap();
969
934
  const stored = getCachedBlob(this.#db, blobId);
970
935
  if (stored === undefined) {
@@ -977,7 +942,7 @@ export class SyncClient {
977
942
  const cap = this.#config.blobCacheMaxBytes;
978
943
  if (cap === undefined)
979
944
  return;
980
- enforceBlobCacheCap(this.#db, cap);
945
+ enforceBlobCacheCap(this.#db, this.#schema, cap);
981
946
  }
982
947
  /** Flush any queued blob uploads (§5.9.7 B4); safe to call standalone. */
983
948
  flushBlobUploads() {
@@ -1446,6 +1411,31 @@ export class SyncClient {
1446
1411
  this.#applyBatch((batch) => {
1447
1412
  this.#db.transaction(() => {
1448
1413
  appendOutboxCommit(this.#db, clientCommitId, operations, this.#now(), this.#captureBeforeImages(operations));
1414
+ for (const operation of operations) {
1415
+ if (operation.op !== 'upsert')
1416
+ continue;
1417
+ const table = this.#schema.tables.get(operation.table);
1418
+ if (table === undefined)
1419
+ continue;
1420
+ for (const column of table.columns) {
1421
+ if (column.type !== 'blob_ref')
1422
+ continue;
1423
+ const raw = operation.values?.[column.name];
1424
+ if (typeof raw !== 'string')
1425
+ continue;
1426
+ let blobId;
1427
+ try {
1428
+ blobId = parseBlobRef(raw).blobId;
1429
+ }
1430
+ catch {
1431
+ continue;
1432
+ }
1433
+ this.#db.exec(`INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
1434
+ SELECT ?, ? WHERE EXISTS (
1435
+ SELECT 1 FROM _syncular_blobs WHERE blob_id = ?
1436
+ )`, [clientCommitId, blobId, blobId]);
1437
+ }
1438
+ }
1449
1439
  this.#applyOperationsLocally(operations, batch);
1450
1440
  batch.status();
1451
1441
  });
@@ -1604,7 +1594,7 @@ export class SyncClient {
1604
1594
  batch.rejections();
1605
1595
  batch.outcomes();
1606
1596
  }
1607
- this.#reconcileBlobs(true);
1597
+ this.#deleteUnreferencedBlobs();
1608
1598
  setMeta(this.#db, metaKey, purge.canonicalPlan);
1609
1599
  return {
1610
1600
  alreadyApplied: false,
@@ -2546,7 +2536,6 @@ export class SyncClient {
2546
2536
  // §7.1: local reads see outbox state applied optimistically — replay
2547
2537
  // the still-pending commits on top of the freshly applied server state.
2548
2538
  this.#replayOutbox();
2549
- this.#reconcileBlobs(false);
2550
2539
  }
2551
2540
  finally {
2552
2541
  this.#endDiagnosticsDeferral();
@@ -2935,7 +2924,7 @@ export class SyncClient {
2935
2924
  batch.rejections();
2936
2925
  batch.outcomes();
2937
2926
  }
2938
- this.#reconcileBlobs(true);
2927
+ this.#deleteUnreferencedBlobs();
2939
2928
  }
2940
2929
  catch (error) {
2941
2930
  if (error instanceof ClientSyncError &&
@@ -3107,18 +3096,10 @@ export class SyncClient {
3107
3096
  });
3108
3097
  });
3109
3098
  }
3110
- /**
3111
- * §5.9.7 B1/B2: recompute blob-cache refcounts from live `blob_ref`
3112
- * columns. No-op unless the schema has blob columns. `deleteOrphans`
3113
- * triggers the revocation-side body deletion (B2).
3114
- */
3115
- #reconcileBlobs(deleteOrphans, blobId) {
3099
+ #deleteUnreferencedBlobs() {
3116
3100
  if (!this.#hasBlobs)
3117
3101
  return;
3118
- reconcileBlobRefcounts(this.#db, this.#schema, {
3119
- deleteOrphans,
3120
- ...(blobId === undefined ? {} : { blobId }),
3121
- });
3102
+ deleteUnreferencedCachedBlobs(this.#db, this.#schema);
3122
3103
  }
3123
3104
  // -- helpers -----------------------------------------------------------------
3124
3105
  #table(name) {
package/dist/outbox.js CHANGED
@@ -62,6 +62,9 @@ export function countOutbox(db) {
62
62
  .count;
63
63
  }
64
64
  export function deleteOutboxCommit(db, clientCommitId) {
65
+ db.exec('DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?', [
66
+ clientCommitId,
67
+ ]);
65
68
  db.exec('DELETE FROM _syncular_outbox_before_images WHERE client_commit_id = ?', [clientCommitId]);
66
69
  db.exec('DELETE FROM _syncular_outbox WHERE client_commit_id = ?', [
67
70
  clientCommitId,
package/dist/schema.js CHANGED
@@ -339,6 +339,9 @@ export function ensureLocalBookkeepingSchema(db) {
339
339
  db.transaction(() => {
340
340
  db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
341
341
  key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
342
+ db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
343
+ commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
344
+ db.exec('CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)');
342
345
  db.exec(`INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`);
343
346
  db.exec(`CREATE TABLE IF NOT EXISTS _syncular_outbox(
344
347
  seq INTEGER PRIMARY KEY AUTOINCREMENT,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/client",
3
- "version": "0.18.0",
3
+ "version": "0.19.0",
4
4
  "description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -89,9 +89,9 @@
89
89
  },
90
90
  "dependencies": {
91
91
  "@sqlite.org/sqlite-wasm": "^3.53.0-build1",
92
- "@syncular/core": "0.18.0"
92
+ "@syncular/core": "0.19.0"
93
93
  },
94
94
  "devDependencies": {
95
- "@syncular/server": "0.18.0"
95
+ "@syncular/server": "0.19.0"
96
96
  }
97
97
  }
package/src/blob.ts CHANGED
@@ -1,14 +1,9 @@
1
1
  /**
2
2
  * Client-side blob cache + transport (SPEC.md §5.9.7).
3
3
  *
4
- * Blob bytes are cached content-addressed by `blobId` and refcounted by the
5
- * local rows whose `blob_ref` columns reference them (B1). The cache is
6
- * derived from live-row references: after any apply/purge, refcounts are
7
- * reconciled from the current `blob_ref` column contents, and a body whose
8
- * only referencing rows were revocation-purged is deleted (B2, evicted ≠
9
- * revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
10
- * row value is the whole download key. Pending uploads are tracked in the
11
- * outbox-adjacent uploads table (B4) and flushed before push.
4
+ * Blob bytes are cached by `blobId`. Retention reads live references directly
5
+ * from `blob_ref` columns and keeps explicit pending-commit dependencies.
6
+ * A small upload row queues each staged body for transfer before push.
12
7
  */
13
8
  import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
14
9
  import type { ClientDatabase } from './database';
@@ -109,36 +104,43 @@ export async function computeBlobId(bytes: Uint8Array): Promise<string> {
109
104
  }
110
105
 
111
106
  export function ensureBlobSchema(db: ClientDatabase): void {
112
- db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blobs(
113
- blob_id TEXT PRIMARY KEY,
114
- bytes BLOB NOT NULL,
115
- byte_length INTEGER NOT NULL,
116
- media_type TEXT,
117
- refcount INTEGER NOT NULL DEFAULT 0,
118
- created_at_ms INTEGER NOT NULL,
119
- last_used_ms INTEGER NOT NULL DEFAULT 0)`);
120
- // Migrate a cache created before the §5.9.7 B1 LRU column: additive,
121
- // idempotent (a duplicate-column error on an already-migrated DB is
122
- // swallowed). last_used_ms drives cap eviction (LRU of zero-ref bodies).
123
- try {
124
- db.exec(
125
- 'ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0',
126
- );
127
- } catch {
128
- // column already exists — the CREATE above included it
107
+ const existing = db.query(
108
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs'",
109
+ );
110
+ if (existing.length > 0) {
111
+ const columns = db
112
+ .query('PRAGMA table_info(_syncular_blobs)')
113
+ .map((row) => row.name)
114
+ .sort();
115
+ const expected = [
116
+ 'blob_id',
117
+ 'byte_length',
118
+ 'bytes',
119
+ 'created_at_ms',
120
+ 'media_type',
121
+ ].sort();
122
+ if (
123
+ columns.length !== expected.length ||
124
+ columns.some((column, index) => column !== expected[index])
125
+ ) {
126
+ throw new ClientSyncError(
127
+ 'sync.schema_mismatch',
128
+ 'Local blob schema is incompatible',
129
+ );
130
+ }
131
+ } else {
132
+ db.exec(`CREATE TABLE _syncular_blobs(
133
+ blob_id TEXT PRIMARY KEY,
134
+ bytes BLOB NOT NULL,
135
+ byte_length INTEGER NOT NULL,
136
+ media_type TEXT,
137
+ created_at_ms INTEGER NOT NULL)`);
129
138
  }
130
139
  db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
131
- blob_id TEXT PRIMARY KEY,
132
- media_type TEXT,
133
- created_at_ms INTEGER NOT NULL)`);
134
- db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
135
- commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
136
- db.exec(
137
- 'CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)',
138
- );
140
+ blob_id TEXT PRIMARY KEY, media_type TEXT, created_at_ms INTEGER NOT NULL)`);
139
141
  }
140
142
 
141
- /** Put bytes into the content-addressed cache (idempotent); touches LRU. */
143
+ /** Put bytes into the content-addressed cache without rewriting an existing body. */
142
144
  export function putCachedBlob(
143
145
  db: ClientDatabase,
144
146
  blobId: string,
@@ -148,17 +150,15 @@ export function putCachedBlob(
148
150
  ): void {
149
151
  db.exec(
150
152
  `INSERT INTO _syncular_blobs(
151
- blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms)
152
- VALUES (?,?,?,?,0,?,?)
153
- ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms`,
154
- [blobId, bytes, bytes.length, mediaType ?? null, nowMs, nowMs],
153
+ blob_id, bytes, byte_length, media_type, created_at_ms)
154
+ VALUES (?,?,?,?,?) ON CONFLICT(blob_id) DO NOTHING`,
155
+ [blobId, bytes, bytes.length, mediaType ?? null, nowMs],
155
156
  );
156
157
  }
157
158
 
158
159
  export function getCachedBlob(
159
160
  db: ClientDatabase,
160
161
  blobId: string,
161
- nowMs?: number,
162
162
  ): CachedBlob | undefined {
163
163
  const rows = db.query(
164
164
  'SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?',
@@ -166,14 +166,6 @@ export function getCachedBlob(
166
166
  );
167
167
  const row = rows[0];
168
168
  if (row === undefined) return undefined;
169
- // §5.9.7 B1 LRU: a cache-hit read touches "recently used" so a hot image
170
- // survives a cap trim. Skipped when no clock is supplied (pure read).
171
- if (nowMs !== undefined) {
172
- db.exec('UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?', [
173
- nowMs,
174
- blobId,
175
- ]);
176
- }
177
169
  return {
178
170
  blobId,
179
171
  bytes: row.bytes as Uint8Array,
@@ -183,17 +175,13 @@ export function getCachedBlob(
183
175
  }
184
176
 
185
177
  /**
186
- * §5.9.7 B1 size cap + LRU eviction. When the sum of cached body sizes exceeds
187
- * `maxBytes`, evict **zero-ref, non-pinned** bodies in least-recently-used
188
- * order until back under the cap. NEVER evicts a referenced body (refcount > 0
189
- * — it must stay resolvable without a re-download) nor a pending-upload-pinned
190
- * body (its bytes are the only copy until push, B4). If every over-cap body is
191
- * referenced or pinned, the cache stays over the cap (correctness beats the
192
- * cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
193
- * any surviving `blob_ref` value. Returns the evicted blobIds.
178
+ * §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
179
+ * evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
180
+ * until back under the cap. Returns the evicted blobIds.
194
181
  */
195
182
  export function enforceBlobCacheCap(
196
183
  db: ClientDatabase,
184
+ schema: CompiledClientSchema,
197
185
  maxBytes: number,
198
186
  ): string[] {
199
187
  const totalRow = db.query(
@@ -201,14 +189,13 @@ export function enforceBlobCacheCap(
201
189
  )[0];
202
190
  let total = Number(totalRow?.total ?? 0);
203
191
  if (total <= maxBytes) return [];
204
- // Eviction candidates: zero-ref AND not pinned by a pending upload, oldest
205
- // (LRU) first, then oldest created as a stable tiebreak.
192
+ // Oldest eligible body first, with blob ID as a stable tie.
206
193
  const candidates = db.query(
207
194
  `SELECT blob_id, byte_length FROM _syncular_blobs
208
- WHERE refcount = 0
209
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
195
+ WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
210
196
  AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
211
- ORDER BY last_used_ms ASC, created_at_ms ASC`,
197
+ AND blob_id NOT IN (${visibleBlobIdsSql(schema)})
198
+ ORDER BY created_at_ms ASC, blob_id ASC`,
212
199
  );
213
200
  const evicted: string[] = [];
214
201
  db.transaction(() => {
@@ -223,20 +210,6 @@ export function enforceBlobCacheCap(
223
210
  return evicted;
224
211
  }
225
212
 
226
- /** Record a pending upload (§5.9.7 B4); flushed before the next push. */
227
- export function recordPendingUpload(
228
- db: ClientDatabase,
229
- blobId: string,
230
- nowMs: number,
231
- mediaType?: string,
232
- ): void {
233
- db.exec(
234
- `INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms)
235
- VALUES (?,?,?)`,
236
- [blobId, mediaType ?? null, nowMs],
237
- );
238
- }
239
-
240
213
  export function listPendingUploads(
241
214
  db: ClientDatabase,
242
215
  ): { blobId: string; mediaType?: string }[] {
@@ -267,13 +240,25 @@ export function listPendingUploads(
267
240
  });
268
241
  }
269
242
 
243
+ export function recordPendingUpload(
244
+ db: ClientDatabase,
245
+ blobId: string,
246
+ nowMs: number,
247
+ mediaType?: string,
248
+ ): void {
249
+ db.exec(
250
+ `INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms)
251
+ VALUES (?,?,?)`,
252
+ [blobId, mediaType ?? null, nowMs],
253
+ );
254
+ }
255
+
270
256
  export function clearPendingUpload(db: ClientDatabase, blobId: string): void {
271
257
  db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id = ?', [blobId]);
272
258
  }
273
259
 
274
260
  /**
275
- * All `blob_ref` column names per table (for refcount reconciliation).
276
- * Blank result ⇒ the schema has no attachments; callers skip reconciliation.
261
+ * All `blob_ref` column names per table.
277
262
  */
278
263
  export function blobRefColumnsBySchema(
279
264
  schema: CompiledClientSchema,
@@ -295,91 +280,35 @@ export function schemaHasBlobs(schema: CompiledClientSchema): boolean {
295
280
  return false;
296
281
  }
297
282
 
298
- /**
299
- * §5.9.7 B1/B2: recompute cache refcounts from the current `blob_ref` column
300
- * contents across all synced tables, then delete cache bodies that dropped
301
- * to zero refs *and* have no pending upload (a pending upload pins its body,
302
- * B4). Called after every apply/purge that may add or remove references.
303
- *
304
- * `blobId` narrows a fresh-download refresh to the inserted cache row.
305
- * `deleteOrphans` distinguishes the two B2 transitions: revocation purge
306
- * passes `true` (drop the now-unauthorized body); a benign apply passes
307
- * `false` (retain zero-ref bodies as LRU cache entries — the shipped
308
- * default). Bodies pinned by a pending upload are always retained.
309
- */
310
- export function reconcileBlobRefcounts(
311
- db: ClientDatabase,
312
- schema: CompiledClientSchema,
313
- options?: {
314
- readonly blobId?: string;
315
- readonly deleteOrphans?: boolean;
316
- },
317
- ): void {
318
- const byTable = blobRefColumnsBySchema(schema);
319
- if (byTable.size === 0) return;
320
- if (options?.blobId !== undefined) {
321
- let count = 0;
322
- for (const [tableName, columns] of byTable) {
323
- for (const column of columns) {
324
- const identifier = quoteIdent(column);
325
- const row = db.query(
326
- `SELECT count(*) AS n FROM ${quoteIdent(tableName)}
327
- WHERE CASE
328
- WHEN typeof(${identifier}) = 'text' AND json_valid(${identifier})
329
- THEN json_extract(${identifier}, '$.blobId')
330
- END = ?`,
331
- [options.blobId],
332
- )[0];
333
- count += Number(row?.n ?? 0);
334
- }
335
- }
336
- db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
337
- count,
338
- options.blobId,
339
- ]);
340
- return;
341
- }
342
- // Count references to each blobId across every blob_ref column.
343
- const counts = new Map<string, number>();
344
- for (const [tableName, columns] of byTable) {
283
+ function visibleBlobIdsSql(schema: CompiledClientSchema): string {
284
+ const selects: string[] = [];
285
+ for (const [tableName, columns] of blobRefColumnsBySchema(schema)) {
345
286
  for (const column of columns) {
346
- const rows = db.query(
347
- `SELECT ${quoteIdent(column)} AS v FROM ${quoteIdent(tableName)}
348
- WHERE ${quoteIdent(column)} IS NOT NULL`,
287
+ const identifier = quoteIdent(column);
288
+ selects.push(
289
+ `SELECT json_extract(${identifier}, '$.blobId') AS blob_id
290
+ FROM ${quoteIdent(tableName)}
291
+ WHERE typeof(${identifier}) = 'text'
292
+ AND json_valid(${identifier})
293
+ AND json_type(${identifier}, '$.blobId') = 'text'`,
349
294
  );
350
- for (const row of rows) {
351
- const raw = row.v;
352
- if (typeof raw !== 'string') continue;
353
- let ref: BlobRef;
354
- try {
355
- ref = parseBlobRef(raw);
356
- } catch {
357
- continue;
358
- }
359
- counts.set(ref.blobId, (counts.get(ref.blobId) ?? 0) + 1);
360
- }
361
295
  }
362
296
  }
363
- db.transaction(() => {
364
- // Reset all refcounts, then apply the recomputed counts.
365
- db.exec('UPDATE _syncular_blobs SET refcount = 0');
366
- for (const [blobId, count] of counts) {
367
- db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
368
- count,
369
- blobId,
370
- ]);
371
- }
372
- if (options?.deleteOrphans === true) {
373
- // §5.9.7 B2 revocation side: delete zero-ref bodies not pinned by a
374
- // pending upload.
375
- db.exec(
376
- `DELETE FROM _syncular_blobs
377
- WHERE refcount = 0
378
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
379
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)`,
380
- );
381
- }
382
- });
297
+ return selects.length === 0
298
+ ? 'SELECT NULL AS blob_id WHERE 0'
299
+ : selects.join(' UNION ');
300
+ }
301
+
302
+ export function deleteUnreferencedCachedBlobs(
303
+ db: ClientDatabase,
304
+ schema: CompiledClientSchema,
305
+ ): void {
306
+ db.exec(
307
+ `DELETE FROM _syncular_blobs
308
+ WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
309
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
310
+ AND blob_id NOT IN (${visibleBlobIdsSql(schema)})`,
311
+ );
383
312
  }
384
313
 
385
314
  export type { BlobRef };
package/src/client.ts CHANGED
@@ -47,13 +47,13 @@ import {
47
47
  type CachedBlob,
48
48
  clearPendingUpload,
49
49
  computeBlobId,
50
+ deleteUnreferencedCachedBlobs,
50
51
  enforceBlobCacheCap,
51
52
  ensureBlobSchema,
52
53
  getCachedBlob,
53
54
  listPendingUploads,
54
55
  parseBlobRef,
55
56
  putCachedBlob,
56
- reconcileBlobRefcounts,
57
57
  recordPendingUpload,
58
58
  schemaHasBlobs,
59
59
  serializeBlobRef,
@@ -302,9 +302,9 @@ export interface SyncClientConfig {
302
302
  readonly blobs?: BlobTransport;
303
303
  /**
304
304
  * §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
305
- * exceeds this, zero-ref, non-pinned bodies are evicted LRU-first after each
306
- * cache write (referenced/pinned bodies are never evicted — correctness
307
- * beats the cap). Absent ⇒ retain until storage pressure (the shipped
305
+ * exceeds this, unreferenced bodies without pending work are evicted in
306
+ * creation timestamp order after each cache write. Referenced bodies remain cached
307
+ * even when that exceeds the cap. Absent means retain until storage pressure (the shipped
308
308
  * default; a referenced body always stays resolvable without a re-download).
309
309
  */
310
310
  readonly blobCacheMaxBytes?: number;
@@ -688,47 +688,7 @@ export class SyncClient {
688
688
  // Do not materialize new app indexes/FTS projections yet: on a version
689
689
  // bump they may reference columns that only exist after the reset.
690
690
  ensureLocalBookkeepingSchema(this.#db);
691
- this.#db.transaction(() => {
692
- for (const trigger of ['insert', 'update', 'delete', 'cache'])
693
- this.#db.exec(
694
- `DROP TRIGGER IF EXISTS _syncular_blob_commit_${trigger}`,
695
- );
696
- if (this.#hasBlobs) {
697
- ensureBlobSchema(this.#db);
698
- const columns = JSON.stringify(
699
- [...this.#schema.tables.values()].flatMap((table) =>
700
- table.columns
701
- .filter((column) => column.type === 'blob_ref')
702
- .map((column) => [table.name, column.name]),
703
- ),
704
- ).replaceAll("'", "''");
705
- const references = `SELECT o.client_commit_id, b.blob_id
706
- FROM _syncular_outbox o JOIN json_each(o.operations) op
707
- JOIN json_each(op.value, '$.values') v JOIN json_each('${columns}') c
708
- JOIN _syncular_blobs b ON b.blob_id = CASE WHEN v.type = 'text' AND json_valid(v.value) THEN json_extract(v.value, '$.blobId') END
709
- WHERE json_extract(op.value, '$.op') = 'upsert'
710
- AND json_extract(op.value, '$.table') = json_extract(c.value, '$[0]')
711
- AND v.key = json_extract(c.value, '$[1]')`;
712
- this.#db
713
- .exec(`CREATE TRIGGER _syncular_blob_commit_insert AFTER INSERT ON _syncular_outbox BEGIN
714
- INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
715
- this.#db
716
- .exec(`CREATE TRIGGER _syncular_blob_commit_update AFTER UPDATE OF operations ON _syncular_outbox BEGIN
717
- DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id;
718
- INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
719
- this.#db
720
- .exec(`CREATE TRIGGER _syncular_blob_commit_delete BEFORE DELETE ON _syncular_outbox BEGIN
721
- DELETE FROM _syncular_blob_uploads WHERE blob_id IN (SELECT blob_id FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id)
722
- AND NOT EXISTS (SELECT 1 FROM _syncular_blob_commit_refs r WHERE r.blob_id = _syncular_blob_uploads.blob_id AND r.commit_id != OLD.client_commit_id);
723
- DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id; END`);
724
- this.#db
725
- .exec(`CREATE TRIGGER _syncular_blob_commit_cache AFTER INSERT ON _syncular_blobs BEGIN
726
- INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND b.blob_id = NEW.blob_id; END`);
727
- this.#db.exec(
728
- `INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references}`,
729
- );
730
- }
731
- });
691
+ if (this.#hasBlobs) ensureBlobSchema(this.#db);
732
692
  this.#db.transaction(() => {
733
693
  pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
734
694
  });
@@ -1510,13 +1470,11 @@ export class SyncClient {
1510
1470
  );
1511
1471
  }
1512
1472
  const blobId = await computeBlobId(bytes);
1473
+ const nowMs = this.#now();
1513
1474
  this.#db.transaction(() => {
1514
- putCachedBlob(this.#db, blobId, bytes, this.#now(), options?.mediaType);
1515
- recordPendingUpload(this.#db, blobId, this.#now(), options?.mediaType);
1475
+ putCachedBlob(this.#db, blobId, bytes, nowMs, options?.mediaType);
1476
+ recordPendingUpload(this.#db, blobId, nowMs, options?.mediaType);
1516
1477
  });
1517
- // §5.9.7 B1: a staged upload is pinned (recordPendingUpload), so the cap
1518
- // trim below will never evict it — but a stage may push other zero-ref
1519
- // bodies over the cap, so run the trim.
1520
1478
  this.#enforceBlobCacheCap();
1521
1479
  return {
1522
1480
  blobId,
@@ -1547,7 +1505,7 @@ export class SyncClient {
1547
1505
  const blobId = blobIdOrRef.startsWith('sha256:')
1548
1506
  ? blobIdOrRef
1549
1507
  : parseBlobRef(blobIdOrRef).blobId;
1550
- const cached = getCachedBlob(this.#db, blobId, this.#now());
1508
+ const cached = getCachedBlob(this.#db, blobId);
1551
1509
  if (cached !== undefined) return cached;
1552
1510
  const transport = this.#config.blobs;
1553
1511
  if (transport === undefined) {
@@ -1595,9 +1553,7 @@ export class SyncClient {
1595
1553
  );
1596
1554
  }
1597
1555
  putCachedBlob(this.#db, blobId, bytes, this.#now());
1598
- // The referencing row can arrive before its body. Pin the new cache entry
1599
- // from current visible references before applying the size cap (§5.9.7 B1).
1600
- this.#reconcileBlobs(false, blobId);
1556
+ this.#db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
1601
1557
  this.#enforceBlobCacheCap();
1602
1558
  const stored = getCachedBlob(this.#db, blobId);
1603
1559
  if (stored === undefined) {
@@ -1613,7 +1569,7 @@ export class SyncClient {
1613
1569
  #enforceBlobCacheCap(): void {
1614
1570
  const cap = this.#config.blobCacheMaxBytes;
1615
1571
  if (cap === undefined) return;
1616
- enforceBlobCacheCap(this.#db, cap);
1572
+ enforceBlobCacheCap(this.#db, this.#schema, cap);
1617
1573
  }
1618
1574
 
1619
1575
  /** Flush any queued blob uploads (§5.9.7 B4); safe to call standalone. */
@@ -2188,6 +2144,29 @@ export class SyncClient {
2188
2144
  this.#now(),
2189
2145
  this.#captureBeforeImages(operations),
2190
2146
  );
2147
+ for (const operation of operations) {
2148
+ if (operation.op !== 'upsert') continue;
2149
+ const table = this.#schema.tables.get(operation.table);
2150
+ if (table === undefined) continue;
2151
+ for (const column of table.columns) {
2152
+ if (column.type !== 'blob_ref') continue;
2153
+ const raw = operation.values?.[column.name];
2154
+ if (typeof raw !== 'string') continue;
2155
+ let blobId: string;
2156
+ try {
2157
+ blobId = parseBlobRef(raw).blobId;
2158
+ } catch {
2159
+ continue;
2160
+ }
2161
+ this.#db.exec(
2162
+ `INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
2163
+ SELECT ?, ? WHERE EXISTS (
2164
+ SELECT 1 FROM _syncular_blobs WHERE blob_id = ?
2165
+ )`,
2166
+ [clientCommitId, blobId, blobId],
2167
+ );
2168
+ }
2169
+ }
2191
2170
  this.#applyOperationsLocally(operations, batch);
2192
2171
  batch.status();
2193
2172
  });
@@ -2386,7 +2365,7 @@ export class SyncClient {
2386
2365
  batch.rejections();
2387
2366
  batch.outcomes();
2388
2367
  }
2389
- this.#reconcileBlobs(true);
2368
+ this.#deleteUnreferencedBlobs();
2390
2369
  setMeta(this.#db, metaKey, purge.canonicalPlan);
2391
2370
  return {
2392
2371
  alreadyApplied: false,
@@ -3564,7 +3543,6 @@ export class SyncClient {
3564
3543
  // §7.1: local reads see outbox state applied optimistically — replay
3565
3544
  // the still-pending commits on top of the freshly applied server state.
3566
3545
  this.#replayOutbox();
3567
- this.#reconcileBlobs(false);
3568
3546
  } finally {
3569
3547
  this.#endDiagnosticsDeferral();
3570
3548
  }
@@ -4047,7 +4025,7 @@ export class SyncClient {
4047
4025
  batch.rejections();
4048
4026
  batch.outcomes();
4049
4027
  }
4050
- this.#reconcileBlobs(true);
4028
+ this.#deleteUnreferencedBlobs();
4051
4029
  } catch (error) {
4052
4030
  if (
4053
4031
  error instanceof ClientSyncError &&
@@ -4265,17 +4243,9 @@ export class SyncClient {
4265
4243
  });
4266
4244
  }
4267
4245
 
4268
- /**
4269
- * §5.9.7 B1/B2: recompute blob-cache refcounts from live `blob_ref`
4270
- * columns. No-op unless the schema has blob columns. `deleteOrphans`
4271
- * triggers the revocation-side body deletion (B2).
4272
- */
4273
- #reconcileBlobs(deleteOrphans: boolean, blobId?: string): void {
4246
+ #deleteUnreferencedBlobs(): void {
4274
4247
  if (!this.#hasBlobs) return;
4275
- reconcileBlobRefcounts(this.#db, this.#schema, {
4276
- deleteOrphans,
4277
- ...(blobId === undefined ? {} : { blobId }),
4278
- });
4248
+ deleteUnreferencedCachedBlobs(this.#db, this.#schema);
4279
4249
  }
4280
4250
 
4281
4251
  // -- helpers -----------------------------------------------------------------
package/src/outbox.ts CHANGED
@@ -135,6 +135,9 @@ export function deleteOutboxCommit(
135
135
  db: ClientDatabase,
136
136
  clientCommitId: string,
137
137
  ): void {
138
+ db.exec('DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?', [
139
+ clientCommitId,
140
+ ]);
138
141
  db.exec(
139
142
  'DELETE FROM _syncular_outbox_before_images WHERE client_commit_id = ?',
140
143
  [clientCommitId],
package/src/schema.ts CHANGED
@@ -501,6 +501,11 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
501
501
  db.transaction(() => {
502
502
  db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
503
503
  key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
504
+ db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
505
+ commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
506
+ db.exec(
507
+ 'CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)',
508
+ );
504
509
  db.exec(
505
510
  `INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`,
506
511
  );