@syncular/client 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -375,7 +375,7 @@ lifecycle with an `EncryptionConfig`; Worker handles use the portable keyring.
375
375
  Within one local SQLite transaction the engine deletes exactly the matching
376
376
  synced rows, lets generated FTS triggers remove their projections, drops every
377
377
  whole pending commit with a matching operation, restores/replays unrelated
378
- optimistic state, reconciles blob references, persists the `purgeId`, and emits
378
+ optimistic state, removes unreferenced cached blob bodies, persists the `purgeId`, and emits
379
379
  one revisioned change batch. A retry with the same canonical plan returns
380
380
  `alreadyApplied: true`; reusing an id with different selectors fails closed.
381
381
  Only bounded, non-empty, code-like values on plaintext string schema columns
@@ -484,9 +484,8 @@ and are cached locally. **Blob bytes live as `BLOB` columns in the client's own
484
484
  SQLite database** (a `_syncular_blobs` cache table), not in a separate OPFS
485
485
  directory or IndexedDB store. This is the pinned decision (SPEC §5.9.7 B1):
486
486
 
487
- - **One storage system.** The bytes are transactional with the refcount rows
488
- that pin them — a refcount adjust and a body insert/delete commit atomically,
489
- so a crash never strands a body against a stale count.
487
+ - **One storage system.** The bytes, upload state, and pending-commit references
488
+ share the client database. SQLite commits each change atomically.
490
489
  - **Survives restarts for free.** The client DB already rides OPFS via the
491
490
  sahpool VFS in the browser (and a plain file under `rusqlite`, `bun:sqlite`,
492
491
  or `node:sqlite` on native runtimes), so there is no second persistence
@@ -495,25 +494,24 @@ directory or IndexedDB store. This is the pinned decision (SPEC §5.9.7 B1):
495
494
  - **SQLite handles multi-MB images fine.** A page-cached `BLOB` read is a memory
496
495
  copy, well within the image/document envelope this targets.
497
496
 
498
- ### Size cap + LRU eviction
497
+ ### Size cap
499
498
 
500
499
  Pass `blobCacheMaxBytes` to cap the on-device cache. When the sum of cached body
501
- sizes exceeds the cap, the client evicts **zero-ref, non-pinned** bodies in
502
- least-recently-used order until back under the cap:
500
+ sizes exceeds the cap, the client evicts bodies with no visible row or pending
501
+ commit by creation timestamp, then blob ID, until back under the cap:
503
502
 
504
503
  ```ts
505
504
  new SyncClient({ /* … */, blobCacheMaxBytes: 256 * 1024 * 1024 }); // 256 MiB
506
505
  ```
507
506
 
508
- - A body **referenced by a live row** (refcount > 0) is **never** evicted — it
507
+ - A body **referenced by a live row** is **never** evicted. It
509
508
  stays resolvable without a re-download.
510
- - A body **pinned by a pending upload** (not yet pushed) is never evicted — its
509
+ - A body **needed by a pending upload** (not yet pushed) is never evicted. Its
511
510
  bytes are the only copy until the commit drains.
512
- - Evicting a zero-ref body only costs a future re-download, never correctness:
511
+ - Evicting an unreferenced body only costs a future re-download, never correctness:
513
512
  any surviving `blob_ref` value re-enables the fetch (§5.9.7 B3). If every
514
513
  over-cap body is referenced or pinned, the cache stays over the cap
515
- (correctness beats the cap). A cache-hit read touches "recently used", so a
516
- hot image survives a trim. Absent `blobCacheMaxBytes` ⇒ retain until storage
514
+ (correctness beats the cap). Absent `blobCacheMaxBytes` ⇒ retain until storage
517
515
  pressure (the default).
518
516
 
519
517
  ### Very large media — the escape hatch
@@ -522,7 +520,7 @@ SQLite is **not** the store for gigabyte video: a single `BLOB` must fit the
522
520
  client's memory and the SQLite row-size envelope. For very large media, run the
523
521
  server with presigned downloads (`blobSignedUrls`) and hand the presigned URL
524
522
  straight to a media element instead of pulling bytes through the cache — the
525
- image-app default (refcounted `BLOB` cache) and the large-media path (presigned
523
+ image-app default (`BLOB` cache) and the large-media path (presigned
526
524
  URL, no byte cache) coexist per attachment.
527
525
 
528
526
  ## Node and Bun SQLite backend (`./sqlite`)
package/dist/blob.d.ts CHANGED
@@ -1,14 +1,9 @@
1
1
  /**
2
2
  * Client-side blob cache + transport (SPEC.md §5.9.7).
3
3
  *
4
- * Blob bytes are cached content-addressed by `blobId` and refcounted by the
5
- * local rows whose `blob_ref` columns reference them (B1). The cache is
6
- * derived from live-row references: after any apply/purge, refcounts are
7
- * reconciled from the current `blob_ref` column contents, and a body whose
8
- * only referencing rows were revocation-purged is deleted (B2, evicted ≠
9
- * revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
10
- * row value is the whole download key. Pending uploads are tracked in the
11
- * outbox-adjacent uploads table (B4) and flushed before push.
4
+ * Blob bytes are cached by `blobId`. Retention reads live references directly
5
+ * from `blob_ref` columns and keeps explicit pending-commit dependencies.
6
+ * A small upload row queues each staged body for transfer before push.
12
7
  */
13
8
  import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
14
9
  import type { ClientDatabase } from './database.js';
@@ -82,46 +77,26 @@ export interface CachedBlob {
82
77
  /** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
83
78
  export declare function computeBlobId(bytes: Uint8Array): Promise<string>;
84
79
  export declare function ensureBlobSchema(db: ClientDatabase): void;
85
- /** Put bytes into the content-addressed cache (idempotent); touches LRU. */
80
+ /** Put bytes into the content-addressed cache without rewriting an existing body. */
86
81
  export declare function putCachedBlob(db: ClientDatabase, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): void;
87
- export declare function getCachedBlob(db: ClientDatabase, blobId: string, nowMs?: number): CachedBlob | undefined;
82
+ export declare function getCachedBlob(db: ClientDatabase, blobId: string): CachedBlob | undefined;
88
83
  /**
89
- * §5.9.7 B1 size cap + LRU eviction. When the sum of cached body sizes exceeds
90
- * `maxBytes`, evict **zero-ref, non-pinned** bodies in least-recently-used
91
- * order until back under the cap. NEVER evicts a referenced body (refcount > 0
92
- * — it must stay resolvable without a re-download) nor a pending-upload-pinned
93
- * body (its bytes are the only copy until push, B4). If every over-cap body is
94
- * referenced or pinned, the cache stays over the cap (correctness beats the
95
- * cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
96
- * any surviving `blob_ref` value. Returns the evicted blobIds.
84
+ * §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
85
+ * evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
86
+ * until back under the cap. Returns the evicted blobIds.
97
87
  */
98
- export declare function enforceBlobCacheCap(db: ClientDatabase, maxBytes: number): string[];
99
- /** Record a pending upload (§5.9.7 B4); flushed before the next push. */
100
- export declare function recordPendingUpload(db: ClientDatabase, blobId: string, nowMs: number, mediaType?: string): void;
88
+ export declare function enforceBlobCacheCap(db: ClientDatabase, schema: CompiledClientSchema, maxBytes: number): string[];
101
89
  export declare function listPendingUploads(db: ClientDatabase): {
102
90
  blobId: string;
103
91
  mediaType?: string;
104
92
  }[];
93
+ export declare function recordPendingUpload(db: ClientDatabase, blobId: string, nowMs: number, mediaType?: string): void;
105
94
  export declare function clearPendingUpload(db: ClientDatabase, blobId: string): void;
106
95
  /**
107
- * All `blob_ref` column names per table (for refcount reconciliation).
108
- * Blank result ⇒ the schema has no attachments; callers skip reconciliation.
96
+ * All `blob_ref` column names per table.
109
97
  */
110
98
  export declare function blobRefColumnsBySchema(schema: CompiledClientSchema): Map<string, string[]>;
111
99
  export declare function schemaHasBlobs(schema: CompiledClientSchema): boolean;
112
- /**
113
- * §5.9.7 B1/B2: recompute cache refcounts from the current `blob_ref` column
114
- * contents across all synced tables, then delete cache bodies that dropped
115
- * to zero refs *and* have no pending upload (a pending upload pins its body,
116
- * B4). Called after every apply/purge that may add or remove references.
117
- *
118
- * `deleteOrphans` distinguishes the two B2 transitions: revocation purge
119
- * passes `true` (drop the now-unauthorized body); a benign apply passes
120
- * `false` (retain zero-ref bodies as LRU cache entries — the shipped
121
- * default). Bodies pinned by a pending upload are always retained.
122
- */
123
- export declare function reconcileBlobRefcounts(db: ClientDatabase, schema: CompiledClientSchema, options?: {
124
- readonly deleteOrphans?: boolean;
125
- }): void;
100
+ export declare function deleteUnreferencedCachedBlobs(db: ClientDatabase, schema: CompiledClientSchema): void;
126
101
  export type { BlobRef };
127
102
  export { parseBlobRef, serializeBlobRef };
package/dist/blob.js CHANGED
@@ -1,68 +1,65 @@
1
1
  /**
2
2
  * Client-side blob cache + transport (SPEC.md §5.9.7).
3
3
  *
4
- * Blob bytes are cached content-addressed by `blobId` and refcounted by the
5
- * local rows whose `blob_ref` columns reference them (B1). The cache is
6
- * derived from live-row references: after any apply/purge, refcounts are
7
- * reconciled from the current `blob_ref` column contents, and a body whose
8
- * only referencing rows were revocation-purged is deleted (B2, evicted ≠
9
- * revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
10
- * row value is the whole download key. Pending uploads are tracked in the
11
- * outbox-adjacent uploads table (B4) and flushed before push.
4
+ * Blob bytes are cached by `blobId`. Retention reads live references directly
5
+ * from `blob_ref` columns and keeps explicit pending-commit dependencies.
6
+ * A small upload row queues each staged body for transfer before push.
12
7
  */
13
8
  import { parseBlobRef, serializeBlobRef } from '@syncular/core';
9
+ import { ClientSyncError } from './errors.js';
14
10
  import { quoteIdent } from './schema.js';
15
11
  /** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
16
12
  export async function computeBlobId(bytes) {
17
- const digest = await crypto.subtle.digest('SHA-256', bytes.slice().buffer);
13
+ const input = bytes.buffer instanceof ArrayBuffer
14
+ ? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
15
+ : bytes.slice();
16
+ const digest = await crypto.subtle.digest('SHA-256', input);
18
17
  const hex = [...new Uint8Array(digest)]
19
18
  .map((b) => b.toString(16).padStart(2, '0'))
20
19
  .join('');
21
20
  return `sha256:${hex}`;
22
21
  }
23
22
  export function ensureBlobSchema(db) {
24
- db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blobs(
25
- blob_id TEXT PRIMARY KEY,
26
- bytes BLOB NOT NULL,
27
- byte_length INTEGER NOT NULL,
28
- media_type TEXT,
29
- refcount INTEGER NOT NULL DEFAULT 0,
30
- created_at_ms INTEGER NOT NULL,
31
- last_used_ms INTEGER NOT NULL DEFAULT 0)`);
32
- // Migrate a cache created before the §5.9.7 B1 LRU column: additive,
33
- // idempotent (a duplicate-column error on an already-migrated DB is
34
- // swallowed). last_used_ms drives cap eviction (LRU of zero-ref bodies).
35
- try {
36
- db.exec('ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0');
23
+ const existing = db.query("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs'");
24
+ if (existing.length > 0) {
25
+ const columns = db
26
+ .query('PRAGMA table_info(_syncular_blobs)')
27
+ .map((row) => row.name)
28
+ .sort();
29
+ const expected = [
30
+ 'blob_id',
31
+ 'byte_length',
32
+ 'bytes',
33
+ 'created_at_ms',
34
+ 'media_type',
35
+ ].sort();
36
+ if (columns.length !== expected.length ||
37
+ columns.some((column, index) => column !== expected[index])) {
38
+ throw new ClientSyncError('sync.schema_mismatch', 'Local blob schema is incompatible');
39
+ }
37
40
  }
38
- catch {
39
- // column already exists — the CREATE above included it
41
+ else {
42
+ db.exec(`CREATE TABLE _syncular_blobs(
43
+ blob_id TEXT PRIMARY KEY,
44
+ bytes BLOB NOT NULL,
45
+ byte_length INTEGER NOT NULL,
46
+ media_type TEXT,
47
+ created_at_ms INTEGER NOT NULL)`);
40
48
  }
41
49
  db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
42
- blob_id TEXT PRIMARY KEY,
43
- media_type TEXT,
44
- created_at_ms INTEGER NOT NULL)`);
50
+ blob_id TEXT PRIMARY KEY, media_type TEXT, created_at_ms INTEGER NOT NULL)`);
45
51
  }
46
- /** Put bytes into the content-addressed cache (idempotent); touches LRU. */
52
+ /** Put bytes into the content-addressed cache without rewriting an existing body. */
47
53
  export function putCachedBlob(db, blobId, bytes, nowMs, mediaType) {
48
54
  db.exec(`INSERT INTO _syncular_blobs(
49
- blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms)
50
- VALUES (?,?,?,?,0,?,?)
51
- ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms`, [blobId, bytes, bytes.length, mediaType ?? null, nowMs, nowMs]);
55
+ blob_id, bytes, byte_length, media_type, created_at_ms)
56
+ VALUES (?,?,?,?,?) ON CONFLICT(blob_id) DO NOTHING`, [blobId, bytes, bytes.length, mediaType ?? null, nowMs]);
52
57
  }
53
- export function getCachedBlob(db, blobId, nowMs) {
58
+ export function getCachedBlob(db, blobId) {
54
59
  const rows = db.query('SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?', [blobId]);
55
60
  const row = rows[0];
56
61
  if (row === undefined)
57
62
  return undefined;
58
- // §5.9.7 B1 LRU: a cache-hit read touches "recently used" so a hot image
59
- // survives a cap trim. Skipped when no clock is supplied (pure read).
60
- if (nowMs !== undefined) {
61
- db.exec('UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?', [
62
- nowMs,
63
- blobId,
64
- ]);
65
- }
66
63
  return {
67
64
  blobId,
68
65
  bytes: row.bytes,
@@ -71,26 +68,21 @@ export function getCachedBlob(db, blobId, nowMs) {
71
68
  };
72
69
  }
73
70
  /**
74
- * §5.9.7 B1 size cap + LRU eviction. When the sum of cached body sizes exceeds
75
- * `maxBytes`, evict **zero-ref, non-pinned** bodies in least-recently-used
76
- * order until back under the cap. NEVER evicts a referenced body (refcount > 0
77
- * — it must stay resolvable without a re-download) nor a pending-upload-pinned
78
- * body (its bytes are the only copy until push, B4). If every over-cap body is
79
- * referenced or pinned, the cache stays over the cap (correctness beats the
80
- * cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
81
- * any surviving `blob_ref` value. Returns the evicted blobIds.
71
+ * §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
72
+ * evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
73
+ * until back under the cap. Returns the evicted blobIds.
82
74
  */
83
- export function enforceBlobCacheCap(db, maxBytes) {
75
+ export function enforceBlobCacheCap(db, schema, maxBytes) {
84
76
  const totalRow = db.query('SELECT COALESCE(SUM(byte_length), 0) AS total FROM _syncular_blobs')[0];
85
77
  let total = Number(totalRow?.total ?? 0);
86
78
  if (total <= maxBytes)
87
79
  return [];
88
- // Eviction candidates: zero-ref AND not pinned by a pending upload, oldest
89
- // (LRU) first, then oldest created as a stable tiebreak.
80
+ // Oldest eligible body first, with blob ID as a stable tie.
90
81
  const candidates = db.query(`SELECT blob_id, byte_length FROM _syncular_blobs
91
- WHERE refcount = 0
92
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
93
- ORDER BY last_used_ms ASC, created_at_ms ASC`);
82
+ WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
83
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
84
+ AND blob_id NOT IN (${visibleBlobIdsSql(schema)})
85
+ ORDER BY created_at_ms ASC, blob_id ASC`);
94
86
  const evicted = [];
95
87
  db.transaction(() => {
96
88
  for (const row of candidates) {
@@ -104,27 +96,35 @@ export function enforceBlobCacheCap(db, maxBytes) {
104
96
  });
105
97
  return evicted;
106
98
  }
107
- /** Record a pending upload (§5.9.7 B4); flushed before the next push. */
99
+ export function listPendingUploads(db) {
100
+ return db
101
+ .query(`SELECT blob_id, media_type FROM (
102
+ SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
103
+ UNION ALL
104
+ SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
105
+ FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
106
+ WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
107
+ ) ORDER BY created_at_ms, blob_id`)
108
+ .map((row) => {
109
+ if (typeof row.blob_id !== 'string' ||
110
+ (row.media_type !== null && typeof row.media_type !== 'string')) {
111
+ throw new ClientSyncError('sync.local_corrupt', 'Pending blob upload metadata is invalid');
112
+ }
113
+ return {
114
+ blobId: row.blob_id,
115
+ ...(row.media_type !== null ? { mediaType: row.media_type } : {}),
116
+ };
117
+ });
118
+ }
108
119
  export function recordPendingUpload(db, blobId, nowMs, mediaType) {
109
120
  db.exec(`INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms)
110
121
  VALUES (?,?,?)`, [blobId, mediaType ?? null, nowMs]);
111
122
  }
112
- export function listPendingUploads(db) {
113
- return db
114
- .query('SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms')
115
- .map((row) => ({
116
- blobId: row.blob_id,
117
- ...(row.media_type !== null
118
- ? { mediaType: row.media_type }
119
- : {}),
120
- }));
121
- }
122
123
  export function clearPendingUpload(db, blobId) {
123
124
  db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id = ?', [blobId]);
124
125
  }
125
126
  /**
126
- * All `blob_ref` column names per table (for refcount reconciliation).
127
- * Blank result ⇒ the schema has no attachments; callers skip reconciliation.
127
+ * All `blob_ref` column names per table.
128
128
  */
129
129
  export function blobRefColumnsBySchema(schema) {
130
130
  const out = new Map();
@@ -144,58 +144,26 @@ export function schemaHasBlobs(schema) {
144
144
  }
145
145
  return false;
146
146
  }
147
- /**
148
- * §5.9.7 B1/B2: recompute cache refcounts from the current `blob_ref` column
149
- * contents across all synced tables, then delete cache bodies that dropped
150
- * to zero refs *and* have no pending upload (a pending upload pins its body,
151
- * B4). Called after every apply/purge that may add or remove references.
152
- *
153
- * `deleteOrphans` distinguishes the two B2 transitions: revocation purge
154
- * passes `true` (drop the now-unauthorized body); a benign apply passes
155
- * `false` (retain zero-ref bodies as LRU cache entries — the shipped
156
- * default). Bodies pinned by a pending upload are always retained.
157
- */
158
- export function reconcileBlobRefcounts(db, schema, options) {
159
- const byTable = blobRefColumnsBySchema(schema);
160
- if (byTable.size === 0)
161
- return;
162
- // Count references to each blobId across every blob_ref column.
163
- const counts = new Map();
164
- for (const [tableName, columns] of byTable) {
147
+ function visibleBlobIdsSql(schema) {
148
+ const selects = [];
149
+ for (const [tableName, columns] of blobRefColumnsBySchema(schema)) {
165
150
  for (const column of columns) {
166
- const rows = db.query(`SELECT ${quoteIdent(column)} AS v FROM ${quoteIdent(tableName)}
167
- WHERE ${quoteIdent(column)} IS NOT NULL`);
168
- for (const row of rows) {
169
- const raw = row.v;
170
- if (typeof raw !== 'string')
171
- continue;
172
- let ref;
173
- try {
174
- ref = parseBlobRef(raw);
175
- }
176
- catch {
177
- continue;
178
- }
179
- counts.set(ref.blobId, (counts.get(ref.blobId) ?? 0) + 1);
180
- }
151
+ const identifier = quoteIdent(column);
152
+ selects.push(`SELECT json_extract(${identifier}, '$.blobId') AS blob_id
153
+ FROM ${quoteIdent(tableName)}
154
+ WHERE typeof(${identifier}) = 'text'
155
+ AND json_valid(${identifier})
156
+ AND json_type(${identifier}, '$.blobId') = 'text'`);
181
157
  }
182
158
  }
183
- db.transaction(() => {
184
- // Reset all refcounts, then apply the recomputed counts.
185
- db.exec('UPDATE _syncular_blobs SET refcount = 0');
186
- for (const [blobId, count] of counts) {
187
- db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
188
- count,
189
- blobId,
190
- ]);
191
- }
192
- if (options?.deleteOrphans === true) {
193
- // §5.9.7 B2 revocation side: delete zero-ref bodies not pinned by a
194
- // pending upload.
195
- db.exec(`DELETE FROM _syncular_blobs
196
- WHERE refcount = 0
197
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)`);
198
- }
199
- });
159
+ return selects.length === 0
160
+ ? 'SELECT NULL AS blob_id WHERE 0'
161
+ : selects.join(' UNION ');
162
+ }
163
+ export function deleteUnreferencedCachedBlobs(db, schema) {
164
+ db.exec(`DELETE FROM _syncular_blobs
165
+ WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
166
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
167
+ AND blob_id NOT IN (${visibleBlobIdsSql(schema)})`);
200
168
  }
201
169
  export { parseBlobRef, serializeBlobRef };
package/dist/client.d.ts CHANGED
@@ -110,9 +110,9 @@ export interface SyncClientConfig {
110
110
  readonly blobs?: BlobTransport;
111
111
  /**
112
112
  * §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
113
- * exceeds this, zero-ref, non-pinned bodies are evicted LRU-first after each
114
- * cache write (referenced/pinned bodies are never evicted — correctness
115
- * beats the cap). Absent ⇒ retain until storage pressure (the shipped
113
+ * exceeds this, unreferenced bodies without pending work are evicted in
114
+ * creation timestamp order after each cache write. Referenced bodies remain cached
115
+ * even when that exceeds the cap. Absent means retain until storage pressure (the shipped
116
116
  * default; a referenced body always stays resolvable without a re-download).
117
117
  */
118
118
  readonly blobCacheMaxBytes?: number;
package/dist/client.js CHANGED
@@ -9,7 +9,7 @@
9
9
  */
10
10
  import { canonicalScopeJson, decodeMessage, decodeRow, decodeRowsSegment, encodeMessage, encodePresencePublish, MessageStreamScanner, PROTOCOL_WIRE_VERSION, parseRealtimeServerEvent, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
11
11
  import { applyCommitFrame, applyRowsSegment, applySqliteSegment, deleteLocalRow, deleteScopedRows, evictScopedRows, upsertLocalRow, } from './apply.js';
12
- import { clearPendingUpload, computeBlobId, enforceBlobCacheCap, ensureBlobSchema, getCachedBlob, listPendingUploads, parseBlobRef, putCachedBlob, reconcileBlobRefcounts, recordPendingUpload, schemaHasBlobs, serializeBlobRef, } from './blob.js';
12
+ import { clearPendingUpload, computeBlobId, deleteUnreferencedCachedBlobs, enforceBlobCacheCap, ensureBlobSchema, getCachedBlob, listPendingUploads, parseBlobRef, putCachedBlob, recordPendingUpload, schemaHasBlobs, serializeBlobRef, } from './blob.js';
13
13
  import { registerDevtools } from './devtools.js';
14
14
  import { CLIENT_DIAGNOSTICS_VERSION, ClientDiagnosticsEmitter, MAX_DIAGNOSTIC_DOMAINS, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS, } from './diagnostics.js';
15
15
  import { ClientSyncError } from './errors.js';
@@ -854,20 +854,19 @@ export class SyncClient {
854
854
  * (via `mutate`) after this call so upload-before-push holds (§5.9.3).
855
855
  */
856
856
  uploadBlob(bytes, options) {
857
- return this.#runProtectedAsync(() => this.#uploadBlob(bytes, options));
857
+ const owned = bytes.slice();
858
+ return this.#runProtectedAsync(() => this.#uploadBlob(owned, options));
858
859
  }
859
860
  async #uploadBlob(bytes, options) {
860
861
  if (this.#config.blobs === undefined) {
861
862
  throw new ClientSyncError('sync.invalid_request', 'uploadBlob requires a blob transport (SyncClientConfig.blobs, §5.9)');
862
863
  }
863
864
  const blobId = await computeBlobId(bytes);
865
+ const nowMs = this.#now();
864
866
  this.#db.transaction(() => {
865
- putCachedBlob(this.#db, blobId, bytes, this.#now(), options?.mediaType);
866
- recordPendingUpload(this.#db, blobId, this.#now(), options?.mediaType);
867
+ putCachedBlob(this.#db, blobId, bytes, nowMs, options?.mediaType);
868
+ recordPendingUpload(this.#db, blobId, nowMs, options?.mediaType);
867
869
  });
868
- // §5.9.7 B1: a staged upload is pinned (recordPendingUpload), so the cap
869
- // trim below will never evict it — but a stage may push other zero-ref
870
- // bodies over the cap, so run the trim.
871
870
  this.#enforceBlobCacheCap();
872
871
  return {
873
872
  blobId,
@@ -895,7 +894,7 @@ export class SyncClient {
895
894
  const blobId = blobIdOrRef.startsWith('sha256:')
896
895
  ? blobIdOrRef
897
896
  : parseBlobRef(blobIdOrRef).blobId;
898
- const cached = getCachedBlob(this.#db, blobId, this.#now());
897
+ const cached = getCachedBlob(this.#db, blobId);
899
898
  if (cached !== undefined)
900
899
  return cached;
901
900
  const transport = this.#config.blobs;
@@ -930,9 +929,7 @@ export class SyncClient {
930
929
  throw new ClientSyncError('sync.invalid_request', `blob content address mismatch for ${blobId} (§5.9.5)`);
931
930
  }
932
931
  putCachedBlob(this.#db, blobId, bytes, this.#now());
933
- // The referencing row can arrive before its body. Pin the new cache entry
934
- // from current visible references before applying the size cap (§5.9.7 B1).
935
- this.#reconcileBlobs(false);
932
+ this.#db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
936
933
  this.#enforceBlobCacheCap();
937
934
  const stored = getCachedBlob(this.#db, blobId);
938
935
  if (stored === undefined) {
@@ -945,7 +942,7 @@ export class SyncClient {
945
942
  const cap = this.#config.blobCacheMaxBytes;
946
943
  if (cap === undefined)
947
944
  return;
948
- enforceBlobCacheCap(this.#db, cap);
945
+ enforceBlobCacheCap(this.#db, this.#schema, cap);
949
946
  }
950
947
  /** Flush any queued blob uploads (§5.9.7 B4); safe to call standalone. */
951
948
  flushBlobUploads() {
@@ -957,11 +954,12 @@ export class SyncClient {
957
954
  return;
958
955
  for (const pending of listPendingUploads(this.#db)) {
959
956
  const cached = getCachedBlob(this.#db, pending.blobId);
960
- if (cached === undefined) {
961
- // The bytes are gone (never happens for a well-behaved client);
962
- // drop the upload so it does not wedge the queue.
963
- clearPendingUpload(this.#db, pending.blobId);
964
- continue;
957
+ if (cached === undefined ||
958
+ !(cached.bytes instanceof Uint8Array) ||
959
+ !Number.isSafeInteger(cached.byteLength) ||
960
+ cached.byteLength !== cached.bytes.byteLength ||
961
+ (await computeBlobId(cached.bytes)) !== pending.blobId) {
962
+ throw new ClientSyncError('sync.local_corrupt', 'Pending blob upload body is missing or corrupt');
965
963
  }
966
964
  await this.#uploadOne(transport, pending.blobId, cached.bytes, pending.mediaType);
967
965
  clearPendingUpload(this.#db, pending.blobId);
@@ -1413,6 +1411,31 @@ export class SyncClient {
1413
1411
  this.#applyBatch((batch) => {
1414
1412
  this.#db.transaction(() => {
1415
1413
  appendOutboxCommit(this.#db, clientCommitId, operations, this.#now(), this.#captureBeforeImages(operations));
1414
+ for (const operation of operations) {
1415
+ if (operation.op !== 'upsert')
1416
+ continue;
1417
+ const table = this.#schema.tables.get(operation.table);
1418
+ if (table === undefined)
1419
+ continue;
1420
+ for (const column of table.columns) {
1421
+ if (column.type !== 'blob_ref')
1422
+ continue;
1423
+ const raw = operation.values?.[column.name];
1424
+ if (typeof raw !== 'string')
1425
+ continue;
1426
+ let blobId;
1427
+ try {
1428
+ blobId = parseBlobRef(raw).blobId;
1429
+ }
1430
+ catch {
1431
+ continue;
1432
+ }
1433
+ this.#db.exec(`INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
1434
+ SELECT ?, ? WHERE EXISTS (
1435
+ SELECT 1 FROM _syncular_blobs WHERE blob_id = ?
1436
+ )`, [clientCommitId, blobId, blobId]);
1437
+ }
1438
+ }
1416
1439
  this.#applyOperationsLocally(operations, batch);
1417
1440
  batch.status();
1418
1441
  });
@@ -1571,7 +1594,7 @@ export class SyncClient {
1571
1594
  batch.rejections();
1572
1595
  batch.outcomes();
1573
1596
  }
1574
- this.#reconcileBlobs(true);
1597
+ this.#deleteUnreferencedBlobs();
1575
1598
  setMeta(this.#db, metaKey, purge.canonicalPlan);
1576
1599
  return {
1577
1600
  alreadyApplied: false,
@@ -2513,7 +2536,6 @@ export class SyncClient {
2513
2536
  // §7.1: local reads see outbox state applied optimistically — replay
2514
2537
  // the still-pending commits on top of the freshly applied server state.
2515
2538
  this.#replayOutbox();
2516
- this.#reconcileBlobs(false);
2517
2539
  }
2518
2540
  finally {
2519
2541
  this.#endDiagnosticsDeferral();
@@ -2902,7 +2924,7 @@ export class SyncClient {
2902
2924
  batch.rejections();
2903
2925
  batch.outcomes();
2904
2926
  }
2905
- this.#reconcileBlobs(true);
2927
+ this.#deleteUnreferencedBlobs();
2906
2928
  }
2907
2929
  catch (error) {
2908
2930
  if (error instanceof ClientSyncError &&
@@ -3074,15 +3096,10 @@ export class SyncClient {
3074
3096
  });
3075
3097
  });
3076
3098
  }
3077
- /**
3078
- * §5.9.7 B1/B2: recompute blob-cache refcounts from live `blob_ref`
3079
- * columns. No-op unless the schema has blob columns. `deleteOrphans`
3080
- * triggers the revocation-side body deletion (B2).
3081
- */
3082
- #reconcileBlobs(deleteOrphans) {
3099
+ #deleteUnreferencedBlobs() {
3083
3100
  if (!this.#hasBlobs)
3084
3101
  return;
3085
- reconcileBlobRefcounts(this.#db, this.#schema, { deleteOrphans });
3102
+ deleteUnreferencedCachedBlobs(this.#db, this.#schema);
3086
3103
  }
3087
3104
  // -- helpers -----------------------------------------------------------------
3088
3105
  #table(name) {
package/dist/outbox.js CHANGED
@@ -62,6 +62,9 @@ export function countOutbox(db) {
62
62
  .count;
63
63
  }
64
64
  export function deleteOutboxCommit(db, clientCommitId) {
65
+ db.exec('DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?', [
66
+ clientCommitId,
67
+ ]);
65
68
  db.exec('DELETE FROM _syncular_outbox_before_images WHERE client_commit_id = ?', [clientCommitId]);
66
69
  db.exec('DELETE FROM _syncular_outbox WHERE client_commit_id = ?', [
67
70
  clientCommitId,
package/dist/schema.js CHANGED
@@ -339,6 +339,9 @@ export function ensureLocalBookkeepingSchema(db) {
339
339
  db.transaction(() => {
340
340
  db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
341
341
  key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
342
+ db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
343
+ commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
344
+ db.exec('CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)');
342
345
  db.exec(`INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`);
343
346
  db.exec(`CREATE TABLE IF NOT EXISTS _syncular_outbox(
344
347
  seq INTEGER PRIMARY KEY AUTOINCREMENT,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/client",
3
- "version": "0.17.0",
3
+ "version": "0.19.0",
4
4
  "description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -89,9 +89,9 @@
89
89
  },
90
90
  "dependencies": {
91
91
  "@sqlite.org/sqlite-wasm": "^3.53.0-build1",
92
- "@syncular/core": "0.17.0"
92
+ "@syncular/core": "0.19.0"
93
93
  },
94
94
  "devDependencies": {
95
- "@syncular/server": "0.17.0"
95
+ "@syncular/server": "0.19.0"
96
96
  }
97
97
  }
package/src/blob.ts CHANGED
@@ -1,17 +1,13 @@
1
1
  /**
2
2
  * Client-side blob cache + transport (SPEC.md §5.9.7).
3
3
  *
4
- * Blob bytes are cached content-addressed by `blobId` and refcounted by the
5
- * local rows whose `blob_ref` columns reference them (B1). The cache is
6
- * derived from live-row references: after any apply/purge, refcounts are
7
- * reconciled from the current `blob_ref` column contents, and a body whose
8
- * only referencing rows were revocation-purged is deleted (B2, evicted ≠
9
- * revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
10
- * row value is the whole download key. Pending uploads are tracked in the
11
- * outbox-adjacent uploads table (B4) and flushed before push.
4
+ * Blob bytes are cached by `blobId`. Retention reads live references directly
5
+ * from `blob_ref` columns and keeps explicit pending-commit dependencies.
6
+ * A small upload row queues each staged body for transfer before push.
12
7
  */
13
8
  import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
14
9
  import type { ClientDatabase } from './database';
10
+ import { ClientSyncError } from './errors';
15
11
  import type { CompiledClientSchema } from './schema';
16
12
  import { quoteIdent } from './schema';
17
13
 
@@ -93,9 +89,13 @@ export interface CachedBlob {
93
89
 
94
90
  /** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
95
91
  export async function computeBlobId(bytes: Uint8Array): Promise<string> {
92
+ const input =
93
+ bytes.buffer instanceof ArrayBuffer
94
+ ? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
95
+ : bytes.slice();
96
96
  const digest = await crypto.subtle.digest(
97
97
  'SHA-256',
98
- bytes.slice().buffer as ArrayBuffer,
98
+ input as Uint8Array<ArrayBuffer>,
99
99
  );
100
100
  const hex = [...new Uint8Array(digest)]
101
101
  .map((b) => b.toString(16).padStart(2, '0'))
@@ -104,31 +104,43 @@ export async function computeBlobId(bytes: Uint8Array): Promise<string> {
104
104
  }
105
105
 
106
106
  export function ensureBlobSchema(db: ClientDatabase): void {
107
- db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blobs(
108
- blob_id TEXT PRIMARY KEY,
109
- bytes BLOB NOT NULL,
110
- byte_length INTEGER NOT NULL,
111
- media_type TEXT,
112
- refcount INTEGER NOT NULL DEFAULT 0,
113
- created_at_ms INTEGER NOT NULL,
114
- last_used_ms INTEGER NOT NULL DEFAULT 0)`);
115
- // Migrate a cache created before the §5.9.7 B1 LRU column: additive,
116
- // idempotent (a duplicate-column error on an already-migrated DB is
117
- // swallowed). last_used_ms drives cap eviction (LRU of zero-ref bodies).
118
- try {
119
- db.exec(
120
- 'ALTER TABLE _syncular_blobs ADD COLUMN last_used_ms INTEGER NOT NULL DEFAULT 0',
121
- );
122
- } catch {
123
- // column already exists — the CREATE above included it
107
+ const existing = db.query(
108
+ "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs'",
109
+ );
110
+ if (existing.length > 0) {
111
+ const columns = db
112
+ .query('PRAGMA table_info(_syncular_blobs)')
113
+ .map((row) => row.name)
114
+ .sort();
115
+ const expected = [
116
+ 'blob_id',
117
+ 'byte_length',
118
+ 'bytes',
119
+ 'created_at_ms',
120
+ 'media_type',
121
+ ].sort();
122
+ if (
123
+ columns.length !== expected.length ||
124
+ columns.some((column, index) => column !== expected[index])
125
+ ) {
126
+ throw new ClientSyncError(
127
+ 'sync.schema_mismatch',
128
+ 'Local blob schema is incompatible',
129
+ );
130
+ }
131
+ } else {
132
+ db.exec(`CREATE TABLE _syncular_blobs(
133
+ blob_id TEXT PRIMARY KEY,
134
+ bytes BLOB NOT NULL,
135
+ byte_length INTEGER NOT NULL,
136
+ media_type TEXT,
137
+ created_at_ms INTEGER NOT NULL)`);
124
138
  }
125
139
  db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
126
- blob_id TEXT PRIMARY KEY,
127
- media_type TEXT,
128
- created_at_ms INTEGER NOT NULL)`);
140
+ blob_id TEXT PRIMARY KEY, media_type TEXT, created_at_ms INTEGER NOT NULL)`);
129
141
  }
130
142
 
131
- /** Put bytes into the content-addressed cache (idempotent); touches LRU. */
143
+ /** Put bytes into the content-addressed cache without rewriting an existing body. */
132
144
  export function putCachedBlob(
133
145
  db: ClientDatabase,
134
146
  blobId: string,
@@ -138,17 +150,15 @@ export function putCachedBlob(
138
150
  ): void {
139
151
  db.exec(
140
152
  `INSERT INTO _syncular_blobs(
141
- blob_id, bytes, byte_length, media_type, refcount, created_at_ms, last_used_ms)
142
- VALUES (?,?,?,?,0,?,?)
143
- ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms`,
144
- [blobId, bytes, bytes.length, mediaType ?? null, nowMs, nowMs],
153
+ blob_id, bytes, byte_length, media_type, created_at_ms)
154
+ VALUES (?,?,?,?,?) ON CONFLICT(blob_id) DO NOTHING`,
155
+ [blobId, bytes, bytes.length, mediaType ?? null, nowMs],
145
156
  );
146
157
  }
147
158
 
148
159
  export function getCachedBlob(
149
160
  db: ClientDatabase,
150
161
  blobId: string,
151
- nowMs?: number,
152
162
  ): CachedBlob | undefined {
153
163
  const rows = db.query(
154
164
  'SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?',
@@ -156,14 +166,6 @@ export function getCachedBlob(
156
166
  );
157
167
  const row = rows[0];
158
168
  if (row === undefined) return undefined;
159
- // §5.9.7 B1 LRU: a cache-hit read touches "recently used" so a hot image
160
- // survives a cap trim. Skipped when no clock is supplied (pure read).
161
- if (nowMs !== undefined) {
162
- db.exec('UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?', [
163
- nowMs,
164
- blobId,
165
- ]);
166
- }
167
169
  return {
168
170
  blobId,
169
171
  bytes: row.bytes as Uint8Array,
@@ -173,17 +175,13 @@ export function getCachedBlob(
173
175
  }
174
176
 
175
177
  /**
176
- * §5.9.7 B1 size cap + LRU eviction. When the sum of cached body sizes exceeds
177
- * `maxBytes`, evict **zero-ref, non-pinned** bodies in least-recently-used
178
- * order until back under the cap. NEVER evicts a referenced body (refcount > 0
179
- * — it must stay resolvable without a re-download) nor a pending-upload-pinned
180
- * body (its bytes are the only copy until push, B4). If every over-cap body is
181
- * referenced or pinned, the cache stays over the cap (correctness beats the
182
- * cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
183
- * any surviving `blob_ref` value. Returns the evicted blobIds.
178
+ * §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
179
+ * evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
180
+ * until back under the cap. Returns the evicted blobIds.
184
181
  */
185
182
  export function enforceBlobCacheCap(
186
183
  db: ClientDatabase,
184
+ schema: CompiledClientSchema,
187
185
  maxBytes: number,
188
186
  ): string[] {
189
187
  const totalRow = db.query(
@@ -191,13 +189,13 @@ export function enforceBlobCacheCap(
191
189
  )[0];
192
190
  let total = Number(totalRow?.total ?? 0);
193
191
  if (total <= maxBytes) return [];
194
- // Eviction candidates: zero-ref AND not pinned by a pending upload, oldest
195
- // (LRU) first, then oldest created as a stable tiebreak.
192
+ // Oldest eligible body first, with blob ID as a stable tie.
196
193
  const candidates = db.query(
197
194
  `SELECT blob_id, byte_length FROM _syncular_blobs
198
- WHERE refcount = 0
199
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
200
- ORDER BY last_used_ms ASC, created_at_ms ASC`,
195
+ WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
196
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
197
+ AND blob_id NOT IN (${visibleBlobIdsSql(schema)})
198
+ ORDER BY created_at_ms ASC, blob_id ASC`,
201
199
  );
202
200
  const evicted: string[] = [];
203
201
  db.transaction(() => {
@@ -212,7 +210,36 @@ export function enforceBlobCacheCap(
212
210
  return evicted;
213
211
  }
214
212
 
215
- /** Record a pending upload (§5.9.7 B4); flushed before the next push. */
213
+ export function listPendingUploads(
214
+ db: ClientDatabase,
215
+ ): { blobId: string; mediaType?: string }[] {
216
+ return db
217
+ .query(
218
+ `SELECT blob_id, media_type FROM (
219
+ SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
220
+ UNION ALL
221
+ SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
222
+ FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
223
+ WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
224
+ ) ORDER BY created_at_ms, blob_id`,
225
+ )
226
+ .map((row) => {
227
+ if (
228
+ typeof row.blob_id !== 'string' ||
229
+ (row.media_type !== null && typeof row.media_type !== 'string')
230
+ ) {
231
+ throw new ClientSyncError(
232
+ 'sync.local_corrupt',
233
+ 'Pending blob upload metadata is invalid',
234
+ );
235
+ }
236
+ return {
237
+ blobId: row.blob_id,
238
+ ...(row.media_type !== null ? { mediaType: row.media_type } : {}),
239
+ };
240
+ });
241
+ }
242
+
216
243
  export function recordPendingUpload(
217
244
  db: ClientDatabase,
218
245
  blobId: string,
@@ -226,28 +253,12 @@ export function recordPendingUpload(
226
253
  );
227
254
  }
228
255
 
229
- export function listPendingUploads(
230
- db: ClientDatabase,
231
- ): { blobId: string; mediaType?: string }[] {
232
- return db
233
- .query(
234
- 'SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms',
235
- )
236
- .map((row) => ({
237
- blobId: row.blob_id as string,
238
- ...(row.media_type !== null
239
- ? { mediaType: row.media_type as string }
240
- : {}),
241
- }));
242
- }
243
-
244
256
  export function clearPendingUpload(db: ClientDatabase, blobId: string): void {
245
257
  db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id = ?', [blobId]);
246
258
  }
247
259
 
248
260
  /**
249
- * All `blob_ref` column names per table (for refcount reconciliation).
250
- * Blank result ⇒ the schema has no attachments; callers skip reconciliation.
261
+ * All `blob_ref` column names per table.
251
262
  */
252
263
  export function blobRefColumnsBySchema(
253
264
  schema: CompiledClientSchema,
@@ -269,64 +280,35 @@ export function schemaHasBlobs(schema: CompiledClientSchema): boolean {
269
280
  return false;
270
281
  }
271
282
 
272
- /**
273
- * §5.9.7 B1/B2: recompute cache refcounts from the current `blob_ref` column
274
- * contents across all synced tables, then delete cache bodies that dropped
275
- * to zero refs *and* have no pending upload (a pending upload pins its body,
276
- * B4). Called after every apply/purge that may add or remove references.
277
- *
278
- * `deleteOrphans` distinguishes the two B2 transitions: revocation purge
279
- * passes `true` (drop the now-unauthorized body); a benign apply passes
280
- * `false` (retain zero-ref bodies as LRU cache entries — the shipped
281
- * default). Bodies pinned by a pending upload are always retained.
282
- */
283
- export function reconcileBlobRefcounts(
284
- db: ClientDatabase,
285
- schema: CompiledClientSchema,
286
- options?: { readonly deleteOrphans?: boolean },
287
- ): void {
288
- const byTable = blobRefColumnsBySchema(schema);
289
- if (byTable.size === 0) return;
290
- // Count references to each blobId across every blob_ref column.
291
- const counts = new Map<string, number>();
292
- for (const [tableName, columns] of byTable) {
283
+ function visibleBlobIdsSql(schema: CompiledClientSchema): string {
284
+ const selects: string[] = [];
285
+ for (const [tableName, columns] of blobRefColumnsBySchema(schema)) {
293
286
  for (const column of columns) {
294
- const rows = db.query(
295
- `SELECT ${quoteIdent(column)} AS v FROM ${quoteIdent(tableName)}
296
- WHERE ${quoteIdent(column)} IS NOT NULL`,
287
+ const identifier = quoteIdent(column);
288
+ selects.push(
289
+ `SELECT json_extract(${identifier}, '$.blobId') AS blob_id
290
+ FROM ${quoteIdent(tableName)}
291
+ WHERE typeof(${identifier}) = 'text'
292
+ AND json_valid(${identifier})
293
+ AND json_type(${identifier}, '$.blobId') = 'text'`,
297
294
  );
298
- for (const row of rows) {
299
- const raw = row.v;
300
- if (typeof raw !== 'string') continue;
301
- let ref: BlobRef;
302
- try {
303
- ref = parseBlobRef(raw);
304
- } catch {
305
- continue;
306
- }
307
- counts.set(ref.blobId, (counts.get(ref.blobId) ?? 0) + 1);
308
- }
309
295
  }
310
296
  }
311
- db.transaction(() => {
312
- // Reset all refcounts, then apply the recomputed counts.
313
- db.exec('UPDATE _syncular_blobs SET refcount = 0');
314
- for (const [blobId, count] of counts) {
315
- db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
316
- count,
317
- blobId,
318
- ]);
319
- }
320
- if (options?.deleteOrphans === true) {
321
- // §5.9.7 B2 revocation side: delete zero-ref bodies not pinned by a
322
- // pending upload.
323
- db.exec(
324
- `DELETE FROM _syncular_blobs
325
- WHERE refcount = 0
326
- AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)`,
327
- );
328
- }
329
- });
297
+ return selects.length === 0
298
+ ? 'SELECT NULL AS blob_id WHERE 0'
299
+ : selects.join(' UNION ');
300
+ }
301
+
302
+ export function deleteUnreferencedCachedBlobs(
303
+ db: ClientDatabase,
304
+ schema: CompiledClientSchema,
305
+ ): void {
306
+ db.exec(
307
+ `DELETE FROM _syncular_blobs
308
+ WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
309
+ AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
310
+ AND blob_id NOT IN (${visibleBlobIdsSql(schema)})`,
311
+ );
330
312
  }
331
313
 
332
314
  export type { BlobRef };
package/src/client.ts CHANGED
@@ -47,13 +47,13 @@ import {
47
47
  type CachedBlob,
48
48
  clearPendingUpload,
49
49
  computeBlobId,
50
+ deleteUnreferencedCachedBlobs,
50
51
  enforceBlobCacheCap,
51
52
  ensureBlobSchema,
52
53
  getCachedBlob,
53
54
  listPendingUploads,
54
55
  parseBlobRef,
55
56
  putCachedBlob,
56
- reconcileBlobRefcounts,
57
57
  recordPendingUpload,
58
58
  schemaHasBlobs,
59
59
  serializeBlobRef,
@@ -302,9 +302,9 @@ export interface SyncClientConfig {
302
302
  readonly blobs?: BlobTransport;
303
303
  /**
304
304
  * §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
305
- * exceeds this, zero-ref, non-pinned bodies are evicted LRU-first after each
306
- * cache write (referenced/pinned bodies are never evicted — correctness
307
- * beats the cap). Absent ⇒ retain until storage pressure (the shipped
305
+ * exceeds this, unreferenced bodies without pending work are evicted in
306
+ * creation timestamp order after each cache write. Referenced bodies remain cached
307
+ * even when that exceeds the cap. Absent means retain until storage pressure (the shipped
308
308
  * default; a referenced body always stays resolvable without a re-download).
309
309
  */
310
310
  readonly blobCacheMaxBytes?: number;
@@ -1455,7 +1455,8 @@ export class SyncClient {
1455
1455
  bytes: Uint8Array,
1456
1456
  options?: { readonly mediaType?: string; readonly name?: string },
1457
1457
  ): Promise<BlobRef> {
1458
- return this.#runProtectedAsync(() => this.#uploadBlob(bytes, options));
1458
+ const owned = bytes.slice();
1459
+ return this.#runProtectedAsync(() => this.#uploadBlob(owned, options));
1459
1460
  }
1460
1461
 
1461
1462
  async #uploadBlob(
@@ -1469,13 +1470,11 @@ export class SyncClient {
1469
1470
  );
1470
1471
  }
1471
1472
  const blobId = await computeBlobId(bytes);
1473
+ const nowMs = this.#now();
1472
1474
  this.#db.transaction(() => {
1473
- putCachedBlob(this.#db, blobId, bytes, this.#now(), options?.mediaType);
1474
- recordPendingUpload(this.#db, blobId, this.#now(), options?.mediaType);
1475
+ putCachedBlob(this.#db, blobId, bytes, nowMs, options?.mediaType);
1476
+ recordPendingUpload(this.#db, blobId, nowMs, options?.mediaType);
1475
1477
  });
1476
- // §5.9.7 B1: a staged upload is pinned (recordPendingUpload), so the cap
1477
- // trim below will never evict it — but a stage may push other zero-ref
1478
- // bodies over the cap, so run the trim.
1479
1478
  this.#enforceBlobCacheCap();
1480
1479
  return {
1481
1480
  blobId,
@@ -1506,7 +1505,7 @@ export class SyncClient {
1506
1505
  const blobId = blobIdOrRef.startsWith('sha256:')
1507
1506
  ? blobIdOrRef
1508
1507
  : parseBlobRef(blobIdOrRef).blobId;
1509
- const cached = getCachedBlob(this.#db, blobId, this.#now());
1508
+ const cached = getCachedBlob(this.#db, blobId);
1510
1509
  if (cached !== undefined) return cached;
1511
1510
  const transport = this.#config.blobs;
1512
1511
  if (transport === undefined) {
@@ -1554,9 +1553,7 @@ export class SyncClient {
1554
1553
  );
1555
1554
  }
1556
1555
  putCachedBlob(this.#db, blobId, bytes, this.#now());
1557
- // The referencing row can arrive before its body. Pin the new cache entry
1558
- // from current visible references before applying the size cap (§5.9.7 B1).
1559
- this.#reconcileBlobs(false);
1556
+ this.#db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
1560
1557
  this.#enforceBlobCacheCap();
1561
1558
  const stored = getCachedBlob(this.#db, blobId);
1562
1559
  if (stored === undefined) {
@@ -1572,7 +1569,7 @@ export class SyncClient {
1572
1569
  #enforceBlobCacheCap(): void {
1573
1570
  const cap = this.#config.blobCacheMaxBytes;
1574
1571
  if (cap === undefined) return;
1575
- enforceBlobCacheCap(this.#db, cap);
1572
+ enforceBlobCacheCap(this.#db, this.#schema, cap);
1576
1573
  }
1577
1574
 
1578
1575
  /** Flush any queued blob uploads (§5.9.7 B4); safe to call standalone. */
@@ -1585,11 +1582,17 @@ export class SyncClient {
1585
1582
  if (transport === undefined || !this.#hasBlobs) return;
1586
1583
  for (const pending of listPendingUploads(this.#db)) {
1587
1584
  const cached = getCachedBlob(this.#db, pending.blobId);
1588
- if (cached === undefined) {
1589
- // The bytes are gone (never happens for a well-behaved client);
1590
- // drop the upload so it does not wedge the queue.
1591
- clearPendingUpload(this.#db, pending.blobId);
1592
- continue;
1585
+ if (
1586
+ cached === undefined ||
1587
+ !(cached.bytes instanceof Uint8Array) ||
1588
+ !Number.isSafeInteger(cached.byteLength) ||
1589
+ cached.byteLength !== cached.bytes.byteLength ||
1590
+ (await computeBlobId(cached.bytes)) !== pending.blobId
1591
+ ) {
1592
+ throw new ClientSyncError(
1593
+ 'sync.local_corrupt',
1594
+ 'Pending blob upload body is missing or corrupt',
1595
+ );
1593
1596
  }
1594
1597
  await this.#uploadOne(
1595
1598
  transport,
@@ -2141,6 +2144,29 @@ export class SyncClient {
2141
2144
  this.#now(),
2142
2145
  this.#captureBeforeImages(operations),
2143
2146
  );
2147
+ for (const operation of operations) {
2148
+ if (operation.op !== 'upsert') continue;
2149
+ const table = this.#schema.tables.get(operation.table);
2150
+ if (table === undefined) continue;
2151
+ for (const column of table.columns) {
2152
+ if (column.type !== 'blob_ref') continue;
2153
+ const raw = operation.values?.[column.name];
2154
+ if (typeof raw !== 'string') continue;
2155
+ let blobId: string;
2156
+ try {
2157
+ blobId = parseBlobRef(raw).blobId;
2158
+ } catch {
2159
+ continue;
2160
+ }
2161
+ this.#db.exec(
2162
+ `INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
2163
+ SELECT ?, ? WHERE EXISTS (
2164
+ SELECT 1 FROM _syncular_blobs WHERE blob_id = ?
2165
+ )`,
2166
+ [clientCommitId, blobId, blobId],
2167
+ );
2168
+ }
2169
+ }
2144
2170
  this.#applyOperationsLocally(operations, batch);
2145
2171
  batch.status();
2146
2172
  });
@@ -2339,7 +2365,7 @@ export class SyncClient {
2339
2365
  batch.rejections();
2340
2366
  batch.outcomes();
2341
2367
  }
2342
- this.#reconcileBlobs(true);
2368
+ this.#deleteUnreferencedBlobs();
2343
2369
  setMeta(this.#db, metaKey, purge.canonicalPlan);
2344
2370
  return {
2345
2371
  alreadyApplied: false,
@@ -3517,7 +3543,6 @@ export class SyncClient {
3517
3543
  // §7.1: local reads see outbox state applied optimistically — replay
3518
3544
  // the still-pending commits on top of the freshly applied server state.
3519
3545
  this.#replayOutbox();
3520
- this.#reconcileBlobs(false);
3521
3546
  } finally {
3522
3547
  this.#endDiagnosticsDeferral();
3523
3548
  }
@@ -4000,7 +4025,7 @@ export class SyncClient {
4000
4025
  batch.rejections();
4001
4026
  batch.outcomes();
4002
4027
  }
4003
- this.#reconcileBlobs(true);
4028
+ this.#deleteUnreferencedBlobs();
4004
4029
  } catch (error) {
4005
4030
  if (
4006
4031
  error instanceof ClientSyncError &&
@@ -4218,14 +4243,9 @@ export class SyncClient {
4218
4243
  });
4219
4244
  }
4220
4245
 
4221
- /**
4222
- * §5.9.7 B1/B2: recompute blob-cache refcounts from live `blob_ref`
4223
- * columns. No-op unless the schema has blob columns. `deleteOrphans`
4224
- * triggers the revocation-side body deletion (B2).
4225
- */
4226
- #reconcileBlobs(deleteOrphans: boolean): void {
4246
+ #deleteUnreferencedBlobs(): void {
4227
4247
  if (!this.#hasBlobs) return;
4228
- reconcileBlobRefcounts(this.#db, this.#schema, { deleteOrphans });
4248
+ deleteUnreferencedCachedBlobs(this.#db, this.#schema);
4229
4249
  }
4230
4250
 
4231
4251
  // -- helpers -----------------------------------------------------------------
package/src/outbox.ts CHANGED
@@ -135,6 +135,9 @@ export function deleteOutboxCommit(
135
135
  db: ClientDatabase,
136
136
  clientCommitId: string,
137
137
  ): void {
138
+ db.exec('DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?', [
139
+ clientCommitId,
140
+ ]);
138
141
  db.exec(
139
142
  'DELETE FROM _syncular_outbox_before_images WHERE client_commit_id = ?',
140
143
  [clientCommitId],
package/src/schema.ts CHANGED
@@ -501,6 +501,11 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
501
501
  db.transaction(() => {
502
502
  db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
503
503
  key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
504
+ db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
505
+ commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
506
+ db.exec(
507
+ 'CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)',
508
+ );
504
509
  db.exec(
505
510
  `INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`,
506
511
  );