@syncular/client 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -13
- package/dist/blob.d.ts +12 -39
- package/dist/blob.js +63 -134
- package/dist/client.d.ts +3 -3
- package/dist/client.js +38 -57
- package/dist/outbox.js +3 -0
- package/dist/schema.js +3 -0
- package/package.json +3 -3
- package/src/blob.ts +86 -157
- package/src/client.ts +38 -68
- package/src/outbox.ts +3 -0
- package/src/schema.ts +5 -0
package/README.md
CHANGED
|
@@ -375,7 +375,7 @@ lifecycle with an `EncryptionConfig`; Worker handles use the portable keyring.
|
|
|
375
375
|
Within one local SQLite transaction the engine deletes exactly the matching
|
|
376
376
|
synced rows, lets generated FTS triggers remove their projections, drops every
|
|
377
377
|
whole pending commit with a matching operation, restores/replays unrelated
|
|
378
|
-
optimistic state,
|
|
378
|
+
optimistic state, removes unreferenced cached blob bodies, persists the `purgeId`, and emits
|
|
379
379
|
one revisioned change batch. A retry with the same canonical plan returns
|
|
380
380
|
`alreadyApplied: true`; reusing an id with different selectors fails closed.
|
|
381
381
|
Only bounded, non-empty, code-like values on plaintext string schema columns
|
|
@@ -484,9 +484,8 @@ and are cached locally. **Blob bytes live as `BLOB` columns in the client's own
|
|
|
484
484
|
SQLite database** (a `_syncular_blobs` cache table), not in a separate OPFS
|
|
485
485
|
directory or IndexedDB store. This is the pinned decision (SPEC §5.9.7 B1):
|
|
486
486
|
|
|
487
|
-
- **One storage system.** The bytes
|
|
488
|
-
|
|
489
|
-
so a crash never strands a body against a stale count.
|
|
487
|
+
- **One storage system.** The bytes, upload state, and pending-commit references
|
|
488
|
+
share the client database. SQLite commits each change atomically.
|
|
490
489
|
- **Survives restarts for free.** The client DB already rides OPFS via the
|
|
491
490
|
sahpool VFS in the browser (and a plain file under `rusqlite`, `bun:sqlite`,
|
|
492
491
|
or `node:sqlite` on native runtimes), so there is no second persistence
|
|
@@ -495,25 +494,24 @@ directory or IndexedDB store. This is the pinned decision (SPEC §5.9.7 B1):
|
|
|
495
494
|
- **SQLite handles multi-MB images fine.** A page-cached `BLOB` read is a memory
|
|
496
495
|
copy, well within the image/document envelope this targets.
|
|
497
496
|
|
|
498
|
-
### Size cap
|
|
497
|
+
### Size cap
|
|
499
498
|
|
|
500
499
|
Pass `blobCacheMaxBytes` to cap the on-device cache. When the sum of cached body
|
|
501
|
-
sizes exceeds the cap, the client evicts
|
|
502
|
-
|
|
500
|
+
sizes exceeds the cap, the client evicts bodies with no visible row or pending
|
|
501
|
+
commit by creation timestamp, then blob ID, until back under the cap:
|
|
503
502
|
|
|
504
503
|
```ts
|
|
505
504
|
new SyncClient({ /* … */, blobCacheMaxBytes: 256 * 1024 * 1024 }); // 256 MiB
|
|
506
505
|
```
|
|
507
506
|
|
|
508
|
-
- A body **referenced by a live row**
|
|
507
|
+
- A body **referenced by a live row** is **never** evicted. It
|
|
509
508
|
stays resolvable without a re-download.
|
|
510
|
-
- A body **
|
|
509
|
+
- A body **needed by a pending upload** (not yet pushed) is never evicted. Its
|
|
511
510
|
bytes are the only copy until the commit drains.
|
|
512
|
-
- Evicting
|
|
511
|
+
- Evicting an unreferenced body only costs a future re-download, never correctness:
|
|
513
512
|
any surviving `blob_ref` value re-enables the fetch (§5.9.7 B3). If every
|
|
514
513
|
over-cap body is referenced or pinned, the cache stays over the cap
|
|
515
|
-
(correctness beats the cap).
|
|
516
|
-
hot image survives a trim. Absent `blobCacheMaxBytes` ⇒ retain until storage
|
|
514
|
+
(correctness beats the cap). Absent `blobCacheMaxBytes` ⇒ retain until storage
|
|
517
515
|
pressure (the default).
|
|
518
516
|
|
|
519
517
|
### Very large media — the escape hatch
|
|
@@ -522,7 +520,7 @@ SQLite is **not** the store for gigabyte video: a single `BLOB` must fit the
|
|
|
522
520
|
client's memory and the SQLite row-size envelope. For very large media, run the
|
|
523
521
|
server with presigned downloads (`blobSignedUrls`) and hand the presigned URL
|
|
524
522
|
straight to a media element instead of pulling bytes through the cache — the
|
|
525
|
-
image-app default (
|
|
523
|
+
image-app default (`BLOB` cache) and the large-media path (presigned
|
|
526
524
|
URL, no byte cache) coexist per attachment.
|
|
527
525
|
|
|
528
526
|
## Node and Bun SQLite backend (`./sqlite`)
|
package/dist/blob.d.ts
CHANGED
|
@@ -1,14 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Client-side blob cache + transport (SPEC.md §5.9.7).
|
|
3
3
|
*
|
|
4
|
-
* Blob bytes are cached
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* reconciled from the current `blob_ref` column contents, and a body whose
|
|
8
|
-
* only referencing rows were revocation-purged is deleted (B2, evicted ≠
|
|
9
|
-
* revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
|
|
10
|
-
* row value is the whole download key. Pending uploads are tracked in the
|
|
11
|
-
* outbox-adjacent uploads table (B4) and flushed before push.
|
|
4
|
+
* Blob bytes are cached by `blobId`. Retention reads live references directly
|
|
5
|
+
* from `blob_ref` columns and keeps explicit pending-commit dependencies.
|
|
6
|
+
* A small upload row queues each staged body for transfer before push.
|
|
12
7
|
*/
|
|
13
8
|
import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
|
|
14
9
|
import type { ClientDatabase } from './database.js';
|
|
@@ -82,48 +77,26 @@ export interface CachedBlob {
|
|
|
82
77
|
/** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
|
|
83
78
|
export declare function computeBlobId(bytes: Uint8Array): Promise<string>;
|
|
84
79
|
export declare function ensureBlobSchema(db: ClientDatabase): void;
|
|
85
|
-
/** Put bytes into the content-addressed cache
|
|
80
|
+
/** Put bytes into the content-addressed cache without rewriting an existing body. */
|
|
86
81
|
export declare function putCachedBlob(db: ClientDatabase, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): void;
|
|
87
|
-
export declare function getCachedBlob(db: ClientDatabase, blobId: string
|
|
82
|
+
export declare function getCachedBlob(db: ClientDatabase, blobId: string): CachedBlob | undefined;
|
|
88
83
|
/**
|
|
89
|
-
* §5.9.7 B1 size cap
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
* — it must stay resolvable without a re-download) nor a pending-upload-pinned
|
|
93
|
-
* body (its bytes are the only copy until push, B4). If every over-cap body is
|
|
94
|
-
* referenced or pinned, the cache stays over the cap (correctness beats the
|
|
95
|
-
* cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
|
|
96
|
-
* any surviving `blob_ref` value. Returns the evicted blobIds.
|
|
84
|
+
* §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
|
|
85
|
+
* evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
|
|
86
|
+
* until back under the cap. Returns the evicted blobIds.
|
|
97
87
|
*/
|
|
98
|
-
export declare function enforceBlobCacheCap(db: ClientDatabase, maxBytes: number): string[];
|
|
99
|
-
/** Record a pending upload (§5.9.7 B4); flushed before the next push. */
|
|
100
|
-
export declare function recordPendingUpload(db: ClientDatabase, blobId: string, nowMs: number, mediaType?: string): void;
|
|
88
|
+
export declare function enforceBlobCacheCap(db: ClientDatabase, schema: CompiledClientSchema, maxBytes: number): string[];
|
|
101
89
|
export declare function listPendingUploads(db: ClientDatabase): {
|
|
102
90
|
blobId: string;
|
|
103
91
|
mediaType?: string;
|
|
104
92
|
}[];
|
|
93
|
+
export declare function recordPendingUpload(db: ClientDatabase, blobId: string, nowMs: number, mediaType?: string): void;
|
|
105
94
|
export declare function clearPendingUpload(db: ClientDatabase, blobId: string): void;
|
|
106
95
|
/**
|
|
107
|
-
* All `blob_ref` column names per table
|
|
108
|
-
* Blank result ⇒ the schema has no attachments; callers skip reconciliation.
|
|
96
|
+
* All `blob_ref` column names per table.
|
|
109
97
|
*/
|
|
110
98
|
export declare function blobRefColumnsBySchema(schema: CompiledClientSchema): Map<string, string[]>;
|
|
111
99
|
export declare function schemaHasBlobs(schema: CompiledClientSchema): boolean;
|
|
112
|
-
|
|
113
|
-
* §5.9.7 B1/B2: recompute cache refcounts from the current `blob_ref` column
|
|
114
|
-
* contents across all synced tables, then delete cache bodies that dropped
|
|
115
|
-
* to zero refs *and* have no pending upload (a pending upload pins its body,
|
|
116
|
-
* B4). Called after every apply/purge that may add or remove references.
|
|
117
|
-
*
|
|
118
|
-
* `blobId` narrows a fresh-download refresh to the inserted cache row.
|
|
119
|
-
* `deleteOrphans` distinguishes the two B2 transitions: revocation purge
|
|
120
|
-
* passes `true` (drop the now-unauthorized body); a benign apply passes
|
|
121
|
-
* `false` (retain zero-ref bodies as LRU cache entries — the shipped
|
|
122
|
-
* default). Bodies pinned by a pending upload are always retained.
|
|
123
|
-
*/
|
|
124
|
-
export declare function reconcileBlobRefcounts(db: ClientDatabase, schema: CompiledClientSchema, options?: {
|
|
125
|
-
readonly blobId?: string;
|
|
126
|
-
readonly deleteOrphans?: boolean;
|
|
127
|
-
}): void;
|
|
100
|
+
export declare function deleteUnreferencedCachedBlobs(db: ClientDatabase, schema: CompiledClientSchema): void;
|
|
128
101
|
export type { BlobRef };
|
|
129
102
|
export { parseBlobRef, serializeBlobRef };
|
package/dist/blob.js
CHANGED
|
@@ -1,14 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Client-side blob cache + transport (SPEC.md §5.9.7).
|
|
3
3
|
*
|
|
4
|
-
* Blob bytes are cached
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* reconciled from the current `blob_ref` column contents, and a body whose
|
|
8
|
-
* only referencing rows were revocation-purged is deleted (B2, evicted ≠
|
|
9
|
-
* revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
|
|
10
|
-
* row value is the whole download key. Pending uploads are tracked in the
|
|
11
|
-
* outbox-adjacent uploads table (B4) and flushed before push.
|
|
4
|
+
* Blob bytes are cached by `blobId`. Retention reads live references directly
|
|
5
|
+
* from `blob_ref` columns and keeps explicit pending-commit dependencies.
|
|
6
|
+
* A small upload row queues each staged body for transfer before push.
|
|
12
7
|
*/
|
|
13
8
|
import { parseBlobRef, serializeBlobRef } from '@syncular/core';
|
|
14
9
|
import { ClientSyncError } from './errors.js';
|
|
@@ -25,51 +20,46 @@ export async function computeBlobId(bytes) {
|
|
|
25
20
|
return `sha256:${hex}`;
|
|
26
21
|
}
|
|
27
22
|
export function ensureBlobSchema(db) {
|
|
28
|
-
db.
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
23
|
+
const existing = db.query("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs'");
|
|
24
|
+
if (existing.length > 0) {
|
|
25
|
+
const columns = db
|
|
26
|
+
.query('PRAGMA table_info(_syncular_blobs)')
|
|
27
|
+
.map((row) => row.name)
|
|
28
|
+
.sort();
|
|
29
|
+
const expected = [
|
|
30
|
+
'blob_id',
|
|
31
|
+
'byte_length',
|
|
32
|
+
'bytes',
|
|
33
|
+
'created_at_ms',
|
|
34
|
+
'media_type',
|
|
35
|
+
].sort();
|
|
36
|
+
if (columns.length !== expected.length ||
|
|
37
|
+
columns.some((column, index) => column !== expected[index])) {
|
|
38
|
+
throw new ClientSyncError('sync.schema_mismatch', 'Local blob schema is incompatible');
|
|
39
|
+
}
|
|
41
40
|
}
|
|
42
|
-
|
|
43
|
-
|
|
41
|
+
else {
|
|
42
|
+
db.exec(`CREATE TABLE _syncular_blobs(
|
|
43
|
+
blob_id TEXT PRIMARY KEY,
|
|
44
|
+
bytes BLOB NOT NULL,
|
|
45
|
+
byte_length INTEGER NOT NULL,
|
|
46
|
+
media_type TEXT,
|
|
47
|
+
created_at_ms INTEGER NOT NULL)`);
|
|
44
48
|
}
|
|
45
49
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
|
|
46
|
-
blob_id TEXT PRIMARY KEY,
|
|
47
|
-
media_type TEXT,
|
|
48
|
-
created_at_ms INTEGER NOT NULL)`);
|
|
49
|
-
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
|
|
50
|
-
commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
|
|
51
|
-
db.exec('CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)');
|
|
50
|
+
blob_id TEXT PRIMARY KEY, media_type TEXT, created_at_ms INTEGER NOT NULL)`);
|
|
52
51
|
}
|
|
53
|
-
/** Put bytes into the content-addressed cache
|
|
52
|
+
/** Put bytes into the content-addressed cache without rewriting an existing body. */
|
|
54
53
|
export function putCachedBlob(db, blobId, bytes, nowMs, mediaType) {
|
|
55
54
|
db.exec(`INSERT INTO _syncular_blobs(
|
|
56
|
-
blob_id, bytes, byte_length, media_type,
|
|
57
|
-
VALUES (
|
|
58
|
-
ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms`, [blobId, bytes, bytes.length, mediaType ?? null, nowMs, nowMs]);
|
|
55
|
+
blob_id, bytes, byte_length, media_type, created_at_ms)
|
|
56
|
+
VALUES (?,?,?,?,?) ON CONFLICT(blob_id) DO NOTHING`, [blobId, bytes, bytes.length, mediaType ?? null, nowMs]);
|
|
59
57
|
}
|
|
60
|
-
export function getCachedBlob(db, blobId
|
|
58
|
+
export function getCachedBlob(db, blobId) {
|
|
61
59
|
const rows = db.query('SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?', [blobId]);
|
|
62
60
|
const row = rows[0];
|
|
63
61
|
if (row === undefined)
|
|
64
62
|
return undefined;
|
|
65
|
-
// §5.9.7 B1 LRU: a cache-hit read touches "recently used" so a hot image
|
|
66
|
-
// survives a cap trim. Skipped when no clock is supplied (pure read).
|
|
67
|
-
if (nowMs !== undefined) {
|
|
68
|
-
db.exec('UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?', [
|
|
69
|
-
nowMs,
|
|
70
|
-
blobId,
|
|
71
|
-
]);
|
|
72
|
-
}
|
|
73
63
|
return {
|
|
74
64
|
blobId,
|
|
75
65
|
bytes: row.bytes,
|
|
@@ -78,27 +68,21 @@ export function getCachedBlob(db, blobId, nowMs) {
|
|
|
78
68
|
};
|
|
79
69
|
}
|
|
80
70
|
/**
|
|
81
|
-
* §5.9.7 B1 size cap
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
* — it must stay resolvable without a re-download) nor a pending-upload-pinned
|
|
85
|
-
* body (its bytes are the only copy until push, B4). If every over-cap body is
|
|
86
|
-
* referenced or pinned, the cache stays over the cap (correctness beats the
|
|
87
|
-
* cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
|
|
88
|
-
* any surviving `blob_ref` value. Returns the evicted blobIds.
|
|
71
|
+
* §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
|
|
72
|
+
* evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
|
|
73
|
+
* until back under the cap. Returns the evicted blobIds.
|
|
89
74
|
*/
|
|
90
|
-
export function enforceBlobCacheCap(db, maxBytes) {
|
|
75
|
+
export function enforceBlobCacheCap(db, schema, maxBytes) {
|
|
91
76
|
const totalRow = db.query('SELECT COALESCE(SUM(byte_length), 0) AS total FROM _syncular_blobs')[0];
|
|
92
77
|
let total = Number(totalRow?.total ?? 0);
|
|
93
78
|
if (total <= maxBytes)
|
|
94
79
|
return [];
|
|
95
|
-
//
|
|
96
|
-
// (LRU) first, then oldest created as a stable tiebreak.
|
|
80
|
+
// Oldest eligible body first, with blob ID as a stable tie.
|
|
97
81
|
const candidates = db.query(`SELECT blob_id, byte_length FROM _syncular_blobs
|
|
98
|
-
WHERE
|
|
99
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
82
|
+
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
100
83
|
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
101
|
-
|
|
84
|
+
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})
|
|
85
|
+
ORDER BY created_at_ms ASC, blob_id ASC`);
|
|
102
86
|
const evicted = [];
|
|
103
87
|
db.transaction(() => {
|
|
104
88
|
for (const row of candidates) {
|
|
@@ -112,11 +96,6 @@ export function enforceBlobCacheCap(db, maxBytes) {
|
|
|
112
96
|
});
|
|
113
97
|
return evicted;
|
|
114
98
|
}
|
|
115
|
-
/** Record a pending upload (§5.9.7 B4); flushed before the next push. */
|
|
116
|
-
export function recordPendingUpload(db, blobId, nowMs, mediaType) {
|
|
117
|
-
db.exec(`INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms)
|
|
118
|
-
VALUES (?,?,?)`, [blobId, mediaType ?? null, nowMs]);
|
|
119
|
-
}
|
|
120
99
|
export function listPendingUploads(db) {
|
|
121
100
|
return db
|
|
122
101
|
.query(`SELECT blob_id, media_type FROM (
|
|
@@ -137,12 +116,15 @@ export function listPendingUploads(db) {
|
|
|
137
116
|
};
|
|
138
117
|
});
|
|
139
118
|
}
|
|
119
|
+
export function recordPendingUpload(db, blobId, nowMs, mediaType) {
|
|
120
|
+
db.exec(`INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms)
|
|
121
|
+
VALUES (?,?,?)`, [blobId, mediaType ?? null, nowMs]);
|
|
122
|
+
}
|
|
140
123
|
export function clearPendingUpload(db, blobId) {
|
|
141
124
|
db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id = ?', [blobId]);
|
|
142
125
|
}
|
|
143
126
|
/**
|
|
144
|
-
* All `blob_ref` column names per table
|
|
145
|
-
* Blank result ⇒ the schema has no attachments; callers skip reconciliation.
|
|
127
|
+
* All `blob_ref` column names per table.
|
|
146
128
|
*/
|
|
147
129
|
export function blobRefColumnsBySchema(schema) {
|
|
148
130
|
const out = new Map();
|
|
@@ -162,79 +144,26 @@ export function schemaHasBlobs(schema) {
|
|
|
162
144
|
}
|
|
163
145
|
return false;
|
|
164
146
|
}
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
* to zero refs *and* have no pending upload (a pending upload pins its body,
|
|
169
|
-
* B4). Called after every apply/purge that may add or remove references.
|
|
170
|
-
*
|
|
171
|
-
* `blobId` narrows a fresh-download refresh to the inserted cache row.
|
|
172
|
-
* `deleteOrphans` distinguishes the two B2 transitions: revocation purge
|
|
173
|
-
* passes `true` (drop the now-unauthorized body); a benign apply passes
|
|
174
|
-
* `false` (retain zero-ref bodies as LRU cache entries — the shipped
|
|
175
|
-
* default). Bodies pinned by a pending upload are always retained.
|
|
176
|
-
*/
|
|
177
|
-
export function reconcileBlobRefcounts(db, schema, options) {
|
|
178
|
-
const byTable = blobRefColumnsBySchema(schema);
|
|
179
|
-
if (byTable.size === 0)
|
|
180
|
-
return;
|
|
181
|
-
if (options?.blobId !== undefined) {
|
|
182
|
-
let count = 0;
|
|
183
|
-
for (const [tableName, columns] of byTable) {
|
|
184
|
-
for (const column of columns) {
|
|
185
|
-
const identifier = quoteIdent(column);
|
|
186
|
-
const row = db.query(`SELECT count(*) AS n FROM ${quoteIdent(tableName)}
|
|
187
|
-
WHERE CASE
|
|
188
|
-
WHEN typeof(${identifier}) = 'text' AND json_valid(${identifier})
|
|
189
|
-
THEN json_extract(${identifier}, '$.blobId')
|
|
190
|
-
END = ?`, [options.blobId])[0];
|
|
191
|
-
count += Number(row?.n ?? 0);
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
|
|
195
|
-
count,
|
|
196
|
-
options.blobId,
|
|
197
|
-
]);
|
|
198
|
-
return;
|
|
199
|
-
}
|
|
200
|
-
// Count references to each blobId across every blob_ref column.
|
|
201
|
-
const counts = new Map();
|
|
202
|
-
for (const [tableName, columns] of byTable) {
|
|
147
|
+
function visibleBlobIdsSql(schema) {
|
|
148
|
+
const selects = [];
|
|
149
|
+
for (const [tableName, columns] of blobRefColumnsBySchema(schema)) {
|
|
203
150
|
for (const column of columns) {
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
let ref;
|
|
211
|
-
try {
|
|
212
|
-
ref = parseBlobRef(raw);
|
|
213
|
-
}
|
|
214
|
-
catch {
|
|
215
|
-
continue;
|
|
216
|
-
}
|
|
217
|
-
counts.set(ref.blobId, (counts.get(ref.blobId) ?? 0) + 1);
|
|
218
|
-
}
|
|
151
|
+
const identifier = quoteIdent(column);
|
|
152
|
+
selects.push(`SELECT json_extract(${identifier}, '$.blobId') AS blob_id
|
|
153
|
+
FROM ${quoteIdent(tableName)}
|
|
154
|
+
WHERE typeof(${identifier}) = 'text'
|
|
155
|
+
AND json_valid(${identifier})
|
|
156
|
+
AND json_type(${identifier}, '$.blobId') = 'text'`);
|
|
219
157
|
}
|
|
220
158
|
}
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
if (options?.deleteOrphans === true) {
|
|
231
|
-
// §5.9.7 B2 revocation side: delete zero-ref bodies not pinned by a
|
|
232
|
-
// pending upload.
|
|
233
|
-
db.exec(`DELETE FROM _syncular_blobs
|
|
234
|
-
WHERE refcount = 0
|
|
235
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
236
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)`);
|
|
237
|
-
}
|
|
238
|
-
});
|
|
159
|
+
return selects.length === 0
|
|
160
|
+
? 'SELECT NULL AS blob_id WHERE 0'
|
|
161
|
+
: selects.join(' UNION ');
|
|
162
|
+
}
|
|
163
|
+
export function deleteUnreferencedCachedBlobs(db, schema) {
|
|
164
|
+
db.exec(`DELETE FROM _syncular_blobs
|
|
165
|
+
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
166
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
167
|
+
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})`);
|
|
239
168
|
}
|
|
240
169
|
export { parseBlobRef, serializeBlobRef };
|
package/dist/client.d.ts
CHANGED
|
@@ -110,9 +110,9 @@ export interface SyncClientConfig {
|
|
|
110
110
|
readonly blobs?: BlobTransport;
|
|
111
111
|
/**
|
|
112
112
|
* §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
|
|
113
|
-
* exceeds this,
|
|
114
|
-
* cache write
|
|
115
|
-
*
|
|
113
|
+
* exceeds this, unreferenced bodies without pending work are evicted in
|
|
114
|
+
* creation timestamp order after each cache write. Referenced bodies remain cached
|
|
115
|
+
* even when that exceeds the cap. Absent means retain until storage pressure (the shipped
|
|
116
116
|
* default; a referenced body always stays resolvable without a re-download).
|
|
117
117
|
*/
|
|
118
118
|
readonly blobCacheMaxBytes?: number;
|
package/dist/client.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { canonicalScopeJson, decodeMessage, decodeRow, decodeRowsSegment, encodeMessage, encodePresencePublish, MessageStreamScanner, PROTOCOL_WIRE_VERSION, parseRealtimeServerEvent, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
|
|
11
11
|
import { applyCommitFrame, applyRowsSegment, applySqliteSegment, deleteLocalRow, deleteScopedRows, evictScopedRows, upsertLocalRow, } from './apply.js';
|
|
12
|
-
import { clearPendingUpload, computeBlobId, enforceBlobCacheCap, ensureBlobSchema, getCachedBlob, listPendingUploads, parseBlobRef, putCachedBlob,
|
|
12
|
+
import { clearPendingUpload, computeBlobId, deleteUnreferencedCachedBlobs, enforceBlobCacheCap, ensureBlobSchema, getCachedBlob, listPendingUploads, parseBlobRef, putCachedBlob, recordPendingUpload, schemaHasBlobs, serializeBlobRef, } from './blob.js';
|
|
13
13
|
import { registerDevtools } from './devtools.js';
|
|
14
14
|
import { CLIENT_DIAGNOSTICS_VERSION, ClientDiagnosticsEmitter, MAX_DIAGNOSTIC_DOMAINS, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS, } from './diagnostics.js';
|
|
15
15
|
import { ClientSyncError } from './errors.js';
|
|
@@ -191,39 +191,8 @@ export class SyncClient {
|
|
|
191
191
|
// Do not materialize new app indexes/FTS projections yet: on a version
|
|
192
192
|
// bump they may reference columns that only exist after the reset.
|
|
193
193
|
ensureLocalBookkeepingSchema(this.#db);
|
|
194
|
-
this.#
|
|
195
|
-
|
|
196
|
-
this.#db.exec(`DROP TRIGGER IF EXISTS _syncular_blob_commit_${trigger}`);
|
|
197
|
-
if (this.#hasBlobs) {
|
|
198
|
-
ensureBlobSchema(this.#db);
|
|
199
|
-
const columns = JSON.stringify([...this.#schema.tables.values()].flatMap((table) => table.columns
|
|
200
|
-
.filter((column) => column.type === 'blob_ref')
|
|
201
|
-
.map((column) => [table.name, column.name]))).replaceAll("'", "''");
|
|
202
|
-
const references = `SELECT o.client_commit_id, b.blob_id
|
|
203
|
-
FROM _syncular_outbox o JOIN json_each(o.operations) op
|
|
204
|
-
JOIN json_each(op.value, '$.values') v JOIN json_each('${columns}') c
|
|
205
|
-
JOIN _syncular_blobs b ON b.blob_id = CASE WHEN v.type = 'text' AND json_valid(v.value) THEN json_extract(v.value, '$.blobId') END
|
|
206
|
-
WHERE json_extract(op.value, '$.op') = 'upsert'
|
|
207
|
-
AND json_extract(op.value, '$.table') = json_extract(c.value, '$[0]')
|
|
208
|
-
AND v.key = json_extract(c.value, '$[1]')`;
|
|
209
|
-
this.#db
|
|
210
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_insert AFTER INSERT ON _syncular_outbox BEGIN
|
|
211
|
-
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
|
|
212
|
-
this.#db
|
|
213
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_update AFTER UPDATE OF operations ON _syncular_outbox BEGIN
|
|
214
|
-
DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id;
|
|
215
|
-
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
|
|
216
|
-
this.#db
|
|
217
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_delete BEFORE DELETE ON _syncular_outbox BEGIN
|
|
218
|
-
DELETE FROM _syncular_blob_uploads WHERE blob_id IN (SELECT blob_id FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id)
|
|
219
|
-
AND NOT EXISTS (SELECT 1 FROM _syncular_blob_commit_refs r WHERE r.blob_id = _syncular_blob_uploads.blob_id AND r.commit_id != OLD.client_commit_id);
|
|
220
|
-
DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id; END`);
|
|
221
|
-
this.#db
|
|
222
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_cache AFTER INSERT ON _syncular_blobs BEGIN
|
|
223
|
-
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND b.blob_id = NEW.blob_id; END`);
|
|
224
|
-
this.#db.exec(`INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references}`);
|
|
225
|
-
}
|
|
226
|
-
});
|
|
194
|
+
if (this.#hasBlobs)
|
|
195
|
+
ensureBlobSchema(this.#db);
|
|
227
196
|
this.#db.transaction(() => {
|
|
228
197
|
pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
|
|
229
198
|
});
|
|
@@ -893,13 +862,11 @@ export class SyncClient {
|
|
|
893
862
|
throw new ClientSyncError('sync.invalid_request', 'uploadBlob requires a blob transport (SyncClientConfig.blobs, §5.9)');
|
|
894
863
|
}
|
|
895
864
|
const blobId = await computeBlobId(bytes);
|
|
865
|
+
const nowMs = this.#now();
|
|
896
866
|
this.#db.transaction(() => {
|
|
897
|
-
putCachedBlob(this.#db, blobId, bytes,
|
|
898
|
-
recordPendingUpload(this.#db, blobId,
|
|
867
|
+
putCachedBlob(this.#db, blobId, bytes, nowMs, options?.mediaType);
|
|
868
|
+
recordPendingUpload(this.#db, blobId, nowMs, options?.mediaType);
|
|
899
869
|
});
|
|
900
|
-
// §5.9.7 B1: a staged upload is pinned (recordPendingUpload), so the cap
|
|
901
|
-
// trim below will never evict it — but a stage may push other zero-ref
|
|
902
|
-
// bodies over the cap, so run the trim.
|
|
903
870
|
this.#enforceBlobCacheCap();
|
|
904
871
|
return {
|
|
905
872
|
blobId,
|
|
@@ -927,7 +894,7 @@ export class SyncClient {
|
|
|
927
894
|
const blobId = blobIdOrRef.startsWith('sha256:')
|
|
928
895
|
? blobIdOrRef
|
|
929
896
|
: parseBlobRef(blobIdOrRef).blobId;
|
|
930
|
-
const cached = getCachedBlob(this.#db, blobId
|
|
897
|
+
const cached = getCachedBlob(this.#db, blobId);
|
|
931
898
|
if (cached !== undefined)
|
|
932
899
|
return cached;
|
|
933
900
|
const transport = this.#config.blobs;
|
|
@@ -962,9 +929,7 @@ export class SyncClient {
|
|
|
962
929
|
throw new ClientSyncError('sync.invalid_request', `blob content address mismatch for ${blobId} (§5.9.5)`);
|
|
963
930
|
}
|
|
964
931
|
putCachedBlob(this.#db, blobId, bytes, this.#now());
|
|
965
|
-
|
|
966
|
-
// from current visible references before applying the size cap (§5.9.7 B1).
|
|
967
|
-
this.#reconcileBlobs(false, blobId);
|
|
932
|
+
this.#db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
968
933
|
this.#enforceBlobCacheCap();
|
|
969
934
|
const stored = getCachedBlob(this.#db, blobId);
|
|
970
935
|
if (stored === undefined) {
|
|
@@ -977,7 +942,7 @@ export class SyncClient {
|
|
|
977
942
|
const cap = this.#config.blobCacheMaxBytes;
|
|
978
943
|
if (cap === undefined)
|
|
979
944
|
return;
|
|
980
|
-
enforceBlobCacheCap(this.#db, cap);
|
|
945
|
+
enforceBlobCacheCap(this.#db, this.#schema, cap);
|
|
981
946
|
}
|
|
982
947
|
/** Flush any queued blob uploads (§5.9.7 B4); safe to call standalone. */
|
|
983
948
|
flushBlobUploads() {
|
|
@@ -1446,6 +1411,31 @@ export class SyncClient {
|
|
|
1446
1411
|
this.#applyBatch((batch) => {
|
|
1447
1412
|
this.#db.transaction(() => {
|
|
1448
1413
|
appendOutboxCommit(this.#db, clientCommitId, operations, this.#now(), this.#captureBeforeImages(operations));
|
|
1414
|
+
for (const operation of operations) {
|
|
1415
|
+
if (operation.op !== 'upsert')
|
|
1416
|
+
continue;
|
|
1417
|
+
const table = this.#schema.tables.get(operation.table);
|
|
1418
|
+
if (table === undefined)
|
|
1419
|
+
continue;
|
|
1420
|
+
for (const column of table.columns) {
|
|
1421
|
+
if (column.type !== 'blob_ref')
|
|
1422
|
+
continue;
|
|
1423
|
+
const raw = operation.values?.[column.name];
|
|
1424
|
+
if (typeof raw !== 'string')
|
|
1425
|
+
continue;
|
|
1426
|
+
let blobId;
|
|
1427
|
+
try {
|
|
1428
|
+
blobId = parseBlobRef(raw).blobId;
|
|
1429
|
+
}
|
|
1430
|
+
catch {
|
|
1431
|
+
continue;
|
|
1432
|
+
}
|
|
1433
|
+
this.#db.exec(`INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
|
|
1434
|
+
SELECT ?, ? WHERE EXISTS (
|
|
1435
|
+
SELECT 1 FROM _syncular_blobs WHERE blob_id = ?
|
|
1436
|
+
)`, [clientCommitId, blobId, blobId]);
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1449
1439
|
this.#applyOperationsLocally(operations, batch);
|
|
1450
1440
|
batch.status();
|
|
1451
1441
|
});
|
|
@@ -1604,7 +1594,7 @@ export class SyncClient {
|
|
|
1604
1594
|
batch.rejections();
|
|
1605
1595
|
batch.outcomes();
|
|
1606
1596
|
}
|
|
1607
|
-
this.#
|
|
1597
|
+
this.#deleteUnreferencedBlobs();
|
|
1608
1598
|
setMeta(this.#db, metaKey, purge.canonicalPlan);
|
|
1609
1599
|
return {
|
|
1610
1600
|
alreadyApplied: false,
|
|
@@ -2546,7 +2536,6 @@ export class SyncClient {
|
|
|
2546
2536
|
// §7.1: local reads see outbox state applied optimistically — replay
|
|
2547
2537
|
// the still-pending commits on top of the freshly applied server state.
|
|
2548
2538
|
this.#replayOutbox();
|
|
2549
|
-
this.#reconcileBlobs(false);
|
|
2550
2539
|
}
|
|
2551
2540
|
finally {
|
|
2552
2541
|
this.#endDiagnosticsDeferral();
|
|
@@ -2935,7 +2924,7 @@ export class SyncClient {
|
|
|
2935
2924
|
batch.rejections();
|
|
2936
2925
|
batch.outcomes();
|
|
2937
2926
|
}
|
|
2938
|
-
this.#
|
|
2927
|
+
this.#deleteUnreferencedBlobs();
|
|
2939
2928
|
}
|
|
2940
2929
|
catch (error) {
|
|
2941
2930
|
if (error instanceof ClientSyncError &&
|
|
@@ -3107,18 +3096,10 @@ export class SyncClient {
|
|
|
3107
3096
|
});
|
|
3108
3097
|
});
|
|
3109
3098
|
}
|
|
3110
|
-
|
|
3111
|
-
* §5.9.7 B1/B2: recompute blob-cache refcounts from live `blob_ref`
|
|
3112
|
-
* columns. No-op unless the schema has blob columns. `deleteOrphans`
|
|
3113
|
-
* triggers the revocation-side body deletion (B2).
|
|
3114
|
-
*/
|
|
3115
|
-
#reconcileBlobs(deleteOrphans, blobId) {
|
|
3099
|
+
#deleteUnreferencedBlobs() {
|
|
3116
3100
|
if (!this.#hasBlobs)
|
|
3117
3101
|
return;
|
|
3118
|
-
|
|
3119
|
-
deleteOrphans,
|
|
3120
|
-
...(blobId === undefined ? {} : { blobId }),
|
|
3121
|
-
});
|
|
3102
|
+
deleteUnreferencedCachedBlobs(this.#db, this.#schema);
|
|
3122
3103
|
}
|
|
3123
3104
|
// -- helpers -----------------------------------------------------------------
|
|
3124
3105
|
#table(name) {
|
package/dist/outbox.js
CHANGED
|
@@ -62,6 +62,9 @@ export function countOutbox(db) {
|
|
|
62
62
|
.count;
|
|
63
63
|
}
|
|
64
64
|
export function deleteOutboxCommit(db, clientCommitId) {
|
|
65
|
+
db.exec('DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?', [
|
|
66
|
+
clientCommitId,
|
|
67
|
+
]);
|
|
65
68
|
db.exec('DELETE FROM _syncular_outbox_before_images WHERE client_commit_id = ?', [clientCommitId]);
|
|
66
69
|
db.exec('DELETE FROM _syncular_outbox WHERE client_commit_id = ?', [
|
|
67
70
|
clientCommitId,
|
package/dist/schema.js
CHANGED
|
@@ -339,6 +339,9 @@ export function ensureLocalBookkeepingSchema(db) {
|
|
|
339
339
|
db.transaction(() => {
|
|
340
340
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
|
|
341
341
|
key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
|
|
342
|
+
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
|
|
343
|
+
commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
|
|
344
|
+
db.exec('CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)');
|
|
342
345
|
db.exec(`INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`);
|
|
343
346
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_outbox(
|
|
344
347
|
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -89,9 +89,9 @@
|
|
|
89
89
|
},
|
|
90
90
|
"dependencies": {
|
|
91
91
|
"@sqlite.org/sqlite-wasm": "^3.53.0-build1",
|
|
92
|
-
"@syncular/core": "0.
|
|
92
|
+
"@syncular/core": "0.19.0"
|
|
93
93
|
},
|
|
94
94
|
"devDependencies": {
|
|
95
|
-
"@syncular/server": "0.
|
|
95
|
+
"@syncular/server": "0.19.0"
|
|
96
96
|
}
|
|
97
97
|
}
|
package/src/blob.ts
CHANGED
|
@@ -1,14 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Client-side blob cache + transport (SPEC.md §5.9.7).
|
|
3
3
|
*
|
|
4
|
-
* Blob bytes are cached
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* reconciled from the current `blob_ref` column contents, and a body whose
|
|
8
|
-
* only referencing rows were revocation-purged is deleted (B2, evicted ≠
|
|
9
|
-
* revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
|
|
10
|
-
* row value is the whole download key. Pending uploads are tracked in the
|
|
11
|
-
* outbox-adjacent uploads table (B4) and flushed before push.
|
|
4
|
+
* Blob bytes are cached by `blobId`. Retention reads live references directly
|
|
5
|
+
* from `blob_ref` columns and keeps explicit pending-commit dependencies.
|
|
6
|
+
* A small upload row queues each staged body for transfer before push.
|
|
12
7
|
*/
|
|
13
8
|
import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
|
|
14
9
|
import type { ClientDatabase } from './database';
|
|
@@ -109,36 +104,43 @@ export async function computeBlobId(bytes: Uint8Array): Promise<string> {
|
|
|
109
104
|
}
|
|
110
105
|
|
|
111
106
|
export function ensureBlobSchema(db: ClientDatabase): void {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
'
|
|
126
|
-
);
|
|
127
|
-
|
|
128
|
-
|
|
107
|
+
const existing = db.query(
|
|
108
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs'",
|
|
109
|
+
);
|
|
110
|
+
if (existing.length > 0) {
|
|
111
|
+
const columns = db
|
|
112
|
+
.query('PRAGMA table_info(_syncular_blobs)')
|
|
113
|
+
.map((row) => row.name)
|
|
114
|
+
.sort();
|
|
115
|
+
const expected = [
|
|
116
|
+
'blob_id',
|
|
117
|
+
'byte_length',
|
|
118
|
+
'bytes',
|
|
119
|
+
'created_at_ms',
|
|
120
|
+
'media_type',
|
|
121
|
+
].sort();
|
|
122
|
+
if (
|
|
123
|
+
columns.length !== expected.length ||
|
|
124
|
+
columns.some((column, index) => column !== expected[index])
|
|
125
|
+
) {
|
|
126
|
+
throw new ClientSyncError(
|
|
127
|
+
'sync.schema_mismatch',
|
|
128
|
+
'Local blob schema is incompatible',
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
} else {
|
|
132
|
+
db.exec(`CREATE TABLE _syncular_blobs(
|
|
133
|
+
blob_id TEXT PRIMARY KEY,
|
|
134
|
+
bytes BLOB NOT NULL,
|
|
135
|
+
byte_length INTEGER NOT NULL,
|
|
136
|
+
media_type TEXT,
|
|
137
|
+
created_at_ms INTEGER NOT NULL)`);
|
|
129
138
|
}
|
|
130
139
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
|
|
131
|
-
blob_id TEXT PRIMARY KEY,
|
|
132
|
-
media_type TEXT,
|
|
133
|
-
created_at_ms INTEGER NOT NULL)`);
|
|
134
|
-
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
|
|
135
|
-
commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
|
|
136
|
-
db.exec(
|
|
137
|
-
'CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)',
|
|
138
|
-
);
|
|
140
|
+
blob_id TEXT PRIMARY KEY, media_type TEXT, created_at_ms INTEGER NOT NULL)`);
|
|
139
141
|
}
|
|
140
142
|
|
|
141
|
-
/** Put bytes into the content-addressed cache
|
|
143
|
+
/** Put bytes into the content-addressed cache without rewriting an existing body. */
|
|
142
144
|
export function putCachedBlob(
|
|
143
145
|
db: ClientDatabase,
|
|
144
146
|
blobId: string,
|
|
@@ -148,17 +150,15 @@ export function putCachedBlob(
|
|
|
148
150
|
): void {
|
|
149
151
|
db.exec(
|
|
150
152
|
`INSERT INTO _syncular_blobs(
|
|
151
|
-
blob_id, bytes, byte_length, media_type,
|
|
152
|
-
VALUES (
|
|
153
|
-
|
|
154
|
-
[blobId, bytes, bytes.length, mediaType ?? null, nowMs, nowMs],
|
|
153
|
+
blob_id, bytes, byte_length, media_type, created_at_ms)
|
|
154
|
+
VALUES (?,?,?,?,?) ON CONFLICT(blob_id) DO NOTHING`,
|
|
155
|
+
[blobId, bytes, bytes.length, mediaType ?? null, nowMs],
|
|
155
156
|
);
|
|
156
157
|
}
|
|
157
158
|
|
|
158
159
|
export function getCachedBlob(
|
|
159
160
|
db: ClientDatabase,
|
|
160
161
|
blobId: string,
|
|
161
|
-
nowMs?: number,
|
|
162
162
|
): CachedBlob | undefined {
|
|
163
163
|
const rows = db.query(
|
|
164
164
|
'SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?',
|
|
@@ -166,14 +166,6 @@ export function getCachedBlob(
|
|
|
166
166
|
);
|
|
167
167
|
const row = rows[0];
|
|
168
168
|
if (row === undefined) return undefined;
|
|
169
|
-
// §5.9.7 B1 LRU: a cache-hit read touches "recently used" so a hot image
|
|
170
|
-
// survives a cap trim. Skipped when no clock is supplied (pure read).
|
|
171
|
-
if (nowMs !== undefined) {
|
|
172
|
-
db.exec('UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?', [
|
|
173
|
-
nowMs,
|
|
174
|
-
blobId,
|
|
175
|
-
]);
|
|
176
|
-
}
|
|
177
169
|
return {
|
|
178
170
|
blobId,
|
|
179
171
|
bytes: row.bytes as Uint8Array,
|
|
@@ -183,17 +175,13 @@ export function getCachedBlob(
|
|
|
183
175
|
}
|
|
184
176
|
|
|
185
177
|
/**
|
|
186
|
-
* §5.9.7 B1 size cap
|
|
187
|
-
*
|
|
188
|
-
*
|
|
189
|
-
* — it must stay resolvable without a re-download) nor a pending-upload-pinned
|
|
190
|
-
* body (its bytes are the only copy until push, B4). If every over-cap body is
|
|
191
|
-
* referenced or pinned, the cache stays over the cap (correctness beats the
|
|
192
|
-
* cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
|
|
193
|
-
* any surviving `blob_ref` value. Returns the evicted blobIds.
|
|
178
|
+
* §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
|
|
179
|
+
* evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
|
|
180
|
+
* until back under the cap. Returns the evicted blobIds.
|
|
194
181
|
*/
|
|
195
182
|
export function enforceBlobCacheCap(
|
|
196
183
|
db: ClientDatabase,
|
|
184
|
+
schema: CompiledClientSchema,
|
|
197
185
|
maxBytes: number,
|
|
198
186
|
): string[] {
|
|
199
187
|
const totalRow = db.query(
|
|
@@ -201,14 +189,13 @@ export function enforceBlobCacheCap(
|
|
|
201
189
|
)[0];
|
|
202
190
|
let total = Number(totalRow?.total ?? 0);
|
|
203
191
|
if (total <= maxBytes) return [];
|
|
204
|
-
//
|
|
205
|
-
// (LRU) first, then oldest created as a stable tiebreak.
|
|
192
|
+
// Oldest eligible body first, with blob ID as a stable tie.
|
|
206
193
|
const candidates = db.query(
|
|
207
194
|
`SELECT blob_id, byte_length FROM _syncular_blobs
|
|
208
|
-
WHERE
|
|
209
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
195
|
+
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
210
196
|
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
211
|
-
|
|
197
|
+
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})
|
|
198
|
+
ORDER BY created_at_ms ASC, blob_id ASC`,
|
|
212
199
|
);
|
|
213
200
|
const evicted: string[] = [];
|
|
214
201
|
db.transaction(() => {
|
|
@@ -223,20 +210,6 @@ export function enforceBlobCacheCap(
|
|
|
223
210
|
return evicted;
|
|
224
211
|
}
|
|
225
212
|
|
|
226
|
-
/** Record a pending upload (§5.9.7 B4); flushed before the next push. */
|
|
227
|
-
export function recordPendingUpload(
|
|
228
|
-
db: ClientDatabase,
|
|
229
|
-
blobId: string,
|
|
230
|
-
nowMs: number,
|
|
231
|
-
mediaType?: string,
|
|
232
|
-
): void {
|
|
233
|
-
db.exec(
|
|
234
|
-
`INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms)
|
|
235
|
-
VALUES (?,?,?)`,
|
|
236
|
-
[blobId, mediaType ?? null, nowMs],
|
|
237
|
-
);
|
|
238
|
-
}
|
|
239
|
-
|
|
240
213
|
export function listPendingUploads(
|
|
241
214
|
db: ClientDatabase,
|
|
242
215
|
): { blobId: string; mediaType?: string }[] {
|
|
@@ -267,13 +240,25 @@ export function listPendingUploads(
|
|
|
267
240
|
});
|
|
268
241
|
}
|
|
269
242
|
|
|
243
|
+
export function recordPendingUpload(
|
|
244
|
+
db: ClientDatabase,
|
|
245
|
+
blobId: string,
|
|
246
|
+
nowMs: number,
|
|
247
|
+
mediaType?: string,
|
|
248
|
+
): void {
|
|
249
|
+
db.exec(
|
|
250
|
+
`INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms)
|
|
251
|
+
VALUES (?,?,?)`,
|
|
252
|
+
[blobId, mediaType ?? null, nowMs],
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
|
|
270
256
|
export function clearPendingUpload(db: ClientDatabase, blobId: string): void {
|
|
271
257
|
db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id = ?', [blobId]);
|
|
272
258
|
}
|
|
273
259
|
|
|
274
260
|
/**
|
|
275
|
-
* All `blob_ref` column names per table
|
|
276
|
-
* Blank result ⇒ the schema has no attachments; callers skip reconciliation.
|
|
261
|
+
* All `blob_ref` column names per table.
|
|
277
262
|
*/
|
|
278
263
|
export function blobRefColumnsBySchema(
|
|
279
264
|
schema: CompiledClientSchema,
|
|
@@ -295,91 +280,35 @@ export function schemaHasBlobs(schema: CompiledClientSchema): boolean {
|
|
|
295
280
|
return false;
|
|
296
281
|
}
|
|
297
282
|
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
* to zero refs *and* have no pending upload (a pending upload pins its body,
|
|
302
|
-
* B4). Called after every apply/purge that may add or remove references.
|
|
303
|
-
*
|
|
304
|
-
* `blobId` narrows a fresh-download refresh to the inserted cache row.
|
|
305
|
-
* `deleteOrphans` distinguishes the two B2 transitions: revocation purge
|
|
306
|
-
* passes `true` (drop the now-unauthorized body); a benign apply passes
|
|
307
|
-
* `false` (retain zero-ref bodies as LRU cache entries — the shipped
|
|
308
|
-
* default). Bodies pinned by a pending upload are always retained.
|
|
309
|
-
*/
|
|
310
|
-
export function reconcileBlobRefcounts(
|
|
311
|
-
db: ClientDatabase,
|
|
312
|
-
schema: CompiledClientSchema,
|
|
313
|
-
options?: {
|
|
314
|
-
readonly blobId?: string;
|
|
315
|
-
readonly deleteOrphans?: boolean;
|
|
316
|
-
},
|
|
317
|
-
): void {
|
|
318
|
-
const byTable = blobRefColumnsBySchema(schema);
|
|
319
|
-
if (byTable.size === 0) return;
|
|
320
|
-
if (options?.blobId !== undefined) {
|
|
321
|
-
let count = 0;
|
|
322
|
-
for (const [tableName, columns] of byTable) {
|
|
323
|
-
for (const column of columns) {
|
|
324
|
-
const identifier = quoteIdent(column);
|
|
325
|
-
const row = db.query(
|
|
326
|
-
`SELECT count(*) AS n FROM ${quoteIdent(tableName)}
|
|
327
|
-
WHERE CASE
|
|
328
|
-
WHEN typeof(${identifier}) = 'text' AND json_valid(${identifier})
|
|
329
|
-
THEN json_extract(${identifier}, '$.blobId')
|
|
330
|
-
END = ?`,
|
|
331
|
-
[options.blobId],
|
|
332
|
-
)[0];
|
|
333
|
-
count += Number(row?.n ?? 0);
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
|
-
db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
|
|
337
|
-
count,
|
|
338
|
-
options.blobId,
|
|
339
|
-
]);
|
|
340
|
-
return;
|
|
341
|
-
}
|
|
342
|
-
// Count references to each blobId across every blob_ref column.
|
|
343
|
-
const counts = new Map<string, number>();
|
|
344
|
-
for (const [tableName, columns] of byTable) {
|
|
283
|
+
function visibleBlobIdsSql(schema: CompiledClientSchema): string {
|
|
284
|
+
const selects: string[] = [];
|
|
285
|
+
for (const [tableName, columns] of blobRefColumnsBySchema(schema)) {
|
|
345
286
|
for (const column of columns) {
|
|
346
|
-
const
|
|
347
|
-
|
|
348
|
-
|
|
287
|
+
const identifier = quoteIdent(column);
|
|
288
|
+
selects.push(
|
|
289
|
+
`SELECT json_extract(${identifier}, '$.blobId') AS blob_id
|
|
290
|
+
FROM ${quoteIdent(tableName)}
|
|
291
|
+
WHERE typeof(${identifier}) = 'text'
|
|
292
|
+
AND json_valid(${identifier})
|
|
293
|
+
AND json_type(${identifier}, '$.blobId') = 'text'`,
|
|
349
294
|
);
|
|
350
|
-
for (const row of rows) {
|
|
351
|
-
const raw = row.v;
|
|
352
|
-
if (typeof raw !== 'string') continue;
|
|
353
|
-
let ref: BlobRef;
|
|
354
|
-
try {
|
|
355
|
-
ref = parseBlobRef(raw);
|
|
356
|
-
} catch {
|
|
357
|
-
continue;
|
|
358
|
-
}
|
|
359
|
-
counts.set(ref.blobId, (counts.get(ref.blobId) ?? 0) + 1);
|
|
360
|
-
}
|
|
361
295
|
}
|
|
362
296
|
}
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
379
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)`,
|
|
380
|
-
);
|
|
381
|
-
}
|
|
382
|
-
});
|
|
297
|
+
return selects.length === 0
|
|
298
|
+
? 'SELECT NULL AS blob_id WHERE 0'
|
|
299
|
+
: selects.join(' UNION ');
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export function deleteUnreferencedCachedBlobs(
|
|
303
|
+
db: ClientDatabase,
|
|
304
|
+
schema: CompiledClientSchema,
|
|
305
|
+
): void {
|
|
306
|
+
db.exec(
|
|
307
|
+
`DELETE FROM _syncular_blobs
|
|
308
|
+
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
309
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
310
|
+
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})`,
|
|
311
|
+
);
|
|
383
312
|
}
|
|
384
313
|
|
|
385
314
|
export type { BlobRef };
|
package/src/client.ts
CHANGED
|
@@ -47,13 +47,13 @@ import {
|
|
|
47
47
|
type CachedBlob,
|
|
48
48
|
clearPendingUpload,
|
|
49
49
|
computeBlobId,
|
|
50
|
+
deleteUnreferencedCachedBlobs,
|
|
50
51
|
enforceBlobCacheCap,
|
|
51
52
|
ensureBlobSchema,
|
|
52
53
|
getCachedBlob,
|
|
53
54
|
listPendingUploads,
|
|
54
55
|
parseBlobRef,
|
|
55
56
|
putCachedBlob,
|
|
56
|
-
reconcileBlobRefcounts,
|
|
57
57
|
recordPendingUpload,
|
|
58
58
|
schemaHasBlobs,
|
|
59
59
|
serializeBlobRef,
|
|
@@ -302,9 +302,9 @@ export interface SyncClientConfig {
|
|
|
302
302
|
readonly blobs?: BlobTransport;
|
|
303
303
|
/**
|
|
304
304
|
* §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
|
|
305
|
-
* exceeds this,
|
|
306
|
-
* cache write
|
|
307
|
-
*
|
|
305
|
+
* exceeds this, unreferenced bodies without pending work are evicted in
|
|
306
|
+
* creation timestamp order after each cache write. Referenced bodies remain cached
|
|
307
|
+
* even when that exceeds the cap. Absent means retain until storage pressure (the shipped
|
|
308
308
|
* default; a referenced body always stays resolvable without a re-download).
|
|
309
309
|
*/
|
|
310
310
|
readonly blobCacheMaxBytes?: number;
|
|
@@ -688,47 +688,7 @@ export class SyncClient {
|
|
|
688
688
|
// Do not materialize new app indexes/FTS projections yet: on a version
|
|
689
689
|
// bump they may reference columns that only exist after the reset.
|
|
690
690
|
ensureLocalBookkeepingSchema(this.#db);
|
|
691
|
-
this.#db
|
|
692
|
-
for (const trigger of ['insert', 'update', 'delete', 'cache'])
|
|
693
|
-
this.#db.exec(
|
|
694
|
-
`DROP TRIGGER IF EXISTS _syncular_blob_commit_${trigger}`,
|
|
695
|
-
);
|
|
696
|
-
if (this.#hasBlobs) {
|
|
697
|
-
ensureBlobSchema(this.#db);
|
|
698
|
-
const columns = JSON.stringify(
|
|
699
|
-
[...this.#schema.tables.values()].flatMap((table) =>
|
|
700
|
-
table.columns
|
|
701
|
-
.filter((column) => column.type === 'blob_ref')
|
|
702
|
-
.map((column) => [table.name, column.name]),
|
|
703
|
-
),
|
|
704
|
-
).replaceAll("'", "''");
|
|
705
|
-
const references = `SELECT o.client_commit_id, b.blob_id
|
|
706
|
-
FROM _syncular_outbox o JOIN json_each(o.operations) op
|
|
707
|
-
JOIN json_each(op.value, '$.values') v JOIN json_each('${columns}') c
|
|
708
|
-
JOIN _syncular_blobs b ON b.blob_id = CASE WHEN v.type = 'text' AND json_valid(v.value) THEN json_extract(v.value, '$.blobId') END
|
|
709
|
-
WHERE json_extract(op.value, '$.op') = 'upsert'
|
|
710
|
-
AND json_extract(op.value, '$.table') = json_extract(c.value, '$[0]')
|
|
711
|
-
AND v.key = json_extract(c.value, '$[1]')`;
|
|
712
|
-
this.#db
|
|
713
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_insert AFTER INSERT ON _syncular_outbox BEGIN
|
|
714
|
-
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
|
|
715
|
-
this.#db
|
|
716
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_update AFTER UPDATE OF operations ON _syncular_outbox BEGIN
|
|
717
|
-
DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id;
|
|
718
|
-
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
|
|
719
|
-
this.#db
|
|
720
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_delete BEFORE DELETE ON _syncular_outbox BEGIN
|
|
721
|
-
DELETE FROM _syncular_blob_uploads WHERE blob_id IN (SELECT blob_id FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id)
|
|
722
|
-
AND NOT EXISTS (SELECT 1 FROM _syncular_blob_commit_refs r WHERE r.blob_id = _syncular_blob_uploads.blob_id AND r.commit_id != OLD.client_commit_id);
|
|
723
|
-
DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id; END`);
|
|
724
|
-
this.#db
|
|
725
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_cache AFTER INSERT ON _syncular_blobs BEGIN
|
|
726
|
-
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND b.blob_id = NEW.blob_id; END`);
|
|
727
|
-
this.#db.exec(
|
|
728
|
-
`INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references}`,
|
|
729
|
-
);
|
|
730
|
-
}
|
|
731
|
-
});
|
|
691
|
+
if (this.#hasBlobs) ensureBlobSchema(this.#db);
|
|
732
692
|
this.#db.transaction(() => {
|
|
733
693
|
pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
|
|
734
694
|
});
|
|
@@ -1510,13 +1470,11 @@ export class SyncClient {
|
|
|
1510
1470
|
);
|
|
1511
1471
|
}
|
|
1512
1472
|
const blobId = await computeBlobId(bytes);
|
|
1473
|
+
const nowMs = this.#now();
|
|
1513
1474
|
this.#db.transaction(() => {
|
|
1514
|
-
putCachedBlob(this.#db, blobId, bytes,
|
|
1515
|
-
recordPendingUpload(this.#db, blobId,
|
|
1475
|
+
putCachedBlob(this.#db, blobId, bytes, nowMs, options?.mediaType);
|
|
1476
|
+
recordPendingUpload(this.#db, blobId, nowMs, options?.mediaType);
|
|
1516
1477
|
});
|
|
1517
|
-
// §5.9.7 B1: a staged upload is pinned (recordPendingUpload), so the cap
|
|
1518
|
-
// trim below will never evict it — but a stage may push other zero-ref
|
|
1519
|
-
// bodies over the cap, so run the trim.
|
|
1520
1478
|
this.#enforceBlobCacheCap();
|
|
1521
1479
|
return {
|
|
1522
1480
|
blobId,
|
|
@@ -1547,7 +1505,7 @@ export class SyncClient {
|
|
|
1547
1505
|
const blobId = blobIdOrRef.startsWith('sha256:')
|
|
1548
1506
|
? blobIdOrRef
|
|
1549
1507
|
: parseBlobRef(blobIdOrRef).blobId;
|
|
1550
|
-
const cached = getCachedBlob(this.#db, blobId
|
|
1508
|
+
const cached = getCachedBlob(this.#db, blobId);
|
|
1551
1509
|
if (cached !== undefined) return cached;
|
|
1552
1510
|
const transport = this.#config.blobs;
|
|
1553
1511
|
if (transport === undefined) {
|
|
@@ -1595,9 +1553,7 @@ export class SyncClient {
|
|
|
1595
1553
|
);
|
|
1596
1554
|
}
|
|
1597
1555
|
putCachedBlob(this.#db, blobId, bytes, this.#now());
|
|
1598
|
-
|
|
1599
|
-
// from current visible references before applying the size cap (§5.9.7 B1).
|
|
1600
|
-
this.#reconcileBlobs(false, blobId);
|
|
1556
|
+
this.#db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
1601
1557
|
this.#enforceBlobCacheCap();
|
|
1602
1558
|
const stored = getCachedBlob(this.#db, blobId);
|
|
1603
1559
|
if (stored === undefined) {
|
|
@@ -1613,7 +1569,7 @@ export class SyncClient {
|
|
|
1613
1569
|
#enforceBlobCacheCap(): void {
|
|
1614
1570
|
const cap = this.#config.blobCacheMaxBytes;
|
|
1615
1571
|
if (cap === undefined) return;
|
|
1616
|
-
enforceBlobCacheCap(this.#db, cap);
|
|
1572
|
+
enforceBlobCacheCap(this.#db, this.#schema, cap);
|
|
1617
1573
|
}
|
|
1618
1574
|
|
|
1619
1575
|
/** Flush any queued blob uploads (§5.9.7 B4); safe to call standalone. */
|
|
@@ -2188,6 +2144,29 @@ export class SyncClient {
|
|
|
2188
2144
|
this.#now(),
|
|
2189
2145
|
this.#captureBeforeImages(operations),
|
|
2190
2146
|
);
|
|
2147
|
+
for (const operation of operations) {
|
|
2148
|
+
if (operation.op !== 'upsert') continue;
|
|
2149
|
+
const table = this.#schema.tables.get(operation.table);
|
|
2150
|
+
if (table === undefined) continue;
|
|
2151
|
+
for (const column of table.columns) {
|
|
2152
|
+
if (column.type !== 'blob_ref') continue;
|
|
2153
|
+
const raw = operation.values?.[column.name];
|
|
2154
|
+
if (typeof raw !== 'string') continue;
|
|
2155
|
+
let blobId: string;
|
|
2156
|
+
try {
|
|
2157
|
+
blobId = parseBlobRef(raw).blobId;
|
|
2158
|
+
} catch {
|
|
2159
|
+
continue;
|
|
2160
|
+
}
|
|
2161
|
+
this.#db.exec(
|
|
2162
|
+
`INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
|
|
2163
|
+
SELECT ?, ? WHERE EXISTS (
|
|
2164
|
+
SELECT 1 FROM _syncular_blobs WHERE blob_id = ?
|
|
2165
|
+
)`,
|
|
2166
|
+
[clientCommitId, blobId, blobId],
|
|
2167
|
+
);
|
|
2168
|
+
}
|
|
2169
|
+
}
|
|
2191
2170
|
this.#applyOperationsLocally(operations, batch);
|
|
2192
2171
|
batch.status();
|
|
2193
2172
|
});
|
|
@@ -2386,7 +2365,7 @@ export class SyncClient {
|
|
|
2386
2365
|
batch.rejections();
|
|
2387
2366
|
batch.outcomes();
|
|
2388
2367
|
}
|
|
2389
|
-
this.#
|
|
2368
|
+
this.#deleteUnreferencedBlobs();
|
|
2390
2369
|
setMeta(this.#db, metaKey, purge.canonicalPlan);
|
|
2391
2370
|
return {
|
|
2392
2371
|
alreadyApplied: false,
|
|
@@ -3564,7 +3543,6 @@ export class SyncClient {
|
|
|
3564
3543
|
// §7.1: local reads see outbox state applied optimistically — replay
|
|
3565
3544
|
// the still-pending commits on top of the freshly applied server state.
|
|
3566
3545
|
this.#replayOutbox();
|
|
3567
|
-
this.#reconcileBlobs(false);
|
|
3568
3546
|
} finally {
|
|
3569
3547
|
this.#endDiagnosticsDeferral();
|
|
3570
3548
|
}
|
|
@@ -4047,7 +4025,7 @@ export class SyncClient {
|
|
|
4047
4025
|
batch.rejections();
|
|
4048
4026
|
batch.outcomes();
|
|
4049
4027
|
}
|
|
4050
|
-
this.#
|
|
4028
|
+
this.#deleteUnreferencedBlobs();
|
|
4051
4029
|
} catch (error) {
|
|
4052
4030
|
if (
|
|
4053
4031
|
error instanceof ClientSyncError &&
|
|
@@ -4265,17 +4243,9 @@ export class SyncClient {
|
|
|
4265
4243
|
});
|
|
4266
4244
|
}
|
|
4267
4245
|
|
|
4268
|
-
|
|
4269
|
-
* §5.9.7 B1/B2: recompute blob-cache refcounts from live `blob_ref`
|
|
4270
|
-
* columns. No-op unless the schema has blob columns. `deleteOrphans`
|
|
4271
|
-
* triggers the revocation-side body deletion (B2).
|
|
4272
|
-
*/
|
|
4273
|
-
#reconcileBlobs(deleteOrphans: boolean, blobId?: string): void {
|
|
4246
|
+
#deleteUnreferencedBlobs(): void {
|
|
4274
4247
|
if (!this.#hasBlobs) return;
|
|
4275
|
-
|
|
4276
|
-
deleteOrphans,
|
|
4277
|
-
...(blobId === undefined ? {} : { blobId }),
|
|
4278
|
-
});
|
|
4248
|
+
deleteUnreferencedCachedBlobs(this.#db, this.#schema);
|
|
4279
4249
|
}
|
|
4280
4250
|
|
|
4281
4251
|
// -- helpers -----------------------------------------------------------------
|
package/src/outbox.ts
CHANGED
|
@@ -135,6 +135,9 @@ export function deleteOutboxCommit(
|
|
|
135
135
|
db: ClientDatabase,
|
|
136
136
|
clientCommitId: string,
|
|
137
137
|
): void {
|
|
138
|
+
db.exec('DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?', [
|
|
139
|
+
clientCommitId,
|
|
140
|
+
]);
|
|
138
141
|
db.exec(
|
|
139
142
|
'DELETE FROM _syncular_outbox_before_images WHERE client_commit_id = ?',
|
|
140
143
|
[clientCommitId],
|
package/src/schema.ts
CHANGED
|
@@ -501,6 +501,11 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
|
|
|
501
501
|
db.transaction(() => {
|
|
502
502
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
|
|
503
503
|
key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
|
|
504
|
+
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
|
|
505
|
+
commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
|
|
506
|
+
db.exec(
|
|
507
|
+
'CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)',
|
|
508
|
+
);
|
|
504
509
|
db.exec(
|
|
505
510
|
`INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`,
|
|
506
511
|
);
|