@syncular/client 0.17.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -13
- package/dist/blob.d.ts +12 -37
- package/dist/blob.js +85 -117
- package/dist/client.d.ts +3 -3
- package/dist/client.js +44 -27
- package/dist/outbox.js +3 -0
- package/dist/schema.js +3 -0
- package/package.json +3 -3
- package/src/blob.ts +110 -128
- package/src/client.ts +50 -30
- package/src/outbox.ts +3 -0
- package/src/schema.ts +5 -0
package/README.md
CHANGED
|
@@ -375,7 +375,7 @@ lifecycle with an `EncryptionConfig`; Worker handles use the portable keyring.
|
|
|
375
375
|
Within one local SQLite transaction the engine deletes exactly the matching
|
|
376
376
|
synced rows, lets generated FTS triggers remove their projections, drops every
|
|
377
377
|
whole pending commit with a matching operation, restores/replays unrelated
|
|
378
|
-
optimistic state,
|
|
378
|
+
optimistic state, removes unreferenced cached blob bodies, persists the `purgeId`, and emits
|
|
379
379
|
one revisioned change batch. A retry with the same canonical plan returns
|
|
380
380
|
`alreadyApplied: true`; reusing an id with different selectors fails closed.
|
|
381
381
|
Only bounded, non-empty, code-like values on plaintext string schema columns
|
|
@@ -484,9 +484,8 @@ and are cached locally. **Blob bytes live as `BLOB` columns in the client's own
|
|
|
484
484
|
SQLite database** (a `_syncular_blobs` cache table), not in a separate OPFS
|
|
485
485
|
directory or IndexedDB store. This is the pinned decision (SPEC §5.9.7 B1):
|
|
486
486
|
|
|
487
|
-
- **One storage system.** The bytes
|
|
488
|
-
|
|
489
|
-
so a crash never strands a body against a stale count.
|
|
487
|
+
- **One storage system.** The bytes, upload state, and pending-commit references
|
|
488
|
+
share the client database. SQLite commits each change atomically.
|
|
490
489
|
- **Survives restarts for free.** The client DB already rides OPFS via the
|
|
491
490
|
sahpool VFS in the browser (and a plain file under `rusqlite`, `bun:sqlite`,
|
|
492
491
|
or `node:sqlite` on native runtimes), so there is no second persistence
|
|
@@ -495,25 +494,24 @@ directory or IndexedDB store. This is the pinned decision (SPEC §5.9.7 B1):
|
|
|
495
494
|
- **SQLite handles multi-MB images fine.** A page-cached `BLOB` read is a memory
|
|
496
495
|
copy, well within the image/document envelope this targets.
|
|
497
496
|
|
|
498
|
-
### Size cap
|
|
497
|
+
### Size cap
|
|
499
498
|
|
|
500
499
|
Pass `blobCacheMaxBytes` to cap the on-device cache. When the sum of cached body
|
|
501
|
-
sizes exceeds the cap, the client evicts
|
|
502
|
-
|
|
500
|
+
sizes exceeds the cap, the client evicts bodies with no visible row or pending
|
|
501
|
+
commit by creation timestamp, then blob ID, until back under the cap:
|
|
503
502
|
|
|
504
503
|
```ts
|
|
505
504
|
new SyncClient({ /* … */, blobCacheMaxBytes: 256 * 1024 * 1024 }); // 256 MiB
|
|
506
505
|
```
|
|
507
506
|
|
|
508
|
-
- A body **referenced by a live row**
|
|
507
|
+
- A body **referenced by a live row** is **never** evicted. It
|
|
509
508
|
stays resolvable without a re-download.
|
|
510
|
-
- A body **
|
|
509
|
+
- A body **needed by a pending upload** (not yet pushed) is never evicted. Its
|
|
511
510
|
bytes are the only copy until the commit drains.
|
|
512
|
-
- Evicting
|
|
511
|
+
- Evicting an unreferenced body only costs a future re-download, never correctness:
|
|
513
512
|
any surviving `blob_ref` value re-enables the fetch (§5.9.7 B3). If every
|
|
514
513
|
over-cap body is referenced or pinned, the cache stays over the cap
|
|
515
|
-
(correctness beats the cap).
|
|
516
|
-
hot image survives a trim. Absent `blobCacheMaxBytes` ⇒ retain until storage
|
|
514
|
+
(correctness beats the cap). Absent `blobCacheMaxBytes` ⇒ retain until storage
|
|
517
515
|
pressure (the default).
|
|
518
516
|
|
|
519
517
|
### Very large media — the escape hatch
|
|
@@ -522,7 +520,7 @@ SQLite is **not** the store for gigabyte video: a single `BLOB` must fit the
|
|
|
522
520
|
client's memory and the SQLite row-size envelope. For very large media, run the
|
|
523
521
|
server with presigned downloads (`blobSignedUrls`) and hand the presigned URL
|
|
524
522
|
straight to a media element instead of pulling bytes through the cache — the
|
|
525
|
-
image-app default (
|
|
523
|
+
image-app default (`BLOB` cache) and the large-media path (presigned
|
|
526
524
|
URL, no byte cache) coexist per attachment.
|
|
527
525
|
|
|
528
526
|
## Node and Bun SQLite backend (`./sqlite`)
|
package/dist/blob.d.ts
CHANGED
|
@@ -1,14 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Client-side blob cache + transport (SPEC.md §5.9.7).
|
|
3
3
|
*
|
|
4
|
-
* Blob bytes are cached
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* reconciled from the current `blob_ref` column contents, and a body whose
|
|
8
|
-
* only referencing rows were revocation-purged is deleted (B2, evicted ≠
|
|
9
|
-
* revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
|
|
10
|
-
* row value is the whole download key. Pending uploads are tracked in the
|
|
11
|
-
* outbox-adjacent uploads table (B4) and flushed before push.
|
|
4
|
+
* Blob bytes are cached by `blobId`. Retention reads live references directly
|
|
5
|
+
* from `blob_ref` columns and keeps explicit pending-commit dependencies.
|
|
6
|
+
* A small upload row queues each staged body for transfer before push.
|
|
12
7
|
*/
|
|
13
8
|
import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
|
|
14
9
|
import type { ClientDatabase } from './database.js';
|
|
@@ -82,46 +77,26 @@ export interface CachedBlob {
|
|
|
82
77
|
/** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
|
|
83
78
|
export declare function computeBlobId(bytes: Uint8Array): Promise<string>;
|
|
84
79
|
export declare function ensureBlobSchema(db: ClientDatabase): void;
|
|
85
|
-
/** Put bytes into the content-addressed cache
|
|
80
|
+
/** Put bytes into the content-addressed cache without rewriting an existing body. */
|
|
86
81
|
export declare function putCachedBlob(db: ClientDatabase, blobId: string, bytes: Uint8Array, nowMs: number, mediaType?: string): void;
|
|
87
|
-
export declare function getCachedBlob(db: ClientDatabase, blobId: string
|
|
82
|
+
export declare function getCachedBlob(db: ClientDatabase, blobId: string): CachedBlob | undefined;
|
|
88
83
|
/**
|
|
89
|
-
* §5.9.7 B1 size cap
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
* — it must stay resolvable without a re-download) nor a pending-upload-pinned
|
|
93
|
-
* body (its bytes are the only copy until push, B4). If every over-cap body is
|
|
94
|
-
* referenced or pinned, the cache stays over the cap (correctness beats the
|
|
95
|
-
* cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
|
|
96
|
-
* any surviving `blob_ref` value. Returns the evicted blobIds.
|
|
84
|
+
* §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
|
|
85
|
+
* evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
|
|
86
|
+
* until back under the cap. Returns the evicted blobIds.
|
|
97
87
|
*/
|
|
98
|
-
export declare function enforceBlobCacheCap(db: ClientDatabase, maxBytes: number): string[];
|
|
99
|
-
/** Record a pending upload (§5.9.7 B4); flushed before the next push. */
|
|
100
|
-
export declare function recordPendingUpload(db: ClientDatabase, blobId: string, nowMs: number, mediaType?: string): void;
|
|
88
|
+
export declare function enforceBlobCacheCap(db: ClientDatabase, schema: CompiledClientSchema, maxBytes: number): string[];
|
|
101
89
|
export declare function listPendingUploads(db: ClientDatabase): {
|
|
102
90
|
blobId: string;
|
|
103
91
|
mediaType?: string;
|
|
104
92
|
}[];
|
|
93
|
+
export declare function recordPendingUpload(db: ClientDatabase, blobId: string, nowMs: number, mediaType?: string): void;
|
|
105
94
|
export declare function clearPendingUpload(db: ClientDatabase, blobId: string): void;
|
|
106
95
|
/**
|
|
107
|
-
* All `blob_ref` column names per table
|
|
108
|
-
* Blank result ⇒ the schema has no attachments; callers skip reconciliation.
|
|
96
|
+
* All `blob_ref` column names per table.
|
|
109
97
|
*/
|
|
110
98
|
export declare function blobRefColumnsBySchema(schema: CompiledClientSchema): Map<string, string[]>;
|
|
111
99
|
export declare function schemaHasBlobs(schema: CompiledClientSchema): boolean;
|
|
112
|
-
|
|
113
|
-
* §5.9.7 B1/B2: recompute cache refcounts from the current `blob_ref` column
|
|
114
|
-
* contents across all synced tables, then delete cache bodies that dropped
|
|
115
|
-
* to zero refs *and* have no pending upload (a pending upload pins its body,
|
|
116
|
-
* B4). Called after every apply/purge that may add or remove references.
|
|
117
|
-
*
|
|
118
|
-
* `deleteOrphans` distinguishes the two B2 transitions: revocation purge
|
|
119
|
-
* passes `true` (drop the now-unauthorized body); a benign apply passes
|
|
120
|
-
* `false` (retain zero-ref bodies as LRU cache entries — the shipped
|
|
121
|
-
* default). Bodies pinned by a pending upload are always retained.
|
|
122
|
-
*/
|
|
123
|
-
export declare function reconcileBlobRefcounts(db: ClientDatabase, schema: CompiledClientSchema, options?: {
|
|
124
|
-
readonly deleteOrphans?: boolean;
|
|
125
|
-
}): void;
|
|
100
|
+
export declare function deleteUnreferencedCachedBlobs(db: ClientDatabase, schema: CompiledClientSchema): void;
|
|
126
101
|
export type { BlobRef };
|
|
127
102
|
export { parseBlobRef, serializeBlobRef };
|
package/dist/blob.js
CHANGED
|
@@ -1,68 +1,65 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Client-side blob cache + transport (SPEC.md §5.9.7).
|
|
3
3
|
*
|
|
4
|
-
* Blob bytes are cached
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* reconciled from the current `blob_ref` column contents, and a body whose
|
|
8
|
-
* only referencing rows were revocation-purged is deleted (B2, evicted ≠
|
|
9
|
-
* revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
|
|
10
|
-
* row value is the whole download key. Pending uploads are tracked in the
|
|
11
|
-
* outbox-adjacent uploads table (B4) and flushed before push.
|
|
4
|
+
* Blob bytes are cached by `blobId`. Retention reads live references directly
|
|
5
|
+
* from `blob_ref` columns and keeps explicit pending-commit dependencies.
|
|
6
|
+
* A small upload row queues each staged body for transfer before push.
|
|
12
7
|
*/
|
|
13
8
|
import { parseBlobRef, serializeBlobRef } from '@syncular/core';
|
|
9
|
+
import { ClientSyncError } from './errors.js';
|
|
14
10
|
import { quoteIdent } from './schema.js';
|
|
15
11
|
/** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
|
|
16
12
|
export async function computeBlobId(bytes) {
|
|
17
|
-
const
|
|
13
|
+
const input = bytes.buffer instanceof ArrayBuffer
|
|
14
|
+
? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
15
|
+
: bytes.slice();
|
|
16
|
+
const digest = await crypto.subtle.digest('SHA-256', input);
|
|
18
17
|
const hex = [...new Uint8Array(digest)]
|
|
19
18
|
.map((b) => b.toString(16).padStart(2, '0'))
|
|
20
19
|
.join('');
|
|
21
20
|
return `sha256:${hex}`;
|
|
22
21
|
}
|
|
23
22
|
export function ensureBlobSchema(db) {
|
|
24
|
-
db.
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
23
|
+
const existing = db.query("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs'");
|
|
24
|
+
if (existing.length > 0) {
|
|
25
|
+
const columns = db
|
|
26
|
+
.query('PRAGMA table_info(_syncular_blobs)')
|
|
27
|
+
.map((row) => row.name)
|
|
28
|
+
.sort();
|
|
29
|
+
const expected = [
|
|
30
|
+
'blob_id',
|
|
31
|
+
'byte_length',
|
|
32
|
+
'bytes',
|
|
33
|
+
'created_at_ms',
|
|
34
|
+
'media_type',
|
|
35
|
+
].sort();
|
|
36
|
+
if (columns.length !== expected.length ||
|
|
37
|
+
columns.some((column, index) => column !== expected[index])) {
|
|
38
|
+
throw new ClientSyncError('sync.schema_mismatch', 'Local blob schema is incompatible');
|
|
39
|
+
}
|
|
37
40
|
}
|
|
38
|
-
|
|
39
|
-
|
|
41
|
+
else {
|
|
42
|
+
db.exec(`CREATE TABLE _syncular_blobs(
|
|
43
|
+
blob_id TEXT PRIMARY KEY,
|
|
44
|
+
bytes BLOB NOT NULL,
|
|
45
|
+
byte_length INTEGER NOT NULL,
|
|
46
|
+
media_type TEXT,
|
|
47
|
+
created_at_ms INTEGER NOT NULL)`);
|
|
40
48
|
}
|
|
41
49
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
|
|
42
|
-
blob_id TEXT PRIMARY KEY,
|
|
43
|
-
media_type TEXT,
|
|
44
|
-
created_at_ms INTEGER NOT NULL)`);
|
|
50
|
+
blob_id TEXT PRIMARY KEY, media_type TEXT, created_at_ms INTEGER NOT NULL)`);
|
|
45
51
|
}
|
|
46
|
-
/** Put bytes into the content-addressed cache
|
|
52
|
+
/** Put bytes into the content-addressed cache without rewriting an existing body. */
|
|
47
53
|
export function putCachedBlob(db, blobId, bytes, nowMs, mediaType) {
|
|
48
54
|
db.exec(`INSERT INTO _syncular_blobs(
|
|
49
|
-
blob_id, bytes, byte_length, media_type,
|
|
50
|
-
VALUES (
|
|
51
|
-
ON CONFLICT(blob_id) DO UPDATE SET last_used_ms = excluded.last_used_ms`, [blobId, bytes, bytes.length, mediaType ?? null, nowMs, nowMs]);
|
|
55
|
+
blob_id, bytes, byte_length, media_type, created_at_ms)
|
|
56
|
+
VALUES (?,?,?,?,?) ON CONFLICT(blob_id) DO NOTHING`, [blobId, bytes, bytes.length, mediaType ?? null, nowMs]);
|
|
52
57
|
}
|
|
53
|
-
export function getCachedBlob(db, blobId
|
|
58
|
+
export function getCachedBlob(db, blobId) {
|
|
54
59
|
const rows = db.query('SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?', [blobId]);
|
|
55
60
|
const row = rows[0];
|
|
56
61
|
if (row === undefined)
|
|
57
62
|
return undefined;
|
|
58
|
-
// §5.9.7 B1 LRU: a cache-hit read touches "recently used" so a hot image
|
|
59
|
-
// survives a cap trim. Skipped when no clock is supplied (pure read).
|
|
60
|
-
if (nowMs !== undefined) {
|
|
61
|
-
db.exec('UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?', [
|
|
62
|
-
nowMs,
|
|
63
|
-
blobId,
|
|
64
|
-
]);
|
|
65
|
-
}
|
|
66
63
|
return {
|
|
67
64
|
blobId,
|
|
68
65
|
bytes: row.bytes,
|
|
@@ -71,26 +68,21 @@ export function getCachedBlob(db, blobId, nowMs) {
|
|
|
71
68
|
};
|
|
72
69
|
}
|
|
73
70
|
/**
|
|
74
|
-
* §5.9.7 B1 size cap
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
* — it must stay resolvable without a re-download) nor a pending-upload-pinned
|
|
78
|
-
* body (its bytes are the only copy until push, B4). If every over-cap body is
|
|
79
|
-
* referenced or pinned, the cache stays over the cap (correctness beats the
|
|
80
|
-
* cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
|
|
81
|
-
* any surviving `blob_ref` value. Returns the evicted blobIds.
|
|
71
|
+
* §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
|
|
72
|
+
* evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
|
|
73
|
+
* until back under the cap. Returns the evicted blobIds.
|
|
82
74
|
*/
|
|
83
|
-
export function enforceBlobCacheCap(db, maxBytes) {
|
|
75
|
+
export function enforceBlobCacheCap(db, schema, maxBytes) {
|
|
84
76
|
const totalRow = db.query('SELECT COALESCE(SUM(byte_length), 0) AS total FROM _syncular_blobs')[0];
|
|
85
77
|
let total = Number(totalRow?.total ?? 0);
|
|
86
78
|
if (total <= maxBytes)
|
|
87
79
|
return [];
|
|
88
|
-
//
|
|
89
|
-
// (LRU) first, then oldest created as a stable tiebreak.
|
|
80
|
+
// Oldest eligible body first, with blob ID as a stable tie.
|
|
90
81
|
const candidates = db.query(`SELECT blob_id, byte_length FROM _syncular_blobs
|
|
91
|
-
WHERE
|
|
92
|
-
AND blob_id NOT IN (SELECT blob_id FROM
|
|
93
|
-
|
|
82
|
+
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
83
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
84
|
+
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})
|
|
85
|
+
ORDER BY created_at_ms ASC, blob_id ASC`);
|
|
94
86
|
const evicted = [];
|
|
95
87
|
db.transaction(() => {
|
|
96
88
|
for (const row of candidates) {
|
|
@@ -104,27 +96,35 @@ export function enforceBlobCacheCap(db, maxBytes) {
|
|
|
104
96
|
});
|
|
105
97
|
return evicted;
|
|
106
98
|
}
|
|
107
|
-
|
|
99
|
+
export function listPendingUploads(db) {
|
|
100
|
+
return db
|
|
101
|
+
.query(`SELECT blob_id, media_type FROM (
|
|
102
|
+
SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
|
|
103
|
+
UNION ALL
|
|
104
|
+
SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
|
|
105
|
+
FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
|
|
106
|
+
WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
|
|
107
|
+
) ORDER BY created_at_ms, blob_id`)
|
|
108
|
+
.map((row) => {
|
|
109
|
+
if (typeof row.blob_id !== 'string' ||
|
|
110
|
+
(row.media_type !== null && typeof row.media_type !== 'string')) {
|
|
111
|
+
throw new ClientSyncError('sync.local_corrupt', 'Pending blob upload metadata is invalid');
|
|
112
|
+
}
|
|
113
|
+
return {
|
|
114
|
+
blobId: row.blob_id,
|
|
115
|
+
...(row.media_type !== null ? { mediaType: row.media_type } : {}),
|
|
116
|
+
};
|
|
117
|
+
});
|
|
118
|
+
}
|
|
108
119
|
export function recordPendingUpload(db, blobId, nowMs, mediaType) {
|
|
109
120
|
db.exec(`INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id, media_type, created_at_ms)
|
|
110
121
|
VALUES (?,?,?)`, [blobId, mediaType ?? null, nowMs]);
|
|
111
122
|
}
|
|
112
|
-
export function listPendingUploads(db) {
|
|
113
|
-
return db
|
|
114
|
-
.query('SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms')
|
|
115
|
-
.map((row) => ({
|
|
116
|
-
blobId: row.blob_id,
|
|
117
|
-
...(row.media_type !== null
|
|
118
|
-
? { mediaType: row.media_type }
|
|
119
|
-
: {}),
|
|
120
|
-
}));
|
|
121
|
-
}
|
|
122
123
|
export function clearPendingUpload(db, blobId) {
|
|
123
124
|
db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id = ?', [blobId]);
|
|
124
125
|
}
|
|
125
126
|
/**
|
|
126
|
-
* All `blob_ref` column names per table
|
|
127
|
-
* Blank result ⇒ the schema has no attachments; callers skip reconciliation.
|
|
127
|
+
* All `blob_ref` column names per table.
|
|
128
128
|
*/
|
|
129
129
|
export function blobRefColumnsBySchema(schema) {
|
|
130
130
|
const out = new Map();
|
|
@@ -144,58 +144,26 @@ export function schemaHasBlobs(schema) {
|
|
|
144
144
|
}
|
|
145
145
|
return false;
|
|
146
146
|
}
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
* to zero refs *and* have no pending upload (a pending upload pins its body,
|
|
151
|
-
* B4). Called after every apply/purge that may add or remove references.
|
|
152
|
-
*
|
|
153
|
-
* `deleteOrphans` distinguishes the two B2 transitions: revocation purge
|
|
154
|
-
* passes `true` (drop the now-unauthorized body); a benign apply passes
|
|
155
|
-
* `false` (retain zero-ref bodies as LRU cache entries — the shipped
|
|
156
|
-
* default). Bodies pinned by a pending upload are always retained.
|
|
157
|
-
*/
|
|
158
|
-
export function reconcileBlobRefcounts(db, schema, options) {
|
|
159
|
-
const byTable = blobRefColumnsBySchema(schema);
|
|
160
|
-
if (byTable.size === 0)
|
|
161
|
-
return;
|
|
162
|
-
// Count references to each blobId across every blob_ref column.
|
|
163
|
-
const counts = new Map();
|
|
164
|
-
for (const [tableName, columns] of byTable) {
|
|
147
|
+
function visibleBlobIdsSql(schema) {
|
|
148
|
+
const selects = [];
|
|
149
|
+
for (const [tableName, columns] of blobRefColumnsBySchema(schema)) {
|
|
165
150
|
for (const column of columns) {
|
|
166
|
-
const
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
let ref;
|
|
173
|
-
try {
|
|
174
|
-
ref = parseBlobRef(raw);
|
|
175
|
-
}
|
|
176
|
-
catch {
|
|
177
|
-
continue;
|
|
178
|
-
}
|
|
179
|
-
counts.set(ref.blobId, (counts.get(ref.blobId) ?? 0) + 1);
|
|
180
|
-
}
|
|
151
|
+
const identifier = quoteIdent(column);
|
|
152
|
+
selects.push(`SELECT json_extract(${identifier}, '$.blobId') AS blob_id
|
|
153
|
+
FROM ${quoteIdent(tableName)}
|
|
154
|
+
WHERE typeof(${identifier}) = 'text'
|
|
155
|
+
AND json_valid(${identifier})
|
|
156
|
+
AND json_type(${identifier}, '$.blobId') = 'text'`);
|
|
181
157
|
}
|
|
182
158
|
}
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
if (options?.deleteOrphans === true) {
|
|
193
|
-
// §5.9.7 B2 revocation side: delete zero-ref bodies not pinned by a
|
|
194
|
-
// pending upload.
|
|
195
|
-
db.exec(`DELETE FROM _syncular_blobs
|
|
196
|
-
WHERE refcount = 0
|
|
197
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)`);
|
|
198
|
-
}
|
|
199
|
-
});
|
|
159
|
+
return selects.length === 0
|
|
160
|
+
? 'SELECT NULL AS blob_id WHERE 0'
|
|
161
|
+
: selects.join(' UNION ');
|
|
162
|
+
}
|
|
163
|
+
export function deleteUnreferencedCachedBlobs(db, schema) {
|
|
164
|
+
db.exec(`DELETE FROM _syncular_blobs
|
|
165
|
+
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
166
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
167
|
+
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})`);
|
|
200
168
|
}
|
|
201
169
|
export { parseBlobRef, serializeBlobRef };
|
package/dist/client.d.ts
CHANGED
|
@@ -110,9 +110,9 @@ export interface SyncClientConfig {
|
|
|
110
110
|
readonly blobs?: BlobTransport;
|
|
111
111
|
/**
|
|
112
112
|
* §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
|
|
113
|
-
* exceeds this,
|
|
114
|
-
* cache write
|
|
115
|
-
*
|
|
113
|
+
* exceeds this, unreferenced bodies without pending work are evicted in
|
|
114
|
+
* creation timestamp order after each cache write. Referenced bodies remain cached
|
|
115
|
+
* even when that exceeds the cap. Absent means retain until storage pressure (the shipped
|
|
116
116
|
* default; a referenced body always stays resolvable without a re-download).
|
|
117
117
|
*/
|
|
118
118
|
readonly blobCacheMaxBytes?: number;
|
package/dist/client.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { canonicalScopeJson, decodeMessage, decodeRow, decodeRowsSegment, encodeMessage, encodePresencePublish, MessageStreamScanner, PROTOCOL_WIRE_VERSION, parseRealtimeServerEvent, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
|
|
11
11
|
import { applyCommitFrame, applyRowsSegment, applySqliteSegment, deleteLocalRow, deleteScopedRows, evictScopedRows, upsertLocalRow, } from './apply.js';
|
|
12
|
-
import { clearPendingUpload, computeBlobId, enforceBlobCacheCap, ensureBlobSchema, getCachedBlob, listPendingUploads, parseBlobRef, putCachedBlob,
|
|
12
|
+
import { clearPendingUpload, computeBlobId, deleteUnreferencedCachedBlobs, enforceBlobCacheCap, ensureBlobSchema, getCachedBlob, listPendingUploads, parseBlobRef, putCachedBlob, recordPendingUpload, schemaHasBlobs, serializeBlobRef, } from './blob.js';
|
|
13
13
|
import { registerDevtools } from './devtools.js';
|
|
14
14
|
import { CLIENT_DIAGNOSTICS_VERSION, ClientDiagnosticsEmitter, MAX_DIAGNOSTIC_DOMAINS, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS, } from './diagnostics.js';
|
|
15
15
|
import { ClientSyncError } from './errors.js';
|
|
@@ -854,20 +854,19 @@ export class SyncClient {
|
|
|
854
854
|
* (via `mutate`) after this call so upload-before-push holds (§5.9.3).
|
|
855
855
|
*/
|
|
856
856
|
uploadBlob(bytes, options) {
|
|
857
|
-
|
|
857
|
+
const owned = bytes.slice();
|
|
858
|
+
return this.#runProtectedAsync(() => this.#uploadBlob(owned, options));
|
|
858
859
|
}
|
|
859
860
|
async #uploadBlob(bytes, options) {
|
|
860
861
|
if (this.#config.blobs === undefined) {
|
|
861
862
|
throw new ClientSyncError('sync.invalid_request', 'uploadBlob requires a blob transport (SyncClientConfig.blobs, §5.9)');
|
|
862
863
|
}
|
|
863
864
|
const blobId = await computeBlobId(bytes);
|
|
865
|
+
const nowMs = this.#now();
|
|
864
866
|
this.#db.transaction(() => {
|
|
865
|
-
putCachedBlob(this.#db, blobId, bytes,
|
|
866
|
-
recordPendingUpload(this.#db, blobId,
|
|
867
|
+
putCachedBlob(this.#db, blobId, bytes, nowMs, options?.mediaType);
|
|
868
|
+
recordPendingUpload(this.#db, blobId, nowMs, options?.mediaType);
|
|
867
869
|
});
|
|
868
|
-
// §5.9.7 B1: a staged upload is pinned (recordPendingUpload), so the cap
|
|
869
|
-
// trim below will never evict it — but a stage may push other zero-ref
|
|
870
|
-
// bodies over the cap, so run the trim.
|
|
871
870
|
this.#enforceBlobCacheCap();
|
|
872
871
|
return {
|
|
873
872
|
blobId,
|
|
@@ -895,7 +894,7 @@ export class SyncClient {
|
|
|
895
894
|
const blobId = blobIdOrRef.startsWith('sha256:')
|
|
896
895
|
? blobIdOrRef
|
|
897
896
|
: parseBlobRef(blobIdOrRef).blobId;
|
|
898
|
-
const cached = getCachedBlob(this.#db, blobId
|
|
897
|
+
const cached = getCachedBlob(this.#db, blobId);
|
|
899
898
|
if (cached !== undefined)
|
|
900
899
|
return cached;
|
|
901
900
|
const transport = this.#config.blobs;
|
|
@@ -930,9 +929,7 @@ export class SyncClient {
|
|
|
930
929
|
throw new ClientSyncError('sync.invalid_request', `blob content address mismatch for ${blobId} (§5.9.5)`);
|
|
931
930
|
}
|
|
932
931
|
putCachedBlob(this.#db, blobId, bytes, this.#now());
|
|
933
|
-
|
|
934
|
-
// from current visible references before applying the size cap (§5.9.7 B1).
|
|
935
|
-
this.#reconcileBlobs(false);
|
|
932
|
+
this.#db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
936
933
|
this.#enforceBlobCacheCap();
|
|
937
934
|
const stored = getCachedBlob(this.#db, blobId);
|
|
938
935
|
if (stored === undefined) {
|
|
@@ -945,7 +942,7 @@ export class SyncClient {
|
|
|
945
942
|
const cap = this.#config.blobCacheMaxBytes;
|
|
946
943
|
if (cap === undefined)
|
|
947
944
|
return;
|
|
948
|
-
enforceBlobCacheCap(this.#db, cap);
|
|
945
|
+
enforceBlobCacheCap(this.#db, this.#schema, cap);
|
|
949
946
|
}
|
|
950
947
|
/** Flush any queued blob uploads (§5.9.7 B4); safe to call standalone. */
|
|
951
948
|
flushBlobUploads() {
|
|
@@ -957,11 +954,12 @@ export class SyncClient {
|
|
|
957
954
|
return;
|
|
958
955
|
for (const pending of listPendingUploads(this.#db)) {
|
|
959
956
|
const cached = getCachedBlob(this.#db, pending.blobId);
|
|
960
|
-
if (cached === undefined
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
957
|
+
if (cached === undefined ||
|
|
958
|
+
!(cached.bytes instanceof Uint8Array) ||
|
|
959
|
+
!Number.isSafeInteger(cached.byteLength) ||
|
|
960
|
+
cached.byteLength !== cached.bytes.byteLength ||
|
|
961
|
+
(await computeBlobId(cached.bytes)) !== pending.blobId) {
|
|
962
|
+
throw new ClientSyncError('sync.local_corrupt', 'Pending blob upload body is missing or corrupt');
|
|
965
963
|
}
|
|
966
964
|
await this.#uploadOne(transport, pending.blobId, cached.bytes, pending.mediaType);
|
|
967
965
|
clearPendingUpload(this.#db, pending.blobId);
|
|
@@ -1413,6 +1411,31 @@ export class SyncClient {
|
|
|
1413
1411
|
this.#applyBatch((batch) => {
|
|
1414
1412
|
this.#db.transaction(() => {
|
|
1415
1413
|
appendOutboxCommit(this.#db, clientCommitId, operations, this.#now(), this.#captureBeforeImages(operations));
|
|
1414
|
+
for (const operation of operations) {
|
|
1415
|
+
if (operation.op !== 'upsert')
|
|
1416
|
+
continue;
|
|
1417
|
+
const table = this.#schema.tables.get(operation.table);
|
|
1418
|
+
if (table === undefined)
|
|
1419
|
+
continue;
|
|
1420
|
+
for (const column of table.columns) {
|
|
1421
|
+
if (column.type !== 'blob_ref')
|
|
1422
|
+
continue;
|
|
1423
|
+
const raw = operation.values?.[column.name];
|
|
1424
|
+
if (typeof raw !== 'string')
|
|
1425
|
+
continue;
|
|
1426
|
+
let blobId;
|
|
1427
|
+
try {
|
|
1428
|
+
blobId = parseBlobRef(raw).blobId;
|
|
1429
|
+
}
|
|
1430
|
+
catch {
|
|
1431
|
+
continue;
|
|
1432
|
+
}
|
|
1433
|
+
this.#db.exec(`INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
|
|
1434
|
+
SELECT ?, ? WHERE EXISTS (
|
|
1435
|
+
SELECT 1 FROM _syncular_blobs WHERE blob_id = ?
|
|
1436
|
+
)`, [clientCommitId, blobId, blobId]);
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1416
1439
|
this.#applyOperationsLocally(operations, batch);
|
|
1417
1440
|
batch.status();
|
|
1418
1441
|
});
|
|
@@ -1571,7 +1594,7 @@ export class SyncClient {
|
|
|
1571
1594
|
batch.rejections();
|
|
1572
1595
|
batch.outcomes();
|
|
1573
1596
|
}
|
|
1574
|
-
this.#
|
|
1597
|
+
this.#deleteUnreferencedBlobs();
|
|
1575
1598
|
setMeta(this.#db, metaKey, purge.canonicalPlan);
|
|
1576
1599
|
return {
|
|
1577
1600
|
alreadyApplied: false,
|
|
@@ -2513,7 +2536,6 @@ export class SyncClient {
|
|
|
2513
2536
|
// §7.1: local reads see outbox state applied optimistically — replay
|
|
2514
2537
|
// the still-pending commits on top of the freshly applied server state.
|
|
2515
2538
|
this.#replayOutbox();
|
|
2516
|
-
this.#reconcileBlobs(false);
|
|
2517
2539
|
}
|
|
2518
2540
|
finally {
|
|
2519
2541
|
this.#endDiagnosticsDeferral();
|
|
@@ -2902,7 +2924,7 @@ export class SyncClient {
|
|
|
2902
2924
|
batch.rejections();
|
|
2903
2925
|
batch.outcomes();
|
|
2904
2926
|
}
|
|
2905
|
-
this.#
|
|
2927
|
+
this.#deleteUnreferencedBlobs();
|
|
2906
2928
|
}
|
|
2907
2929
|
catch (error) {
|
|
2908
2930
|
if (error instanceof ClientSyncError &&
|
|
@@ -3074,15 +3096,10 @@ export class SyncClient {
|
|
|
3074
3096
|
});
|
|
3075
3097
|
});
|
|
3076
3098
|
}
|
|
3077
|
-
|
|
3078
|
-
* §5.9.7 B1/B2: recompute blob-cache refcounts from live `blob_ref`
|
|
3079
|
-
* columns. No-op unless the schema has blob columns. `deleteOrphans`
|
|
3080
|
-
* triggers the revocation-side body deletion (B2).
|
|
3081
|
-
*/
|
|
3082
|
-
#reconcileBlobs(deleteOrphans) {
|
|
3099
|
+
#deleteUnreferencedBlobs() {
|
|
3083
3100
|
if (!this.#hasBlobs)
|
|
3084
3101
|
return;
|
|
3085
|
-
|
|
3102
|
+
deleteUnreferencedCachedBlobs(this.#db, this.#schema);
|
|
3086
3103
|
}
|
|
3087
3104
|
// -- helpers -----------------------------------------------------------------
|
|
3088
3105
|
#table(name) {
|
package/dist/outbox.js
CHANGED
|
@@ -62,6 +62,9 @@ export function countOutbox(db) {
|
|
|
62
62
|
.count;
|
|
63
63
|
}
|
|
64
64
|
export function deleteOutboxCommit(db, clientCommitId) {
|
|
65
|
+
db.exec('DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?', [
|
|
66
|
+
clientCommitId,
|
|
67
|
+
]);
|
|
65
68
|
db.exec('DELETE FROM _syncular_outbox_before_images WHERE client_commit_id = ?', [clientCommitId]);
|
|
66
69
|
db.exec('DELETE FROM _syncular_outbox WHERE client_commit_id = ?', [
|
|
67
70
|
clientCommitId,
|
package/dist/schema.js
CHANGED
|
@@ -339,6 +339,9 @@ export function ensureLocalBookkeepingSchema(db) {
|
|
|
339
339
|
db.transaction(() => {
|
|
340
340
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
|
|
341
341
|
key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
|
|
342
|
+
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
|
|
343
|
+
commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
|
|
344
|
+
db.exec('CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)');
|
|
342
345
|
db.exec(`INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`);
|
|
343
346
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_outbox(
|
|
344
347
|
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -89,9 +89,9 @@
|
|
|
89
89
|
},
|
|
90
90
|
"dependencies": {
|
|
91
91
|
"@sqlite.org/sqlite-wasm": "^3.53.0-build1",
|
|
92
|
-
"@syncular/core": "0.
|
|
92
|
+
"@syncular/core": "0.19.0"
|
|
93
93
|
},
|
|
94
94
|
"devDependencies": {
|
|
95
|
-
"@syncular/server": "0.
|
|
95
|
+
"@syncular/server": "0.19.0"
|
|
96
96
|
}
|
|
97
97
|
}
|
package/src/blob.ts
CHANGED
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Client-side blob cache + transport (SPEC.md §5.9.7).
|
|
3
3
|
*
|
|
4
|
-
* Blob bytes are cached
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* reconciled from the current `blob_ref` column contents, and a body whose
|
|
8
|
-
* only referencing rows were revocation-purged is deleted (B2, evicted ≠
|
|
9
|
-
* revoked). BlobRefs stay resolvable at any time (B3): the `blobId` in the
|
|
10
|
-
* row value is the whole download key. Pending uploads are tracked in the
|
|
11
|
-
* outbox-adjacent uploads table (B4) and flushed before push.
|
|
4
|
+
* Blob bytes are cached by `blobId`. Retention reads live references directly
|
|
5
|
+
* from `blob_ref` columns and keeps explicit pending-commit dependencies.
|
|
6
|
+
* A small upload row queues each staged body for transfer before push.
|
|
12
7
|
*/
|
|
13
8
|
import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
|
|
14
9
|
import type { ClientDatabase } from './database';
|
|
10
|
+
import { ClientSyncError } from './errors';
|
|
15
11
|
import type { CompiledClientSchema } from './schema';
|
|
16
12
|
import { quoteIdent } from './schema';
|
|
17
13
|
|
|
@@ -93,9 +89,13 @@ export interface CachedBlob {
|
|
|
93
89
|
|
|
94
90
|
/** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
|
|
95
91
|
export async function computeBlobId(bytes: Uint8Array): Promise<string> {
|
|
92
|
+
const input =
|
|
93
|
+
bytes.buffer instanceof ArrayBuffer
|
|
94
|
+
? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
95
|
+
: bytes.slice();
|
|
96
96
|
const digest = await crypto.subtle.digest(
|
|
97
97
|
'SHA-256',
|
|
98
|
-
|
|
98
|
+
input as Uint8Array<ArrayBuffer>,
|
|
99
99
|
);
|
|
100
100
|
const hex = [...new Uint8Array(digest)]
|
|
101
101
|
.map((b) => b.toString(16).padStart(2, '0'))
|
|
@@ -104,31 +104,43 @@ export async function computeBlobId(bytes: Uint8Array): Promise<string> {
|
|
|
104
104
|
}
|
|
105
105
|
|
|
106
106
|
export function ensureBlobSchema(db: ClientDatabase): void {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
'
|
|
121
|
-
);
|
|
122
|
-
|
|
123
|
-
|
|
107
|
+
const existing = db.query(
|
|
108
|
+
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '_syncular_blobs'",
|
|
109
|
+
);
|
|
110
|
+
if (existing.length > 0) {
|
|
111
|
+
const columns = db
|
|
112
|
+
.query('PRAGMA table_info(_syncular_blobs)')
|
|
113
|
+
.map((row) => row.name)
|
|
114
|
+
.sort();
|
|
115
|
+
const expected = [
|
|
116
|
+
'blob_id',
|
|
117
|
+
'byte_length',
|
|
118
|
+
'bytes',
|
|
119
|
+
'created_at_ms',
|
|
120
|
+
'media_type',
|
|
121
|
+
].sort();
|
|
122
|
+
if (
|
|
123
|
+
columns.length !== expected.length ||
|
|
124
|
+
columns.some((column, index) => column !== expected[index])
|
|
125
|
+
) {
|
|
126
|
+
throw new ClientSyncError(
|
|
127
|
+
'sync.schema_mismatch',
|
|
128
|
+
'Local blob schema is incompatible',
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
} else {
|
|
132
|
+
db.exec(`CREATE TABLE _syncular_blobs(
|
|
133
|
+
blob_id TEXT PRIMARY KEY,
|
|
134
|
+
bytes BLOB NOT NULL,
|
|
135
|
+
byte_length INTEGER NOT NULL,
|
|
136
|
+
media_type TEXT,
|
|
137
|
+
created_at_ms INTEGER NOT NULL)`);
|
|
124
138
|
}
|
|
125
139
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(
|
|
126
|
-
blob_id TEXT PRIMARY KEY,
|
|
127
|
-
media_type TEXT,
|
|
128
|
-
created_at_ms INTEGER NOT NULL)`);
|
|
140
|
+
blob_id TEXT PRIMARY KEY, media_type TEXT, created_at_ms INTEGER NOT NULL)`);
|
|
129
141
|
}
|
|
130
142
|
|
|
131
|
-
/** Put bytes into the content-addressed cache
|
|
143
|
+
/** Put bytes into the content-addressed cache without rewriting an existing body. */
|
|
132
144
|
export function putCachedBlob(
|
|
133
145
|
db: ClientDatabase,
|
|
134
146
|
blobId: string,
|
|
@@ -138,17 +150,15 @@ export function putCachedBlob(
|
|
|
138
150
|
): void {
|
|
139
151
|
db.exec(
|
|
140
152
|
`INSERT INTO _syncular_blobs(
|
|
141
|
-
blob_id, bytes, byte_length, media_type,
|
|
142
|
-
VALUES (
|
|
143
|
-
|
|
144
|
-
[blobId, bytes, bytes.length, mediaType ?? null, nowMs, nowMs],
|
|
153
|
+
blob_id, bytes, byte_length, media_type, created_at_ms)
|
|
154
|
+
VALUES (?,?,?,?,?) ON CONFLICT(blob_id) DO NOTHING`,
|
|
155
|
+
[blobId, bytes, bytes.length, mediaType ?? null, nowMs],
|
|
145
156
|
);
|
|
146
157
|
}
|
|
147
158
|
|
|
148
159
|
export function getCachedBlob(
|
|
149
160
|
db: ClientDatabase,
|
|
150
161
|
blobId: string,
|
|
151
|
-
nowMs?: number,
|
|
152
162
|
): CachedBlob | undefined {
|
|
153
163
|
const rows = db.query(
|
|
154
164
|
'SELECT bytes, byte_length, media_type FROM _syncular_blobs WHERE blob_id = ?',
|
|
@@ -156,14 +166,6 @@ export function getCachedBlob(
|
|
|
156
166
|
);
|
|
157
167
|
const row = rows[0];
|
|
158
168
|
if (row === undefined) return undefined;
|
|
159
|
-
// §5.9.7 B1 LRU: a cache-hit read touches "recently used" so a hot image
|
|
160
|
-
// survives a cap trim. Skipped when no clock is supplied (pure read).
|
|
161
|
-
if (nowMs !== undefined) {
|
|
162
|
-
db.exec('UPDATE _syncular_blobs SET last_used_ms = ? WHERE blob_id = ?', [
|
|
163
|
-
nowMs,
|
|
164
|
-
blobId,
|
|
165
|
-
]);
|
|
166
|
-
}
|
|
167
169
|
return {
|
|
168
170
|
blobId,
|
|
169
171
|
bytes: row.bytes as Uint8Array,
|
|
@@ -173,17 +175,13 @@ export function getCachedBlob(
|
|
|
173
175
|
}
|
|
174
176
|
|
|
175
177
|
/**
|
|
176
|
-
* §5.9.7 B1 size cap
|
|
177
|
-
*
|
|
178
|
-
*
|
|
179
|
-
* — it must stay resolvable without a re-download) nor a pending-upload-pinned
|
|
180
|
-
* body (its bytes are the only copy until push, B4). If every over-cap body is
|
|
181
|
-
* referenced or pinned, the cache stays over the cap (correctness beats the
|
|
182
|
-
* cap). Evicting a zero-ref body is always safe: B3 re-enables the fetch from
|
|
183
|
-
* any surviving `blob_ref` value. Returns the evicted blobIds.
|
|
178
|
+
* §5.9.7 B1 size cap. When the sum of cached body sizes exceeds `maxBytes`,
|
|
179
|
+
* evict unreferenced, non-pinned bodies by creation timestamp, then blob ID,
|
|
180
|
+
* until back under the cap. Returns the evicted blobIds.
|
|
184
181
|
*/
|
|
185
182
|
export function enforceBlobCacheCap(
|
|
186
183
|
db: ClientDatabase,
|
|
184
|
+
schema: CompiledClientSchema,
|
|
187
185
|
maxBytes: number,
|
|
188
186
|
): string[] {
|
|
189
187
|
const totalRow = db.query(
|
|
@@ -191,13 +189,13 @@ export function enforceBlobCacheCap(
|
|
|
191
189
|
)[0];
|
|
192
190
|
let total = Number(totalRow?.total ?? 0);
|
|
193
191
|
if (total <= maxBytes) return [];
|
|
194
|
-
//
|
|
195
|
-
// (LRU) first, then oldest created as a stable tiebreak.
|
|
192
|
+
// Oldest eligible body first, with blob ID as a stable tie.
|
|
196
193
|
const candidates = db.query(
|
|
197
194
|
`SELECT blob_id, byte_length FROM _syncular_blobs
|
|
198
|
-
WHERE
|
|
199
|
-
AND blob_id NOT IN (SELECT blob_id FROM
|
|
200
|
-
|
|
195
|
+
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
196
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
197
|
+
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})
|
|
198
|
+
ORDER BY created_at_ms ASC, blob_id ASC`,
|
|
201
199
|
);
|
|
202
200
|
const evicted: string[] = [];
|
|
203
201
|
db.transaction(() => {
|
|
@@ -212,7 +210,36 @@ export function enforceBlobCacheCap(
|
|
|
212
210
|
return evicted;
|
|
213
211
|
}
|
|
214
212
|
|
|
215
|
-
|
|
213
|
+
export function listPendingUploads(
|
|
214
|
+
db: ClientDatabase,
|
|
215
|
+
): { blobId: string; mediaType?: string }[] {
|
|
216
|
+
return db
|
|
217
|
+
.query(
|
|
218
|
+
`SELECT blob_id, media_type FROM (
|
|
219
|
+
SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
|
|
220
|
+
UNION ALL
|
|
221
|
+
SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
|
|
222
|
+
FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
|
|
223
|
+
WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
|
|
224
|
+
) ORDER BY created_at_ms, blob_id`,
|
|
225
|
+
)
|
|
226
|
+
.map((row) => {
|
|
227
|
+
if (
|
|
228
|
+
typeof row.blob_id !== 'string' ||
|
|
229
|
+
(row.media_type !== null && typeof row.media_type !== 'string')
|
|
230
|
+
) {
|
|
231
|
+
throw new ClientSyncError(
|
|
232
|
+
'sync.local_corrupt',
|
|
233
|
+
'Pending blob upload metadata is invalid',
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
return {
|
|
237
|
+
blobId: row.blob_id,
|
|
238
|
+
...(row.media_type !== null ? { mediaType: row.media_type } : {}),
|
|
239
|
+
};
|
|
240
|
+
});
|
|
241
|
+
}
|
|
242
|
+
|
|
216
243
|
export function recordPendingUpload(
|
|
217
244
|
db: ClientDatabase,
|
|
218
245
|
blobId: string,
|
|
@@ -226,28 +253,12 @@ export function recordPendingUpload(
|
|
|
226
253
|
);
|
|
227
254
|
}
|
|
228
255
|
|
|
229
|
-
export function listPendingUploads(
|
|
230
|
-
db: ClientDatabase,
|
|
231
|
-
): { blobId: string; mediaType?: string }[] {
|
|
232
|
-
return db
|
|
233
|
-
.query(
|
|
234
|
-
'SELECT blob_id, media_type FROM _syncular_blob_uploads ORDER BY created_at_ms',
|
|
235
|
-
)
|
|
236
|
-
.map((row) => ({
|
|
237
|
-
blobId: row.blob_id as string,
|
|
238
|
-
...(row.media_type !== null
|
|
239
|
-
? { mediaType: row.media_type as string }
|
|
240
|
-
: {}),
|
|
241
|
-
}));
|
|
242
|
-
}
|
|
243
|
-
|
|
244
256
|
export function clearPendingUpload(db: ClientDatabase, blobId: string): void {
|
|
245
257
|
db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id = ?', [blobId]);
|
|
246
258
|
}
|
|
247
259
|
|
|
248
260
|
/**
|
|
249
|
-
* All `blob_ref` column names per table
|
|
250
|
-
* Blank result ⇒ the schema has no attachments; callers skip reconciliation.
|
|
261
|
+
* All `blob_ref` column names per table.
|
|
251
262
|
*/
|
|
252
263
|
export function blobRefColumnsBySchema(
|
|
253
264
|
schema: CompiledClientSchema,
|
|
@@ -269,64 +280,35 @@ export function schemaHasBlobs(schema: CompiledClientSchema): boolean {
|
|
|
269
280
|
return false;
|
|
270
281
|
}
|
|
271
282
|
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
* to zero refs *and* have no pending upload (a pending upload pins its body,
|
|
276
|
-
* B4). Called after every apply/purge that may add or remove references.
|
|
277
|
-
*
|
|
278
|
-
* `deleteOrphans` distinguishes the two B2 transitions: revocation purge
|
|
279
|
-
* passes `true` (drop the now-unauthorized body); a benign apply passes
|
|
280
|
-
* `false` (retain zero-ref bodies as LRU cache entries — the shipped
|
|
281
|
-
* default). Bodies pinned by a pending upload are always retained.
|
|
282
|
-
*/
|
|
283
|
-
export function reconcileBlobRefcounts(
|
|
284
|
-
db: ClientDatabase,
|
|
285
|
-
schema: CompiledClientSchema,
|
|
286
|
-
options?: { readonly deleteOrphans?: boolean },
|
|
287
|
-
): void {
|
|
288
|
-
const byTable = blobRefColumnsBySchema(schema);
|
|
289
|
-
if (byTable.size === 0) return;
|
|
290
|
-
// Count references to each blobId across every blob_ref column.
|
|
291
|
-
const counts = new Map<string, number>();
|
|
292
|
-
for (const [tableName, columns] of byTable) {
|
|
283
|
+
function visibleBlobIdsSql(schema: CompiledClientSchema): string {
|
|
284
|
+
const selects: string[] = [];
|
|
285
|
+
for (const [tableName, columns] of blobRefColumnsBySchema(schema)) {
|
|
293
286
|
for (const column of columns) {
|
|
294
|
-
const
|
|
295
|
-
|
|
296
|
-
|
|
287
|
+
const identifier = quoteIdent(column);
|
|
288
|
+
selects.push(
|
|
289
|
+
`SELECT json_extract(${identifier}, '$.blobId') AS blob_id
|
|
290
|
+
FROM ${quoteIdent(tableName)}
|
|
291
|
+
WHERE typeof(${identifier}) = 'text'
|
|
292
|
+
AND json_valid(${identifier})
|
|
293
|
+
AND json_type(${identifier}, '$.blobId') = 'text'`,
|
|
297
294
|
);
|
|
298
|
-
for (const row of rows) {
|
|
299
|
-
const raw = row.v;
|
|
300
|
-
if (typeof raw !== 'string') continue;
|
|
301
|
-
let ref: BlobRef;
|
|
302
|
-
try {
|
|
303
|
-
ref = parseBlobRef(raw);
|
|
304
|
-
} catch {
|
|
305
|
-
continue;
|
|
306
|
-
}
|
|
307
|
-
counts.set(ref.blobId, (counts.get(ref.blobId) ?? 0) + 1);
|
|
308
|
-
}
|
|
309
295
|
}
|
|
310
296
|
}
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)`,
|
|
327
|
-
);
|
|
328
|
-
}
|
|
329
|
-
});
|
|
297
|
+
return selects.length === 0
|
|
298
|
+
? 'SELECT NULL AS blob_id WHERE 0'
|
|
299
|
+
: selects.join(' UNION ');
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
export function deleteUnreferencedCachedBlobs(
|
|
303
|
+
db: ClientDatabase,
|
|
304
|
+
schema: CompiledClientSchema,
|
|
305
|
+
): void {
|
|
306
|
+
db.exec(
|
|
307
|
+
`DELETE FROM _syncular_blobs
|
|
308
|
+
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
309
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
310
|
+
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})`,
|
|
311
|
+
);
|
|
330
312
|
}
|
|
331
313
|
|
|
332
314
|
export type { BlobRef };
|
package/src/client.ts
CHANGED
|
@@ -47,13 +47,13 @@ import {
|
|
|
47
47
|
type CachedBlob,
|
|
48
48
|
clearPendingUpload,
|
|
49
49
|
computeBlobId,
|
|
50
|
+
deleteUnreferencedCachedBlobs,
|
|
50
51
|
enforceBlobCacheCap,
|
|
51
52
|
ensureBlobSchema,
|
|
52
53
|
getCachedBlob,
|
|
53
54
|
listPendingUploads,
|
|
54
55
|
parseBlobRef,
|
|
55
56
|
putCachedBlob,
|
|
56
|
-
reconcileBlobRefcounts,
|
|
57
57
|
recordPendingUpload,
|
|
58
58
|
schemaHasBlobs,
|
|
59
59
|
serializeBlobRef,
|
|
@@ -302,9 +302,9 @@ export interface SyncClientConfig {
|
|
|
302
302
|
readonly blobs?: BlobTransport;
|
|
303
303
|
/**
|
|
304
304
|
* §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
|
|
305
|
-
* exceeds this,
|
|
306
|
-
* cache write
|
|
307
|
-
*
|
|
305
|
+
* exceeds this, unreferenced bodies without pending work are evicted in
|
|
306
|
+
* creation timestamp order after each cache write. Referenced bodies remain cached
|
|
307
|
+
* even when that exceeds the cap. Absent means retain until storage pressure (the shipped
|
|
308
308
|
* default; a referenced body always stays resolvable without a re-download).
|
|
309
309
|
*/
|
|
310
310
|
readonly blobCacheMaxBytes?: number;
|
|
@@ -1455,7 +1455,8 @@ export class SyncClient {
|
|
|
1455
1455
|
bytes: Uint8Array,
|
|
1456
1456
|
options?: { readonly mediaType?: string; readonly name?: string },
|
|
1457
1457
|
): Promise<BlobRef> {
|
|
1458
|
-
|
|
1458
|
+
const owned = bytes.slice();
|
|
1459
|
+
return this.#runProtectedAsync(() => this.#uploadBlob(owned, options));
|
|
1459
1460
|
}
|
|
1460
1461
|
|
|
1461
1462
|
async #uploadBlob(
|
|
@@ -1469,13 +1470,11 @@ export class SyncClient {
|
|
|
1469
1470
|
);
|
|
1470
1471
|
}
|
|
1471
1472
|
const blobId = await computeBlobId(bytes);
|
|
1473
|
+
const nowMs = this.#now();
|
|
1472
1474
|
this.#db.transaction(() => {
|
|
1473
|
-
putCachedBlob(this.#db, blobId, bytes,
|
|
1474
|
-
recordPendingUpload(this.#db, blobId,
|
|
1475
|
+
putCachedBlob(this.#db, blobId, bytes, nowMs, options?.mediaType);
|
|
1476
|
+
recordPendingUpload(this.#db, blobId, nowMs, options?.mediaType);
|
|
1475
1477
|
});
|
|
1476
|
-
// §5.9.7 B1: a staged upload is pinned (recordPendingUpload), so the cap
|
|
1477
|
-
// trim below will never evict it — but a stage may push other zero-ref
|
|
1478
|
-
// bodies over the cap, so run the trim.
|
|
1479
1478
|
this.#enforceBlobCacheCap();
|
|
1480
1479
|
return {
|
|
1481
1480
|
blobId,
|
|
@@ -1506,7 +1505,7 @@ export class SyncClient {
|
|
|
1506
1505
|
const blobId = blobIdOrRef.startsWith('sha256:')
|
|
1507
1506
|
? blobIdOrRef
|
|
1508
1507
|
: parseBlobRef(blobIdOrRef).blobId;
|
|
1509
|
-
const cached = getCachedBlob(this.#db, blobId
|
|
1508
|
+
const cached = getCachedBlob(this.#db, blobId);
|
|
1510
1509
|
if (cached !== undefined) return cached;
|
|
1511
1510
|
const transport = this.#config.blobs;
|
|
1512
1511
|
if (transport === undefined) {
|
|
@@ -1554,9 +1553,7 @@ export class SyncClient {
|
|
|
1554
1553
|
);
|
|
1555
1554
|
}
|
|
1556
1555
|
putCachedBlob(this.#db, blobId, bytes, this.#now());
|
|
1557
|
-
|
|
1558
|
-
// from current visible references before applying the size cap (§5.9.7 B1).
|
|
1559
|
-
this.#reconcileBlobs(false);
|
|
1556
|
+
this.#db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
1560
1557
|
this.#enforceBlobCacheCap();
|
|
1561
1558
|
const stored = getCachedBlob(this.#db, blobId);
|
|
1562
1559
|
if (stored === undefined) {
|
|
@@ -1572,7 +1569,7 @@ export class SyncClient {
|
|
|
1572
1569
|
#enforceBlobCacheCap(): void {
|
|
1573
1570
|
const cap = this.#config.blobCacheMaxBytes;
|
|
1574
1571
|
if (cap === undefined) return;
|
|
1575
|
-
enforceBlobCacheCap(this.#db, cap);
|
|
1572
|
+
enforceBlobCacheCap(this.#db, this.#schema, cap);
|
|
1576
1573
|
}
|
|
1577
1574
|
|
|
1578
1575
|
/** Flush any queued blob uploads (§5.9.7 B4); safe to call standalone. */
|
|
@@ -1585,11 +1582,17 @@ export class SyncClient {
|
|
|
1585
1582
|
if (transport === undefined || !this.#hasBlobs) return;
|
|
1586
1583
|
for (const pending of listPendingUploads(this.#db)) {
|
|
1587
1584
|
const cached = getCachedBlob(this.#db, pending.blobId);
|
|
1588
|
-
if (
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1585
|
+
if (
|
|
1586
|
+
cached === undefined ||
|
|
1587
|
+
!(cached.bytes instanceof Uint8Array) ||
|
|
1588
|
+
!Number.isSafeInteger(cached.byteLength) ||
|
|
1589
|
+
cached.byteLength !== cached.bytes.byteLength ||
|
|
1590
|
+
(await computeBlobId(cached.bytes)) !== pending.blobId
|
|
1591
|
+
) {
|
|
1592
|
+
throw new ClientSyncError(
|
|
1593
|
+
'sync.local_corrupt',
|
|
1594
|
+
'Pending blob upload body is missing or corrupt',
|
|
1595
|
+
);
|
|
1593
1596
|
}
|
|
1594
1597
|
await this.#uploadOne(
|
|
1595
1598
|
transport,
|
|
@@ -2141,6 +2144,29 @@ export class SyncClient {
|
|
|
2141
2144
|
this.#now(),
|
|
2142
2145
|
this.#captureBeforeImages(operations),
|
|
2143
2146
|
);
|
|
2147
|
+
for (const operation of operations) {
|
|
2148
|
+
if (operation.op !== 'upsert') continue;
|
|
2149
|
+
const table = this.#schema.tables.get(operation.table);
|
|
2150
|
+
if (table === undefined) continue;
|
|
2151
|
+
for (const column of table.columns) {
|
|
2152
|
+
if (column.type !== 'blob_ref') continue;
|
|
2153
|
+
const raw = operation.values?.[column.name];
|
|
2154
|
+
if (typeof raw !== 'string') continue;
|
|
2155
|
+
let blobId: string;
|
|
2156
|
+
try {
|
|
2157
|
+
blobId = parseBlobRef(raw).blobId;
|
|
2158
|
+
} catch {
|
|
2159
|
+
continue;
|
|
2160
|
+
}
|
|
2161
|
+
this.#db.exec(
|
|
2162
|
+
`INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
|
|
2163
|
+
SELECT ?, ? WHERE EXISTS (
|
|
2164
|
+
SELECT 1 FROM _syncular_blobs WHERE blob_id = ?
|
|
2165
|
+
)`,
|
|
2166
|
+
[clientCommitId, blobId, blobId],
|
|
2167
|
+
);
|
|
2168
|
+
}
|
|
2169
|
+
}
|
|
2144
2170
|
this.#applyOperationsLocally(operations, batch);
|
|
2145
2171
|
batch.status();
|
|
2146
2172
|
});
|
|
@@ -2339,7 +2365,7 @@ export class SyncClient {
|
|
|
2339
2365
|
batch.rejections();
|
|
2340
2366
|
batch.outcomes();
|
|
2341
2367
|
}
|
|
2342
|
-
this.#
|
|
2368
|
+
this.#deleteUnreferencedBlobs();
|
|
2343
2369
|
setMeta(this.#db, metaKey, purge.canonicalPlan);
|
|
2344
2370
|
return {
|
|
2345
2371
|
alreadyApplied: false,
|
|
@@ -3517,7 +3543,6 @@ export class SyncClient {
|
|
|
3517
3543
|
// §7.1: local reads see outbox state applied optimistically — replay
|
|
3518
3544
|
// the still-pending commits on top of the freshly applied server state.
|
|
3519
3545
|
this.#replayOutbox();
|
|
3520
|
-
this.#reconcileBlobs(false);
|
|
3521
3546
|
} finally {
|
|
3522
3547
|
this.#endDiagnosticsDeferral();
|
|
3523
3548
|
}
|
|
@@ -4000,7 +4025,7 @@ export class SyncClient {
|
|
|
4000
4025
|
batch.rejections();
|
|
4001
4026
|
batch.outcomes();
|
|
4002
4027
|
}
|
|
4003
|
-
this.#
|
|
4028
|
+
this.#deleteUnreferencedBlobs();
|
|
4004
4029
|
} catch (error) {
|
|
4005
4030
|
if (
|
|
4006
4031
|
error instanceof ClientSyncError &&
|
|
@@ -4218,14 +4243,9 @@ export class SyncClient {
|
|
|
4218
4243
|
});
|
|
4219
4244
|
}
|
|
4220
4245
|
|
|
4221
|
-
|
|
4222
|
-
* §5.9.7 B1/B2: recompute blob-cache refcounts from live `blob_ref`
|
|
4223
|
-
* columns. No-op unless the schema has blob columns. `deleteOrphans`
|
|
4224
|
-
* triggers the revocation-side body deletion (B2).
|
|
4225
|
-
*/
|
|
4226
|
-
#reconcileBlobs(deleteOrphans: boolean): void {
|
|
4246
|
+
#deleteUnreferencedBlobs(): void {
|
|
4227
4247
|
if (!this.#hasBlobs) return;
|
|
4228
|
-
|
|
4248
|
+
deleteUnreferencedCachedBlobs(this.#db, this.#schema);
|
|
4229
4249
|
}
|
|
4230
4250
|
|
|
4231
4251
|
// -- helpers -----------------------------------------------------------------
|
package/src/outbox.ts
CHANGED
|
@@ -135,6 +135,9 @@ export function deleteOutboxCommit(
|
|
|
135
135
|
db: ClientDatabase,
|
|
136
136
|
clientCommitId: string,
|
|
137
137
|
): void {
|
|
138
|
+
db.exec('DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?', [
|
|
139
|
+
clientCommitId,
|
|
140
|
+
]);
|
|
138
141
|
db.exec(
|
|
139
142
|
'DELETE FROM _syncular_outbox_before_images WHERE client_commit_id = ?',
|
|
140
143
|
[clientCommitId],
|
package/src/schema.ts
CHANGED
|
@@ -501,6 +501,11 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
|
|
|
501
501
|
db.transaction(() => {
|
|
502
502
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
|
|
503
503
|
key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
|
|
504
|
+
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
|
|
505
|
+
commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
|
|
506
|
+
db.exec(
|
|
507
|
+
'CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)',
|
|
508
|
+
);
|
|
504
509
|
db.exec(
|
|
505
510
|
`INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`,
|
|
506
511
|
);
|