@syncular/client 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blob.d.ts +2 -0
- package/dist/blob.js +48 -9
- package/dist/client.js +47 -11
- package/package.json +3 -3
- package/src/blob.ts +63 -10
- package/src/client.ts +60 -10
package/dist/blob.d.ts
CHANGED
|
@@ -115,12 +115,14 @@ export declare function schemaHasBlobs(schema: CompiledClientSchema): boolean;
|
|
|
115
115
|
* to zero refs *and* have no pending upload (a pending upload pins its body,
|
|
116
116
|
* B4). Called after every apply/purge that may add or remove references.
|
|
117
117
|
*
|
|
118
|
+
* `blobId` narrows a fresh-download refresh to the inserted cache row.
|
|
118
119
|
* `deleteOrphans` distinguishes the two B2 transitions: revocation purge
|
|
119
120
|
* passes `true` (drop the now-unauthorized body); a benign apply passes
|
|
120
121
|
* `false` (retain zero-ref bodies as LRU cache entries — the shipped
|
|
121
122
|
* default). Bodies pinned by a pending upload are always retained.
|
|
122
123
|
*/
|
|
123
124
|
export declare function reconcileBlobRefcounts(db: ClientDatabase, schema: CompiledClientSchema, options?: {
|
|
125
|
+
readonly blobId?: string;
|
|
124
126
|
readonly deleteOrphans?: boolean;
|
|
125
127
|
}): void;
|
|
126
128
|
export type { BlobRef };
|
package/dist/blob.js
CHANGED
|
@@ -11,10 +11,14 @@
|
|
|
11
11
|
* outbox-adjacent uploads table (B4) and flushed before push.
|
|
12
12
|
*/
|
|
13
13
|
import { parseBlobRef, serializeBlobRef } from '@syncular/core';
|
|
14
|
+
import { ClientSyncError } from './errors.js';
|
|
14
15
|
import { quoteIdent } from './schema.js';
|
|
15
16
|
/** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
|
|
16
17
|
export async function computeBlobId(bytes) {
|
|
17
|
-
const
|
|
18
|
+
const input = bytes.buffer instanceof ArrayBuffer
|
|
19
|
+
? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
20
|
+
: bytes.slice();
|
|
21
|
+
const digest = await crypto.subtle.digest('SHA-256', input);
|
|
18
22
|
const hex = [...new Uint8Array(digest)]
|
|
19
23
|
.map((b) => b.toString(16).padStart(2, '0'))
|
|
20
24
|
.join('');
|
|
@@ -42,6 +46,9 @@ export function ensureBlobSchema(db) {
|
|
|
42
46
|
blob_id TEXT PRIMARY KEY,
|
|
43
47
|
media_type TEXT,
|
|
44
48
|
created_at_ms INTEGER NOT NULL)`);
|
|
49
|
+
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
|
|
50
|
+
commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
|
|
51
|
+
db.exec('CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)');
|
|
45
52
|
}
|
|
46
53
|
/** Put bytes into the content-addressed cache (idempotent); touches LRU. */
|
|
47
54
|
export function putCachedBlob(db, blobId, bytes, nowMs, mediaType) {
|
|
@@ -90,6 +97,7 @@ export function enforceBlobCacheCap(db, maxBytes) {
|
|
|
90
97
|
const candidates = db.query(`SELECT blob_id, byte_length FROM _syncular_blobs
|
|
91
98
|
WHERE refcount = 0
|
|
92
99
|
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
100
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
93
101
|
ORDER BY last_used_ms ASC, created_at_ms ASC`);
|
|
94
102
|
const evicted = [];
|
|
95
103
|
db.transaction(() => {
|
|
@@ -111,13 +119,23 @@ export function recordPendingUpload(db, blobId, nowMs, mediaType) {
|
|
|
111
119
|
}
|
|
112
120
|
export function listPendingUploads(db) {
|
|
113
121
|
return db
|
|
114
|
-
.query(
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
122
|
+
.query(`SELECT blob_id, media_type FROM (
|
|
123
|
+
SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
|
|
124
|
+
UNION ALL
|
|
125
|
+
SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
|
|
126
|
+
FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
|
|
127
|
+
WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
|
|
128
|
+
) ORDER BY created_at_ms, blob_id`)
|
|
129
|
+
.map((row) => {
|
|
130
|
+
if (typeof row.blob_id !== 'string' ||
|
|
131
|
+
(row.media_type !== null && typeof row.media_type !== 'string')) {
|
|
132
|
+
throw new ClientSyncError('sync.local_corrupt', 'Pending blob upload metadata is invalid');
|
|
133
|
+
}
|
|
134
|
+
return {
|
|
135
|
+
blobId: row.blob_id,
|
|
136
|
+
...(row.media_type !== null ? { mediaType: row.media_type } : {}),
|
|
137
|
+
};
|
|
138
|
+
});
|
|
121
139
|
}
|
|
122
140
|
export function clearPendingUpload(db, blobId) {
|
|
123
141
|
db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id = ?', [blobId]);
|
|
@@ -150,6 +168,7 @@ export function schemaHasBlobs(schema) {
|
|
|
150
168
|
* to zero refs *and* have no pending upload (a pending upload pins its body,
|
|
151
169
|
* B4). Called after every apply/purge that may add or remove references.
|
|
152
170
|
*
|
|
171
|
+
* `blobId` narrows a fresh-download refresh to the inserted cache row.
|
|
153
172
|
* `deleteOrphans` distinguishes the two B2 transitions: revocation purge
|
|
154
173
|
* passes `true` (drop the now-unauthorized body); a benign apply passes
|
|
155
174
|
* `false` (retain zero-ref bodies as LRU cache entries — the shipped
|
|
@@ -159,6 +178,25 @@ export function reconcileBlobRefcounts(db, schema, options) {
|
|
|
159
178
|
const byTable = blobRefColumnsBySchema(schema);
|
|
160
179
|
if (byTable.size === 0)
|
|
161
180
|
return;
|
|
181
|
+
if (options?.blobId !== undefined) {
|
|
182
|
+
let count = 0;
|
|
183
|
+
for (const [tableName, columns] of byTable) {
|
|
184
|
+
for (const column of columns) {
|
|
185
|
+
const identifier = quoteIdent(column);
|
|
186
|
+
const row = db.query(`SELECT count(*) AS n FROM ${quoteIdent(tableName)}
|
|
187
|
+
WHERE CASE
|
|
188
|
+
WHEN typeof(${identifier}) = 'text' AND json_valid(${identifier})
|
|
189
|
+
THEN json_extract(${identifier}, '$.blobId')
|
|
190
|
+
END = ?`, [options.blobId])[0];
|
|
191
|
+
count += Number(row?.n ?? 0);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
|
|
195
|
+
count,
|
|
196
|
+
options.blobId,
|
|
197
|
+
]);
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
162
200
|
// Count references to each blobId across every blob_ref column.
|
|
163
201
|
const counts = new Map();
|
|
164
202
|
for (const [tableName, columns] of byTable) {
|
|
@@ -194,7 +232,8 @@ export function reconcileBlobRefcounts(db, schema, options) {
|
|
|
194
232
|
// pending upload.
|
|
195
233
|
db.exec(`DELETE FROM _syncular_blobs
|
|
196
234
|
WHERE refcount = 0
|
|
197
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
235
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
236
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)`);
|
|
198
237
|
}
|
|
199
238
|
});
|
|
200
239
|
}
|
package/dist/client.js
CHANGED
|
@@ -191,8 +191,39 @@ export class SyncClient {
|
|
|
191
191
|
// Do not materialize new app indexes/FTS projections yet: on a version
|
|
192
192
|
// bump they may reference columns that only exist after the reset.
|
|
193
193
|
ensureLocalBookkeepingSchema(this.#db);
|
|
194
|
-
|
|
195
|
-
|
|
194
|
+
this.#db.transaction(() => {
|
|
195
|
+
for (const trigger of ['insert', 'update', 'delete', 'cache'])
|
|
196
|
+
this.#db.exec(`DROP TRIGGER IF EXISTS _syncular_blob_commit_${trigger}`);
|
|
197
|
+
if (this.#hasBlobs) {
|
|
198
|
+
ensureBlobSchema(this.#db);
|
|
199
|
+
const columns = JSON.stringify([...this.#schema.tables.values()].flatMap((table) => table.columns
|
|
200
|
+
.filter((column) => column.type === 'blob_ref')
|
|
201
|
+
.map((column) => [table.name, column.name]))).replaceAll("'", "''");
|
|
202
|
+
const references = `SELECT o.client_commit_id, b.blob_id
|
|
203
|
+
FROM _syncular_outbox o JOIN json_each(o.operations) op
|
|
204
|
+
JOIN json_each(op.value, '$.values') v JOIN json_each('${columns}') c
|
|
205
|
+
JOIN _syncular_blobs b ON b.blob_id = CASE WHEN v.type = 'text' AND json_valid(v.value) THEN json_extract(v.value, '$.blobId') END
|
|
206
|
+
WHERE json_extract(op.value, '$.op') = 'upsert'
|
|
207
|
+
AND json_extract(op.value, '$.table') = json_extract(c.value, '$[0]')
|
|
208
|
+
AND v.key = json_extract(c.value, '$[1]')`;
|
|
209
|
+
this.#db
|
|
210
|
+
.exec(`CREATE TRIGGER _syncular_blob_commit_insert AFTER INSERT ON _syncular_outbox BEGIN
|
|
211
|
+
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
|
|
212
|
+
this.#db
|
|
213
|
+
.exec(`CREATE TRIGGER _syncular_blob_commit_update AFTER UPDATE OF operations ON _syncular_outbox BEGIN
|
|
214
|
+
DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id;
|
|
215
|
+
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
|
|
216
|
+
this.#db
|
|
217
|
+
.exec(`CREATE TRIGGER _syncular_blob_commit_delete BEFORE DELETE ON _syncular_outbox BEGIN
|
|
218
|
+
DELETE FROM _syncular_blob_uploads WHERE blob_id IN (SELECT blob_id FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id)
|
|
219
|
+
AND NOT EXISTS (SELECT 1 FROM _syncular_blob_commit_refs r WHERE r.blob_id = _syncular_blob_uploads.blob_id AND r.commit_id != OLD.client_commit_id);
|
|
220
|
+
DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id; END`);
|
|
221
|
+
this.#db
|
|
222
|
+
.exec(`CREATE TRIGGER _syncular_blob_commit_cache AFTER INSERT ON _syncular_blobs BEGIN
|
|
223
|
+
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND b.blob_id = NEW.blob_id; END`);
|
|
224
|
+
this.#db.exec(`INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references}`);
|
|
225
|
+
}
|
|
226
|
+
});
|
|
196
227
|
this.#db.transaction(() => {
|
|
197
228
|
pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
|
|
198
229
|
});
|
|
@@ -854,7 +885,8 @@ export class SyncClient {
|
|
|
854
885
|
* (via `mutate`) after this call so upload-before-push holds (§5.9.3).
|
|
855
886
|
*/
|
|
856
887
|
uploadBlob(bytes, options) {
|
|
857
|
-
|
|
888
|
+
const owned = bytes.slice();
|
|
889
|
+
return this.#runProtectedAsync(() => this.#uploadBlob(owned, options));
|
|
858
890
|
}
|
|
859
891
|
async #uploadBlob(bytes, options) {
|
|
860
892
|
if (this.#config.blobs === undefined) {
|
|
@@ -932,7 +964,7 @@ export class SyncClient {
|
|
|
932
964
|
putCachedBlob(this.#db, blobId, bytes, this.#now());
|
|
933
965
|
// The referencing row can arrive before its body. Pin the new cache entry
|
|
934
966
|
// from current visible references before applying the size cap (§5.9.7 B1).
|
|
935
|
-
this.#reconcileBlobs(false);
|
|
967
|
+
this.#reconcileBlobs(false, blobId);
|
|
936
968
|
this.#enforceBlobCacheCap();
|
|
937
969
|
const stored = getCachedBlob(this.#db, blobId);
|
|
938
970
|
if (stored === undefined) {
|
|
@@ -957,11 +989,12 @@ export class SyncClient {
|
|
|
957
989
|
return;
|
|
958
990
|
for (const pending of listPendingUploads(this.#db)) {
|
|
959
991
|
const cached = getCachedBlob(this.#db, pending.blobId);
|
|
960
|
-
if (cached === undefined
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
992
|
+
if (cached === undefined ||
|
|
993
|
+
!(cached.bytes instanceof Uint8Array) ||
|
|
994
|
+
!Number.isSafeInteger(cached.byteLength) ||
|
|
995
|
+
cached.byteLength !== cached.bytes.byteLength ||
|
|
996
|
+
(await computeBlobId(cached.bytes)) !== pending.blobId) {
|
|
997
|
+
throw new ClientSyncError('sync.local_corrupt', 'Pending blob upload body is missing or corrupt');
|
|
965
998
|
}
|
|
966
999
|
await this.#uploadOne(transport, pending.blobId, cached.bytes, pending.mediaType);
|
|
967
1000
|
clearPendingUpload(this.#db, pending.blobId);
|
|
@@ -3079,10 +3112,13 @@ export class SyncClient {
|
|
|
3079
3112
|
* columns. No-op unless the schema has blob columns. `deleteOrphans`
|
|
3080
3113
|
* triggers the revocation-side body deletion (B2).
|
|
3081
3114
|
*/
|
|
3082
|
-
#reconcileBlobs(deleteOrphans) {
|
|
3115
|
+
#reconcileBlobs(deleteOrphans, blobId) {
|
|
3083
3116
|
if (!this.#hasBlobs)
|
|
3084
3117
|
return;
|
|
3085
|
-
reconcileBlobRefcounts(this.#db, this.#schema, {
|
|
3118
|
+
reconcileBlobRefcounts(this.#db, this.#schema, {
|
|
3119
|
+
deleteOrphans,
|
|
3120
|
+
...(blobId === undefined ? {} : { blobId }),
|
|
3121
|
+
});
|
|
3086
3122
|
}
|
|
3087
3123
|
// -- helpers -----------------------------------------------------------------
|
|
3088
3124
|
#table(name) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.18.0",
|
|
4
4
|
"description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -89,9 +89,9 @@
|
|
|
89
89
|
},
|
|
90
90
|
"dependencies": {
|
|
91
91
|
"@sqlite.org/sqlite-wasm": "^3.53.0-build1",
|
|
92
|
-
"@syncular/core": "0.
|
|
92
|
+
"@syncular/core": "0.18.0"
|
|
93
93
|
},
|
|
94
94
|
"devDependencies": {
|
|
95
|
-
"@syncular/server": "0.
|
|
95
|
+
"@syncular/server": "0.18.0"
|
|
96
96
|
}
|
|
97
97
|
}
|
package/src/blob.ts
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
*/
|
|
13
13
|
import { type BlobRef, parseBlobRef, serializeBlobRef } from '@syncular/core';
|
|
14
14
|
import type { ClientDatabase } from './database';
|
|
15
|
+
import { ClientSyncError } from './errors';
|
|
15
16
|
import type { CompiledClientSchema } from './schema';
|
|
16
17
|
import { quoteIdent } from './schema';
|
|
17
18
|
|
|
@@ -93,9 +94,13 @@ export interface CachedBlob {
|
|
|
93
94
|
|
|
94
95
|
/** `"sha256:" + hex` of the bytes — the content address (§5.9.1). */
|
|
95
96
|
export async function computeBlobId(bytes: Uint8Array): Promise<string> {
|
|
97
|
+
const input =
|
|
98
|
+
bytes.buffer instanceof ArrayBuffer
|
|
99
|
+
? new Uint8Array(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
100
|
+
: bytes.slice();
|
|
96
101
|
const digest = await crypto.subtle.digest(
|
|
97
102
|
'SHA-256',
|
|
98
|
-
|
|
103
|
+
input as Uint8Array<ArrayBuffer>,
|
|
99
104
|
);
|
|
100
105
|
const hex = [...new Uint8Array(digest)]
|
|
101
106
|
.map((b) => b.toString(16).padStart(2, '0'))
|
|
@@ -126,6 +131,11 @@ export function ensureBlobSchema(db: ClientDatabase): void {
|
|
|
126
131
|
blob_id TEXT PRIMARY KEY,
|
|
127
132
|
media_type TEXT,
|
|
128
133
|
created_at_ms INTEGER NOT NULL)`);
|
|
134
|
+
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
|
|
135
|
+
commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
|
|
136
|
+
db.exec(
|
|
137
|
+
'CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)',
|
|
138
|
+
);
|
|
129
139
|
}
|
|
130
140
|
|
|
131
141
|
/** Put bytes into the content-addressed cache (idempotent); touches LRU. */
|
|
@@ -197,6 +207,7 @@ export function enforceBlobCacheCap(
|
|
|
197
207
|
`SELECT blob_id, byte_length FROM _syncular_blobs
|
|
198
208
|
WHERE refcount = 0
|
|
199
209
|
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
210
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
200
211
|
ORDER BY last_used_ms ASC, created_at_ms ASC`,
|
|
201
212
|
);
|
|
202
213
|
const evicted: string[] = [];
|
|
@@ -231,14 +242,29 @@ export function listPendingUploads(
|
|
|
231
242
|
): { blobId: string; mediaType?: string }[] {
|
|
232
243
|
return db
|
|
233
244
|
.query(
|
|
234
|
-
|
|
245
|
+
`SELECT blob_id, media_type FROM (
|
|
246
|
+
SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
|
|
247
|
+
UNION ALL
|
|
248
|
+
SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
|
|
249
|
+
FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
|
|
250
|
+
WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
|
|
251
|
+
) ORDER BY created_at_ms, blob_id`,
|
|
235
252
|
)
|
|
236
|
-
.map((row) =>
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
253
|
+
.map((row) => {
|
|
254
|
+
if (
|
|
255
|
+
typeof row.blob_id !== 'string' ||
|
|
256
|
+
(row.media_type !== null && typeof row.media_type !== 'string')
|
|
257
|
+
) {
|
|
258
|
+
throw new ClientSyncError(
|
|
259
|
+
'sync.local_corrupt',
|
|
260
|
+
'Pending blob upload metadata is invalid',
|
|
261
|
+
);
|
|
262
|
+
}
|
|
263
|
+
return {
|
|
264
|
+
blobId: row.blob_id,
|
|
265
|
+
...(row.media_type !== null ? { mediaType: row.media_type } : {}),
|
|
266
|
+
};
|
|
267
|
+
});
|
|
242
268
|
}
|
|
243
269
|
|
|
244
270
|
export function clearPendingUpload(db: ClientDatabase, blobId: string): void {
|
|
@@ -275,6 +301,7 @@ export function schemaHasBlobs(schema: CompiledClientSchema): boolean {
|
|
|
275
301
|
* to zero refs *and* have no pending upload (a pending upload pins its body,
|
|
276
302
|
* B4). Called after every apply/purge that may add or remove references.
|
|
277
303
|
*
|
|
304
|
+
* `blobId` narrows a fresh-download refresh to the inserted cache row.
|
|
278
305
|
* `deleteOrphans` distinguishes the two B2 transitions: revocation purge
|
|
279
306
|
* passes `true` (drop the now-unauthorized body); a benign apply passes
|
|
280
307
|
* `false` (retain zero-ref bodies as LRU cache entries — the shipped
|
|
@@ -283,10 +310,35 @@ export function schemaHasBlobs(schema: CompiledClientSchema): boolean {
|
|
|
283
310
|
export function reconcileBlobRefcounts(
|
|
284
311
|
db: ClientDatabase,
|
|
285
312
|
schema: CompiledClientSchema,
|
|
286
|
-
options?: {
|
|
313
|
+
options?: {
|
|
314
|
+
readonly blobId?: string;
|
|
315
|
+
readonly deleteOrphans?: boolean;
|
|
316
|
+
},
|
|
287
317
|
): void {
|
|
288
318
|
const byTable = blobRefColumnsBySchema(schema);
|
|
289
319
|
if (byTable.size === 0) return;
|
|
320
|
+
if (options?.blobId !== undefined) {
|
|
321
|
+
let count = 0;
|
|
322
|
+
for (const [tableName, columns] of byTable) {
|
|
323
|
+
for (const column of columns) {
|
|
324
|
+
const identifier = quoteIdent(column);
|
|
325
|
+
const row = db.query(
|
|
326
|
+
`SELECT count(*) AS n FROM ${quoteIdent(tableName)}
|
|
327
|
+
WHERE CASE
|
|
328
|
+
WHEN typeof(${identifier}) = 'text' AND json_valid(${identifier})
|
|
329
|
+
THEN json_extract(${identifier}, '$.blobId')
|
|
330
|
+
END = ?`,
|
|
331
|
+
[options.blobId],
|
|
332
|
+
)[0];
|
|
333
|
+
count += Number(row?.n ?? 0);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
db.exec('UPDATE _syncular_blobs SET refcount = ? WHERE blob_id = ?', [
|
|
337
|
+
count,
|
|
338
|
+
options.blobId,
|
|
339
|
+
]);
|
|
340
|
+
return;
|
|
341
|
+
}
|
|
290
342
|
// Count references to each blobId across every blob_ref column.
|
|
291
343
|
const counts = new Map<string, number>();
|
|
292
344
|
for (const [tableName, columns] of byTable) {
|
|
@@ -323,7 +375,8 @@ export function reconcileBlobRefcounts(
|
|
|
323
375
|
db.exec(
|
|
324
376
|
`DELETE FROM _syncular_blobs
|
|
325
377
|
WHERE refcount = 0
|
|
326
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
378
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
379
|
+
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)`,
|
|
327
380
|
);
|
|
328
381
|
}
|
|
329
382
|
});
|
package/src/client.ts
CHANGED
|
@@ -688,7 +688,47 @@ export class SyncClient {
|
|
|
688
688
|
// Do not materialize new app indexes/FTS projections yet: on a version
|
|
689
689
|
// bump they may reference columns that only exist after the reset.
|
|
690
690
|
ensureLocalBookkeepingSchema(this.#db);
|
|
691
|
-
|
|
691
|
+
this.#db.transaction(() => {
|
|
692
|
+
for (const trigger of ['insert', 'update', 'delete', 'cache'])
|
|
693
|
+
this.#db.exec(
|
|
694
|
+
`DROP TRIGGER IF EXISTS _syncular_blob_commit_${trigger}`,
|
|
695
|
+
);
|
|
696
|
+
if (this.#hasBlobs) {
|
|
697
|
+
ensureBlobSchema(this.#db);
|
|
698
|
+
const columns = JSON.stringify(
|
|
699
|
+
[...this.#schema.tables.values()].flatMap((table) =>
|
|
700
|
+
table.columns
|
|
701
|
+
.filter((column) => column.type === 'blob_ref')
|
|
702
|
+
.map((column) => [table.name, column.name]),
|
|
703
|
+
),
|
|
704
|
+
).replaceAll("'", "''");
|
|
705
|
+
const references = `SELECT o.client_commit_id, b.blob_id
|
|
706
|
+
FROM _syncular_outbox o JOIN json_each(o.operations) op
|
|
707
|
+
JOIN json_each(op.value, '$.values') v JOIN json_each('${columns}') c
|
|
708
|
+
JOIN _syncular_blobs b ON b.blob_id = CASE WHEN v.type = 'text' AND json_valid(v.value) THEN json_extract(v.value, '$.blobId') END
|
|
709
|
+
WHERE json_extract(op.value, '$.op') = 'upsert'
|
|
710
|
+
AND json_extract(op.value, '$.table') = json_extract(c.value, '$[0]')
|
|
711
|
+
AND v.key = json_extract(c.value, '$[1]')`;
|
|
712
|
+
this.#db
|
|
713
|
+
.exec(`CREATE TRIGGER _syncular_blob_commit_insert AFTER INSERT ON _syncular_outbox BEGIN
|
|
714
|
+
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
|
|
715
|
+
this.#db
|
|
716
|
+
.exec(`CREATE TRIGGER _syncular_blob_commit_update AFTER UPDATE OF operations ON _syncular_outbox BEGIN
|
|
717
|
+
DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id;
|
|
718
|
+
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
|
|
719
|
+
this.#db
|
|
720
|
+
.exec(`CREATE TRIGGER _syncular_blob_commit_delete BEFORE DELETE ON _syncular_outbox BEGIN
|
|
721
|
+
DELETE FROM _syncular_blob_uploads WHERE blob_id IN (SELECT blob_id FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id)
|
|
722
|
+
AND NOT EXISTS (SELECT 1 FROM _syncular_blob_commit_refs r WHERE r.blob_id = _syncular_blob_uploads.blob_id AND r.commit_id != OLD.client_commit_id);
|
|
723
|
+
DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id; END`);
|
|
724
|
+
this.#db
|
|
725
|
+
.exec(`CREATE TRIGGER _syncular_blob_commit_cache AFTER INSERT ON _syncular_blobs BEGIN
|
|
726
|
+
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND b.blob_id = NEW.blob_id; END`);
|
|
727
|
+
this.#db.exec(
|
|
728
|
+
`INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references}`,
|
|
729
|
+
);
|
|
730
|
+
}
|
|
731
|
+
});
|
|
692
732
|
this.#db.transaction(() => {
|
|
693
733
|
pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
|
|
694
734
|
});
|
|
@@ -1455,7 +1495,8 @@ export class SyncClient {
|
|
|
1455
1495
|
bytes: Uint8Array,
|
|
1456
1496
|
options?: { readonly mediaType?: string; readonly name?: string },
|
|
1457
1497
|
): Promise<BlobRef> {
|
|
1458
|
-
|
|
1498
|
+
const owned = bytes.slice();
|
|
1499
|
+
return this.#runProtectedAsync(() => this.#uploadBlob(owned, options));
|
|
1459
1500
|
}
|
|
1460
1501
|
|
|
1461
1502
|
async #uploadBlob(
|
|
@@ -1556,7 +1597,7 @@ export class SyncClient {
|
|
|
1556
1597
|
putCachedBlob(this.#db, blobId, bytes, this.#now());
|
|
1557
1598
|
// The referencing row can arrive before its body. Pin the new cache entry
|
|
1558
1599
|
// from current visible references before applying the size cap (§5.9.7 B1).
|
|
1559
|
-
this.#reconcileBlobs(false);
|
|
1600
|
+
this.#reconcileBlobs(false, blobId);
|
|
1560
1601
|
this.#enforceBlobCacheCap();
|
|
1561
1602
|
const stored = getCachedBlob(this.#db, blobId);
|
|
1562
1603
|
if (stored === undefined) {
|
|
@@ -1585,11 +1626,17 @@ export class SyncClient {
|
|
|
1585
1626
|
if (transport === undefined || !this.#hasBlobs) return;
|
|
1586
1627
|
for (const pending of listPendingUploads(this.#db)) {
|
|
1587
1628
|
const cached = getCachedBlob(this.#db, pending.blobId);
|
|
1588
|
-
if (
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1629
|
+
if (
|
|
1630
|
+
cached === undefined ||
|
|
1631
|
+
!(cached.bytes instanceof Uint8Array) ||
|
|
1632
|
+
!Number.isSafeInteger(cached.byteLength) ||
|
|
1633
|
+
cached.byteLength !== cached.bytes.byteLength ||
|
|
1634
|
+
(await computeBlobId(cached.bytes)) !== pending.blobId
|
|
1635
|
+
) {
|
|
1636
|
+
throw new ClientSyncError(
|
|
1637
|
+
'sync.local_corrupt',
|
|
1638
|
+
'Pending blob upload body is missing or corrupt',
|
|
1639
|
+
);
|
|
1593
1640
|
}
|
|
1594
1641
|
await this.#uploadOne(
|
|
1595
1642
|
transport,
|
|
@@ -4223,9 +4270,12 @@ export class SyncClient {
|
|
|
4223
4270
|
* columns. No-op unless the schema has blob columns. `deleteOrphans`
|
|
4224
4271
|
* triggers the revocation-side body deletion (B2).
|
|
4225
4272
|
*/
|
|
4226
|
-
#reconcileBlobs(deleteOrphans: boolean): void {
|
|
4273
|
+
#reconcileBlobs(deleteOrphans: boolean, blobId?: string): void {
|
|
4227
4274
|
if (!this.#hasBlobs) return;
|
|
4228
|
-
reconcileBlobRefcounts(this.#db, this.#schema, {
|
|
4275
|
+
reconcileBlobRefcounts(this.#db, this.#schema, {
|
|
4276
|
+
deleteOrphans,
|
|
4277
|
+
...(blobId === undefined ? {} : { blobId }),
|
|
4278
|
+
});
|
|
4229
4279
|
}
|
|
4230
4280
|
|
|
4231
4281
|
// -- helpers -----------------------------------------------------------------
|