@syncular/client 0.30.10 → 0.30.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -0
- package/dist/apply.d.ts +5 -1
- package/dist/apply.js +58 -58
- package/dist/blob.js +13 -39
- package/dist/client.js +178 -161
- package/dist/failed-overlay.d.ts +10 -5
- package/dist/failed-overlay.js +118 -36
- package/dist/outbox.d.ts +2 -1
- package/dist/outbox.js +20 -40
- package/dist/outcomes.d.ts +8 -0
- package/dist/outcomes.js +22 -30
- package/dist/previous-version.js +14 -28
- package/dist/schema.d.ts +3 -0
- package/dist/schema.js +89 -133
- package/dist/state.d.ts +6 -9
- package/dist/state.js +66 -32
- package/dist/wasm-database.js +40 -24
- package/dist/worker-host.js +37 -11
- package/package.json +3 -3
- package/src/apply.ts +95 -72
- package/src/blob.ts +17 -39
- package/src/client.ts +312 -214
- package/src/failed-overlay.ts +197 -48
- package/src/outbox.ts +25 -44
- package/src/outcomes.ts +40 -35
- package/src/previous-version.ts +14 -28
- package/src/schema.ts +154 -166
- package/src/state.ts +90 -33
- package/src/wasm-database.ts +56 -29
- package/src/worker-host.ts +34 -12
package/dist/wasm-database.js
CHANGED
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
* `ClientDatabase`. Browser-only — exercised by the demo, never imported
|
|
23
23
|
* by bun tests (subpath export `@syncular/client/wasm`).
|
|
24
24
|
*/
|
|
25
|
+
import { webLocksLeaderLock } from './leader-lock.js';
|
|
25
26
|
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
|
26
27
|
import { assertImageAlias, runTransaction, } from './database.js';
|
|
27
28
|
import { ClientSyncError, invalidRequest, STORAGE_BUSY_CODE, STORAGE_UNAVAILABLE_CODE, } from './errors.js';
|
|
@@ -37,6 +38,7 @@ class WasmClientDatabase {
|
|
|
37
38
|
#sqlite3;
|
|
38
39
|
/** RFC 0005: present only for a persistent OPFS database. */
|
|
39
40
|
#sah;
|
|
41
|
+
#closed = false;
|
|
40
42
|
#tx = { depth: 0 };
|
|
41
43
|
constructor(db, sqlite3, sah) {
|
|
42
44
|
this.#db = db;
|
|
@@ -128,7 +130,14 @@ class WasmClientDatabase {
|
|
|
128
130
|
}
|
|
129
131
|
}
|
|
130
132
|
close() {
|
|
133
|
+
if (this.#closed)
|
|
134
|
+
return;
|
|
131
135
|
this.#db.close();
|
|
136
|
+
// DB.close() leaves the SAH pool live. Close siblings first, then pause the
|
|
137
|
+
// VFS without deleting files, before releasing this worker's physical lock.
|
|
138
|
+
this.#sah?.util.pauseVfs();
|
|
139
|
+
this.#closed = true;
|
|
140
|
+
void this.#sah?.lease.release();
|
|
132
141
|
}
|
|
133
142
|
}
|
|
134
143
|
let sqlite3Init;
|
|
@@ -249,35 +258,42 @@ export async function openPersistentWasmDatabase(name, options) {
|
|
|
249
258
|
'requires OPFS (~2023+ browsers). There is no IndexedDB or ' +
|
|
250
259
|
'in-memory fallback for persistent mode.');
|
|
251
260
|
}
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
let pool = sahPools.get(directory);
|
|
255
|
-
if (pool === undefined) {
|
|
256
|
-
pool = sqlite3
|
|
257
|
-
.installOpfsSAHPoolVfs({
|
|
258
|
-
// VFS registration names must be unique per directory.
|
|
259
|
-
name: `syncular-sahpool-${directory.replace(/[^A-Za-z0-9]+/g, '-')}`,
|
|
260
|
-
directory,
|
|
261
|
-
// sqlite-wasm caches a rejected initialization promise by VFS name.
|
|
262
|
-
// Syncular removes its own rejected entry below, so allow a later
|
|
263
|
-
// open in the same worker to make a real attempt as well.
|
|
264
|
-
forceReinitIfPreviouslyFailed: true,
|
|
265
|
-
initialCapacity: Math.max(options?.initialCapacity ?? SAH_POOL_DEFAULT_CAPACITY, SAH_POOL_MIN_CAPACITY),
|
|
266
|
-
})
|
|
267
|
-
.catch((error) => {
|
|
268
|
-
sahPools.delete(directory);
|
|
269
|
-
throw opfsSahPoolError(error, directory);
|
|
270
|
-
});
|
|
271
|
-
sahPools.set(directory, pool);
|
|
261
|
+
if (!navigator.locks) {
|
|
262
|
+
throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, 'Persistent OPFS storage requires Web Locks');
|
|
272
263
|
}
|
|
273
|
-
const
|
|
274
|
-
|
|
264
|
+
const directory = (options?.directory ?? `.syncular/${name}`).replace(/^\/+|\/+$/g, '');
|
|
265
|
+
// The document's leader lease can end before its worker's SAHs close. This
|
|
266
|
+
// worker-owned lock has exactly the same lifetime as the physical pool.
|
|
267
|
+
const lease = await webLocksLeaderLock().acquire(`syncular-opfs/${directory}`);
|
|
268
|
+
let util;
|
|
269
|
+
let db;
|
|
275
270
|
try {
|
|
271
|
+
const sqlite3 = await initSqlite3();
|
|
272
|
+
let pool = sahPools.get(directory);
|
|
273
|
+
if (pool === undefined) {
|
|
274
|
+
pool = sqlite3
|
|
275
|
+
.installOpfsSAHPoolVfs({
|
|
276
|
+
name: `syncular-sahpool-${directory.replace(/[^A-Za-z0-9]+/g, '-')}`,
|
|
277
|
+
directory,
|
|
278
|
+
forceReinitIfPreviouslyFailed: true,
|
|
279
|
+
initialCapacity: Math.max(options?.initialCapacity ?? SAH_POOL_DEFAULT_CAPACITY, SAH_POOL_MIN_CAPACITY),
|
|
280
|
+
})
|
|
281
|
+
.catch((error) => {
|
|
282
|
+
sahPools.delete(directory);
|
|
283
|
+
throw opfsSahPoolError(error, directory);
|
|
284
|
+
});
|
|
285
|
+
sahPools.set(directory, pool);
|
|
286
|
+
}
|
|
287
|
+
util = await pool;
|
|
288
|
+
await util.unpauseVfs();
|
|
289
|
+
db = new util.OpfsSAHPoolDb(`/${name}.db`);
|
|
276
290
|
configureSahCrashRecovery(db, sqlite3);
|
|
277
|
-
return new WasmClientDatabase(db, sqlite3, { util });
|
|
291
|
+
return new WasmClientDatabase(db, sqlite3, { util, lease });
|
|
278
292
|
}
|
|
279
293
|
catch (error) {
|
|
280
|
-
db
|
|
294
|
+
db?.close();
|
|
295
|
+
util?.pauseVfs();
|
|
296
|
+
await lease.release();
|
|
281
297
|
throw error;
|
|
282
298
|
}
|
|
283
299
|
}
|
package/dist/worker-host.js
CHANGED
|
@@ -437,7 +437,29 @@ async function startWorkerCore(options) {
|
|
|
437
437
|
const worker = config.worker();
|
|
438
438
|
const pending = new Map();
|
|
439
439
|
const nextId = { value: 1 };
|
|
440
|
+
let terminated = false;
|
|
441
|
+
let rejectReady;
|
|
442
|
+
const closedError = new ClientSyncError(WORKER_FAILED_CODE, 'the handle was closed');
|
|
443
|
+
const terminate = () => {
|
|
444
|
+
if (terminated)
|
|
445
|
+
return;
|
|
446
|
+
terminated = true;
|
|
447
|
+
globalThis.removeEventListener?.('pagehide', terminate);
|
|
448
|
+
bridge?.close();
|
|
449
|
+
worker.terminate();
|
|
450
|
+
rejectReady?.(closedError);
|
|
451
|
+
for (const entry of pending.values())
|
|
452
|
+
entry.reject(closedError);
|
|
453
|
+
pending.clear();
|
|
454
|
+
void lease.release();
|
|
455
|
+
};
|
|
456
|
+
// Installed before bootstrap starts: pagehide must also terminate a worker
|
|
457
|
+
// whose first open/import is still pending. Termination closes its SAHs and
|
|
458
|
+
// releases its physical Web Lock; the next document waits for that lock.
|
|
459
|
+
globalThis.addEventListener?.('pagehide', terminate);
|
|
440
460
|
const invoke = (method, args) => {
|
|
461
|
+
if (terminated)
|
|
462
|
+
return Promise.reject(closedError);
|
|
441
463
|
const id = nextId.value++;
|
|
442
464
|
return new Promise((resolve, reject) => {
|
|
443
465
|
pending.set(id, { resolve, reject });
|
|
@@ -451,6 +473,7 @@ async function startWorkerCore(options) {
|
|
|
451
473
|
};
|
|
452
474
|
let bridge;
|
|
453
475
|
const ready = new Promise((resolve, reject) => {
|
|
476
|
+
rejectReady = reject;
|
|
454
477
|
const onMessage = (event) => {
|
|
455
478
|
const message = event.data;
|
|
456
479
|
switch (message.t) {
|
|
@@ -507,27 +530,30 @@ async function startWorkerCore(options) {
|
|
|
507
530
|
invoke,
|
|
508
531
|
bridge,
|
|
509
532
|
lease,
|
|
510
|
-
close: async (
|
|
533
|
+
close: async (terminateWorker) => {
|
|
534
|
+
if (terminated)
|
|
535
|
+
return;
|
|
511
536
|
bridge?.close();
|
|
512
537
|
try {
|
|
513
538
|
await invoke('close', []);
|
|
514
539
|
}
|
|
515
540
|
catch {
|
|
516
|
-
//
|
|
541
|
+
// A terminated or failed worker has no further close RPC to service.
|
|
542
|
+
}
|
|
543
|
+
if (terminateWorker)
|
|
544
|
+
terminate();
|
|
545
|
+
else {
|
|
546
|
+
globalThis.removeEventListener?.('pagehide', terminate);
|
|
547
|
+
for (const entry of pending.values())
|
|
548
|
+
entry.reject(closedError);
|
|
549
|
+
pending.clear();
|
|
550
|
+
await lease.release();
|
|
517
551
|
}
|
|
518
|
-
if (terminate)
|
|
519
|
-
worker.terminate();
|
|
520
|
-
const closedError = new ClientSyncError(WORKER_FAILED_CODE, 'the handle was closed');
|
|
521
|
-
for (const entry of pending.values())
|
|
522
|
-
entry.reject(closedError);
|
|
523
|
-
pending.clear();
|
|
524
|
-
await lease.release();
|
|
525
552
|
},
|
|
526
553
|
};
|
|
527
554
|
}
|
|
528
555
|
catch (error) {
|
|
529
|
-
|
|
530
|
-
await lease.release();
|
|
556
|
+
terminate();
|
|
531
557
|
throw error;
|
|
532
558
|
}
|
|
533
559
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/client",
|
|
3
|
-
"version": "0.30.
|
|
3
|
+
"version": "0.30.12",
|
|
4
4
|
"description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -89,10 +89,10 @@
|
|
|
89
89
|
},
|
|
90
90
|
"dependencies": {
|
|
91
91
|
"@sqlite.org/sqlite-wasm": "^3.53.0-build1",
|
|
92
|
-
"@syncular/core": "0.30.
|
|
92
|
+
"@syncular/core": "0.30.12"
|
|
93
93
|
},
|
|
94
94
|
"devDependencies": {
|
|
95
|
-
"@syncular/server": "0.30.
|
|
95
|
+
"@syncular/server": "0.30.12",
|
|
96
96
|
"playwright": "1.63.0",
|
|
97
97
|
"@electric-sql/pglite": "^0.5.4"
|
|
98
98
|
}
|
package/src/apply.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { RETAINED_ROWS } from './failed-overlay';
|
|
2
|
+
import { retainedBaseWrite, restoreFailedBases } from './failed-overlay';
|
|
2
3
|
/**
|
|
3
4
|
* Local application of server data: `COMMIT` frames (§4.5), rows segments
|
|
4
5
|
* (§5.2, §5.6), and the scope-matched delete shared by the §3.3 purge
|
|
@@ -19,6 +20,8 @@ import {
|
|
|
19
20
|
type CompiledClientSchema,
|
|
20
21
|
type CompiledClientTable,
|
|
21
22
|
quoteIdent,
|
|
23
|
+
matchesLocalScopes,
|
|
24
|
+
type JsonRowValue,
|
|
22
25
|
SYNC_VERSION_COLUMN,
|
|
23
26
|
toSqlValue,
|
|
24
27
|
rowValueToJson,
|
|
@@ -37,8 +40,7 @@ function upsertSql(table: CompiledClientTable): string {
|
|
|
37
40
|
.filter((name) => name !== quoteIdent(table.primaryKey))
|
|
38
41
|
.map((name) => `${name}=excluded.${name}`)
|
|
39
42
|
.join(', ');
|
|
40
|
-
return `INSERT INTO ${quoteIdent(table.name)}
|
|
41
|
-
ON CONFLICT (${quoteIdent(table.primaryKey)}) DO UPDATE SET ${updates}`;
|
|
43
|
+
return `INSERT INTO ${quoteIdent(table.name)}(${names.join(', ')})VALUES(${placeholders})ON CONFLICT(${quoteIdent(table.primaryKey)})DO UPDATE SET ${updates}`;
|
|
42
44
|
}
|
|
43
45
|
|
|
44
46
|
export function upsertLocalRow(
|
|
@@ -48,10 +50,11 @@ export function upsertLocalRow(
|
|
|
48
50
|
syncVersion: number,
|
|
49
51
|
serverWrite = true,
|
|
50
52
|
): void {
|
|
53
|
+
db.exec(upsertSql(table), [...values.map(toSqlValue), syncVersion]);
|
|
51
54
|
if (serverWrite)
|
|
52
55
|
retainedBaseWrite(
|
|
53
56
|
db,
|
|
54
|
-
table
|
|
57
|
+
table,
|
|
55
58
|
String(values[table.primaryKeyIndex]),
|
|
56
59
|
Object.fromEntries(
|
|
57
60
|
table.columns.map((column, index) => [
|
|
@@ -61,7 +64,6 @@ export function upsertLocalRow(
|
|
|
61
64
|
),
|
|
62
65
|
syncVersion,
|
|
63
66
|
);
|
|
64
|
-
db.exec(upsertSql(table), [...values.map(toSqlValue), syncVersion]);
|
|
65
67
|
}
|
|
66
68
|
|
|
67
69
|
export function deleteLocalRow(
|
|
@@ -70,9 +72,9 @@ export function deleteLocalRow(
|
|
|
70
72
|
rowId: string,
|
|
71
73
|
serverWrite = true,
|
|
72
74
|
): void {
|
|
73
|
-
if (serverWrite) retainedBaseWrite(db, table
|
|
75
|
+
if (serverWrite) retainedBaseWrite(db, table, rowId);
|
|
74
76
|
db.exec(
|
|
75
|
-
`DELETE FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(table.primaryKey)}
|
|
77
|
+
`DELETE FROM ${quoteIdent(table.name)} WHERE ${quoteIdent(table.primaryKey)}=?`,
|
|
76
78
|
[rowId],
|
|
77
79
|
);
|
|
78
80
|
}
|
|
@@ -94,7 +96,8 @@ export async function applyCommitFrame(
|
|
|
94
96
|
schema: CompiledClientSchema,
|
|
95
97
|
frame: CommitFrame,
|
|
96
98
|
encryption?: EncryptionConfig,
|
|
97
|
-
transaction:
|
|
99
|
+
transaction: (fn: () => readonly string[] | undefined) => unknown = (fn) =>
|
|
100
|
+
db.transaction(fn),
|
|
98
101
|
): Promise<void> {
|
|
99
102
|
type Resolved =
|
|
100
103
|
| { op: 'delete'; table: CompiledClientTable; rowId: string }
|
|
@@ -139,13 +142,31 @@ export async function applyCommitFrame(
|
|
|
139
142
|
});
|
|
140
143
|
}
|
|
141
144
|
transaction(() => {
|
|
142
|
-
|
|
145
|
+
const rowKeys = frame.tables.some((name) =>
|
|
146
|
+
schema.tables.get(name)?.indexes.some((index) => index.unique),
|
|
147
|
+
)
|
|
148
|
+
? undefined
|
|
149
|
+
: frame.changes.map((change) =>
|
|
150
|
+
JSON.stringify([frame.tables[change.tableIndex], change.rowId]),
|
|
151
|
+
);
|
|
152
|
+
restoreFailedBases(db, schema, false, rowKeys);
|
|
153
|
+
for (const [index, change] of resolved.entries()) {
|
|
154
|
+
const rowId = frame.changes[index]!.rowId;
|
|
155
|
+
db.exec(
|
|
156
|
+
"UPDATE _syncular_outbox_before_images SET delivery_seq=MAX(COALESCE(delivery_seq,0),?) WHERE (client_commit_id,op_index) IN(SELECT client_commit_id,json_each.key FROM _syncular_outbox,json_each(operations) WHERE json_extract(value,'$.table')=? AND json_extract(value,'$.rowId')=?)",
|
|
157
|
+
[frame.commitSeq, change.table.name, rowId],
|
|
158
|
+
);
|
|
159
|
+
db.exec(
|
|
160
|
+
`DELETE FROM ${RETAINED_ROWS} WHERE tbl=? AND id=? AND commit_seq<=?`,
|
|
161
|
+
[change.table.name, rowId, frame.commitSeq],
|
|
162
|
+
);
|
|
143
163
|
if (change.op === 'delete') {
|
|
144
164
|
deleteLocalRow(db, change.table, change.rowId);
|
|
145
165
|
} else {
|
|
146
166
|
upsertLocalRow(db, change.table, change.values, change.rowVersion);
|
|
147
167
|
}
|
|
148
168
|
}
|
|
169
|
+
return rowKeys;
|
|
149
170
|
});
|
|
150
171
|
}
|
|
151
172
|
|
|
@@ -195,39 +216,43 @@ export function validateSegmentColumns(
|
|
|
195
216
|
* (`sync.scope_revoked`) when the table has no local mapping for a key:
|
|
196
217
|
* precision or nothing, never clear-the-table.
|
|
197
218
|
*/
|
|
219
|
+
export function localScopePredicate(
|
|
220
|
+
table: CompiledClientTable,
|
|
221
|
+
effective: ScopeMap,
|
|
222
|
+
): { sql: string; params: string[] } | undefined {
|
|
223
|
+
const entries = Object.entries(effective);
|
|
224
|
+
if (!entries.length || entries.some(([, values]) => !values.length)) return;
|
|
225
|
+
const params: string[] = [];
|
|
226
|
+
const sql = entries
|
|
227
|
+
.map(([variable, values]) => {
|
|
228
|
+
const column = table.scopeColumnByVariable.get(variable);
|
|
229
|
+
if (column === undefined)
|
|
230
|
+
throw new ClientSyncError(
|
|
231
|
+
'sync.scope_revoked',
|
|
232
|
+
'table has no local scope-column mapping',
|
|
233
|
+
);
|
|
234
|
+
params.push(...values);
|
|
235
|
+
return `${quoteIdent(column)} IN (${values.map(() => '?').join(', ')})`;
|
|
236
|
+
})
|
|
237
|
+
.join(' AND ');
|
|
238
|
+
return { sql, params };
|
|
239
|
+
}
|
|
240
|
+
|
|
198
241
|
export function deleteScopedRows(
|
|
199
242
|
db: ClientDatabase,
|
|
200
243
|
table: CompiledClientTable,
|
|
201
244
|
effective: ScopeMap,
|
|
202
245
|
): void {
|
|
203
|
-
const
|
|
204
|
-
if (
|
|
205
|
-
const
|
|
206
|
-
const params: string[] = [];
|
|
207
|
-
for (const [variable, values] of entries) {
|
|
208
|
-
const column = table.scopeColumnByVariable.get(variable);
|
|
209
|
-
if (column === undefined) {
|
|
210
|
-
throw new ClientSyncError(
|
|
211
|
-
'sync.scope_revoked',
|
|
212
|
-
`table ${JSON.stringify(table.name)} has no local scope-column mapping for ${JSON.stringify(variable)} (§3.3 fail-closed)`,
|
|
213
|
-
);
|
|
214
|
-
}
|
|
215
|
-
if (values.length === 0) return;
|
|
216
|
-
clauses.push(
|
|
217
|
-
`${quoteIdent(column)} IN (${values.map(() => '?').join(', ')})`,
|
|
218
|
-
);
|
|
219
|
-
params.push(...values);
|
|
220
|
-
}
|
|
246
|
+
const scope = localScopePredicate(table, effective);
|
|
247
|
+
if (!scope) return;
|
|
248
|
+
const { sql, params } = scope;
|
|
221
249
|
for (const row of db.query(
|
|
222
|
-
`SELECT ${quoteIdent(table.primaryKey)} AS id FROM ${quoteIdent(table.name)} WHERE ${
|
|
250
|
+
`SELECT ${quoteIdent(table.primaryKey)} AS id FROM ${quoteIdent(table.name)} WHERE ${sql}`,
|
|
223
251
|
params,
|
|
224
252
|
)) {
|
|
225
|
-
retainedBaseWrite(db, table
|
|
253
|
+
retainedBaseWrite(db, table, String(row.id));
|
|
226
254
|
}
|
|
227
|
-
db.exec(
|
|
228
|
-
`DELETE FROM ${quoteIdent(table.name)} WHERE ${clauses.join(' AND ')}`,
|
|
229
|
-
params,
|
|
230
|
-
);
|
|
255
|
+
db.exec(`DELETE FROM ${quoteIdent(table.name)} WHERE ${sql}`, params);
|
|
231
256
|
}
|
|
232
257
|
|
|
233
258
|
/**
|
|
@@ -246,24 +271,9 @@ export function evictScopedRows(
|
|
|
246
271
|
effective: ScopeMap,
|
|
247
272
|
pinnedRowIds: ReadonlySet<string>,
|
|
248
273
|
): { remaining: boolean; deferred: boolean } {
|
|
249
|
-
const
|
|
250
|
-
if (
|
|
251
|
-
const
|
|
252
|
-
const params: string[] = [];
|
|
253
|
-
for (const [variable, values] of entries) {
|
|
254
|
-
const column = table.scopeColumnByVariable.get(variable);
|
|
255
|
-
if (column === undefined) {
|
|
256
|
-
throw new ClientSyncError(
|
|
257
|
-
'sync.scope_revoked',
|
|
258
|
-
`table ${JSON.stringify(table.name)} has no local scope-column mapping for ${JSON.stringify(variable)} (§4.8/§3.3 fail-closed)`,
|
|
259
|
-
);
|
|
260
|
-
}
|
|
261
|
-
if (values.length === 0) return { remaining: false, deferred: false };
|
|
262
|
-
clauses.push(
|
|
263
|
-
`${quoteIdent(column)} IN (${values.map(() => '?').join(', ')})`,
|
|
264
|
-
);
|
|
265
|
-
params.push(...values);
|
|
266
|
-
}
|
|
274
|
+
const scope = localScopePredicate(table, effective);
|
|
275
|
+
if (!scope) return { remaining: false, deferred: false };
|
|
276
|
+
const { sql, params } = scope;
|
|
267
277
|
const pk = quoteIdent(table.primaryKey);
|
|
268
278
|
let pinnedClause = '';
|
|
269
279
|
if (pinnedRowIds.size > 0) {
|
|
@@ -272,21 +282,40 @@ export function evictScopedRows(
|
|
|
272
282
|
params.push(...ids);
|
|
273
283
|
}
|
|
274
284
|
const target = quoteIdent(table.name);
|
|
285
|
+
for (const row of db.query(
|
|
286
|
+
`SELECT id,intent,base FROM ${RETAINED_ROWS} WHERE commit_seq IS NOT NULL AND tbl=?`,
|
|
287
|
+
[table.name],
|
|
288
|
+
)) {
|
|
289
|
+
if (
|
|
290
|
+
!pinnedRowIds.has(row.id as string) &&
|
|
291
|
+
[row.intent, row.base].some(
|
|
292
|
+
(json) =>
|
|
293
|
+
json !== null &&
|
|
294
|
+
matchesLocalScopes(
|
|
295
|
+
table,
|
|
296
|
+
effective,
|
|
297
|
+
JSON.parse(json as string) as Record<string, JsonRowValue>,
|
|
298
|
+
),
|
|
299
|
+
)
|
|
300
|
+
)
|
|
301
|
+
db.exec(
|
|
302
|
+
`DELETE FROM ${RETAINED_ROWS} WHERE tbl=? AND id=? AND commit_seq IS NOT NULL`,
|
|
303
|
+
[table.name, row.id as string],
|
|
304
|
+
);
|
|
305
|
+
}
|
|
275
306
|
db.exec(
|
|
276
|
-
`DELETE FROM ${target} WHERE ${pk} IN
|
|
277
|
-
SELECT ${pk} FROM ${target} WHERE ${clauses.join(' AND ')}${pinnedClause} LIMIT 1024
|
|
278
|
-
)`,
|
|
307
|
+
`DELETE FROM ${target} WHERE ${pk} IN(SELECT ${pk} FROM ${target} WHERE ${sql}${pinnedClause} LIMIT 1024)`,
|
|
279
308
|
params,
|
|
280
309
|
);
|
|
281
310
|
const remaining =
|
|
282
311
|
db.query(
|
|
283
|
-
`SELECT 1 FROM ${target} WHERE ${
|
|
312
|
+
`SELECT 1 FROM ${target} WHERE ${sql}${pinnedClause} LIMIT 1`,
|
|
284
313
|
params,
|
|
285
314
|
).length > 0;
|
|
286
315
|
const deferred =
|
|
287
316
|
remaining ||
|
|
288
317
|
db.query(
|
|
289
|
-
`SELECT 1 FROM ${target} WHERE ${
|
|
318
|
+
`SELECT 1 FROM ${target} WHERE ${sql} LIMIT 1`,
|
|
290
319
|
params.slice(0, params.length - pinnedRowIds.size),
|
|
291
320
|
).length > 0;
|
|
292
321
|
return { remaining, deferred };
|
|
@@ -331,7 +360,7 @@ export async function applySqliteSegment(
|
|
|
331
360
|
const withImage = db.withSqliteImage?.bind(db);
|
|
332
361
|
if (withImage === undefined) {
|
|
333
362
|
throw invalidRequest(
|
|
334
|
-
'
|
|
363
|
+
'database cannot import images; disable accept bit 2 (§4.2)',
|
|
335
364
|
);
|
|
336
365
|
}
|
|
337
366
|
if (descriptor.table !== table.name) {
|
|
@@ -343,10 +372,7 @@ export async function applySqliteSegment(
|
|
|
343
372
|
let meta: ReturnType<ClientDatabase['query']>;
|
|
344
373
|
try {
|
|
345
374
|
meta = db.query(
|
|
346
|
-
`SELECT format,
|
|
347
|
-
"asOfCommitSeq" AS pin, "scopeDigest" AS sd,
|
|
348
|
-
"rowCount" AS rc
|
|
349
|
-
FROM ${IMAGE_ALIAS}."_syncular_segment"`,
|
|
375
|
+
`SELECT format,"table" AS tbl,"schemaVersion" AS sv,"asOfCommitSeq" AS pin,"scopeDigest" AS sd,"rowCount" AS rc FROM ${IMAGE_ALIAS}."_syncular_segment"`,
|
|
350
376
|
);
|
|
351
377
|
} catch {
|
|
352
378
|
imageInvalid('bytes are not a SQLite image with _syncular_segment');
|
|
@@ -396,7 +422,7 @@ export async function applySqliteSegment(
|
|
|
396
422
|
if (primaryKeys.length !== 1 || primaryKeys[0]?.name !== table.primaryKey)
|
|
397
423
|
imageInvalid('image primary key does not match generated schema');
|
|
398
424
|
const source = `${IMAGE_ALIAS}.${quoteIdent(table.name)}`;
|
|
399
|
-
const count = Number(db.query(`SELECT count(*)
|
|
425
|
+
const count = Number(db.query(`SELECT count(*)AS n FROM ${source}`)[0]?.n);
|
|
400
426
|
if (count !== descriptor.rowCount)
|
|
401
427
|
imageInvalid('image row count does not match descriptor');
|
|
402
428
|
const names = table.columns.map((column) => quoteIdent(column.name));
|
|
@@ -411,9 +437,7 @@ export async function applySqliteSegment(
|
|
|
411
437
|
for (;;) {
|
|
412
438
|
if (options.isCurrent?.() === false) return applied;
|
|
413
439
|
const boundary = db.query(
|
|
414
|
-
`SELECT ${primaryKey} AS boundary FROM ${source}
|
|
415
|
-
${after === undefined ? '' : `WHERE ${primaryKey} > ?`}
|
|
416
|
-
ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`,
|
|
440
|
+
`SELECT ${primaryKey} AS boundary FROM ${source} ${after === undefined ? '' : `WHERE ${primaryKey} > ?`} ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`,
|
|
417
441
|
after === undefined ? [] : [after],
|
|
418
442
|
)[0]?.boundary;
|
|
419
443
|
if (boundary === null) imageInvalid('image primary key must not be null');
|
|
@@ -428,23 +452,21 @@ export async function applySqliteSegment(
|
|
|
428
452
|
params.push(boundary);
|
|
429
453
|
}
|
|
430
454
|
(options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
455
|
+
restoreFailedBases(db, schema);
|
|
431
456
|
if (after === undefined && options.clearFirst)
|
|
432
457
|
deleteScopedRows(db, table, options.effective);
|
|
433
458
|
db.exec(
|
|
434
|
-
`INSERT INTO ${quoteIdent(table.name)}
|
|
435
|
-
SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')}
|
|
436
|
-
FROM ${source} WHERE ${predicates.join(' AND ') || 'true'}
|
|
437
|
-
ON CONFLICT (${primaryKey}) DO UPDATE SET ${updates}`,
|
|
459
|
+
`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${predicates.join(' AND ') || 'true'} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates}`,
|
|
438
460
|
params,
|
|
439
461
|
);
|
|
440
|
-
applied += Number(db.query('SELECT changes()
|
|
462
|
+
applied += Number(db.query('SELECT changes()AS n')[0]?.n ?? 0);
|
|
441
463
|
for (const row of db.query(
|
|
442
464
|
`SELECT * FROM ${source} WHERE ${predicates.join(' AND ') || 'true'}`,
|
|
443
465
|
params,
|
|
444
466
|
)) {
|
|
445
467
|
retainedBaseWrite(
|
|
446
468
|
db,
|
|
447
|
-
table
|
|
469
|
+
table,
|
|
448
470
|
String(row[table.primaryKey]),
|
|
449
471
|
Object.fromEntries(
|
|
450
472
|
table.columns.map((column) => [
|
|
@@ -515,11 +537,12 @@ export async function applyRowsSegment(
|
|
|
515
537
|
first = false;
|
|
516
538
|
if (!clearThisBlock && rows.length === 0) continue;
|
|
517
539
|
(options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
540
|
+
const protectedRows = restoreFailedBases(db, schema);
|
|
518
541
|
if (clearThisBlock) {
|
|
519
542
|
deleteScopedRows(db, table, options.effective);
|
|
520
543
|
}
|
|
521
544
|
for (const row of rows) {
|
|
522
|
-
upsertLocalRow(db, table, row.values, row.serverVersion);
|
|
545
|
+
upsertLocalRow(db, table, row.values, row.serverVersion, protectedRows);
|
|
523
546
|
applied += 1;
|
|
524
547
|
if (applied % 1024 === 0) options.onProgress?.(applied);
|
|
525
548
|
}
|
package/src/blob.ts
CHANGED
|
@@ -105,7 +105,7 @@ export async function computeBlobId(bytes: Uint8Array): Promise<string> {
|
|
|
105
105
|
|
|
106
106
|
export function ensureBlobSchema(db: ClientDatabase): void {
|
|
107
107
|
const existing = db.query(
|
|
108
|
-
"SELECT 1 FROM sqlite_master WHERE type
|
|
108
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='_syncular_blobs'",
|
|
109
109
|
);
|
|
110
110
|
if (existing.length > 0) {
|
|
111
111
|
const columns = db
|
|
@@ -129,15 +129,13 @@ export function ensureBlobSchema(db: ClientDatabase): void {
|
|
|
129
129
|
);
|
|
130
130
|
}
|
|
131
131
|
} else {
|
|
132
|
-
db.exec(
|
|
133
|
-
blob_id TEXT PRIMARY KEY,
|
|
134
|
-
|
|
135
|
-
byte_length INTEGER NOT NULL,
|
|
136
|
-
media_type TEXT,
|
|
137
|
-
created_at_ms INTEGER NOT NULL)`);
|
|
132
|
+
db.exec(
|
|
133
|
+
'CREATE TABLE _syncular_blobs(blob_id TEXT PRIMARY KEY,bytes BLOB NOT NULL,byte_length INTEGER NOT NULL,media_type TEXT,created_at_ms INTEGER NOT NULL)',
|
|
134
|
+
);
|
|
138
135
|
}
|
|
139
|
-
db.exec(
|
|
140
|
-
blob_id TEXT PRIMARY KEY,
|
|
136
|
+
db.exec(
|
|
137
|
+
'CREATE TABLE IF NOT EXISTS _syncular_blob_uploads(blob_id TEXT PRIMARY KEY,media_type TEXT,created_at_ms INTEGER NOT NULL)',
|
|
138
|
+
);
|
|
141
139
|
}
|
|
142
140
|
|
|
143
141
|
/** Put bytes into the content-addressed cache without rewriting an existing body. */
|
|
@@ -149,9 +147,7 @@ export function putCachedBlob(
|
|
|
149
147
|
mediaType?: string,
|
|
150
148
|
): void {
|
|
151
149
|
db.exec(
|
|
152
|
-
|
|
153
|
-
blob_id, bytes, byte_length, media_type, created_at_ms)
|
|
154
|
-
VALUES (?,?,?,?,?) ON CONFLICT(blob_id) DO NOTHING`,
|
|
150
|
+
'INSERT INTO _syncular_blobs(blob_id,bytes,byte_length,media_type,created_at_ms)VALUES(?,?,?,?,?)ON CONFLICT(blob_id)DO NOTHING',
|
|
155
151
|
[blobId, bytes, bytes.length, mediaType ?? null, nowMs],
|
|
156
152
|
);
|
|
157
153
|
}
|
|
@@ -161,7 +157,7 @@ export function getCachedBlob(
|
|
|
161
157
|
blobId: string,
|
|
162
158
|
): CachedBlob | undefined {
|
|
163
159
|
const rows = db.query(
|
|
164
|
-
'SELECT bytes,
|
|
160
|
+
'SELECT bytes,byte_length,media_type FROM _syncular_blobs WHERE blob_id=?',
|
|
165
161
|
[blobId],
|
|
166
162
|
);
|
|
167
163
|
const row = rows[0];
|
|
@@ -185,24 +181,20 @@ export function enforceBlobCacheCap(
|
|
|
185
181
|
maxBytes: number,
|
|
186
182
|
): string[] {
|
|
187
183
|
const totalRow = db.query(
|
|
188
|
-
'SELECT COALESCE(SUM(byte_length),
|
|
184
|
+
'SELECT COALESCE(SUM(byte_length),0)AS total FROM _syncular_blobs',
|
|
189
185
|
)[0];
|
|
190
186
|
let total = Number(totalRow?.total ?? 0);
|
|
191
187
|
if (total <= maxBytes) return [];
|
|
192
188
|
// Oldest eligible body first, with blob ID as a stable tie.
|
|
193
189
|
const candidates = db.query(
|
|
194
|
-
`SELECT blob_id,
|
|
195
|
-
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
196
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
197
|
-
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})
|
|
198
|
-
ORDER BY created_at_ms ASC, blob_id ASC`,
|
|
190
|
+
`SELECT blob_id,byte_length FROM _syncular_blobs WHERE blob_id NOT IN(SELECT blob_id FROM _syncular_blob_uploads)AND blob_id NOT IN(SELECT blob_id FROM _syncular_blob_commit_refs)AND blob_id NOT IN(${visibleBlobIdsSql(schema)})ORDER BY created_at_ms ASC,blob_id ASC`,
|
|
199
191
|
);
|
|
200
192
|
const evicted: string[] = [];
|
|
201
193
|
db.transaction(() => {
|
|
202
194
|
for (const row of candidates) {
|
|
203
195
|
if (total <= maxBytes) break;
|
|
204
196
|
const blobId = row.blob_id as string;
|
|
205
|
-
db.exec('DELETE FROM _syncular_blobs WHERE blob_id
|
|
197
|
+
db.exec('DELETE FROM _syncular_blobs WHERE blob_id=?', [blobId]);
|
|
206
198
|
total -= Number(row.byte_length);
|
|
207
199
|
evicted.push(blobId);
|
|
208
200
|
}
|
|
@@ -215,13 +207,7 @@ export function listPendingUploads(
|
|
|
215
207
|
): { blobId: string; mediaType?: string }[] {
|
|
216
208
|
return db
|
|
217
209
|
.query(
|
|
218
|
-
|
|
219
|
-
SELECT blob_id, media_type, created_at_ms FROM _syncular_blob_uploads
|
|
220
|
-
UNION ALL
|
|
221
|
-
SELECT DISTINCT r.blob_id, b.media_type, 9223372036854775807 AS created_at_ms
|
|
222
|
-
FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id = r.blob_id
|
|
223
|
-
WHERE NOT EXISTS (SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id = r.blob_id)
|
|
224
|
-
) ORDER BY created_at_ms, blob_id`,
|
|
210
|
+
'SELECT blob_id,media_type FROM(SELECT blob_id,media_type,created_at_ms FROM _syncular_blob_uploads UNION ALL SELECT DISTINCT r.blob_id,b.media_type,9223372036854775807 AS created_at_ms FROM _syncular_blob_commit_refs r LEFT JOIN _syncular_blobs b ON b.blob_id=r.blob_id WHERE NOT EXISTS(SELECT 1 FROM _syncular_blob_uploads u WHERE u.blob_id=r.blob_id))ORDER BY created_at_ms,blob_id',
|
|
225
211
|
)
|
|
226
212
|
.map((row) => {
|
|
227
213
|
if (
|
|
@@ -247,14 +233,13 @@ export function recordPendingUpload(
|
|
|
247
233
|
mediaType?: string,
|
|
248
234
|
): void {
|
|
249
235
|
db.exec(
|
|
250
|
-
|
|
251
|
-
VALUES (?,?,?)`,
|
|
236
|
+
'INSERT OR IGNORE INTO _syncular_blob_uploads(blob_id,media_type,created_at_ms)VALUES(?,?,?)',
|
|
252
237
|
[blobId, mediaType ?? null, nowMs],
|
|
253
238
|
);
|
|
254
239
|
}
|
|
255
240
|
|
|
256
241
|
export function clearPendingUpload(db: ClientDatabase, blobId: string): void {
|
|
257
|
-
db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id
|
|
242
|
+
db.exec('DELETE FROM _syncular_blob_uploads WHERE blob_id=?', [blobId]);
|
|
258
243
|
}
|
|
259
244
|
|
|
260
245
|
/**
|
|
@@ -286,11 +271,7 @@ function visibleBlobIdsSql(schema: CompiledClientSchema): string {
|
|
|
286
271
|
for (const column of columns) {
|
|
287
272
|
const identifier = quoteIdent(column);
|
|
288
273
|
selects.push(
|
|
289
|
-
`SELECT json_extract(${identifier},
|
|
290
|
-
FROM ${quoteIdent(tableName)}
|
|
291
|
-
WHERE typeof(${identifier}) = 'text'
|
|
292
|
-
AND json_valid(${identifier})
|
|
293
|
-
AND json_type(${identifier}, '$.blobId') = 'text'`,
|
|
274
|
+
`SELECT json_extract(${identifier},'$.blobId')AS blob_id FROM ${quoteIdent(tableName)} WHERE typeof(${identifier})='text' AND json_valid(${identifier})AND json_type(${identifier},'$.blobId')='text'`,
|
|
294
275
|
);
|
|
295
276
|
}
|
|
296
277
|
}
|
|
@@ -304,10 +285,7 @@ export function deleteUnreferencedCachedBlobs(
|
|
|
304
285
|
schema: CompiledClientSchema,
|
|
305
286
|
): void {
|
|
306
287
|
db.exec(
|
|
307
|
-
`DELETE FROM _syncular_blobs
|
|
308
|
-
WHERE blob_id NOT IN (SELECT blob_id FROM _syncular_blob_uploads)
|
|
309
|
-
AND blob_id NOT IN (SELECT blob_id FROM _syncular_blob_commit_refs)
|
|
310
|
-
AND blob_id NOT IN (${visibleBlobIdsSql(schema)})`,
|
|
288
|
+
`DELETE FROM _syncular_blobs WHERE blob_id NOT IN(SELECT blob_id FROM _syncular_blob_uploads)AND blob_id NOT IN(SELECT blob_id FROM _syncular_blob_commit_refs)AND blob_id NOT IN(${visibleBlobIdsSql(schema)})`,
|
|
311
289
|
);
|
|
312
290
|
}
|
|
313
291
|
|