@syncular/client 0.65.2 → 0.66.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -2
- package/dist/apply.d.ts +4 -4
- package/dist/apply.js +55 -66
- package/dist/client.js +29 -9
- package/dist/progress.d.ts +4 -0
- package/dist/wasm-database.js +9 -20
- package/package.json +3 -3
- package/src/apply.ts +77 -77
- package/src/client.ts +24 -3
- package/src/progress.ts +4 -0
- package/src/wasm-database.ts +11 -25
package/README.md
CHANGED
|
@@ -848,8 +848,12 @@ through the core, without intermediate download updates.
|
|
|
848
848
|
|
|
849
849
|
Import reports every 1,024 rows and its final count. `rowsProcessed` includes work
|
|
850
850
|
inside the current transaction; a failed import can roll those rows back. SQLite
|
|
851
|
-
images commit in
|
|
852
|
-
|
|
851
|
+
images commit in transactions of 1,024 rows, then twice as many per transaction
|
|
852
|
+
up to 65,536, and yield after each commit. Earlier committed transactions
|
|
853
|
+
survive a later interruption. A SQLite image payload also reports
|
|
854
|
+
`importWriteMs` (time before COMMIT: row statements with index maintenance)
|
|
855
|
+
and `importCommitMs` (COMMIT and revision publication), each summed over the
|
|
856
|
+
payload's transactions. `complete` follows checkpoint persistence
|
|
853
857
|
and optimistic-state reconciliation for the round. More bootstrap pages can
|
|
854
858
|
remain; use window coverage to determine whether a query is complete.
|
|
855
859
|
Listener exceptions do not fail sync. Listeners should hand UI work off and avoid
|
package/dist/apply.d.ts
CHANGED
|
@@ -71,10 +71,10 @@ export interface SqliteSegmentDescriptor {
|
|
|
71
71
|
readonly scopeDigest: string;
|
|
72
72
|
}
|
|
73
73
|
/**
|
|
74
|
-
* Apply a §5.3 sqlite image in committed
|
|
75
|
-
* the in-file metadata against the descriptor, validate the
|
|
76
|
-
* column names/order against the generated schema, run the
|
|
77
|
-
* first-page clear when fresh, then bulk-copy
|
|
74
|
+
* Apply a §5.3 sqlite image in committed transactions of 1,024 to 65,536
|
|
75
|
+
* rows: validate the in-file metadata against the descriptor, validate the
|
|
76
|
+
* data table's column names/order against the generated schema, run the
|
|
77
|
+
* §5.6 first-page clear when fresh, then bulk-copy 1,024-row ranges through a
|
|
78
78
|
* primary-key upsert — `_syncular_version` lands in
|
|
79
79
|
* `_sync_version` exactly like a rows segment's per-row `serverVersion`.
|
|
80
80
|
*/
|
package/dist/apply.js
CHANGED
|
@@ -125,29 +125,15 @@ export async function applyCommitFrame(db, schema, frame, encryption, transactio
|
|
|
125
125
|
* fatal (`sync.schema_mismatch`): the descriptor validates, never infers.
|
|
126
126
|
*/
|
|
127
127
|
export function validateSegmentColumns(schema, table, segment) {
|
|
128
|
-
const
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
if (segment.columns.length !== table.columns.length) {
|
|
138
|
-
mismatch(`column count ${segment.columns.length}`);
|
|
139
|
-
}
|
|
140
|
-
for (let i = 0; i < table.columns.length; i++) {
|
|
141
|
-
const expected = table.columns[i];
|
|
142
|
-
const actual = segment.columns[i];
|
|
143
|
-
if (expected === undefined || actual === undefined)
|
|
144
|
-
continue;
|
|
145
|
-
if (expected.name !== actual.name ||
|
|
146
|
-
expected.type !== actual.type ||
|
|
147
|
-
expected.nullable !== actual.nullable) {
|
|
148
|
-
mismatch(`column ${i} (${actual.name})`);
|
|
149
|
-
}
|
|
150
|
-
}
|
|
128
|
+
const column = segment.columns.length !== table.columns.length
|
|
129
|
+
? 'count'
|
|
130
|
+
: table.columns.findIndex((expected, i) => expected.name !== segment.columns[i]?.name ||
|
|
131
|
+
expected.type !== segment.columns[i]?.type ||
|
|
132
|
+
expected.nullable !== segment.columns[i]?.nullable);
|
|
133
|
+
if (segment.table !== table.name ||
|
|
134
|
+
segment.schemaVersion !== schema.version ||
|
|
135
|
+
column !== -1)
|
|
136
|
+
throw new ClientSyncError('sync.schema_mismatch', 'rows segment columns do not match the generated schema (§5.2)', false, column === -1 ? { table: table.name } : { table: table.name, column });
|
|
151
137
|
}
|
|
152
138
|
/**
|
|
153
139
|
* Scope-matched local delete (§3.3 purge / §4.8 eviction / §5.6 first-page
|
|
@@ -252,10 +238,10 @@ function imageInvalid(detail) {
|
|
|
252
238
|
throw invalidRequest(`sqlite segment rejected: ${detail} (§5.3)`);
|
|
253
239
|
}
|
|
254
240
|
/**
|
|
255
|
-
* Apply a §5.3 sqlite image in committed
|
|
256
|
-
* the in-file metadata against the descriptor, validate the
|
|
257
|
-
* column names/order against the generated schema, run the
|
|
258
|
-
* first-page clear when fresh, then bulk-copy
|
|
241
|
+
* Apply a §5.3 sqlite image in committed transactions of 1,024 to 65,536
|
|
242
|
+
* rows: validate the in-file metadata against the descriptor, validate the
|
|
243
|
+
* data table's column names/order against the generated schema, run the
|
|
244
|
+
* §5.6 first-page clear when fresh, then bulk-copy 1,024-row ranges through a
|
|
259
245
|
* primary-key upsert — `_syncular_version` lands in
|
|
260
246
|
* `_sync_version` exactly like a rows segment's per-row `serverVersion`.
|
|
261
247
|
*/
|
|
@@ -285,23 +271,13 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
285
271
|
if (meta.length !== 1 || record === undefined) {
|
|
286
272
|
imageInvalid('_syncular_segment must contain exactly one row');
|
|
287
273
|
}
|
|
288
|
-
if (record.format !== 1
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
imageInvalid(
|
|
295
|
-
}
|
|
296
|
-
if (Number(record.pin) !== descriptor.asOfCommitSeq) {
|
|
297
|
-
imageInvalid(`asOfCommitSeq ${String(record.pin)}`);
|
|
298
|
-
}
|
|
299
|
-
if (record.sd !== descriptor.scopeDigest) {
|
|
300
|
-
imageInvalid('scopeDigest mismatch');
|
|
301
|
-
}
|
|
302
|
-
if (Number(record.rc) !== descriptor.rowCount) {
|
|
303
|
-
imageInvalid(`rowCount ${String(record.rc)}`);
|
|
304
|
-
}
|
|
274
|
+
if (record.format !== 1 ||
|
|
275
|
+
record.tbl !== table.name ||
|
|
276
|
+
Number(record.sv) !== schema.version ||
|
|
277
|
+
Number(record.pin) !== descriptor.asOfCommitSeq ||
|
|
278
|
+
record.sd !== descriptor.scopeDigest ||
|
|
279
|
+
Number(record.rc) !== descriptor.rowCount)
|
|
280
|
+
imageInvalid('_syncular_segment does not match the descriptor');
|
|
305
281
|
// 2. Column names and order vs the generated schema (§5.3 rule 3 —
|
|
306
282
|
// sync.schema_mismatch, the §5.2 rule specialized).
|
|
307
283
|
const info = db.query(`PRAGMA ${IMAGE_ALIAS}.table_info(${quoteIdent(table.name)})`);
|
|
@@ -312,7 +288,7 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
312
288
|
const actual = info.map((row) => String(row.name));
|
|
313
289
|
if (actual.length !== expected.length ||
|
|
314
290
|
expected.some((name, index) => actual[index] !== name)) {
|
|
315
|
-
throw new ClientSyncError('sync.schema_mismatch',
|
|
291
|
+
throw new ClientSyncError('sync.schema_mismatch', 'sqlite segment columns do not match the generated schema (§5.3)', false, { table: table.name });
|
|
316
292
|
}
|
|
317
293
|
const primaryKeys = info.filter((column) => Number(column.pk) > 0);
|
|
318
294
|
if (primaryKeys.length !== 1 || primaryKeys[0]?.name !== table.primaryKey)
|
|
@@ -329,50 +305,62 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
329
305
|
.map((name) => `${name}=excluded.${name}`)
|
|
330
306
|
.join(', ');
|
|
331
307
|
const version = quoteIdent(SYNC_VERSION_COLUMN);
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
const boundary = db.query(`SELECT ${primaryKey} AS
|
|
308
|
+
// A statement copies the 1,024 rows after `from`; NULL keys cannot hide
|
|
309
|
+
// between statements.
|
|
310
|
+
const range = (from) => {
|
|
311
|
+
const params = from === undefined ? [] : [from];
|
|
312
|
+
const predicates = from === undefined ? [] : [`${primaryKey}>?`];
|
|
313
|
+
const boundary = db.query(`SELECT ${primaryKey} AS b FROM ${source} WHERE ${predicates[0] ?? 'true'} ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`, params)[0]?.b;
|
|
338
314
|
if (boundary === null)
|
|
339
315
|
imageInvalid('image primary key must not be null');
|
|
340
|
-
const predicates = [];
|
|
341
|
-
const params = [];
|
|
342
|
-
if (after !== undefined) {
|
|
343
|
-
predicates.push(`${primaryKey} > ?`);
|
|
344
|
-
params.push(after);
|
|
345
|
-
}
|
|
346
316
|
if (boundary !== undefined) {
|
|
347
|
-
predicates.push(`${primaryKey}
|
|
317
|
+
predicates.push(`${primaryKey}<=?`);
|
|
348
318
|
params.push(boundary);
|
|
349
319
|
}
|
|
320
|
+
return [boundary, predicates.join(' AND ') || 'true', params];
|
|
321
|
+
};
|
|
322
|
+
let [boundary, where, params] = range();
|
|
323
|
+
let applied = 0;
|
|
324
|
+
// §5.3: a transaction holds one statement at first and doubles after each
|
|
325
|
+
// commit up to 64; a replaying transaction holds one.
|
|
326
|
+
for (let statements = 1;; statements = Math.min(statements * 2, 64)) {
|
|
327
|
+
if (options.isCurrent?.() === false)
|
|
328
|
+
return applied;
|
|
329
|
+
const first = applied === 0;
|
|
350
330
|
const replay = countOutbox(db) ||
|
|
351
331
|
db.query(`SELECT 1 FROM ${RETAINED_ROWS} WHERE tbl=? LIMIT 1`, [
|
|
352
332
|
table.name,
|
|
353
333
|
]).length;
|
|
354
334
|
const rows = replay
|
|
355
|
-
? db.query(`SELECT * FROM ${source} WHERE ${
|
|
335
|
+
? db.query(`SELECT * FROM ${source} WHERE ${where}`, params)
|
|
356
336
|
: [];
|
|
357
337
|
(options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
358
338
|
const scope = !replay
|
|
359
339
|
? []
|
|
360
340
|
: [
|
|
361
341
|
...(hasUniqueIndex(table) ? [JSON.stringify([table.name])] : []),
|
|
362
|
-
...(
|
|
342
|
+
...(first && options.clearFirst
|
|
363
343
|
? bootstrapScope(db, table, options.effective, options.owners)
|
|
364
344
|
: []),
|
|
365
345
|
...rows.map((row) => JSON.stringify([table.name, String(row[table.primaryKey])])),
|
|
366
346
|
];
|
|
367
347
|
const protectedRows = restoreFailedBases(db, schema, scope);
|
|
368
|
-
if (
|
|
348
|
+
if (first && options.clearFirst)
|
|
369
349
|
deleteScopedRows(db, table, options.effective, options.owners);
|
|
370
350
|
const kept = new Set();
|
|
371
351
|
const skipped = new Set();
|
|
372
352
|
const landing = rows.filter((row) => lands(db, table, String(row[table.primaryKey]), Number(row._syncular_version), kept, skipped));
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
353
|
+
for (let statement = 1;; statement++) {
|
|
354
|
+
db.exec(`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates} WHERE excluded.${version}>${quoteIdent(table.name)}.${version}`, params);
|
|
355
|
+
applied += Number(db.query(`SELECT count(*)AS n FROM ${source} WHERE ${where}`, params)[0]?.n);
|
|
356
|
+
options.onProgress?.(applied);
|
|
357
|
+
if (replay ||
|
|
358
|
+
statement === statements ||
|
|
359
|
+
boundary === undefined ||
|
|
360
|
+
applied === count)
|
|
361
|
+
break;
|
|
362
|
+
[boundary, where, params] = range(boundary);
|
|
363
|
+
}
|
|
376
364
|
if (protectedRows)
|
|
377
365
|
for (const row of landing) {
|
|
378
366
|
retainedBaseWrite(db, table, String(row[table.primaryKey]), Object.fromEntries(table.columns.map((column) => [
|
|
@@ -382,14 +370,15 @@ export async function applySqliteSegment(db, schema, table, bytes, descriptor, o
|
|
|
382
370
|
}
|
|
383
371
|
return rebaseScope(scope, kept, skipped);
|
|
384
372
|
});
|
|
385
|
-
options.onProgress?.(applied);
|
|
386
373
|
await yieldToHost();
|
|
387
374
|
if (boundary === undefined || applied === count)
|
|
388
375
|
break;
|
|
389
|
-
|
|
376
|
+
[boundary, where, params] = range(boundary);
|
|
390
377
|
}
|
|
391
378
|
if (applied !== count)
|
|
392
379
|
imageInvalid('image applied row count does not match descriptor');
|
|
380
|
+
// A file replica's WAL keeps its high-water size until truncated.
|
|
381
|
+
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
393
382
|
return applied;
|
|
394
383
|
});
|
|
395
384
|
}
|
package/dist/client.js
CHANGED
|
@@ -3915,6 +3915,8 @@ export class SyncClient {
|
|
|
3915
3915
|
bytesTotal: frame.byteLength,
|
|
3916
3916
|
rowsProcessed: 0,
|
|
3917
3917
|
rowsTotal: frame.rowCount,
|
|
3918
|
+
importWriteMs: undefined,
|
|
3919
|
+
importCommitMs: undefined,
|
|
3918
3920
|
});
|
|
3919
3921
|
prefetch();
|
|
3920
3922
|
const pending = prefetched.get(frame);
|
|
@@ -3947,11 +3949,13 @@ export class SyncClient {
|
|
|
3947
3949
|
bytesTotal: bytes.byteLength,
|
|
3948
3950
|
rowsProcessed: 0,
|
|
3949
3951
|
rowsTotal: segment.blocks.reduce((n, block) => n + block.length, 0),
|
|
3952
|
+
importWriteMs: undefined,
|
|
3953
|
+
importCommitMs: undefined,
|
|
3950
3954
|
});
|
|
3951
3955
|
await this.#applySegmentOrFail(section, summary, image?.table ?? segment.table, (table, clearFirst, effective, owners) => {
|
|
3952
3956
|
const options = this.#segmentApplyOptions(table, clearFirst, effective, owners, image !== undefined
|
|
3953
3957
|
? image.rowCount > 0
|
|
3954
|
-
: segment.blocks.some((block) => block.length > 0), resetEpoch);
|
|
3958
|
+
: segment.blocks.some((block) => block.length > 0), resetEpoch, image !== undefined);
|
|
3955
3959
|
return image !== undefined
|
|
3956
3960
|
? applySqliteSegment(this.#db, this.#schema, table, bytes, image, options)
|
|
3957
3961
|
: applyRowsSegment(this.#db, this.#schema, table, segment, options, this.#encryption);
|
|
@@ -4355,21 +4359,37 @@ export class SyncClient {
|
|
|
4355
4359
|
this.#db.query(`SELECT 1 FROM ${quoteIdent(table.name)} WHERE ${scope.sql} LIMIT 1`, scope.params).length > 0);
|
|
4356
4360
|
}
|
|
4357
4361
|
/** Shared transaction, reset fence and progress for both segment formats. */
|
|
4358
|
-
#segmentApplyOptions(table, clearFirst, effective, owners, hasRows, resetEpoch) {
|
|
4362
|
+
#segmentApplyOptions(table, clearFirst, effective, owners, hasRows, resetEpoch, image) {
|
|
4363
|
+
let writeMs = 0;
|
|
4364
|
+
let commitMs = 0;
|
|
4359
4365
|
return {
|
|
4360
4366
|
clearFirst,
|
|
4361
4367
|
effective,
|
|
4362
4368
|
owners,
|
|
4363
4369
|
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
4364
4370
|
onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
|
|
4365
|
-
transaction: (fn) =>
|
|
4366
|
-
|
|
4367
|
-
|
|
4368
|
-
|
|
4369
|
-
|
|
4370
|
-
|
|
4371
|
+
transaction: (fn) => {
|
|
4372
|
+
const started = performance.now();
|
|
4373
|
+
let written = started;
|
|
4374
|
+
const result = this.#applyBatch((batch) => {
|
|
4375
|
+
if (hasRows ||
|
|
4376
|
+
(clearFirst && this.#scopedRowsExist(table, effective, owners)))
|
|
4377
|
+
batch.table(table.name);
|
|
4378
|
+
const result = fn();
|
|
4379
|
+
this.#replayOutbox(result, false);
|
|
4380
|
+
written = performance.now();
|
|
4381
|
+
return result;
|
|
4382
|
+
});
|
|
4383
|
+
// §7.7: image payloads report their import cost.
|
|
4384
|
+
writeMs += written - started;
|
|
4385
|
+
commitMs += performance.now() - written;
|
|
4386
|
+
if (image)
|
|
4387
|
+
this.#progress.update({
|
|
4388
|
+
importWriteMs: Math.round(writeMs),
|
|
4389
|
+
importCommitMs: Math.round(commitMs),
|
|
4390
|
+
});
|
|
4371
4391
|
return result;
|
|
4372
|
-
}
|
|
4392
|
+
},
|
|
4373
4393
|
};
|
|
4374
4394
|
}
|
|
4375
4395
|
/**
|
package/dist/progress.d.ts
CHANGED
|
@@ -11,6 +11,10 @@ export interface SyncProgress {
|
|
|
11
11
|
readonly bytesTotal?: number;
|
|
12
12
|
readonly rowsProcessed: number;
|
|
13
13
|
readonly rowsTotal?: number;
|
|
14
|
+
/** Time in this payload's import transactions before COMMIT, in ms. */
|
|
15
|
+
readonly importWriteMs?: number | undefined;
|
|
16
|
+
/** Time committing them and publishing their changes, in ms. */
|
|
17
|
+
readonly importCommitMs?: number | undefined;
|
|
14
18
|
readonly errorCode?: string;
|
|
15
19
|
readonly causeKind?: TransportCauseKind;
|
|
16
20
|
readonly httpStatus?: number;
|
package/dist/wasm-database.js
CHANGED
|
@@ -241,11 +241,7 @@ function opfsSahPoolError(error, directory) {
|
|
|
241
241
|
normalized.includes('opfs api is too old')) {
|
|
242
242
|
return new ClientSyncError(STORAGE_UNAVAILABLE_CODE, `Persistent OPFS storage is unavailable: ${detail}`);
|
|
243
243
|
}
|
|
244
|
-
return new ClientSyncError(STORAGE_BUSY_CODE, '
|
|
245
|
-
`${JSON.stringify(directory)}. Another live engine may still own its ` +
|
|
246
|
-
'SAH pool, or the browser may still be releasing it after a reload. ' +
|
|
247
|
-
'Close the other instance or retry after a short delay; do not delete ' +
|
|
248
|
-
`or rename the directory. Underlying error: ${detail}`, true);
|
|
244
|
+
return new ClientSyncError(STORAGE_BUSY_CODE, 'Another engine holds the OPFS storage directory, or the browser is still releasing it; retry after it closes', true, { directory, cause: detail });
|
|
249
245
|
}
|
|
250
246
|
/** sqlite-wasm's default SAH pool capacity. */
|
|
251
247
|
const SAH_POOL_DEFAULT_CAPACITY = 6;
|
|
@@ -269,14 +265,12 @@ function configureSahCrashRecovery(db, sqlite3) {
|
|
|
269
265
|
const stack = wasm.pstack.pointer;
|
|
270
266
|
try {
|
|
271
267
|
const output = wasm.pstack.allocPtr();
|
|
272
|
-
|
|
273
|
-
capi.sqlite3_file_control(db.pointer, 'main', capi.SQLITE_FCNTL_FILE_POINTER, output)
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
const methodsPointer = file.$pMethods;
|
|
279
|
-
file.dispose(); // Borrowed struct view; SQLite still owns the file.
|
|
268
|
+
const filePointer = db.pointer !== undefined &&
|
|
269
|
+
capi.sqlite3_file_control(db.pointer, 'main', capi.SQLITE_FCNTL_FILE_POINTER, output) === 0 &&
|
|
270
|
+
wasm.peekPtr(output);
|
|
271
|
+
const file = filePointer ? new capi.sqlite3_file(filePointer) : undefined;
|
|
272
|
+
const methodsPointer = file?.$pMethods;
|
|
273
|
+
file?.dispose(); // Borrowed struct view; SQLite still owns the file.
|
|
280
274
|
if (!methodsPointer) {
|
|
281
275
|
throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, 'Could not configure OPFS crash recovery');
|
|
282
276
|
}
|
|
@@ -310,16 +304,11 @@ export async function openPersistentWasmDatabase(name, options) {
|
|
|
310
304
|
throw invalidRequest(`invalid persistent database name ${JSON.stringify(name)}`);
|
|
311
305
|
}
|
|
312
306
|
if (!inWorkerContext()) {
|
|
313
|
-
throw invalidRequest('openPersistentWasmDatabase() must run inside a Web Worker
|
|
314
|
-
'persistent mode is whole-core-in-a-worker by design (Direction ' +
|
|
315
|
-
'decision 2, 2026-07-03). Use the worker handle from the main ' +
|
|
316
|
-
'thread, or openWasmDatabase() for explicitly ephemeral state.');
|
|
307
|
+
throw invalidRequest('openPersistentWasmDatabase() must run inside a Web Worker; use the worker handle on the main thread, or openWasmDatabase() for ephemeral state');
|
|
317
308
|
}
|
|
318
309
|
const storage = globalThis.navigator?.storage;
|
|
319
310
|
if (typeof storage?.getDirectory !== 'function') {
|
|
320
|
-
throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, '
|
|
321
|
-
'requires OPFS (~2023+ browsers). There is no IndexedDB or ' +
|
|
322
|
-
'in-memory fallback for persistent mode.');
|
|
311
|
+
throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, 'Persistent storage requires OPFS, which this browser lacks');
|
|
323
312
|
}
|
|
324
313
|
if (!navigator.locks) {
|
|
325
314
|
throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, 'Persistent OPFS storage requires Web Locks');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.66.1",
|
|
4
4
|
"description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -121,11 +121,11 @@
|
|
|
121
121
|
},
|
|
122
122
|
"dependencies": {
|
|
123
123
|
"@sqlite.org/sqlite-wasm": "^3.53.0-build1",
|
|
124
|
-
"@syncular/core": "0.
|
|
124
|
+
"@syncular/core": "0.66.1"
|
|
125
125
|
},
|
|
126
126
|
"devDependencies": {
|
|
127
127
|
"@electric-sql/pglite": "^0.5.4",
|
|
128
|
-
"@syncular/server": "0.
|
|
128
|
+
"@syncular/server": "0.66.1",
|
|
129
129
|
"playwright": "1.63.0",
|
|
130
130
|
"vite": "8.3.4"
|
|
131
131
|
}
|
package/src/apply.ts
CHANGED
|
@@ -208,33 +208,26 @@ export function validateSegmentColumns(
|
|
|
208
208
|
table: CompiledClientTable,
|
|
209
209
|
segment: RowsSegment,
|
|
210
210
|
): void {
|
|
211
|
-
const
|
|
211
|
+
const column =
|
|
212
|
+
segment.columns.length !== table.columns.length
|
|
213
|
+
? 'count'
|
|
214
|
+
: table.columns.findIndex(
|
|
215
|
+
(expected, i) =>
|
|
216
|
+
expected.name !== segment.columns[i]?.name ||
|
|
217
|
+
expected.type !== segment.columns[i]?.type ||
|
|
218
|
+
expected.nullable !== segment.columns[i]?.nullable,
|
|
219
|
+
);
|
|
220
|
+
if (
|
|
221
|
+
segment.table !== table.name ||
|
|
222
|
+
segment.schemaVersion !== schema.version ||
|
|
223
|
+
column !== -1
|
|
224
|
+
)
|
|
212
225
|
throw new ClientSyncError(
|
|
213
226
|
'sync.schema_mismatch',
|
|
214
|
-
|
|
227
|
+
'rows segment columns do not match the generated schema (§5.2)',
|
|
228
|
+
false,
|
|
229
|
+
column === -1 ? { table: table.name } : { table: table.name, column },
|
|
215
230
|
);
|
|
216
|
-
};
|
|
217
|
-
if (segment.table !== table.name) {
|
|
218
|
-
mismatch(`segment table ${JSON.stringify(segment.table)}`);
|
|
219
|
-
}
|
|
220
|
-
if (segment.schemaVersion !== schema.version) {
|
|
221
|
-
mismatch(`segment schemaVersion ${segment.schemaVersion}`);
|
|
222
|
-
}
|
|
223
|
-
if (segment.columns.length !== table.columns.length) {
|
|
224
|
-
mismatch(`column count ${segment.columns.length}`);
|
|
225
|
-
}
|
|
226
|
-
for (let i = 0; i < table.columns.length; i++) {
|
|
227
|
-
const expected = table.columns[i];
|
|
228
|
-
const actual = segment.columns[i];
|
|
229
|
-
if (expected === undefined || actual === undefined) continue;
|
|
230
|
-
if (
|
|
231
|
-
expected.name !== actual.name ||
|
|
232
|
-
expected.type !== actual.type ||
|
|
233
|
-
expected.nullable !== actual.nullable
|
|
234
|
-
) {
|
|
235
|
-
mismatch(`column ${i} (${actual.name})`);
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
231
|
}
|
|
239
232
|
|
|
240
233
|
/**
|
|
@@ -427,10 +420,10 @@ function imageInvalid(detail: string): never {
|
|
|
427
420
|
}
|
|
428
421
|
|
|
429
422
|
/**
|
|
430
|
-
* Apply a §5.3 sqlite image in committed
|
|
431
|
-
* the in-file metadata against the descriptor, validate the
|
|
432
|
-
* column names/order against the generated schema, run the
|
|
433
|
-
* first-page clear when fresh, then bulk-copy
|
|
423
|
+
* Apply a §5.3 sqlite image in committed transactions of 1,024 to 65,536
|
|
424
|
+
* rows: validate the in-file metadata against the descriptor, validate the
|
|
425
|
+
* data table's column names/order against the generated schema, run the
|
|
426
|
+
* §5.6 first-page clear when fresh, then bulk-copy 1,024-row ranges through a
|
|
434
427
|
* primary-key upsert — `_syncular_version` lands in
|
|
435
428
|
* `_sync_version` exactly like a rows segment's per-row `serverVersion`.
|
|
436
429
|
*/
|
|
@@ -477,22 +470,15 @@ export async function applySqliteSegment(
|
|
|
477
470
|
if (meta.length !== 1 || record === undefined) {
|
|
478
471
|
imageInvalid('_syncular_segment must contain exactly one row');
|
|
479
472
|
}
|
|
480
|
-
if (
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
imageInvalid(
|
|
489
|
-
}
|
|
490
|
-
if (record.sd !== descriptor.scopeDigest) {
|
|
491
|
-
imageInvalid('scopeDigest mismatch');
|
|
492
|
-
}
|
|
493
|
-
if (Number(record.rc) !== descriptor.rowCount) {
|
|
494
|
-
imageInvalid(`rowCount ${String(record.rc)}`);
|
|
495
|
-
}
|
|
473
|
+
if (
|
|
474
|
+
record.format !== 1 ||
|
|
475
|
+
record.tbl !== table.name ||
|
|
476
|
+
Number(record.sv) !== schema.version ||
|
|
477
|
+
Number(record.pin) !== descriptor.asOfCommitSeq ||
|
|
478
|
+
record.sd !== descriptor.scopeDigest ||
|
|
479
|
+
Number(record.rc) !== descriptor.rowCount
|
|
480
|
+
)
|
|
481
|
+
imageInvalid('_syncular_segment does not match the descriptor');
|
|
496
482
|
|
|
497
483
|
// 2. Column names and order vs the generated schema (§5.3 rule 3 —
|
|
498
484
|
// sync.schema_mismatch, the §5.2 rule specialized).
|
|
@@ -510,7 +496,9 @@ export async function applySqliteSegment(
|
|
|
510
496
|
) {
|
|
511
497
|
throw new ClientSyncError(
|
|
512
498
|
'sync.schema_mismatch',
|
|
513
|
-
|
|
499
|
+
'sqlite segment columns do not match the generated schema (§5.3)',
|
|
500
|
+
false,
|
|
501
|
+
{ table: table.name },
|
|
514
502
|
);
|
|
515
503
|
}
|
|
516
504
|
|
|
@@ -529,42 +517,43 @@ export async function applySqliteSegment(
|
|
|
529
517
|
.map((name) => `${name}=excluded.${name}`)
|
|
530
518
|
.join(', ');
|
|
531
519
|
const version = quoteIdent(SYNC_VERSION_COLUMN);
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
520
|
+
// A statement copies the 1,024 rows after `from`; NULL keys cannot hide
|
|
521
|
+
// between statements.
|
|
522
|
+
const range = (from?: SqlValue) => {
|
|
523
|
+
const params = from === undefined ? [] : [from];
|
|
524
|
+
const predicates = from === undefined ? [] : [`${primaryKey}>?`];
|
|
536
525
|
const boundary = db.query(
|
|
537
|
-
`SELECT ${primaryKey} AS
|
|
538
|
-
|
|
539
|
-
)[0]?.
|
|
526
|
+
`SELECT ${primaryKey} AS b FROM ${source} WHERE ${predicates[0] ?? 'true'} ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`,
|
|
527
|
+
params,
|
|
528
|
+
)[0]?.b;
|
|
540
529
|
if (boundary === null) imageInvalid('image primary key must not be null');
|
|
541
|
-
const predicates: string[] = [];
|
|
542
|
-
const params: SqlValue[] = [];
|
|
543
|
-
if (after !== undefined) {
|
|
544
|
-
predicates.push(`${primaryKey} > ?`);
|
|
545
|
-
params.push(after);
|
|
546
|
-
}
|
|
547
530
|
if (boundary !== undefined) {
|
|
548
|
-
predicates.push(`${primaryKey}
|
|
531
|
+
predicates.push(`${primaryKey}<=?`);
|
|
549
532
|
params.push(boundary);
|
|
550
533
|
}
|
|
534
|
+
return [boundary, predicates.join(' AND ') || 'true', params] as const;
|
|
535
|
+
};
|
|
536
|
+
let [boundary, where, params] = range();
|
|
537
|
+
let applied = 0;
|
|
538
|
+
// §5.3: a transaction holds one statement at first and doubles after each
|
|
539
|
+
// commit up to 64; a replaying transaction holds one.
|
|
540
|
+
for (let statements = 1; ; statements = Math.min(statements * 2, 64)) {
|
|
541
|
+
if (options.isCurrent?.() === false) return applied;
|
|
542
|
+
const first = applied === 0;
|
|
551
543
|
const replay =
|
|
552
544
|
countOutbox(db) ||
|
|
553
545
|
db.query(`SELECT 1 FROM ${RETAINED_ROWS} WHERE tbl=? LIMIT 1`, [
|
|
554
546
|
table.name,
|
|
555
547
|
]).length;
|
|
556
548
|
const rows = replay
|
|
557
|
-
? db.query(
|
|
558
|
-
`SELECT * FROM ${source} WHERE ${predicates.join(' AND ') || 'true'}`,
|
|
559
|
-
params,
|
|
560
|
-
)
|
|
549
|
+
? db.query(`SELECT * FROM ${source} WHERE ${where}`, params)
|
|
561
550
|
: [];
|
|
562
551
|
(options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
563
552
|
const scope = !replay
|
|
564
553
|
? []
|
|
565
554
|
: [
|
|
566
555
|
...(hasUniqueIndex(table) ? [JSON.stringify([table.name])] : []),
|
|
567
|
-
...(
|
|
556
|
+
...(first && options.clearFirst
|
|
568
557
|
? bootstrapScope(db, table, options.effective, options.owners)
|
|
569
558
|
: []),
|
|
570
559
|
...rows.map((row) =>
|
|
@@ -572,7 +561,7 @@ export async function applySqliteSegment(
|
|
|
572
561
|
),
|
|
573
562
|
];
|
|
574
563
|
const protectedRows = restoreFailedBases(db, schema, scope);
|
|
575
|
-
if (
|
|
564
|
+
if (first && options.clearFirst)
|
|
576
565
|
deleteScopedRows(db, table, options.effective, options.owners);
|
|
577
566
|
const kept = new Set<string>();
|
|
578
567
|
const skipped = new Set<string>();
|
|
@@ -586,17 +575,27 @@ export async function applySqliteSegment(
|
|
|
586
575
|
skipped,
|
|
587
576
|
),
|
|
588
577
|
);
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
params,
|
|
593
|
-
);
|
|
594
|
-
applied += Number(
|
|
595
|
-
db.query(
|
|
596
|
-
`SELECT count(*)AS n FROM ${source} WHERE ${where}`,
|
|
578
|
+
for (let statement = 1; ; statement++) {
|
|
579
|
+
db.exec(
|
|
580
|
+
`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates} WHERE excluded.${version}>${quoteIdent(table.name)}.${version}`,
|
|
597
581
|
params,
|
|
598
|
-
)
|
|
599
|
-
|
|
582
|
+
);
|
|
583
|
+
applied += Number(
|
|
584
|
+
db.query(
|
|
585
|
+
`SELECT count(*)AS n FROM ${source} WHERE ${where}`,
|
|
586
|
+
params,
|
|
587
|
+
)[0]?.n,
|
|
588
|
+
);
|
|
589
|
+
options.onProgress?.(applied);
|
|
590
|
+
if (
|
|
591
|
+
replay ||
|
|
592
|
+
statement === statements ||
|
|
593
|
+
boundary === undefined ||
|
|
594
|
+
applied === count
|
|
595
|
+
)
|
|
596
|
+
break;
|
|
597
|
+
[boundary, where, params] = range(boundary);
|
|
598
|
+
}
|
|
600
599
|
if (protectedRows)
|
|
601
600
|
for (const row of landing) {
|
|
602
601
|
retainedBaseWrite(
|
|
@@ -614,13 +613,14 @@ export async function applySqliteSegment(
|
|
|
614
613
|
}
|
|
615
614
|
return rebaseScope(scope, kept, skipped);
|
|
616
615
|
});
|
|
617
|
-
options.onProgress?.(applied);
|
|
618
616
|
await yieldToHost();
|
|
619
617
|
if (boundary === undefined || applied === count) break;
|
|
620
|
-
|
|
618
|
+
[boundary, where, params] = range(boundary);
|
|
621
619
|
}
|
|
622
620
|
if (applied !== count)
|
|
623
621
|
imageInvalid('image applied row count does not match descriptor');
|
|
622
|
+
// A file replica's WAL keeps its high-water size until truncated.
|
|
623
|
+
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
624
624
|
return applied;
|
|
625
625
|
});
|
|
626
626
|
}
|
package/src/client.ts
CHANGED
|
@@ -5432,6 +5432,8 @@ export class SyncClient {
|
|
|
5432
5432
|
bytesTotal: frame.byteLength,
|
|
5433
5433
|
rowsProcessed: 0,
|
|
5434
5434
|
rowsTotal: frame.rowCount,
|
|
5435
|
+
importWriteMs: undefined,
|
|
5436
|
+
importCommitMs: undefined,
|
|
5435
5437
|
});
|
|
5436
5438
|
prefetch();
|
|
5437
5439
|
const pending = prefetched.get(frame);
|
|
@@ -5476,6 +5478,8 @@ export class SyncClient {
|
|
|
5476
5478
|
(n, block) => n + block.length,
|
|
5477
5479
|
0,
|
|
5478
5480
|
),
|
|
5481
|
+
importWriteMs: undefined,
|
|
5482
|
+
importCommitMs: undefined,
|
|
5479
5483
|
});
|
|
5480
5484
|
await this.#applySegmentOrFail(
|
|
5481
5485
|
section,
|
|
@@ -5491,6 +5495,7 @@ export class SyncClient {
|
|
|
5491
5495
|
? image.rowCount > 0
|
|
5492
5496
|
: segment!.blocks.some((block) => block.length > 0),
|
|
5493
5497
|
resetEpoch,
|
|
5498
|
+
image !== undefined,
|
|
5494
5499
|
);
|
|
5495
5500
|
return image !== undefined
|
|
5496
5501
|
? applySqliteSegment(
|
|
@@ -6001,15 +6006,20 @@ export class SyncClient {
|
|
|
6001
6006
|
owners: readonly ScopeMap[],
|
|
6002
6007
|
hasRows: boolean,
|
|
6003
6008
|
resetEpoch: number,
|
|
6009
|
+
image: boolean,
|
|
6004
6010
|
): Parameters<typeof applyRowsSegment>[4] {
|
|
6011
|
+
let writeMs = 0;
|
|
6012
|
+
let commitMs = 0;
|
|
6005
6013
|
return {
|
|
6006
6014
|
clearFirst,
|
|
6007
6015
|
effective,
|
|
6008
6016
|
owners,
|
|
6009
6017
|
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
6010
6018
|
onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
|
|
6011
|
-
transaction: (fn) =>
|
|
6012
|
-
|
|
6019
|
+
transaction: (fn) => {
|
|
6020
|
+
const started = performance.now();
|
|
6021
|
+
let written = started;
|
|
6022
|
+
const result = this.#applyBatch((batch) => {
|
|
6013
6023
|
if (
|
|
6014
6024
|
hasRows ||
|
|
6015
6025
|
(clearFirst && this.#scopedRowsExist(table, effective, owners))
|
|
@@ -6017,8 +6027,19 @@ export class SyncClient {
|
|
|
6017
6027
|
batch.table(table.name);
|
|
6018
6028
|
const result = fn();
|
|
6019
6029
|
this.#replayOutbox(result, false);
|
|
6030
|
+
written = performance.now();
|
|
6020
6031
|
return result;
|
|
6021
|
-
})
|
|
6032
|
+
});
|
|
6033
|
+
// §7.7: image payloads report their import cost.
|
|
6034
|
+
writeMs += written - started;
|
|
6035
|
+
commitMs += performance.now() - written;
|
|
6036
|
+
if (image)
|
|
6037
|
+
this.#progress.update({
|
|
6038
|
+
importWriteMs: Math.round(writeMs),
|
|
6039
|
+
importCommitMs: Math.round(commitMs),
|
|
6040
|
+
});
|
|
6041
|
+
return result;
|
|
6042
|
+
},
|
|
6022
6043
|
};
|
|
6023
6044
|
}
|
|
6024
6045
|
|
package/src/progress.ts
CHANGED
|
@@ -12,6 +12,10 @@ export interface SyncProgress {
|
|
|
12
12
|
readonly bytesTotal?: number;
|
|
13
13
|
readonly rowsProcessed: number;
|
|
14
14
|
readonly rowsTotal?: number;
|
|
15
|
+
/** Time in this payload's import transactions before COMMIT, in ms. */
|
|
16
|
+
readonly importWriteMs?: number | undefined;
|
|
17
|
+
/** Time committing them and publishing their changes, in ms. */
|
|
18
|
+
readonly importCommitMs?: number | undefined;
|
|
15
19
|
readonly errorCode?: string;
|
|
16
20
|
readonly causeKind?: TransportCauseKind;
|
|
17
21
|
readonly httpStatus?: number;
|
package/src/wasm-database.ts
CHANGED
|
@@ -419,12 +419,9 @@ function opfsSahPoolError(error: unknown, directory: string): ClientSyncError {
|
|
|
419
419
|
}
|
|
420
420
|
return new ClientSyncError(
|
|
421
421
|
STORAGE_BUSY_CODE,
|
|
422
|
-
'
|
|
423
|
-
`${JSON.stringify(directory)}. Another live engine may still own its ` +
|
|
424
|
-
'SAH pool, or the browser may still be releasing it after a reload. ' +
|
|
425
|
-
'Close the other instance or retry after a short delay; do not delete ' +
|
|
426
|
-
`or rename the directory. Underlying error: ${detail}`,
|
|
422
|
+
'Another engine holds the OPFS storage directory, or the browser is still releasing it; retry after it closes',
|
|
427
423
|
true,
|
|
424
|
+
{ directory, cause: detail },
|
|
428
425
|
);
|
|
429
426
|
}
|
|
430
427
|
|
|
@@ -454,24 +451,18 @@ function configureSahCrashRecovery(
|
|
|
454
451
|
const stack = wasm.pstack.pointer;
|
|
455
452
|
try {
|
|
456
453
|
const output = wasm.pstack.allocPtr();
|
|
457
|
-
|
|
458
|
-
db.pointer
|
|
454
|
+
const filePointer =
|
|
455
|
+
db.pointer !== undefined &&
|
|
459
456
|
capi.sqlite3_file_control(
|
|
460
457
|
db.pointer,
|
|
461
458
|
'main',
|
|
462
459
|
capi.SQLITE_FCNTL_FILE_POINTER,
|
|
463
460
|
output,
|
|
464
|
-
)
|
|
465
|
-
|
|
466
|
-
)
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
'Could not configure OPFS crash recovery',
|
|
470
|
-
);
|
|
471
|
-
}
|
|
472
|
-
const file = new capi.sqlite3_file(wasm.peekPtr(output));
|
|
473
|
-
const methodsPointer = file.$pMethods;
|
|
474
|
-
file.dispose(); // Borrowed struct view; SQLite still owns the file.
|
|
461
|
+
) === 0 &&
|
|
462
|
+
wasm.peekPtr(output);
|
|
463
|
+
const file = filePointer ? new capi.sqlite3_file(filePointer) : undefined;
|
|
464
|
+
const methodsPointer = file?.$pMethods;
|
|
465
|
+
file?.dispose(); // Borrowed struct view; SQLite still owns the file.
|
|
475
466
|
if (!methodsPointer) {
|
|
476
467
|
throw new ClientSyncError(
|
|
477
468
|
STORAGE_UNAVAILABLE_CODE,
|
|
@@ -514,10 +505,7 @@ export async function openPersistentWasmDatabase(
|
|
|
514
505
|
}
|
|
515
506
|
if (!inWorkerContext()) {
|
|
516
507
|
throw invalidRequest(
|
|
517
|
-
'openPersistentWasmDatabase() must run inside a Web Worker
|
|
518
|
-
'persistent mode is whole-core-in-a-worker by design (Direction ' +
|
|
519
|
-
'decision 2, 2026-07-03). Use the worker handle from the main ' +
|
|
520
|
-
'thread, or openWasmDatabase() for explicitly ephemeral state.',
|
|
508
|
+
'openPersistentWasmDatabase() must run inside a Web Worker; use the worker handle on the main thread, or openWasmDatabase() for ephemeral state',
|
|
521
509
|
);
|
|
522
510
|
}
|
|
523
511
|
const storage = (
|
|
@@ -526,9 +514,7 @@ export async function openPersistentWasmDatabase(
|
|
|
526
514
|
if (typeof storage?.getDirectory !== 'function') {
|
|
527
515
|
throw new ClientSyncError(
|
|
528
516
|
STORAGE_UNAVAILABLE_CODE,
|
|
529
|
-
'
|
|
530
|
-
'requires OPFS (~2023+ browsers). There is no IndexedDB or ' +
|
|
531
|
-
'in-memory fallback for persistent mode.',
|
|
517
|
+
'Persistent storage requires OPFS, which this browser lacks',
|
|
532
518
|
);
|
|
533
519
|
}
|
|
534
520
|
if (!navigator.locks) {
|