@syncular/client 0.65.1 → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -7
- package/dist/apply.d.ts +4 -4
- package/dist/apply.js +55 -66
- package/dist/client.js +29 -9
- package/dist/progress.d.ts +4 -0
- package/dist/realtime-supervisor.d.ts +47 -13
- package/dist/realtime-supervisor.js +374 -277
- package/dist/wasm-database.js +9 -20
- package/package.json +3 -3
- package/src/apply.ts +77 -77
- package/src/client.ts +24 -3
- package/src/progress.ts +4 -0
- package/src/realtime-supervisor.ts +408 -303
- package/src/wasm-database.ts +11 -25
package/dist/wasm-database.js
CHANGED
|
@@ -241,11 +241,7 @@ function opfsSahPoolError(error, directory) {
|
|
|
241
241
|
normalized.includes('opfs api is too old')) {
|
|
242
242
|
return new ClientSyncError(STORAGE_UNAVAILABLE_CODE, `Persistent OPFS storage is unavailable: ${detail}`);
|
|
243
243
|
}
|
|
244
|
-
return new ClientSyncError(STORAGE_BUSY_CODE, '
|
|
245
|
-
`${JSON.stringify(directory)}. Another live engine may still own its ` +
|
|
246
|
-
'SAH pool, or the browser may still be releasing it after a reload. ' +
|
|
247
|
-
'Close the other instance or retry after a short delay; do not delete ' +
|
|
248
|
-
`or rename the directory. Underlying error: ${detail}`, true);
|
|
244
|
+
return new ClientSyncError(STORAGE_BUSY_CODE, 'Another engine holds the OPFS storage directory, or the browser is still releasing it; retry after it closes', true, { directory, cause: detail });
|
|
249
245
|
}
|
|
250
246
|
/** sqlite-wasm's default SAH pool capacity. */
|
|
251
247
|
const SAH_POOL_DEFAULT_CAPACITY = 6;
|
|
@@ -269,14 +265,12 @@ function configureSahCrashRecovery(db, sqlite3) {
|
|
|
269
265
|
const stack = wasm.pstack.pointer;
|
|
270
266
|
try {
|
|
271
267
|
const output = wasm.pstack.allocPtr();
|
|
272
|
-
|
|
273
|
-
capi.sqlite3_file_control(db.pointer, 'main', capi.SQLITE_FCNTL_FILE_POINTER, output)
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
const methodsPointer = file.$pMethods;
|
|
279
|
-
file.dispose(); // Borrowed struct view; SQLite still owns the file.
|
|
268
|
+
const filePointer = db.pointer !== undefined &&
|
|
269
|
+
capi.sqlite3_file_control(db.pointer, 'main', capi.SQLITE_FCNTL_FILE_POINTER, output) === 0 &&
|
|
270
|
+
wasm.peekPtr(output);
|
|
271
|
+
const file = filePointer ? new capi.sqlite3_file(filePointer) : undefined;
|
|
272
|
+
const methodsPointer = file?.$pMethods;
|
|
273
|
+
file?.dispose(); // Borrowed struct view; SQLite still owns the file.
|
|
280
274
|
if (!methodsPointer) {
|
|
281
275
|
throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, 'Could not configure OPFS crash recovery');
|
|
282
276
|
}
|
|
@@ -310,16 +304,11 @@ export async function openPersistentWasmDatabase(name, options) {
|
|
|
310
304
|
throw invalidRequest(`invalid persistent database name ${JSON.stringify(name)}`);
|
|
311
305
|
}
|
|
312
306
|
if (!inWorkerContext()) {
|
|
313
|
-
throw invalidRequest('openPersistentWasmDatabase() must run inside a Web Worker
|
|
314
|
-
'persistent mode is whole-core-in-a-worker by design (Direction ' +
|
|
315
|
-
'decision 2, 2026-07-03). Use the worker handle from the main ' +
|
|
316
|
-
'thread, or openWasmDatabase() for explicitly ephemeral state.');
|
|
307
|
+
throw invalidRequest('openPersistentWasmDatabase() must run inside a Web Worker; use the worker handle on the main thread, or openWasmDatabase() for ephemeral state');
|
|
317
308
|
}
|
|
318
309
|
const storage = globalThis.navigator?.storage;
|
|
319
310
|
if (typeof storage?.getDirectory !== 'function') {
|
|
320
|
-
throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, '
|
|
321
|
-
'requires OPFS (~2023+ browsers). There is no IndexedDB or ' +
|
|
322
|
-
'in-memory fallback for persistent mode.');
|
|
311
|
+
throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, 'Persistent storage requires OPFS, which this browser lacks');
|
|
323
312
|
}
|
|
324
313
|
if (!navigator.locks) {
|
|
325
314
|
throw new ClientSyncError(STORAGE_UNAVAILABLE_CODE, 'Persistent OPFS storage requires Web Locks');
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/client",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.66.0",
|
|
4
4
|
"description": "Syncular TypeScript client core — offline-first sync over SQLite (WASM/OPFS, Bun, Node)",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -121,11 +121,11 @@
|
|
|
121
121
|
},
|
|
122
122
|
"dependencies": {
|
|
123
123
|
"@sqlite.org/sqlite-wasm": "^3.53.0-build1",
|
|
124
|
-
"@syncular/core": "0.
|
|
124
|
+
"@syncular/core": "0.66.0"
|
|
125
125
|
},
|
|
126
126
|
"devDependencies": {
|
|
127
127
|
"@electric-sql/pglite": "^0.5.4",
|
|
128
|
-
"@syncular/server": "0.
|
|
128
|
+
"@syncular/server": "0.66.0",
|
|
129
129
|
"playwright": "1.63.0",
|
|
130
130
|
"vite": "8.3.4"
|
|
131
131
|
}
|
package/src/apply.ts
CHANGED
|
@@ -208,33 +208,26 @@ export function validateSegmentColumns(
|
|
|
208
208
|
table: CompiledClientTable,
|
|
209
209
|
segment: RowsSegment,
|
|
210
210
|
): void {
|
|
211
|
-
const
|
|
211
|
+
const column =
|
|
212
|
+
segment.columns.length !== table.columns.length
|
|
213
|
+
? 'count'
|
|
214
|
+
: table.columns.findIndex(
|
|
215
|
+
(expected, i) =>
|
|
216
|
+
expected.name !== segment.columns[i]?.name ||
|
|
217
|
+
expected.type !== segment.columns[i]?.type ||
|
|
218
|
+
expected.nullable !== segment.columns[i]?.nullable,
|
|
219
|
+
);
|
|
220
|
+
if (
|
|
221
|
+
segment.table !== table.name ||
|
|
222
|
+
segment.schemaVersion !== schema.version ||
|
|
223
|
+
column !== -1
|
|
224
|
+
)
|
|
212
225
|
throw new ClientSyncError(
|
|
213
226
|
'sync.schema_mismatch',
|
|
214
|
-
|
|
227
|
+
'rows segment columns do not match the generated schema (§5.2)',
|
|
228
|
+
false,
|
|
229
|
+
column === -1 ? { table: table.name } : { table: table.name, column },
|
|
215
230
|
);
|
|
216
|
-
};
|
|
217
|
-
if (segment.table !== table.name) {
|
|
218
|
-
mismatch(`segment table ${JSON.stringify(segment.table)}`);
|
|
219
|
-
}
|
|
220
|
-
if (segment.schemaVersion !== schema.version) {
|
|
221
|
-
mismatch(`segment schemaVersion ${segment.schemaVersion}`);
|
|
222
|
-
}
|
|
223
|
-
if (segment.columns.length !== table.columns.length) {
|
|
224
|
-
mismatch(`column count ${segment.columns.length}`);
|
|
225
|
-
}
|
|
226
|
-
for (let i = 0; i < table.columns.length; i++) {
|
|
227
|
-
const expected = table.columns[i];
|
|
228
|
-
const actual = segment.columns[i];
|
|
229
|
-
if (expected === undefined || actual === undefined) continue;
|
|
230
|
-
if (
|
|
231
|
-
expected.name !== actual.name ||
|
|
232
|
-
expected.type !== actual.type ||
|
|
233
|
-
expected.nullable !== actual.nullable
|
|
234
|
-
) {
|
|
235
|
-
mismatch(`column ${i} (${actual.name})`);
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
231
|
}
|
|
239
232
|
|
|
240
233
|
/**
|
|
@@ -427,10 +420,10 @@ function imageInvalid(detail: string): never {
|
|
|
427
420
|
}
|
|
428
421
|
|
|
429
422
|
/**
|
|
430
|
-
* Apply a §5.3 sqlite image in committed
|
|
431
|
-
* the in-file metadata against the descriptor, validate the
|
|
432
|
-
* column names/order against the generated schema, run the
|
|
433
|
-
* first-page clear when fresh, then bulk-copy
|
|
423
|
+
* Apply a §5.3 sqlite image in committed transactions of 1,024 to 65,536
|
|
424
|
+
* rows: validate the in-file metadata against the descriptor, validate the
|
|
425
|
+
* data table's column names/order against the generated schema, run the
|
|
426
|
+
* §5.6 first-page clear when fresh, then bulk-copy 1,024-row ranges through a
|
|
434
427
|
* primary-key upsert — `_syncular_version` lands in
|
|
435
428
|
* `_sync_version` exactly like a rows segment's per-row `serverVersion`.
|
|
436
429
|
*/
|
|
@@ -477,22 +470,15 @@ export async function applySqliteSegment(
|
|
|
477
470
|
if (meta.length !== 1 || record === undefined) {
|
|
478
471
|
imageInvalid('_syncular_segment must contain exactly one row');
|
|
479
472
|
}
|
|
480
|
-
if (
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
imageInvalid(
|
|
489
|
-
}
|
|
490
|
-
if (record.sd !== descriptor.scopeDigest) {
|
|
491
|
-
imageInvalid('scopeDigest mismatch');
|
|
492
|
-
}
|
|
493
|
-
if (Number(record.rc) !== descriptor.rowCount) {
|
|
494
|
-
imageInvalid(`rowCount ${String(record.rc)}`);
|
|
495
|
-
}
|
|
473
|
+
if (
|
|
474
|
+
record.format !== 1 ||
|
|
475
|
+
record.tbl !== table.name ||
|
|
476
|
+
Number(record.sv) !== schema.version ||
|
|
477
|
+
Number(record.pin) !== descriptor.asOfCommitSeq ||
|
|
478
|
+
record.sd !== descriptor.scopeDigest ||
|
|
479
|
+
Number(record.rc) !== descriptor.rowCount
|
|
480
|
+
)
|
|
481
|
+
imageInvalid('_syncular_segment does not match the descriptor');
|
|
496
482
|
|
|
497
483
|
// 2. Column names and order vs the generated schema (§5.3 rule 3 —
|
|
498
484
|
// sync.schema_mismatch, the §5.2 rule specialized).
|
|
@@ -510,7 +496,9 @@ export async function applySqliteSegment(
|
|
|
510
496
|
) {
|
|
511
497
|
throw new ClientSyncError(
|
|
512
498
|
'sync.schema_mismatch',
|
|
513
|
-
|
|
499
|
+
'sqlite segment columns do not match the generated schema (§5.3)',
|
|
500
|
+
false,
|
|
501
|
+
{ table: table.name },
|
|
514
502
|
);
|
|
515
503
|
}
|
|
516
504
|
|
|
@@ -529,42 +517,43 @@ export async function applySqliteSegment(
|
|
|
529
517
|
.map((name) => `${name}=excluded.${name}`)
|
|
530
518
|
.join(', ');
|
|
531
519
|
const version = quoteIdent(SYNC_VERSION_COLUMN);
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
520
|
+
// A statement copies the 1,024 rows after `from`; NULL keys cannot hide
|
|
521
|
+
// between statements.
|
|
522
|
+
const range = (from?: SqlValue) => {
|
|
523
|
+
const params = from === undefined ? [] : [from];
|
|
524
|
+
const predicates = from === undefined ? [] : [`${primaryKey}>?`];
|
|
536
525
|
const boundary = db.query(
|
|
537
|
-
`SELECT ${primaryKey} AS
|
|
538
|
-
|
|
539
|
-
)[0]?.
|
|
526
|
+
`SELECT ${primaryKey} AS b FROM ${source} WHERE ${predicates[0] ?? 'true'} ORDER BY ${primaryKey} LIMIT 1 OFFSET 1023`,
|
|
527
|
+
params,
|
|
528
|
+
)[0]?.b;
|
|
540
529
|
if (boundary === null) imageInvalid('image primary key must not be null');
|
|
541
|
-
const predicates: string[] = [];
|
|
542
|
-
const params: SqlValue[] = [];
|
|
543
|
-
if (after !== undefined) {
|
|
544
|
-
predicates.push(`${primaryKey} > ?`);
|
|
545
|
-
params.push(after);
|
|
546
|
-
}
|
|
547
530
|
if (boundary !== undefined) {
|
|
548
|
-
predicates.push(`${primaryKey}
|
|
531
|
+
predicates.push(`${primaryKey}<=?`);
|
|
549
532
|
params.push(boundary);
|
|
550
533
|
}
|
|
534
|
+
return [boundary, predicates.join(' AND ') || 'true', params] as const;
|
|
535
|
+
};
|
|
536
|
+
let [boundary, where, params] = range();
|
|
537
|
+
let applied = 0;
|
|
538
|
+
// §5.3: a transaction holds one statement at first and doubles after each
|
|
539
|
+
// commit up to 64; a replaying transaction holds one.
|
|
540
|
+
for (let statements = 1; ; statements = Math.min(statements * 2, 64)) {
|
|
541
|
+
if (options.isCurrent?.() === false) return applied;
|
|
542
|
+
const first = applied === 0;
|
|
551
543
|
const replay =
|
|
552
544
|
countOutbox(db) ||
|
|
553
545
|
db.query(`SELECT 1 FROM ${RETAINED_ROWS} WHERE tbl=? LIMIT 1`, [
|
|
554
546
|
table.name,
|
|
555
547
|
]).length;
|
|
556
548
|
const rows = replay
|
|
557
|
-
? db.query(
|
|
558
|
-
`SELECT * FROM ${source} WHERE ${predicates.join(' AND ') || 'true'}`,
|
|
559
|
-
params,
|
|
560
|
-
)
|
|
549
|
+
? db.query(`SELECT * FROM ${source} WHERE ${where}`, params)
|
|
561
550
|
: [];
|
|
562
551
|
(options.transaction ?? ((fn) => db.transaction(fn)))(() => {
|
|
563
552
|
const scope = !replay
|
|
564
553
|
? []
|
|
565
554
|
: [
|
|
566
555
|
...(hasUniqueIndex(table) ? [JSON.stringify([table.name])] : []),
|
|
567
|
-
...(
|
|
556
|
+
...(first && options.clearFirst
|
|
568
557
|
? bootstrapScope(db, table, options.effective, options.owners)
|
|
569
558
|
: []),
|
|
570
559
|
...rows.map((row) =>
|
|
@@ -572,7 +561,7 @@ export async function applySqliteSegment(
|
|
|
572
561
|
),
|
|
573
562
|
];
|
|
574
563
|
const protectedRows = restoreFailedBases(db, schema, scope);
|
|
575
|
-
if (
|
|
564
|
+
if (first && options.clearFirst)
|
|
576
565
|
deleteScopedRows(db, table, options.effective, options.owners);
|
|
577
566
|
const kept = new Set<string>();
|
|
578
567
|
const skipped = new Set<string>();
|
|
@@ -586,17 +575,27 @@ export async function applySqliteSegment(
|
|
|
586
575
|
skipped,
|
|
587
576
|
),
|
|
588
577
|
);
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
params,
|
|
593
|
-
);
|
|
594
|
-
applied += Number(
|
|
595
|
-
db.query(
|
|
596
|
-
`SELECT count(*)AS n FROM ${source} WHERE ${where}`,
|
|
578
|
+
for (let statement = 1; ; statement++) {
|
|
579
|
+
db.exec(
|
|
580
|
+
`INSERT INTO ${quoteIdent(table.name)}(${insertNames.join(', ')})SELECT ${[...names, quoteIdent('_syncular_version')].join(', ')} FROM ${source} WHERE ${where} ON CONFLICT(${primaryKey})DO UPDATE SET ${updates} WHERE excluded.${version}>${quoteIdent(table.name)}.${version}`,
|
|
597
581
|
params,
|
|
598
|
-
)
|
|
599
|
-
|
|
582
|
+
);
|
|
583
|
+
applied += Number(
|
|
584
|
+
db.query(
|
|
585
|
+
`SELECT count(*)AS n FROM ${source} WHERE ${where}`,
|
|
586
|
+
params,
|
|
587
|
+
)[0]?.n,
|
|
588
|
+
);
|
|
589
|
+
options.onProgress?.(applied);
|
|
590
|
+
if (
|
|
591
|
+
replay ||
|
|
592
|
+
statement === statements ||
|
|
593
|
+
boundary === undefined ||
|
|
594
|
+
applied === count
|
|
595
|
+
)
|
|
596
|
+
break;
|
|
597
|
+
[boundary, where, params] = range(boundary);
|
|
598
|
+
}
|
|
600
599
|
if (protectedRows)
|
|
601
600
|
for (const row of landing) {
|
|
602
601
|
retainedBaseWrite(
|
|
@@ -614,13 +613,14 @@ export async function applySqliteSegment(
|
|
|
614
613
|
}
|
|
615
614
|
return rebaseScope(scope, kept, skipped);
|
|
616
615
|
});
|
|
617
|
-
options.onProgress?.(applied);
|
|
618
616
|
await yieldToHost();
|
|
619
617
|
if (boundary === undefined || applied === count) break;
|
|
620
|
-
|
|
618
|
+
[boundary, where, params] = range(boundary);
|
|
621
619
|
}
|
|
622
620
|
if (applied !== count)
|
|
623
621
|
imageInvalid('image applied row count does not match descriptor');
|
|
622
|
+
// A file replica's WAL keeps its high-water size until truncated.
|
|
623
|
+
db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
624
624
|
return applied;
|
|
625
625
|
});
|
|
626
626
|
}
|
package/src/client.ts
CHANGED
|
@@ -5432,6 +5432,8 @@ export class SyncClient {
|
|
|
5432
5432
|
bytesTotal: frame.byteLength,
|
|
5433
5433
|
rowsProcessed: 0,
|
|
5434
5434
|
rowsTotal: frame.rowCount,
|
|
5435
|
+
importWriteMs: undefined,
|
|
5436
|
+
importCommitMs: undefined,
|
|
5435
5437
|
});
|
|
5436
5438
|
prefetch();
|
|
5437
5439
|
const pending = prefetched.get(frame);
|
|
@@ -5476,6 +5478,8 @@ export class SyncClient {
|
|
|
5476
5478
|
(n, block) => n + block.length,
|
|
5477
5479
|
0,
|
|
5478
5480
|
),
|
|
5481
|
+
importWriteMs: undefined,
|
|
5482
|
+
importCommitMs: undefined,
|
|
5479
5483
|
});
|
|
5480
5484
|
await this.#applySegmentOrFail(
|
|
5481
5485
|
section,
|
|
@@ -5491,6 +5495,7 @@ export class SyncClient {
|
|
|
5491
5495
|
? image.rowCount > 0
|
|
5492
5496
|
: segment!.blocks.some((block) => block.length > 0),
|
|
5493
5497
|
resetEpoch,
|
|
5498
|
+
image !== undefined,
|
|
5494
5499
|
);
|
|
5495
5500
|
return image !== undefined
|
|
5496
5501
|
? applySqliteSegment(
|
|
@@ -6001,15 +6006,20 @@ export class SyncClient {
|
|
|
6001
6006
|
owners: readonly ScopeMap[],
|
|
6002
6007
|
hasRows: boolean,
|
|
6003
6008
|
resetEpoch: number,
|
|
6009
|
+
image: boolean,
|
|
6004
6010
|
): Parameters<typeof applyRowsSegment>[4] {
|
|
6011
|
+
let writeMs = 0;
|
|
6012
|
+
let commitMs = 0;
|
|
6005
6013
|
return {
|
|
6006
6014
|
clearFirst,
|
|
6007
6015
|
effective,
|
|
6008
6016
|
owners,
|
|
6009
6017
|
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
6010
6018
|
onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
|
|
6011
|
-
transaction: (fn) =>
|
|
6012
|
-
|
|
6019
|
+
transaction: (fn) => {
|
|
6020
|
+
const started = performance.now();
|
|
6021
|
+
let written = started;
|
|
6022
|
+
const result = this.#applyBatch((batch) => {
|
|
6013
6023
|
if (
|
|
6014
6024
|
hasRows ||
|
|
6015
6025
|
(clearFirst && this.#scopedRowsExist(table, effective, owners))
|
|
@@ -6017,8 +6027,19 @@ export class SyncClient {
|
|
|
6017
6027
|
batch.table(table.name);
|
|
6018
6028
|
const result = fn();
|
|
6019
6029
|
this.#replayOutbox(result, false);
|
|
6030
|
+
written = performance.now();
|
|
6020
6031
|
return result;
|
|
6021
|
-
})
|
|
6032
|
+
});
|
|
6033
|
+
// §7.7: image payloads report their import cost.
|
|
6034
|
+
writeMs += written - started;
|
|
6035
|
+
commitMs += performance.now() - written;
|
|
6036
|
+
if (image)
|
|
6037
|
+
this.#progress.update({
|
|
6038
|
+
importWriteMs: Math.round(writeMs),
|
|
6039
|
+
importCommitMs: Math.round(commitMs),
|
|
6040
|
+
});
|
|
6041
|
+
return result;
|
|
6042
|
+
},
|
|
6022
6043
|
};
|
|
6023
6044
|
}
|
|
6024
6045
|
|
package/src/progress.ts
CHANGED
|
@@ -12,6 +12,10 @@ export interface SyncProgress {
|
|
|
12
12
|
readonly bytesTotal?: number;
|
|
13
13
|
readonly rowsProcessed: number;
|
|
14
14
|
readonly rowsTotal?: number;
|
|
15
|
+
/** Time in this payload's import transactions before COMMIT, in ms. */
|
|
16
|
+
readonly importWriteMs?: number | undefined;
|
|
17
|
+
/** Time committing them and publishing their changes, in ms. */
|
|
18
|
+
readonly importCommitMs?: number | undefined;
|
|
15
19
|
readonly errorCode?: string;
|
|
16
20
|
readonly causeKind?: TransportCauseKind;
|
|
17
21
|
readonly httpStatus?: number;
|