@syncular/client 0.19.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +118 -6
- package/dist/apply.d.ts +16 -7
- package/dist/apply.js +83 -37
- package/dist/bun-database.d.ts +1 -1
- package/dist/bun-database.js +2 -2
- package/dist/client.d.ts +7 -4
- package/dist/client.js +121 -35
- package/dist/database.d.ts +3 -2
- package/dist/http.d.ts +0 -8
- package/dist/http.js +36 -3
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/node-database.d.ts +1 -1
- package/dist/node-database.js +2 -2
- package/dist/progress.d.ts +24 -0
- package/dist/progress.js +42 -0
- package/dist/reactive-store.js +49 -8
- package/dist/schema.js +15 -4
- package/dist/transport.d.ts +2 -1
- package/dist/wasm-database.js +44 -4
- package/dist/worker-entry.d.ts +2 -0
- package/dist/worker-entry.js +34 -7
- package/dist/worker-host.d.ts +3 -0
- package/dist/worker-host.js +13 -1
- package/dist/worker-protocol.d.ts +4 -0
- package/package.json +4 -3
- package/src/apply.ts +99 -47
- package/src/bun-database.ts +6 -2
- package/src/client.ts +153 -38
- package/src/database.ts +7 -2
- package/src/http.ts +41 -3
- package/src/index.ts +6 -0
- package/src/node-database.ts +6 -2
- package/src/progress.ts +62 -0
- package/src/reactive-store.ts +58 -8
- package/src/schema.ts +22 -4
- package/src/transport.ts +5 -1
- package/src/wasm-database.ts +86 -4
- package/src/worker-entry.ts +38 -7
- package/src/worker-host.ts +17 -1
- package/src/worker-protocol.ts +2 -0
package/dist/client.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { ProgressEmitter, } from './progress.js';
|
|
1
2
|
/**
|
|
2
3
|
* SyncClient implements the client side of SPEC.md §§3–8.
|
|
3
4
|
*
|
|
@@ -8,7 +9,7 @@
|
|
|
8
9
|
* `sync()` round (§7.2); local reads go straight to the database.
|
|
9
10
|
*/
|
|
10
11
|
import { canonicalScopeJson, decodeMessage, decodeRow, decodeRowsSegment, encodeMessage, encodePresencePublish, MessageStreamScanner, PROTOCOL_WIRE_VERSION, parseRealtimeServerEvent, REALTIME_TAG_DELTA, REALTIME_TAG_ROUND, } from '@syncular/core';
|
|
11
|
-
import { applyCommitFrame, applyRowsSegment, applySqliteSegment, deleteLocalRow, deleteScopedRows, evictScopedRows, upsertLocalRow, } from './apply.js';
|
|
12
|
+
import { applyCommitFrame, applyRowsSegment, applySqliteSegment, deleteLocalRow, deleteScopedRows, evictScopedRows, yieldToHost, upsertLocalRow, } from './apply.js';
|
|
12
13
|
import { clearPendingUpload, computeBlobId, deleteUnreferencedCachedBlobs, enforceBlobCacheCap, ensureBlobSchema, getCachedBlob, listPendingUploads, parseBlobRef, putCachedBlob, recordPendingUpload, schemaHasBlobs, serializeBlobRef, } from './blob.js';
|
|
13
14
|
import { registerDevtools } from './devtools.js';
|
|
14
15
|
import { CLIENT_DIAGNOSTICS_VERSION, ClientDiagnosticsEmitter, MAX_DIAGNOSTIC_DOMAINS, MAX_DIAGNOSTIC_EXPECTED_SUBSCRIPTIONS, } from './diagnostics.js';
|
|
@@ -102,6 +103,8 @@ export class SyncClient {
|
|
|
102
103
|
#realtimeGeneration = 0;
|
|
103
104
|
#pendingRound;
|
|
104
105
|
#needsPull = false;
|
|
106
|
+
#progress = new ProgressEmitter();
|
|
107
|
+
#progressAttempt = 0;
|
|
105
108
|
#syncing = false;
|
|
106
109
|
/**
|
|
107
110
|
* True from the synchronous entry of `sync()` until its serialized round
|
|
@@ -233,6 +236,7 @@ export class SyncClient {
|
|
|
233
236
|
// an application-issued sync() call.
|
|
234
237
|
const startupWork = this.#schemaFloor === undefined &&
|
|
235
238
|
(countOutbox(this.#db) > 0 ||
|
|
239
|
+
loadPendingEvictions(this.#db).length > 0 ||
|
|
236
240
|
subscriptions.some((sub) => sub.status === 'active'));
|
|
237
241
|
if (startupWork && this.#securityLifecycle === 'active') {
|
|
238
242
|
this.#needsPull = true;
|
|
@@ -359,6 +363,7 @@ export class SyncClient {
|
|
|
359
363
|
await this.#lease?.release();
|
|
360
364
|
this.#lease = undefined;
|
|
361
365
|
this.#started = false;
|
|
366
|
+
this.#progress.clear();
|
|
362
367
|
this.#syncNeededListeners.clear();
|
|
363
368
|
this.#syncIntentListeners.clear();
|
|
364
369
|
}
|
|
@@ -537,6 +542,12 @@ export class SyncClient {
|
|
|
537
542
|
return this.#invalidation.on(listener);
|
|
538
543
|
}
|
|
539
544
|
/** Subscribe to exact revisioned observer transactions (SPEC §7.5). */
|
|
545
|
+
onProgress(listener) {
|
|
546
|
+
return this.#progress.on(listener);
|
|
547
|
+
}
|
|
548
|
+
progressSnapshot() {
|
|
549
|
+
return this.#progress.snapshot();
|
|
550
|
+
}
|
|
540
551
|
onChange(listener) {
|
|
541
552
|
return this.#changes.on(listener);
|
|
542
553
|
}
|
|
@@ -1116,22 +1127,17 @@ export class SyncClient {
|
|
|
1116
1127
|
}
|
|
1117
1128
|
socket.send(encodePresencePublish(scopeKey, doc));
|
|
1118
1129
|
}
|
|
1119
|
-
/**
|
|
1120
|
-
* §4.2 accept mask: the configured override, or the rows baseline plus
|
|
1121
|
-
* bit 2 when the backend can import sqlite images (§5.3) and a segment
|
|
1122
|
-
* downloader exists (sqlite segments are never inline, §5.7), plus
|
|
1123
|
-
* bit 3 when the downloader exposes a direct URL fetch (§5.4
|
|
1124
|
-
* capability negotiation).
|
|
1125
|
-
*/
|
|
1130
|
+
/** Advertise supported segment formats; all imports yield between commits. */
|
|
1126
1131
|
#acceptMask() {
|
|
1127
1132
|
const configured = this.#config.limits?.accept;
|
|
1128
1133
|
if (configured !== undefined)
|
|
1129
1134
|
return configured;
|
|
1130
1135
|
const segments = this.#config.segments;
|
|
1131
|
-
const sqliteCapable = typeof this.#db.withSqliteImage === 'function' && segments !== undefined;
|
|
1132
1136
|
const urlCapable = typeof segments?.fetchUrl === 'function';
|
|
1133
1137
|
return (ACCEPT_ROWS_BASELINE |
|
|
1134
|
-
(
|
|
1138
|
+
(typeof this.#db.withSqliteImage === 'function' && segments !== undefined
|
|
1139
|
+
? ACCEPT_SQLITE
|
|
1140
|
+
: 0) |
|
|
1135
1141
|
(urlCapable ? ACCEPT_SIGNED_URLS : 0));
|
|
1136
1142
|
}
|
|
1137
1143
|
subscriptions() {
|
|
@@ -1189,8 +1195,8 @@ export class SyncClient {
|
|
|
1189
1195
|
* Set the live window units for a base (§4.8): a value-sharded family of
|
|
1190
1196
|
* subscriptions, one per unit. Computes the diff against the registry —
|
|
1191
1197
|
* added units get fresh subscriptions (image-lane bootstrap on the next
|
|
1192
|
-
* sync); removed units are unsubscribed
|
|
1193
|
-
* transaction (E1–E4). Idempotent: calling with the same units is a
|
|
1198
|
+
* sync); removed units are unsubscribed with their first eviction chunk
|
|
1199
|
+
* in one local transaction (E1–E4). Idempotent: calling with the same units is a
|
|
1194
1200
|
* no-op. Re-entry (a unit removed then re-added) cancels any deferred
|
|
1195
1201
|
* eviction and fresh-bootstraps.
|
|
1196
1202
|
*
|
|
@@ -1248,7 +1254,7 @@ export class SyncClient {
|
|
|
1248
1254
|
for (const { unit, subId } of live) {
|
|
1249
1255
|
if (wanted.has(unit))
|
|
1250
1256
|
continue;
|
|
1251
|
-
this.#evictUnit(baseKey, base, unit, subId);
|
|
1257
|
+
await this.#evictUnit(baseKey, base, unit, subId);
|
|
1252
1258
|
changed = true;
|
|
1253
1259
|
}
|
|
1254
1260
|
});
|
|
@@ -1287,8 +1293,8 @@ export class SyncClient {
|
|
|
1287
1293
|
return { units: live.map((u) => u.unit), pending };
|
|
1288
1294
|
}
|
|
1289
1295
|
/**
|
|
1290
|
-
* §4.8 E1–E4: evict one departing unit
|
|
1291
|
-
*
|
|
1296
|
+
* §4.8 E1–E4: evict one departing unit in committed chunks, fusing
|
|
1297
|
+
* its first chunk with unsubscription. Deletes the unit's rows EXCEPT those pinned by a
|
|
1292
1298
|
* pending outbox commit (E1); if any pin remains, records a deferred
|
|
1293
1299
|
* eviction retried on the next outbox drain. Discards the subscription's
|
|
1294
1300
|
* cursor/resume/effective-echo (E3) and its version state with the rows
|
|
@@ -1296,19 +1302,21 @@ export class SyncClient {
|
|
|
1296
1302
|
* with no local scope-column mapping, surfaces a configuration error and
|
|
1297
1303
|
* evicts nothing (§4.8/§3.3).
|
|
1298
1304
|
*/
|
|
1299
|
-
#evictUnit(baseKey, base, unit, subId) {
|
|
1305
|
+
async #evictUnit(baseKey, base, unit, subId) {
|
|
1300
1306
|
const table = this.#table(base.table);
|
|
1301
1307
|
const sub = getSubscription(this.#db, subId);
|
|
1302
1308
|
// The rows a unit holds live under its LAST effective scopes if it ever
|
|
1303
1309
|
// synced; before first sync, the requested unit scopes are the match.
|
|
1304
1310
|
const effective = sub?.effectiveScopes ?? unitScopes(base, unit);
|
|
1305
1311
|
const pinned = this.#pinnedRowIds(base.table);
|
|
1312
|
+
let remaining = false;
|
|
1306
1313
|
this.#applyBatch((batch) => {
|
|
1307
1314
|
this.#db.transaction(() => {
|
|
1308
|
-
const
|
|
1315
|
+
const result = evictScopedRows(this.#db, table, effective, pinned);
|
|
1316
|
+
remaining = result.remaining;
|
|
1309
1317
|
deleteWindowUnit(this.#db, baseKey, unit);
|
|
1310
1318
|
deleteSubscription(this.#db, subId);
|
|
1311
|
-
if (deferred) {
|
|
1319
|
+
if (result.deferred) {
|
|
1312
1320
|
savePendingEviction(this.#db, subId, base.table, effective);
|
|
1313
1321
|
}
|
|
1314
1322
|
else {
|
|
@@ -1320,13 +1328,17 @@ export class SyncClient {
|
|
|
1320
1328
|
this.#needsPull = true;
|
|
1321
1329
|
batch.status();
|
|
1322
1330
|
});
|
|
1331
|
+
if (remaining) {
|
|
1332
|
+
await yieldToHost();
|
|
1333
|
+
await this.#drainPendingEvictions();
|
|
1334
|
+
}
|
|
1323
1335
|
}
|
|
1324
1336
|
/**
|
|
1325
1337
|
* §4.8 E1: retry deferred evictions after the outbox drains. A pinned
|
|
1326
1338
|
* unit's rows are removed once no pending commit references them; a unit
|
|
1327
1339
|
* that re-entered the window in the meantime has no pending record left.
|
|
1328
1340
|
*/
|
|
1329
|
-
#drainPendingEvictions() {
|
|
1341
|
+
async #drainPendingEvictions() {
|
|
1330
1342
|
const pending = loadPendingEvictions(this.#db);
|
|
1331
1343
|
if (pending.length === 0)
|
|
1332
1344
|
return;
|
|
@@ -1336,16 +1348,23 @@ export class SyncClient {
|
|
|
1336
1348
|
deletePendingEviction(this.#db, entry.subId);
|
|
1337
1349
|
continue;
|
|
1338
1350
|
}
|
|
1339
|
-
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1351
|
+
let remaining;
|
|
1352
|
+
do {
|
|
1353
|
+
if (!loadPendingEvictions(this.#db).some((item) => item.subId === entry.subId))
|
|
1354
|
+
break;
|
|
1355
|
+
// Local writes may add pins while the worker yields between chunks.
|
|
1356
|
+
const pinned = this.#pinnedRowIds(entry.table);
|
|
1357
|
+
remaining = false;
|
|
1358
|
+
this.#applyBatch((batch) => {
|
|
1359
|
+
const result = evictScopedRows(this.#db, table, entry.effective, pinned);
|
|
1360
|
+
remaining = result.remaining;
|
|
1361
|
+
if (!result.deferred)
|
|
1345
1362
|
deletePendingEviction(this.#db, entry.subId);
|
|
1363
|
+
batch.scopeMap(table, entry.effective);
|
|
1346
1364
|
});
|
|
1347
|
-
|
|
1348
|
-
|
|
1365
|
+
if (remaining)
|
|
1366
|
+
await yieldToHost();
|
|
1367
|
+
} while (remaining);
|
|
1349
1368
|
}
|
|
1350
1369
|
}
|
|
1351
1370
|
/**
|
|
@@ -1837,6 +1856,12 @@ export class SyncClient {
|
|
|
1837
1856
|
}, (error) => {
|
|
1838
1857
|
const completedAtMs = this.#now();
|
|
1839
1858
|
const code = error.code;
|
|
1859
|
+
this.#progress.update({
|
|
1860
|
+
state: 'failed',
|
|
1861
|
+
errorCode: typeof code === 'string'
|
|
1862
|
+
? this.#diagnosticCode(code)
|
|
1863
|
+
: 'client.unknown_failure',
|
|
1864
|
+
});
|
|
1840
1865
|
this.#lastRound = {
|
|
1841
1866
|
status: 'failed',
|
|
1842
1867
|
startedAtMs,
|
|
@@ -1883,7 +1908,15 @@ export class SyncClient {
|
|
|
1883
1908
|
: 'client.unknown_failure';
|
|
1884
1909
|
}
|
|
1885
1910
|
async #runSync() {
|
|
1911
|
+
this.#progress.emit({
|
|
1912
|
+
attempt: ++this.#progressAttempt,
|
|
1913
|
+
state: 'running',
|
|
1914
|
+
phase: 'request',
|
|
1915
|
+
bytesReceived: 0,
|
|
1916
|
+
rowsProcessed: 0,
|
|
1917
|
+
});
|
|
1886
1918
|
if (this.#schemaFloor !== undefined) {
|
|
1919
|
+
this.#progress.update({ state: 'complete' });
|
|
1887
1920
|
return {
|
|
1888
1921
|
...emptySummary(0),
|
|
1889
1922
|
bootstrapping: [],
|
|
@@ -1896,6 +1929,7 @@ export class SyncClient {
|
|
|
1896
1929
|
// survive it — the reference server keeps no replay buffer (§8.2).
|
|
1897
1930
|
this.#setSyncNeeded(false);
|
|
1898
1931
|
try {
|
|
1932
|
+
await this.#drainPendingEvictions();
|
|
1899
1933
|
const logEpoch = getMeta(this.#db, LOG_EPOCH_META_KEY);
|
|
1900
1934
|
// §5.9.7 B4: upload pending blobs BEFORE pushing rows that reference
|
|
1901
1935
|
// them, so the server-side existence check (§6.6) passes.
|
|
@@ -1957,7 +1991,10 @@ export class SyncClient {
|
|
|
1957
1991
|
const summary = await this.#processResponse(message, outbox, subs, 'pull', resetEpoch);
|
|
1958
1992
|
// §4.8 E1: the push half may have drained commits that pinned rows of
|
|
1959
1993
|
// a shrunk window unit — retry any deferred evictions now.
|
|
1960
|
-
this.#drainPendingEvictions();
|
|
1994
|
+
await this.#drainPendingEvictions();
|
|
1995
|
+
this.#progress.update(summary.failed.length > 0
|
|
1996
|
+
? { state: 'failed', errorCode: 'sync.scope_revoked' }
|
|
1997
|
+
: { state: 'complete' });
|
|
1961
1998
|
this.#retryDelayMs = 250;
|
|
1962
1999
|
if (deferred > 0) {
|
|
1963
2000
|
// §6.1 splitBatch remainder: more queued commits than this request
|
|
@@ -2001,10 +2038,21 @@ export class SyncClient {
|
|
|
2001
2038
|
* Pull repeatedly until quiescent: no commits delivered, no bootstrap
|
|
2002
2039
|
* pages pending, no resets to recover (§4.5 "pull again" SHOULD).
|
|
2003
2040
|
*/
|
|
2004
|
-
async syncUntilIdle(maxRounds
|
|
2041
|
+
async syncUntilIdle(maxRounds) {
|
|
2042
|
+
const budget = maxRounds ?? 20;
|
|
2005
2043
|
let last;
|
|
2006
|
-
for (let round = 0; round <
|
|
2044
|
+
for (let round = 0; round < budget; round++) {
|
|
2045
|
+
const before = JSON.stringify(loadSubscriptions(this.#db).map((sub) => [sub.id, sub.bootstrapState]));
|
|
2007
2046
|
last = await this.sync();
|
|
2047
|
+
if (maxRounds === undefined &&
|
|
2048
|
+
last.segmentRowsApplied > 0 &&
|
|
2049
|
+
last.bootstrapping.length > 0 &&
|
|
2050
|
+
before !==
|
|
2051
|
+
JSON.stringify(loadSubscriptions(this.#db).map((sub) => [
|
|
2052
|
+
sub.id,
|
|
2053
|
+
sub.bootstrapState,
|
|
2054
|
+
])))
|
|
2055
|
+
round = -1;
|
|
2008
2056
|
if (last.schemaFloor !== undefined)
|
|
2009
2057
|
return last;
|
|
2010
2058
|
if (last.commitsApplied === 0 &&
|
|
@@ -2016,7 +2064,7 @@ export class SyncClient {
|
|
|
2016
2064
|
return last;
|
|
2017
2065
|
}
|
|
2018
2066
|
}
|
|
2019
|
-
throw new ClientSyncError('sync.invalid_request',
|
|
2067
|
+
throw new ClientSyncError('sync.invalid_request', 'sync did not reach idle within the round budget');
|
|
2020
2068
|
}
|
|
2021
2069
|
/**
|
|
2022
2070
|
* One request/response round trip (§8.7): over the socket whenever it
|
|
@@ -2430,8 +2478,20 @@ export class SyncClient {
|
|
|
2430
2478
|
break;
|
|
2431
2479
|
}
|
|
2432
2480
|
const segment = decodeRowsSegment(frame.payload);
|
|
2481
|
+
this.#progress.update({
|
|
2482
|
+
phase: 'import',
|
|
2483
|
+
subscriptionId: section.sub.id,
|
|
2484
|
+
table: section.sub.table,
|
|
2485
|
+
segmentId: undefined,
|
|
2486
|
+
bytesReceived: frame.payload.byteLength,
|
|
2487
|
+
bytesTotal: frame.payload.byteLength,
|
|
2488
|
+
rowsProcessed: 0,
|
|
2489
|
+
rowsTotal: segment.blocks.reduce((n, block) => n + block.length, 0),
|
|
2490
|
+
});
|
|
2433
2491
|
await this.#applySegmentOrFail(section, summary, (table, clearFirst, effective) => applyRowsSegment(this.#db, this.#schema, table, segment, {
|
|
2434
2492
|
clearFirst,
|
|
2493
|
+
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
2494
|
+
onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
|
|
2435
2495
|
effective,
|
|
2436
2496
|
transaction: (fn) => this.#applyBatch((batch) => {
|
|
2437
2497
|
if (segment.blocks.some((block) => block.length > 0) ||
|
|
@@ -2439,7 +2499,9 @@ export class SyncClient {
|
|
|
2439
2499
|
this.#scopedRowsExist(table, effective))) {
|
|
2440
2500
|
batch.table(table.name);
|
|
2441
2501
|
}
|
|
2442
|
-
|
|
2502
|
+
const result = fn();
|
|
2503
|
+
this.#replayOutbox();
|
|
2504
|
+
return result;
|
|
2443
2505
|
}),
|
|
2444
2506
|
}, this.#encryption), section.fresh && !section.cleared);
|
|
2445
2507
|
break;
|
|
@@ -2457,6 +2519,7 @@ export class SyncClient {
|
|
|
2457
2519
|
throw new ClientSyncError('sync.invalid_request', 'SEGMENT_REF mediaType sqlite was not advertised in accept (§4.2)');
|
|
2458
2520
|
}
|
|
2459
2521
|
const bytes = await this.#downloadSegment(frame, section.sub);
|
|
2522
|
+
this.#progress.update({ phase: 'import' });
|
|
2460
2523
|
if (frame.mediaType === 'sqlite') {
|
|
2461
2524
|
// §5.3: images are whole-table — a paged descriptor is
|
|
2462
2525
|
// invalid, and the image is always its table's first page.
|
|
@@ -2471,6 +2534,8 @@ export class SyncClient {
|
|
|
2471
2534
|
scopeDigest: frame.scopeDigest,
|
|
2472
2535
|
}, {
|
|
2473
2536
|
clearFirst,
|
|
2537
|
+
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
2538
|
+
onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
|
|
2474
2539
|
effective,
|
|
2475
2540
|
transaction: (fn) => this.#applyBatch((batch) => {
|
|
2476
2541
|
if (frame.rowCount > 0 ||
|
|
@@ -2478,7 +2543,9 @@ export class SyncClient {
|
|
|
2478
2543
|
this.#scopedRowsExist(table, effective))) {
|
|
2479
2544
|
batch.table(table.name);
|
|
2480
2545
|
}
|
|
2481
|
-
|
|
2546
|
+
const result = fn();
|
|
2547
|
+
this.#replayOutbox();
|
|
2548
|
+
return result;
|
|
2482
2549
|
}),
|
|
2483
2550
|
}), section.fresh && !section.cleared);
|
|
2484
2551
|
}
|
|
@@ -2486,6 +2553,8 @@ export class SyncClient {
|
|
|
2486
2553
|
const segment = decodeRowsSegment(bytes);
|
|
2487
2554
|
await this.#applySegmentOrFail(section, summary, (table, clearFirst, effective) => applyRowsSegment(this.#db, this.#schema, table, segment, {
|
|
2488
2555
|
clearFirst,
|
|
2556
|
+
isCurrent: () => this.#localResetEpoch === resetEpoch,
|
|
2557
|
+
onProgress: (rowsProcessed) => this.#progress.update({ rowsProcessed }),
|
|
2489
2558
|
effective,
|
|
2490
2559
|
transaction: (fn) => this.#applyBatch((batch) => {
|
|
2491
2560
|
if (segment.blocks.some((block) => block.length > 0) ||
|
|
@@ -2493,7 +2562,9 @@ export class SyncClient {
|
|
|
2493
2562
|
this.#scopedRowsExist(table, effective))) {
|
|
2494
2563
|
batch.table(table.name);
|
|
2495
2564
|
}
|
|
2496
|
-
|
|
2565
|
+
const result = fn();
|
|
2566
|
+
this.#replayOutbox();
|
|
2567
|
+
return result;
|
|
2497
2568
|
}),
|
|
2498
2569
|
}, this.#encryption), section.fresh &&
|
|
2499
2570
|
!section.cleared &&
|
|
@@ -2790,6 +2861,19 @@ export class SyncClient {
|
|
|
2790
2861
|
if (downloader === undefined) {
|
|
2791
2862
|
throw new ClientSyncError('sync.invalid_request', 'received SEGMENT_REF but no segment downloader is configured');
|
|
2792
2863
|
}
|
|
2864
|
+
this.#progress.update({
|
|
2865
|
+
phase: 'download',
|
|
2866
|
+
subscriptionId: sub.id,
|
|
2867
|
+
table: frame.table,
|
|
2868
|
+
segmentId: frame.segmentId,
|
|
2869
|
+
bytesReceived: 0,
|
|
2870
|
+
bytesTotal: frame.byteLength,
|
|
2871
|
+
rowsProcessed: 0,
|
|
2872
|
+
rowsTotal: frame.rowCount,
|
|
2873
|
+
});
|
|
2874
|
+
const onProgress = (bytesReceived) => {
|
|
2875
|
+
this.#progress.update({ bytesReceived });
|
|
2876
|
+
};
|
|
2793
2877
|
let bytes;
|
|
2794
2878
|
if (frame.url !== undefined) {
|
|
2795
2879
|
// §5.4: a url-carrying descriptor MUST be fetched from that URL —
|
|
@@ -2806,18 +2890,20 @@ export class SyncClient {
|
|
|
2806
2890
|
// §5.4: MUST NOT start a fetch at/past expiry.
|
|
2807
2891
|
throw new ClientSyncError('sync.segment_expired', `signed URL for segment ${frame.segmentId} expired before fetch — re-pull mints fresh descriptors (§5.4)`, true);
|
|
2808
2892
|
}
|
|
2809
|
-
bytes = await fetchUrl(frame.url);
|
|
2893
|
+
bytes = await fetchUrl(frame.url, onProgress);
|
|
2810
2894
|
}
|
|
2811
2895
|
else {
|
|
2812
2896
|
bytes = await downloader({
|
|
2813
2897
|
segmentId: frame.segmentId,
|
|
2814
2898
|
table: frame.table,
|
|
2815
2899
|
requestedScopesJson: canonicalScopeJson(sub.scopes),
|
|
2900
|
+
onProgress,
|
|
2816
2901
|
});
|
|
2817
2902
|
}
|
|
2818
2903
|
// §5.1: verify the content address before applying; on mismatch the
|
|
2819
2904
|
// segment is discarded and the cursor/resume token stay unpersisted,
|
|
2820
2905
|
// so the next pull re-delivers.
|
|
2906
|
+
onProgress(bytes.byteLength);
|
|
2821
2907
|
const hash = await sha256Hex(bytes);
|
|
2822
2908
|
if (`sha256:${hash}` !== frame.segmentId) {
|
|
2823
2909
|
throw new ClientSyncError('sync.invalid_request', `segment ${frame.segmentId} failed content-address verification (§5.1)`, true);
|
package/dist/database.d.ts
CHANGED
|
@@ -20,11 +20,12 @@ export interface ClientDatabase {
|
|
|
20
20
|
/**
|
|
21
21
|
* Optional sqlite-image capability (SPEC §5.3): expose `bytes` as an
|
|
22
22
|
* attached read-only database schema named `alias` for the duration of
|
|
23
|
-
* `fn`,
|
|
23
|
+
* `fn`, awaiting its result before detaching. No transaction spans its awaits.
|
|
24
|
+
* Presence of this method is what makes the client
|
|
24
25
|
* advertise `accept` bit 2 (§4.2). MUST be called outside any open
|
|
25
26
|
* transaction (SQLite cannot ATTACH inside one); `fn` may open its own.
|
|
26
27
|
*/
|
|
27
|
-
withSqliteImage?<T>(bytes: Uint8Array, alias: string, fn: () => T): T
|
|
28
|
+
withSqliteImage?<T>(bytes: Uint8Array, alias: string, fn: () => T | Promise<T>): Promise<T>;
|
|
28
29
|
close(): void;
|
|
29
30
|
}
|
|
30
31
|
/** Attach-schema aliases are code-chosen, never user input — enforce it. */
|
package/dist/http.d.ts
CHANGED
|
@@ -16,14 +16,6 @@ export declare function httpSyncTransport(syncUrl: string, options?: HttpTranspo
|
|
|
16
16
|
export declare function httpRemoteOperationTransport(operationsUrl: string, options?: HttpTransportOptions): RemoteOperationTransport;
|
|
17
17
|
/** WebSocket connector for registered query snapshots. */
|
|
18
18
|
export declare function webSocketRemoteOperationConnector(realtimeUrl: string): RemoteOperationRealtimeConnector;
|
|
19
|
-
/**
|
|
20
|
-
* §5.5 direct endpoint with the `X-Syncular-Scopes` re-authorization
|
|
21
|
-
* header, plus the §5.4 `fetchUrl` capability (advertises accept bit 3).
|
|
22
|
-
* `fetchUrl` sends NO headers at all — the signed URL is the entire
|
|
23
|
-
* grant, and host auth must never leak to CDN/object hosts (§5.4).
|
|
24
|
-
* Resolution (which path a descriptor takes, expiry, no fall-through)
|
|
25
|
-
* lives in the client core, not here.
|
|
26
|
-
*/
|
|
27
19
|
export declare function httpSegmentDownloader(segmentsBaseUrl: string, options?: HttpTransportOptions): SegmentDownloader;
|
|
28
20
|
/**
|
|
29
21
|
* §5.9.3/§5.9.5 blob transport: host-authenticated `PUT`/`GET
|
package/dist/http.js
CHANGED
|
@@ -97,6 +97,39 @@ export function webSocketRemoteOperationConnector(realtimeUrl) {
|
|
|
97
97
|
* Resolution (which path a descriptor takes, expiry, no fall-through)
|
|
98
98
|
* lives in the client core, not here.
|
|
99
99
|
*/
|
|
100
|
+
async function readSegmentBody(response, onProgress) {
|
|
101
|
+
const chunks = [];
|
|
102
|
+
let received = 0;
|
|
103
|
+
let reported = 0;
|
|
104
|
+
const reader = response.body?.getReader();
|
|
105
|
+
if (reader !== undefined) {
|
|
106
|
+
try {
|
|
107
|
+
for (;;) {
|
|
108
|
+
const { done, value } = await reader.read();
|
|
109
|
+
if (done)
|
|
110
|
+
break;
|
|
111
|
+
chunks.push(value);
|
|
112
|
+
received += value.byteLength;
|
|
113
|
+
if (received - reported >= 64 * 1024) {
|
|
114
|
+
onProgress?.(received);
|
|
115
|
+
reported = received;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
finally {
|
|
120
|
+
reader.releaseLock();
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
if (received !== reported)
|
|
124
|
+
onProgress?.(received);
|
|
125
|
+
const bytes = new Uint8Array(received);
|
|
126
|
+
let offset = 0;
|
|
127
|
+
for (const chunk of chunks) {
|
|
128
|
+
bytes.set(chunk, offset);
|
|
129
|
+
offset += chunk.byteLength;
|
|
130
|
+
}
|
|
131
|
+
return bytes;
|
|
132
|
+
}
|
|
100
133
|
export function httpSegmentDownloader(segmentsBaseUrl, options) {
|
|
101
134
|
const doFetch = options?.fetch ?? fetch;
|
|
102
135
|
const direct = async (request) => {
|
|
@@ -108,15 +141,15 @@ export function httpSegmentDownloader(segmentsBaseUrl, options) {
|
|
|
108
141
|
});
|
|
109
142
|
if (!response.ok)
|
|
110
143
|
await throwHttpError(response);
|
|
111
|
-
return
|
|
144
|
+
return readSegmentBody(response, request.onProgress);
|
|
112
145
|
};
|
|
113
|
-
const fetchUrl = async (url) => {
|
|
146
|
+
const fetchUrl = async (url, onProgress) => {
|
|
114
147
|
// Deliberately headerless: the URL is the bearer grant (§5.4).
|
|
115
148
|
const response = await doFetch(url);
|
|
116
149
|
if (!response.ok) {
|
|
117
150
|
throw new ClientSyncError('sync.transport_failed', `signed-URL fetch failed with HTTP ${response.status} (§5.4: descriptor invalidated, re-pull to recover)`, true);
|
|
118
151
|
}
|
|
119
|
-
return
|
|
152
|
+
return readSegmentBody(response, onProgress);
|
|
120
153
|
};
|
|
121
154
|
return Object.assign(direct, { fetchUrl });
|
|
122
155
|
}
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
package/dist/node-database.d.ts
CHANGED
|
@@ -12,7 +12,7 @@ export declare class NodeClientDatabase implements ClientDatabase {
|
|
|
12
12
|
query(sql: string, params?: readonly SqlValue[]): SqlRow[];
|
|
13
13
|
transaction<T>(fn: () => T): T;
|
|
14
14
|
/** §5.3 image import through a private file attached for one callback. */
|
|
15
|
-
withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T): T
|
|
15
|
+
withSqliteImage<T>(bytes: Uint8Array, alias: string, fn: () => T | Promise<T>): Promise<T>;
|
|
16
16
|
close(): void;
|
|
17
17
|
}
|
|
18
18
|
export declare function openNodeDatabase(path?: string): ClientDatabase;
|
package/dist/node-database.js
CHANGED
|
@@ -52,7 +52,7 @@ export class NodeClientDatabase {
|
|
|
52
52
|
return runTransaction(this.#tx, (sql) => this.db.exec(sql), fn);
|
|
53
53
|
}
|
|
54
54
|
/** §5.3 image import through a private file attached for one callback. */
|
|
55
|
-
withSqliteImage(bytes, alias, fn) {
|
|
55
|
+
async withSqliteImage(bytes, alias, fn) {
|
|
56
56
|
assertImageAlias(alias);
|
|
57
57
|
const dir = mkdtempSync(join(tmpdir(), 'syncular-image-'));
|
|
58
58
|
const path = join(dir, 'segment.db');
|
|
@@ -60,7 +60,7 @@ export class NodeClientDatabase {
|
|
|
60
60
|
writeFileSync(path, bytes);
|
|
61
61
|
this.db.prepare(`ATTACH DATABASE ? AS ${alias}`).run(path);
|
|
62
62
|
try {
|
|
63
|
-
return fn();
|
|
63
|
+
return await fn();
|
|
64
64
|
}
|
|
65
65
|
finally {
|
|
66
66
|
this.db.prepare(`DETACH DATABASE ${alias}`).run();
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/** Ephemeral sync work, independent of committed change notifications (SPEC §7.6). */
|
|
2
|
+
export interface SyncProgress {
|
|
3
|
+
readonly attempt: number;
|
|
4
|
+
readonly state: 'running' | 'complete' | 'failed';
|
|
5
|
+
readonly phase: 'request' | 'download' | 'import';
|
|
6
|
+
readonly subscriptionId?: string;
|
|
7
|
+
readonly table?: string;
|
|
8
|
+
readonly segmentId?: string | undefined;
|
|
9
|
+
readonly bytesReceived: number;
|
|
10
|
+
readonly bytesTotal?: number;
|
|
11
|
+
readonly rowsProcessed: number;
|
|
12
|
+
readonly rowsTotal?: number;
|
|
13
|
+
readonly errorCode?: string;
|
|
14
|
+
}
|
|
15
|
+
export type SyncProgressListener = (progress: SyncProgress) => void;
|
|
16
|
+
/** Shared by the direct client and event-forwarding handles. */
|
|
17
|
+
export declare class ProgressEmitter {
|
|
18
|
+
#private;
|
|
19
|
+
clear(): void;
|
|
20
|
+
snapshot(): SyncProgress | undefined;
|
|
21
|
+
on(listener: SyncProgressListener): () => void;
|
|
22
|
+
emit(progress: SyncProgress): void;
|
|
23
|
+
update(update: Partial<SyncProgress>): void;
|
|
24
|
+
}
|
package/dist/progress.js
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/** Shared by the direct client and event-forwarding handles. */
|
|
2
|
+
export class ProgressEmitter {
|
|
3
|
+
#snapshot;
|
|
4
|
+
#listeners = new Set();
|
|
5
|
+
clear() {
|
|
6
|
+
this.#listeners.clear();
|
|
7
|
+
this.#snapshot = undefined;
|
|
8
|
+
}
|
|
9
|
+
snapshot() {
|
|
10
|
+
return this.#snapshot;
|
|
11
|
+
}
|
|
12
|
+
on(listener) {
|
|
13
|
+
this.#listeners.add(listener);
|
|
14
|
+
if (this.#snapshot !== undefined) {
|
|
15
|
+
try {
|
|
16
|
+
listener(this.#snapshot);
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
/* Observers cannot interrupt sync. */
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
return () => {
|
|
23
|
+
this.#listeners.delete(listener);
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
emit(progress) {
|
|
27
|
+
const snapshot = Object.freeze({ ...progress });
|
|
28
|
+
this.#snapshot = snapshot;
|
|
29
|
+
for (const listener of [...this.#listeners]) {
|
|
30
|
+
try {
|
|
31
|
+
listener(snapshot);
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
/* Observers cannot interrupt sync. */
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
update(update) {
|
|
39
|
+
if (this.#snapshot?.state === 'running')
|
|
40
|
+
this.emit({ ...this.#snapshot, ...update });
|
|
41
|
+
}
|
|
42
|
+
}
|