@syncular/client 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +112 -19
- package/dist/apply.d.ts +2 -0
- package/dist/apply.js +32 -8
- package/dist/blob.d.ts +12 -39
- package/dist/blob.js +63 -134
- package/dist/client.d.ts +6 -3
- package/dist/client.js +95 -58
- package/dist/http.d.ts +0 -8
- package/dist/http.js +36 -3
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/outbox.js +3 -0
- package/dist/progress.d.ts +24 -0
- package/dist/progress.js +42 -0
- package/dist/schema.js +3 -0
- package/dist/transport.d.ts +2 -1
- package/dist/wasm-database.js +41 -1
- package/dist/worker-entry.d.ts +2 -0
- package/dist/worker-entry.js +34 -7
- package/dist/worker-host.d.ts +3 -0
- package/dist/worker-host.js +13 -1
- package/dist/worker-protocol.d.ts +4 -0
- package/package.json +4 -3
- package/src/apply.ts +39 -13
- package/src/blob.ts +86 -157
- package/src/client.ts +110 -69
- package/src/http.ts +41 -3
- package/src/index.ts +6 -0
- package/src/outbox.ts +3 -0
- package/src/progress.ts +62 -0
- package/src/schema.ts +5 -0
- package/src/transport.ts +5 -1
- package/src/wasm-database.ts +79 -1
- package/src/worker-entry.ts +38 -7
- package/src/worker-host.ts +17 -1
- package/src/worker-protocol.ts +2 -0
package/src/client.ts
CHANGED
|
@@ -1,3 +1,8 @@
|
|
|
1
|
+
import {
|
|
2
|
+
ProgressEmitter,
|
|
3
|
+
type SyncProgress,
|
|
4
|
+
type SyncProgressListener,
|
|
5
|
+
} from './progress';
|
|
1
6
|
/**
|
|
2
7
|
* SyncClient implements the client side of SPEC.md §§3–8.
|
|
3
8
|
*
|
|
@@ -47,13 +52,13 @@ import {
|
|
|
47
52
|
type CachedBlob,
|
|
48
53
|
clearPendingUpload,
|
|
49
54
|
computeBlobId,
|
|
55
|
+
deleteUnreferencedCachedBlobs,
|
|
50
56
|
enforceBlobCacheCap,
|
|
51
57
|
ensureBlobSchema,
|
|
52
58
|
getCachedBlob,
|
|
53
59
|
listPendingUploads,
|
|
54
60
|
parseBlobRef,
|
|
55
61
|
putCachedBlob,
|
|
56
|
-
reconcileBlobRefcounts,
|
|
57
62
|
recordPendingUpload,
|
|
58
63
|
schemaHasBlobs,
|
|
59
64
|
serializeBlobRef,
|
|
@@ -302,9 +307,9 @@ export interface SyncClientConfig {
|
|
|
302
307
|
readonly blobs?: BlobTransport;
|
|
303
308
|
/**
|
|
304
309
|
* §5.9.7 B1 blob-cache size cap (bytes). When the sum of cached body sizes
|
|
305
|
-
* exceeds this,
|
|
306
|
-
* cache write
|
|
307
|
-
*
|
|
310
|
+
* exceeds this, unreferenced bodies without pending work are evicted in
|
|
311
|
+
* creation timestamp order after each cache write. Referenced bodies remain cached
|
|
312
|
+
* even when that exceeds the cap. Absent means retain until storage pressure (the shipped
|
|
308
313
|
* default; a referenced body always stays resolvable without a re-download).
|
|
309
314
|
*/
|
|
310
315
|
readonly blobCacheMaxBytes?: number;
|
|
@@ -584,6 +589,8 @@ export class SyncClient {
|
|
|
584
589
|
#realtimeGeneration = 0;
|
|
585
590
|
#pendingRound: PendingRound | undefined;
|
|
586
591
|
#needsPull = false;
|
|
592
|
+
readonly #progress = new ProgressEmitter();
|
|
593
|
+
#progressAttempt = 0;
|
|
587
594
|
#syncing = false;
|
|
588
595
|
/**
|
|
589
596
|
* True from the synchronous entry of `sync()` until its serialized round
|
|
@@ -688,47 +695,7 @@ export class SyncClient {
|
|
|
688
695
|
// Do not materialize new app indexes/FTS projections yet: on a version
|
|
689
696
|
// bump they may reference columns that only exist after the reset.
|
|
690
697
|
ensureLocalBookkeepingSchema(this.#db);
|
|
691
|
-
this.#db
|
|
692
|
-
for (const trigger of ['insert', 'update', 'delete', 'cache'])
|
|
693
|
-
this.#db.exec(
|
|
694
|
-
`DROP TRIGGER IF EXISTS _syncular_blob_commit_${trigger}`,
|
|
695
|
-
);
|
|
696
|
-
if (this.#hasBlobs) {
|
|
697
|
-
ensureBlobSchema(this.#db);
|
|
698
|
-
const columns = JSON.stringify(
|
|
699
|
-
[...this.#schema.tables.values()].flatMap((table) =>
|
|
700
|
-
table.columns
|
|
701
|
-
.filter((column) => column.type === 'blob_ref')
|
|
702
|
-
.map((column) => [table.name, column.name]),
|
|
703
|
-
),
|
|
704
|
-
).replaceAll("'", "''");
|
|
705
|
-
const references = `SELECT o.client_commit_id, b.blob_id
|
|
706
|
-
FROM _syncular_outbox o JOIN json_each(o.operations) op
|
|
707
|
-
JOIN json_each(op.value, '$.values') v JOIN json_each('${columns}') c
|
|
708
|
-
JOIN _syncular_blobs b ON b.blob_id = CASE WHEN v.type = 'text' AND json_valid(v.value) THEN json_extract(v.value, '$.blobId') END
|
|
709
|
-
WHERE json_extract(op.value, '$.op') = 'upsert'
|
|
710
|
-
AND json_extract(op.value, '$.table') = json_extract(c.value, '$[0]')
|
|
711
|
-
AND v.key = json_extract(c.value, '$[1]')`;
|
|
712
|
-
this.#db
|
|
713
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_insert AFTER INSERT ON _syncular_outbox BEGIN
|
|
714
|
-
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
|
|
715
|
-
this.#db
|
|
716
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_update AFTER UPDATE OF operations ON _syncular_outbox BEGIN
|
|
717
|
-
DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id;
|
|
718
|
-
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND o.client_commit_id = NEW.client_commit_id; END`);
|
|
719
|
-
this.#db
|
|
720
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_delete BEFORE DELETE ON _syncular_outbox BEGIN
|
|
721
|
-
DELETE FROM _syncular_blob_uploads WHERE blob_id IN (SELECT blob_id FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id)
|
|
722
|
-
AND NOT EXISTS (SELECT 1 FROM _syncular_blob_commit_refs r WHERE r.blob_id = _syncular_blob_uploads.blob_id AND r.commit_id != OLD.client_commit_id);
|
|
723
|
-
DELETE FROM _syncular_blob_commit_refs WHERE commit_id = OLD.client_commit_id; END`);
|
|
724
|
-
this.#db
|
|
725
|
-
.exec(`CREATE TRIGGER _syncular_blob_commit_cache AFTER INSERT ON _syncular_blobs BEGIN
|
|
726
|
-
INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references} AND b.blob_id = NEW.blob_id; END`);
|
|
727
|
-
this.#db.exec(
|
|
728
|
-
`INSERT OR IGNORE INTO _syncular_blob_commit_refs ${references}`,
|
|
729
|
-
);
|
|
730
|
-
}
|
|
731
|
-
});
|
|
698
|
+
if (this.#hasBlobs) ensureBlobSchema(this.#db);
|
|
732
699
|
this.#db.transaction(() => {
|
|
733
700
|
pruneCommitOutcomes(this.#db, this.#outcomeRetentionMaxEntries);
|
|
734
701
|
});
|
|
@@ -913,6 +880,7 @@ export class SyncClient {
|
|
|
913
880
|
await this.#lease?.release();
|
|
914
881
|
this.#lease = undefined;
|
|
915
882
|
this.#started = false;
|
|
883
|
+
this.#progress.clear();
|
|
916
884
|
this.#syncNeededListeners.clear();
|
|
917
885
|
this.#syncIntentListeners.clear();
|
|
918
886
|
}
|
|
@@ -1113,6 +1081,14 @@ export class SyncClient {
|
|
|
1113
1081
|
}
|
|
1114
1082
|
|
|
1115
1083
|
/** Subscribe to exact revisioned observer transactions (SPEC §7.5). */
|
|
1084
|
+
onProgress(listener: SyncProgressListener): () => void {
|
|
1085
|
+
return this.#progress.on(listener);
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
progressSnapshot(): SyncProgress | undefined {
|
|
1089
|
+
return this.#progress.snapshot();
|
|
1090
|
+
}
|
|
1091
|
+
|
|
1116
1092
|
onChange(listener: ClientChangeListener): () => void {
|
|
1117
1093
|
return this.#changes.on(listener);
|
|
1118
1094
|
}
|
|
@@ -1510,13 +1486,11 @@ export class SyncClient {
|
|
|
1510
1486
|
);
|
|
1511
1487
|
}
|
|
1512
1488
|
const blobId = await computeBlobId(bytes);
|
|
1489
|
+
const nowMs = this.#now();
|
|
1513
1490
|
this.#db.transaction(() => {
|
|
1514
|
-
putCachedBlob(this.#db, blobId, bytes,
|
|
1515
|
-
recordPendingUpload(this.#db, blobId,
|
|
1491
|
+
putCachedBlob(this.#db, blobId, bytes, nowMs, options?.mediaType);
|
|
1492
|
+
recordPendingUpload(this.#db, blobId, nowMs, options?.mediaType);
|
|
1516
1493
|
});
|
|
1517
|
-
// §5.9.7 B1: a staged upload is pinned (recordPendingUpload), so the cap
|
|
1518
|
-
// trim below will never evict it — but a stage may push other zero-ref
|
|
1519
|
-
// bodies over the cap, so run the trim.
|
|
1520
1494
|
this.#enforceBlobCacheCap();
|
|
1521
1495
|
return {
|
|
1522
1496
|
blobId,
|
|
@@ -1547,7 +1521,7 @@ export class SyncClient {
|
|
|
1547
1521
|
const blobId = blobIdOrRef.startsWith('sha256:')
|
|
1548
1522
|
? blobIdOrRef
|
|
1549
1523
|
: parseBlobRef(blobIdOrRef).blobId;
|
|
1550
|
-
const cached = getCachedBlob(this.#db, blobId
|
|
1524
|
+
const cached = getCachedBlob(this.#db, blobId);
|
|
1551
1525
|
if (cached !== undefined) return cached;
|
|
1552
1526
|
const transport = this.#config.blobs;
|
|
1553
1527
|
if (transport === undefined) {
|
|
@@ -1595,9 +1569,7 @@ export class SyncClient {
|
|
|
1595
1569
|
);
|
|
1596
1570
|
}
|
|
1597
1571
|
putCachedBlob(this.#db, blobId, bytes, this.#now());
|
|
1598
|
-
|
|
1599
|
-
// from current visible references before applying the size cap (§5.9.7 B1).
|
|
1600
|
-
this.#reconcileBlobs(false, blobId);
|
|
1572
|
+
this.#db.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
1601
1573
|
this.#enforceBlobCacheCap();
|
|
1602
1574
|
const stored = getCachedBlob(this.#db, blobId);
|
|
1603
1575
|
if (stored === undefined) {
|
|
@@ -1613,7 +1585,7 @@ export class SyncClient {
|
|
|
1613
1585
|
#enforceBlobCacheCap(): void {
|
|
1614
1586
|
const cap = this.#config.blobCacheMaxBytes;
|
|
1615
1587
|
if (cap === undefined) return;
|
|
1616
|
-
enforceBlobCacheCap(this.#db, cap);
|
|
1588
|
+
enforceBlobCacheCap(this.#db, this.#schema, cap);
|
|
1617
1589
|
}
|
|
1618
1590
|
|
|
1619
1591
|
/** Flush any queued blob uploads (§5.9.7 B4); safe to call standalone. */
|
|
@@ -2188,6 +2160,29 @@ export class SyncClient {
|
|
|
2188
2160
|
this.#now(),
|
|
2189
2161
|
this.#captureBeforeImages(operations),
|
|
2190
2162
|
);
|
|
2163
|
+
for (const operation of operations) {
|
|
2164
|
+
if (operation.op !== 'upsert') continue;
|
|
2165
|
+
const table = this.#schema.tables.get(operation.table);
|
|
2166
|
+
if (table === undefined) continue;
|
|
2167
|
+
for (const column of table.columns) {
|
|
2168
|
+
if (column.type !== 'blob_ref') continue;
|
|
2169
|
+
const raw = operation.values?.[column.name];
|
|
2170
|
+
if (typeof raw !== 'string') continue;
|
|
2171
|
+
let blobId: string;
|
|
2172
|
+
try {
|
|
2173
|
+
blobId = parseBlobRef(raw).blobId;
|
|
2174
|
+
} catch {
|
|
2175
|
+
continue;
|
|
2176
|
+
}
|
|
2177
|
+
this.#db.exec(
|
|
2178
|
+
`INSERT OR IGNORE INTO _syncular_blob_commit_refs(commit_id, blob_id)
|
|
2179
|
+
SELECT ?, ? WHERE EXISTS (
|
|
2180
|
+
SELECT 1 FROM _syncular_blobs WHERE blob_id = ?
|
|
2181
|
+
)`,
|
|
2182
|
+
[clientCommitId, blobId, blobId],
|
|
2183
|
+
);
|
|
2184
|
+
}
|
|
2185
|
+
}
|
|
2191
2186
|
this.#applyOperationsLocally(operations, batch);
|
|
2192
2187
|
batch.status();
|
|
2193
2188
|
});
|
|
@@ -2386,7 +2381,7 @@ export class SyncClient {
|
|
|
2386
2381
|
batch.rejections();
|
|
2387
2382
|
batch.outcomes();
|
|
2388
2383
|
}
|
|
2389
|
-
this.#
|
|
2384
|
+
this.#deleteUnreferencedBlobs();
|
|
2390
2385
|
setMeta(this.#db, metaKey, purge.canonicalPlan);
|
|
2391
2386
|
return {
|
|
2392
2387
|
alreadyApplied: false,
|
|
@@ -2676,6 +2671,13 @@ export class SyncClient {
|
|
|
2676
2671
|
(error: unknown) => {
|
|
2677
2672
|
const completedAtMs = this.#now();
|
|
2678
2673
|
const code = (error as { code?: unknown }).code;
|
|
2674
|
+
this.#progress.update({
|
|
2675
|
+
state: 'failed',
|
|
2676
|
+
errorCode:
|
|
2677
|
+
typeof code === 'string'
|
|
2678
|
+
? this.#diagnosticCode(code)
|
|
2679
|
+
: 'client.unknown_failure',
|
|
2680
|
+
});
|
|
2679
2681
|
this.#lastRound = {
|
|
2680
2682
|
status: 'failed',
|
|
2681
2683
|
startedAtMs,
|
|
@@ -2730,7 +2732,15 @@ export class SyncClient {
|
|
|
2730
2732
|
}
|
|
2731
2733
|
|
|
2732
2734
|
async #runSync(): Promise<SyncSummary> {
|
|
2735
|
+
this.#progress.emit({
|
|
2736
|
+
attempt: ++this.#progressAttempt,
|
|
2737
|
+
state: 'running',
|
|
2738
|
+
phase: 'request',
|
|
2739
|
+
bytesReceived: 0,
|
|
2740
|
+
rowsProcessed: 0,
|
|
2741
|
+
});
|
|
2733
2742
|
if (this.#schemaFloor !== undefined) {
|
|
2743
|
+
this.#progress.update({ state: 'complete' });
|
|
2734
2744
|
return {
|
|
2735
2745
|
...emptySummary(0),
|
|
2736
2746
|
bootstrapping: [],
|
|
@@ -2821,6 +2831,11 @@ export class SyncClient {
|
|
|
2821
2831
|
// §4.8 E1: the push half may have drained commits that pinned rows of
|
|
2822
2832
|
// a shrunk window unit — retry any deferred evictions now.
|
|
2823
2833
|
this.#drainPendingEvictions();
|
|
2834
|
+
this.#progress.update(
|
|
2835
|
+
summary.failed.length > 0
|
|
2836
|
+
? { state: 'failed', errorCode: 'sync.scope_revoked' }
|
|
2837
|
+
: { state: 'complete' },
|
|
2838
|
+
);
|
|
2824
2839
|
this.#retryDelayMs = 250;
|
|
2825
2840
|
if (deferred > 0) {
|
|
2826
2841
|
// §6.1 splitBatch remainder: more queued commits than this request
|
|
@@ -3389,6 +3404,19 @@ export class SyncClient {
|
|
|
3389
3404
|
break;
|
|
3390
3405
|
}
|
|
3391
3406
|
const segment = decodeRowsSegment(frame.payload);
|
|
3407
|
+
this.#progress.update({
|
|
3408
|
+
phase: 'import',
|
|
3409
|
+
subscriptionId: section.sub.id,
|
|
3410
|
+
table: section.sub.table,
|
|
3411
|
+
segmentId: undefined,
|
|
3412
|
+
bytesReceived: frame.payload.byteLength,
|
|
3413
|
+
bytesTotal: frame.payload.byteLength,
|
|
3414
|
+
rowsProcessed: 0,
|
|
3415
|
+
rowsTotal: segment.blocks.reduce(
|
|
3416
|
+
(n, block) => n + block.length,
|
|
3417
|
+
0,
|
|
3418
|
+
),
|
|
3419
|
+
});
|
|
3392
3420
|
await this.#applySegmentOrFail(
|
|
3393
3421
|
section,
|
|
3394
3422
|
summary,
|
|
@@ -3400,6 +3428,8 @@ export class SyncClient {
|
|
|
3400
3428
|
segment,
|
|
3401
3429
|
{
|
|
3402
3430
|
clearFirst,
|
|
3431
|
+
onProgress: (rowsProcessed) =>
|
|
3432
|
+
this.#progress.update({ rowsProcessed }),
|
|
3403
3433
|
effective,
|
|
3404
3434
|
transaction: (fn) =>
|
|
3405
3435
|
this.#applyBatch((batch) => {
|
|
@@ -3439,6 +3469,7 @@ export class SyncClient {
|
|
|
3439
3469
|
);
|
|
3440
3470
|
}
|
|
3441
3471
|
const bytes = await this.#downloadSegment(frame, section.sub);
|
|
3472
|
+
this.#progress.update({ phase: 'import' });
|
|
3442
3473
|
if (frame.mediaType === 'sqlite') {
|
|
3443
3474
|
// §5.3: images are whole-table — a paged descriptor is
|
|
3444
3475
|
// invalid, and the image is always its table's first page.
|
|
@@ -3468,6 +3499,8 @@ export class SyncClient {
|
|
|
3468
3499
|
},
|
|
3469
3500
|
{
|
|
3470
3501
|
clearFirst,
|
|
3502
|
+
onProgress: (rowsProcessed) =>
|
|
3503
|
+
this.#progress.update({ rowsProcessed }),
|
|
3471
3504
|
effective,
|
|
3472
3505
|
transaction: (fn) =>
|
|
3473
3506
|
this.#applyBatch((batch) => {
|
|
@@ -3497,6 +3530,8 @@ export class SyncClient {
|
|
|
3497
3530
|
segment,
|
|
3498
3531
|
{
|
|
3499
3532
|
clearFirst,
|
|
3533
|
+
onProgress: (rowsProcessed) =>
|
|
3534
|
+
this.#progress.update({ rowsProcessed }),
|
|
3500
3535
|
effective,
|
|
3501
3536
|
transaction: (fn) =>
|
|
3502
3537
|
this.#applyBatch((batch) => {
|
|
@@ -3564,7 +3599,6 @@ export class SyncClient {
|
|
|
3564
3599
|
// §7.1: local reads see outbox state applied optimistically — replay
|
|
3565
3600
|
// the still-pending commits on top of the freshly applied server state.
|
|
3566
3601
|
this.#replayOutbox();
|
|
3567
|
-
this.#reconcileBlobs(false);
|
|
3568
3602
|
} finally {
|
|
3569
3603
|
this.#endDiagnosticsDeferral();
|
|
3570
3604
|
}
|
|
@@ -3879,6 +3913,19 @@ export class SyncClient {
|
|
|
3879
3913
|
'received SEGMENT_REF but no segment downloader is configured',
|
|
3880
3914
|
);
|
|
3881
3915
|
}
|
|
3916
|
+
this.#progress.update({
|
|
3917
|
+
phase: 'download',
|
|
3918
|
+
subscriptionId: sub.id,
|
|
3919
|
+
table: frame.table,
|
|
3920
|
+
segmentId: frame.segmentId,
|
|
3921
|
+
bytesReceived: 0,
|
|
3922
|
+
bytesTotal: frame.byteLength,
|
|
3923
|
+
rowsProcessed: 0,
|
|
3924
|
+
rowsTotal: frame.rowCount,
|
|
3925
|
+
});
|
|
3926
|
+
const onProgress = (bytesReceived: number): void => {
|
|
3927
|
+
this.#progress.update({ bytesReceived });
|
|
3928
|
+
};
|
|
3882
3929
|
let bytes: Uint8Array;
|
|
3883
3930
|
if (frame.url !== undefined) {
|
|
3884
3931
|
// §5.4: a url-carrying descriptor MUST be fetched from that URL —
|
|
@@ -3906,17 +3953,19 @@ export class SyncClient {
|
|
|
3906
3953
|
true,
|
|
3907
3954
|
);
|
|
3908
3955
|
}
|
|
3909
|
-
bytes = await fetchUrl(frame.url);
|
|
3956
|
+
bytes = await fetchUrl(frame.url, onProgress);
|
|
3910
3957
|
} else {
|
|
3911
3958
|
bytes = await downloader({
|
|
3912
3959
|
segmentId: frame.segmentId,
|
|
3913
3960
|
table: frame.table,
|
|
3914
3961
|
requestedScopesJson: canonicalScopeJson(sub.scopes),
|
|
3962
|
+
onProgress,
|
|
3915
3963
|
});
|
|
3916
3964
|
}
|
|
3917
3965
|
// §5.1: verify the content address before applying; on mismatch the
|
|
3918
3966
|
// segment is discarded and the cursor/resume token stay unpersisted,
|
|
3919
3967
|
// so the next pull re-delivers.
|
|
3968
|
+
onProgress(bytes.byteLength);
|
|
3920
3969
|
const hash = await sha256Hex(bytes);
|
|
3921
3970
|
if (`sha256:${hash}` !== frame.segmentId) {
|
|
3922
3971
|
throw new ClientSyncError(
|
|
@@ -4047,7 +4096,7 @@ export class SyncClient {
|
|
|
4047
4096
|
batch.rejections();
|
|
4048
4097
|
batch.outcomes();
|
|
4049
4098
|
}
|
|
4050
|
-
this.#
|
|
4099
|
+
this.#deleteUnreferencedBlobs();
|
|
4051
4100
|
} catch (error) {
|
|
4052
4101
|
if (
|
|
4053
4102
|
error instanceof ClientSyncError &&
|
|
@@ -4265,17 +4314,9 @@ export class SyncClient {
|
|
|
4265
4314
|
});
|
|
4266
4315
|
}
|
|
4267
4316
|
|
|
4268
|
-
|
|
4269
|
-
* §5.9.7 B1/B2: recompute blob-cache refcounts from live `blob_ref`
|
|
4270
|
-
* columns. No-op unless the schema has blob columns. `deleteOrphans`
|
|
4271
|
-
* triggers the revocation-side body deletion (B2).
|
|
4272
|
-
*/
|
|
4273
|
-
#reconcileBlobs(deleteOrphans: boolean, blobId?: string): void {
|
|
4317
|
+
#deleteUnreferencedBlobs(): void {
|
|
4274
4318
|
if (!this.#hasBlobs) return;
|
|
4275
|
-
|
|
4276
|
-
deleteOrphans,
|
|
4277
|
-
...(blobId === undefined ? {} : { blobId }),
|
|
4278
|
-
});
|
|
4319
|
+
deleteUnreferencedCachedBlobs(this.#db, this.#schema);
|
|
4279
4320
|
}
|
|
4280
4321
|
|
|
4281
4322
|
// -- helpers -----------------------------------------------------------------
|
package/src/http.ts
CHANGED
|
@@ -139,6 +139,40 @@ export function webSocketRemoteOperationConnector(
|
|
|
139
139
|
* Resolution (which path a descriptor takes, expiry, no fall-through)
|
|
140
140
|
* lives in the client core, not here.
|
|
141
141
|
*/
|
|
142
|
+
async function readSegmentBody(
|
|
143
|
+
response: Response,
|
|
144
|
+
onProgress?: (bytesReceived: number) => void,
|
|
145
|
+
): Promise<Uint8Array> {
|
|
146
|
+
const chunks: Uint8Array[] = [];
|
|
147
|
+
let received = 0;
|
|
148
|
+
let reported = 0;
|
|
149
|
+
const reader = response.body?.getReader();
|
|
150
|
+
if (reader !== undefined) {
|
|
151
|
+
try {
|
|
152
|
+
for (;;) {
|
|
153
|
+
const { done, value } = await reader.read();
|
|
154
|
+
if (done) break;
|
|
155
|
+
chunks.push(value);
|
|
156
|
+
received += value.byteLength;
|
|
157
|
+
if (received - reported >= 64 * 1024) {
|
|
158
|
+
onProgress?.(received);
|
|
159
|
+
reported = received;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
} finally {
|
|
163
|
+
reader.releaseLock();
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
if (received !== reported) onProgress?.(received);
|
|
167
|
+
const bytes = new Uint8Array(received);
|
|
168
|
+
let offset = 0;
|
|
169
|
+
for (const chunk of chunks) {
|
|
170
|
+
bytes.set(chunk, offset);
|
|
171
|
+
offset += chunk.byteLength;
|
|
172
|
+
}
|
|
173
|
+
return bytes;
|
|
174
|
+
}
|
|
175
|
+
|
|
142
176
|
export function httpSegmentDownloader(
|
|
143
177
|
segmentsBaseUrl: string,
|
|
144
178
|
options?: HttpTransportOptions,
|
|
@@ -147,6 +181,7 @@ export function httpSegmentDownloader(
|
|
|
147
181
|
const direct = async (request: {
|
|
148
182
|
readonly segmentId: string;
|
|
149
183
|
readonly requestedScopesJson: string;
|
|
184
|
+
readonly onProgress?: (bytesReceived: number) => void;
|
|
150
185
|
}) => {
|
|
151
186
|
const response = await doFetch(
|
|
152
187
|
`${segmentsBaseUrl}/${encodeURIComponent(request.segmentId)}`,
|
|
@@ -158,9 +193,12 @@ export function httpSegmentDownloader(
|
|
|
158
193
|
},
|
|
159
194
|
);
|
|
160
195
|
if (!response.ok) await throwHttpError(response);
|
|
161
|
-
return
|
|
196
|
+
return readSegmentBody(response, request.onProgress);
|
|
162
197
|
};
|
|
163
|
-
const fetchUrl = async (
|
|
198
|
+
const fetchUrl = async (
|
|
199
|
+
url: string,
|
|
200
|
+
onProgress?: (bytesReceived: number) => void,
|
|
201
|
+
) => {
|
|
164
202
|
// Deliberately headerless: the URL is the bearer grant (§5.4).
|
|
165
203
|
const response = await doFetch(url);
|
|
166
204
|
if (!response.ok) {
|
|
@@ -170,7 +208,7 @@ export function httpSegmentDownloader(
|
|
|
170
208
|
true,
|
|
171
209
|
);
|
|
172
210
|
}
|
|
173
|
-
return
|
|
211
|
+
return readSegmentBody(response, onProgress);
|
|
174
212
|
};
|
|
175
213
|
return Object.assign(direct, { fetchUrl });
|
|
176
214
|
}
|
package/src/index.ts
CHANGED
package/src/outbox.ts
CHANGED
|
@@ -135,6 +135,9 @@ export function deleteOutboxCommit(
|
|
|
135
135
|
db: ClientDatabase,
|
|
136
136
|
clientCommitId: string,
|
|
137
137
|
): void {
|
|
138
|
+
db.exec('DELETE FROM _syncular_blob_commit_refs WHERE commit_id = ?', [
|
|
139
|
+
clientCommitId,
|
|
140
|
+
]);
|
|
138
141
|
db.exec(
|
|
139
142
|
'DELETE FROM _syncular_outbox_before_images WHERE client_commit_id = ?',
|
|
140
143
|
[clientCommitId],
|
package/src/progress.ts
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/** Ephemeral sync work, independent of committed change notifications (SPEC §7.6). */
|
|
2
|
+
export interface SyncProgress {
|
|
3
|
+
readonly attempt: number;
|
|
4
|
+
readonly state: 'running' | 'complete' | 'failed';
|
|
5
|
+
readonly phase: 'request' | 'download' | 'import';
|
|
6
|
+
readonly subscriptionId?: string;
|
|
7
|
+
readonly table?: string;
|
|
8
|
+
readonly segmentId?: string | undefined;
|
|
9
|
+
readonly bytesReceived: number;
|
|
10
|
+
readonly bytesTotal?: number;
|
|
11
|
+
readonly rowsProcessed: number;
|
|
12
|
+
readonly rowsTotal?: number;
|
|
13
|
+
readonly errorCode?: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export type SyncProgressListener = (progress: SyncProgress) => void;
|
|
17
|
+
|
|
18
|
+
/** Shared by the direct client and event-forwarding handles. */
|
|
19
|
+
export class ProgressEmitter {
|
|
20
|
+
#snapshot: SyncProgress | undefined;
|
|
21
|
+
readonly #listeners = new Set<SyncProgressListener>();
|
|
22
|
+
|
|
23
|
+
clear(): void {
|
|
24
|
+
this.#listeners.clear();
|
|
25
|
+
this.#snapshot = undefined;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
snapshot(): SyncProgress | undefined {
|
|
29
|
+
return this.#snapshot;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
on(listener: SyncProgressListener): () => void {
|
|
33
|
+
this.#listeners.add(listener);
|
|
34
|
+
if (this.#snapshot !== undefined) {
|
|
35
|
+
try {
|
|
36
|
+
listener(this.#snapshot);
|
|
37
|
+
} catch {
|
|
38
|
+
/* Observers cannot interrupt sync. */
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return () => {
|
|
42
|
+
this.#listeners.delete(listener);
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
emit(progress: SyncProgress): void {
|
|
47
|
+
const snapshot = Object.freeze({ ...progress });
|
|
48
|
+
this.#snapshot = snapshot;
|
|
49
|
+
for (const listener of [...this.#listeners]) {
|
|
50
|
+
try {
|
|
51
|
+
listener(snapshot);
|
|
52
|
+
} catch {
|
|
53
|
+
/* Observers cannot interrupt sync. */
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
update(update: Partial<SyncProgress>): void {
|
|
59
|
+
if (this.#snapshot?.state === 'running')
|
|
60
|
+
this.emit({ ...this.#snapshot, ...update });
|
|
61
|
+
}
|
|
62
|
+
}
|
package/src/schema.ts
CHANGED
|
@@ -501,6 +501,11 @@ export function ensureLocalBookkeepingSchema(db: ClientDatabase): void {
|
|
|
501
501
|
db.transaction(() => {
|
|
502
502
|
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_meta(
|
|
503
503
|
key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
|
|
504
|
+
db.exec(`CREATE TABLE IF NOT EXISTS _syncular_blob_commit_refs(
|
|
505
|
+
commit_id TEXT NOT NULL, blob_id TEXT NOT NULL, PRIMARY KEY(commit_id, blob_id))`);
|
|
506
|
+
db.exec(
|
|
507
|
+
'CREATE INDEX IF NOT EXISTS _syncular_blob_commit_refs_body ON _syncular_blob_commit_refs(blob_id)',
|
|
508
|
+
);
|
|
504
509
|
db.exec(
|
|
505
510
|
`INSERT OR IGNORE INTO _syncular_meta(key, value) VALUES ('localRevision', '0')`,
|
|
506
511
|
);
|
package/src/transport.ts
CHANGED
|
@@ -32,6 +32,7 @@ export interface SegmentFetchRequest {
|
|
|
32
32
|
readonly table: string;
|
|
33
33
|
/** Canonical JSON (§11.2) of the requested scope map (§5.5 header). */
|
|
34
34
|
readonly requestedScopesJson: string;
|
|
35
|
+
readonly onProgress?: (bytesReceived: number) => void;
|
|
35
36
|
}
|
|
36
37
|
|
|
37
38
|
/**
|
|
@@ -44,7 +45,10 @@ export interface SegmentFetchRequest {
|
|
|
44
45
|
*/
|
|
45
46
|
export interface SegmentDownloader {
|
|
46
47
|
(request: SegmentFetchRequest): Promise<Uint8Array>;
|
|
47
|
-
readonly fetchUrl?: (
|
|
48
|
+
readonly fetchUrl?: (
|
|
49
|
+
url: string,
|
|
50
|
+
onProgress?: (bytesReceived: number) => void,
|
|
51
|
+
) => Promise<Uint8Array>;
|
|
48
52
|
}
|
|
49
53
|
|
|
50
54
|
export interface RealtimeHandlers {
|
package/src/wasm-database.ts
CHANGED
|
@@ -53,6 +53,13 @@ interface SahPoolUtil {
|
|
|
53
53
|
OpfsSAHPoolDb: new (filename: string) => Oo1Database;
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
interface SqliteIoMethods {
|
|
57
|
+
installMethod(
|
|
58
|
+
name: 'xCheckReservedLock',
|
|
59
|
+
callback: (file: number, output: number) => number,
|
|
60
|
+
): unknown;
|
|
61
|
+
}
|
|
62
|
+
|
|
56
63
|
interface Sqlite3Static {
|
|
57
64
|
oo1: {
|
|
58
65
|
DB: new (filename: string, flags?: string) => Oo1Database;
|
|
@@ -60,6 +67,18 @@ interface Sqlite3Static {
|
|
|
60
67
|
capi: {
|
|
61
68
|
SQLITE_DESERIALIZE_FREEONCLOSE: number;
|
|
62
69
|
SQLITE_DESERIALIZE_READONLY: number;
|
|
70
|
+
SQLITE_FCNTL_FILE_POINTER: number;
|
|
71
|
+
sqlite3_file_control(
|
|
72
|
+
db: number,
|
|
73
|
+
schema: string,
|
|
74
|
+
operation: number,
|
|
75
|
+
output: number,
|
|
76
|
+
): number;
|
|
77
|
+
sqlite3_file: new (pointer: number) => {
|
|
78
|
+
readonly $pMethods: number;
|
|
79
|
+
dispose(): void;
|
|
80
|
+
};
|
|
81
|
+
sqlite3_io_methods: new (pointer: number) => SqliteIoMethods;
|
|
63
82
|
sqlite3_deserialize(
|
|
64
83
|
db: number,
|
|
65
84
|
schema: string,
|
|
@@ -71,6 +90,13 @@ interface Sqlite3Static {
|
|
|
71
90
|
};
|
|
72
91
|
wasm: {
|
|
73
92
|
allocFromTypedArray(bytes: Uint8Array): number;
|
|
93
|
+
peekPtr(pointer: number): number;
|
|
94
|
+
poke32(pointer: number, value: number): unknown;
|
|
95
|
+
pstack: {
|
|
96
|
+
readonly pointer: number;
|
|
97
|
+
allocPtr(): number;
|
|
98
|
+
restore(pointer: number): void;
|
|
99
|
+
};
|
|
74
100
|
};
|
|
75
101
|
installOpfsSAHPoolVfs(options: {
|
|
76
102
|
name?: string;
|
|
@@ -217,6 +243,10 @@ function inWorkerContext(): boolean {
|
|
|
217
243
|
/** One registered VFS per pool directory, reused across opens. */
|
|
218
244
|
const sahPools = new Map<string, Promise<SahPoolUtil>>();
|
|
219
245
|
|
|
246
|
+
// SAH pools share an I/O method table within the WASM runtime. Keep the
|
|
247
|
+
// installed callback alive for that runtime, including across database closes.
|
|
248
|
+
const sahIoMethods = new Map<number, SqliteIoMethods>();
|
|
249
|
+
|
|
220
250
|
function opfsSahPoolError(error: unknown, directory: string): ClientSyncError {
|
|
221
251
|
const detail = error instanceof Error ? error.message : String(error);
|
|
222
252
|
const normalized = detail.toLowerCase();
|
|
@@ -306,5 +336,53 @@ export async function openPersistentWasmDatabase(
|
|
|
306
336
|
sahPools.set(directory, pool);
|
|
307
337
|
}
|
|
308
338
|
const util = await pool;
|
|
309
|
-
|
|
339
|
+
const db = new util.OpfsSAHPoolDb(`/${name}.db`);
|
|
340
|
+
const { capi, wasm } = sqlite3;
|
|
341
|
+
const stack = wasm.pstack.pointer;
|
|
342
|
+
try {
|
|
343
|
+
// Do this before the first SQL statement, when SQLite checks for a hot
|
|
344
|
+
// journal. The SAH-pool callback in sqlite-wasm 3.53 reports a reserved
|
|
345
|
+
// lock unconditionally, which suppresses crash rollback. The pool already
|
|
346
|
+
// excludes other owners; report no competing writer, as the other OPFS
|
|
347
|
+
// VFS does. Keep DELETE/FULL and the existing database/journal files.
|
|
348
|
+
const output = wasm.pstack.allocPtr();
|
|
349
|
+
if (
|
|
350
|
+
db.pointer === undefined ||
|
|
351
|
+
capi.sqlite3_file_control(
|
|
352
|
+
db.pointer,
|
|
353
|
+
'main',
|
|
354
|
+
capi.SQLITE_FCNTL_FILE_POINTER,
|
|
355
|
+
output,
|
|
356
|
+
) !== 0 ||
|
|
357
|
+
!wasm.peekPtr(output)
|
|
358
|
+
) {
|
|
359
|
+
throw new ClientSyncError(
|
|
360
|
+
STORAGE_UNAVAILABLE_CODE,
|
|
361
|
+
'Could not configure OPFS crash recovery',
|
|
362
|
+
);
|
|
363
|
+
}
|
|
364
|
+
const file = new capi.sqlite3_file(wasm.peekPtr(output));
|
|
365
|
+
const methodsPointer = file.$pMethods;
|
|
366
|
+
file.dispose(); // Borrowed struct view; SQLite still owns the file.
|
|
367
|
+
if (!methodsPointer) {
|
|
368
|
+
throw new ClientSyncError(
|
|
369
|
+
STORAGE_UNAVAILABLE_CODE,
|
|
370
|
+
'Could not configure OPFS crash recovery',
|
|
371
|
+
);
|
|
372
|
+
}
|
|
373
|
+
if (!sahIoMethods.has(methodsPointer)) {
|
|
374
|
+
const methods = new capi.sqlite3_io_methods(methodsPointer);
|
|
375
|
+
methods.installMethod('xCheckReservedLock', (_file, reserved) => {
|
|
376
|
+
wasm.poke32(reserved, 0);
|
|
377
|
+
return 0;
|
|
378
|
+
});
|
|
379
|
+
sahIoMethods.set(methodsPointer, methods);
|
|
380
|
+
}
|
|
381
|
+
return new WasmClientDatabase(db, sqlite3);
|
|
382
|
+
} catch (error) {
|
|
383
|
+
db.close();
|
|
384
|
+
throw error;
|
|
385
|
+
} finally {
|
|
386
|
+
wasm.pstack.restore(stack);
|
|
387
|
+
}
|
|
310
388
|
}
|