@terreno/syncdb 57.13.0 → 57.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -3
- package/dist/client.d.ts +12 -0
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +304 -56
- package/dist/client.js.map +1 -1
- package/dist/client.test.js +255 -0
- package/dist/client.test.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.d.ts +4 -3
- package/dist/persisters/encryptedIndexedDbPersister.d.ts.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.js +21 -16
- package/dist/persisters/encryptedIndexedDbPersister.js.map +1 -1
- package/dist/persisters/encryptedIndexedDbPersister.test.js +40 -1
- package/dist/persisters/encryptedIndexedDbPersister.test.js.map +1 -1
- package/dist/sync/bootstrap.d.ts +34 -3
- package/dist/sync/bootstrap.d.ts.map +1 -1
- package/dist/sync/bootstrap.js +224 -71
- package/dist/sync/bootstrap.js.map +1 -1
- package/dist/sync/bootstrap.test.js +210 -2
- package/dist/sync/bootstrap.test.js.map +1 -1
- package/dist/sync/httpChannel.d.ts +14 -1
- package/dist/sync/httpChannel.d.ts.map +1 -1
- package/dist/sync/httpChannel.js +34 -0
- package/dist/sync/httpChannel.js.map +1 -1
- package/dist/sync/httpChannel.test.js +54 -0
- package/dist/sync/httpChannel.test.js.map +1 -1
- package/dist/types.d.ts +14 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/client.test.ts +314 -1
- package/src/client.ts +362 -54
- package/src/index.ts +1 -0
- package/src/persisters/encryptedIndexedDbPersister.test.ts +44 -1
- package/src/persisters/encryptedIndexedDbPersister.ts +24 -17
- package/src/sync/bootstrap.test.ts +269 -3
- package/src/sync/bootstrap.ts +336 -82
- package/src/sync/httpChannel.test.ts +67 -0
- package/src/sync/httpChannel.ts +50 -0
- package/src/types.ts +12 -0
package/README.md
CHANGED
|
@@ -25,7 +25,7 @@ Local-first data layer for Terreno apps. A TinyBase `MergeableStore` (encrypted
|
|
|
25
25
|
│ sync:mutate ─────────────────────────────►│ │
|
|
26
26
|
│ sync:ack / sync:nack ◄────────────────────┘ │
|
|
27
27
|
│ sync:delta ◄──────────────────────────────────────────────┘
|
|
28
|
-
│
|
|
28
|
+
│ POST /sync/snapshot/batch (bootstrap + catch-up, HTTP)
|
|
29
29
|
└── POST /sync/mutate (fallback while the socket is down)
|
|
30
30
|
```
|
|
31
31
|
|
|
@@ -102,7 +102,7 @@ const todoRouter = modelRouter("/todos", Todo, {
|
|
|
102
102
|
|
|
103
103
|
new TerrenoApp({userModel: User})
|
|
104
104
|
.register(todoRouter)
|
|
105
|
-
.register(new SyncApp()) // GET /sync/snapshot, POST /sync/mutate, GET /sync/key
|
|
105
|
+
.register(new SyncApp()) // POST /sync/snapshot/batch, GET /sync/snapshot, POST /sync/mutate, GET /sync/key
|
|
106
106
|
.register(new RealtimeApp()) // Socket.io server; installs sync:subscribe/sync:mutate handlers
|
|
107
107
|
.start();
|
|
108
108
|
```
|
|
@@ -168,6 +168,7 @@ const client = createSyncDb({
|
|
|
168
168
|
seqJumpReconcileMinIntervalMs?, // seq-jump reconcile rate limit (default 30s)
|
|
169
169
|
batchSize?, // max mutations per batched drain send (default 50; server caps at 100)
|
|
170
170
|
haltQueueOnConflict?, // conflict policy — see "Conflict handling modes" below (default false)
|
|
171
|
+
snapshotConcurrency?, // per-stream GET /sync/snapshot passes at once (default 6; 1 = one at a time)
|
|
171
172
|
onDecryptFailure?, // override the default wipe+re-bootstrap on undecryptable data (web)
|
|
172
173
|
tombstoneRetentionMs?, // client-side tombstone compaction window (default 90 days; 0 disables)
|
|
173
174
|
onAuthRequired?, // fires once per auth-pause episode — prompt for re-login
|
|
@@ -322,6 +323,7 @@ Sequencing guarantees: validation failures never consume a seq (the claim happen
|
|
|
322
323
|
| Endpoint | Purpose | Auth |
|
|
323
324
|
|---|---|---|
|
|
324
325
|
| `GET /sync/snapshot?collection=&cursor=&limit=` | Bootstrap + catch-up. Returns `{entities: [{id, data, seq, deleted}], cursor, hasMore}`. `cursor=0` = full snapshot (legacy docs without `_syncSeq` arrive in the first page with seq 0). Default page 100, max 100. | Model `list` permissions + server-enforced scope filter |
|
|
326
|
+
| `POST /sync/snapshot/batch` | Bootstrap + catch-up for many streams in one round trip. Body `{streams: [{stream, cursor?, legacyCursor?}], limit?}` (max 500 streams; `limit` is the entity budget for the whole response, default 5,000, max 10,000). Returns `{pages}`, one `GET /sync/snapshot`-shaped page per stream in request order. | Same as `GET /sync/snapshot`, per stream; any denied stream fails the request |
|
|
325
327
|
| `POST /sync/mutate` | HTTP fallback for outbox replay (same handler as the socket channel). Body: `{mutationId, collection, operation, id?, data?, baseVersion?}`. Returns `{ack}` or `{nack}` with status 409 (conflict), 403 (unauthorized), 422 (validation), 429 (rate_limited, carries `retryAfterMs`), 500 (error). | modelRouter create/update/delete write path |
|
|
326
328
|
| `POST /sync/mutate/batch` | HTTP fallback for batched outbox replay. Body: `{mutations: SyncMutateRequest[]}` (max 100; intra-batch duplicate `mutationId`s rejected up front). Returns `{results: ({type:"ack",ack}\|{type:"nack",nack})[]}` — applied strictly in array order, stopping at the first non-ack (a shorter `results` array than the request means everything after it was never attempted). | modelRouter create/update/delete write path, per mutation |
|
|
327
329
|
| `GET /sync/key` | Caller's per-user key material for the server key provider (32 random bytes, base64; created on first call). | Own key only |
|
|
@@ -449,8 +451,9 @@ This writes `store/syncDbSdk.ts` with `SYNC_COLLECTIONS`, entity types, and hook
|
|
|
449
451
|
- **Multi-tab web**: two tabs of the same user share one IndexedDB blob; concurrent persister saves are last-writer-wins at the blob level and can drop the other tab's queued outbox rows. Single-writer coordination (Web Locks) is not implemented yet — avoid relying on offline writes from multiple simultaneous tabs.
|
|
450
452
|
- **`Model.bulkWrite` is unavailable on synced models**: it skips Mongoose middleware, so it can never stamp seqs. `registerSync` replaces the static with one that throws — bulk updates to a synced collection have to loop per document.
|
|
451
453
|
- **Native plaintext by design**: no SQLCipher; the OS sandbox is deemed sufficient.
|
|
452
|
-
- **Whole-store persistence**: each save serializes and (on web) encrypts the full store — cost scales with store size, not change size. Bound it by scoping which collections sync; saves are debounced.
|
|
454
|
+
- **Whole-store persistence**: each save serializes and (on web) encrypts the full store — cost scales with store size, not change size. Bound it by scoping which collections sync; saves are debounced, and a burst of transactions costs one serialization per write, not one per transaction.
|
|
453
455
|
- **`realtime` + `sync` coexistence**: a model may enable both (distinct events, `sync` vs `sync:delta`, so clients never double-apply), at the cost of double emission work. `modelRouter` `realtime` is **deprecated and will be removed in Terreno 58**; use `sync` only.
|
|
456
|
+
- **Cold start scales with row count, not stream count**: streams page together through `POST /sync/snapshot/batch`, one round trip per round of up to 5,000 rows. Against a server without that route, every stream costs at least one `GET /sync/snapshot` round trip (`snapshotConcurrency` at a time, default 6). Sync only the collections the client reads. See [Use syncdb correctly](https://github.com/TerrenoLabs/terreno/blob/master/docs/how-to/use-syncdb-correctly.md) for this and the write-path pitfalls apps must handle.
|
|
454
457
|
- **Seq counter write amplification**: every synced write does an atomic `$inc` on a per-stream counter doc. Acceptable at current scale; Redis-based counters are the documented upgrade path.
|
|
455
458
|
|
|
456
459
|
## Documentation
|
package/dist/client.d.ts
CHANGED
|
@@ -12,6 +12,12 @@ import type { AuthProvider, ConflictResolutionStrategy, SyncMutationOperation, S
|
|
|
12
12
|
export declare const DEFAULT_RECONCILE_INTERVAL_MS: number;
|
|
13
13
|
/** Minimum interval between seq-jump-triggered reconciles per stream (30s). */
|
|
14
14
|
export declare const DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS = 30000;
|
|
15
|
+
/**
|
|
16
|
+
* Streams snapshot-paged at once by reconcile, subscribe catch-up, force resync and
|
|
17
|
+
* wipe re-bootstrap. Every stream costs at least one `GET /sync/snapshot` round trip,
|
|
18
|
+
* so paging them one at a time made cold start scale with collections × scopes.
|
|
19
|
+
*/
|
|
20
|
+
export declare const DEFAULT_SNAPSHOT_CONCURRENCY = 6;
|
|
15
21
|
/** E5: default local tombstone retention window (90 days), matching the server's (C7). */
|
|
16
22
|
export declare const DEFAULT_TOMBSTONE_RETENTION_MS: number;
|
|
17
23
|
/**
|
|
@@ -105,6 +111,12 @@ export interface SyncDbConfig {
|
|
|
105
111
|
wipeOnSignOut?: boolean;
|
|
106
112
|
/** Max mutations per batched drain send (B3, default 50; server caps at 100). */
|
|
107
113
|
batchSize?: number;
|
|
114
|
+
/**
|
|
115
|
+
* Max streams snapshot-paged at once (default {@link DEFAULT_SNAPSHOT_CONCURRENCY}).
|
|
116
|
+
* Each stream still pages its own pages in order; this bounds how many streams page
|
|
117
|
+
* side by side. `1` restores one-stream-at-a-time paging.
|
|
118
|
+
*/
|
|
119
|
+
snapshotConcurrency?: number;
|
|
108
120
|
/**
|
|
109
121
|
* B4: when true, a conflict halts the ENTIRE drain instead of just blocking
|
|
110
122
|
* its entity. Default false (per-entity blocking — see README "Conflict
|
package/dist/client.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,gBAAgB,CAAC;AAEhD,OAAO,EAAkB,KAAK,YAAY,EAAE,KAAK,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AAG9F,OAAO,EAAmC,KAAK,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAGjF,OAAO,KAAK,EAAC,6BAA6B,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AACxF,OAAO,EAAqB,KAAK,YAAY,EAAC,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAkB,KAAK,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAOhE,OAAO,EAAuC,KAAK,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAG1F,OAAO,KAAK,EAA8C,aAAa,EAAC,MAAM,kBAAkB,CAAC;AACjG,OAAO,EAAC,KAAK,2BAA2B,EAAwB,MAAM,sBAAsB,CAAC;AAC7F,OAAO,KAAK,EACV,YAAY,EACZ,0BAA0B,EAM1B,qBAAqB,EACrB,UAAU,EACX,MAAM,SAAS,CAAC;AAEjB,mEAAmE;AACnE,eAAO,MAAM,6BAA6B,QAAa,CAAC;AAExD,+EAA+E;AAC/E,eAAO,MAAM,0CAA0C,QAAS,CAAC;AAEjE,0FAA0F;AAC1F,eAAO,MAAM,8BAA8B,QAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,gBAAgB,CAAC;AAEhD,OAAO,EAAkB,KAAK,YAAY,EAAE,KAAK,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AAG9F,OAAO,EAAmC,KAAK,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAGjF,OAAO,KAAK,EAAC,6BAA6B,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AACxF,OAAO,EAAqB,KAAK,YAAY,EAAC,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAkB,KAAK,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAOhE,OAAO,EAAuC,KAAK,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAG1F,OAAO,KAAK,EAA8C,aAAa,EAAC,MAAM,kBAAkB,CAAC;AACjG,OAAO,EAAC,KAAK,2BAA2B,EAAwB,MAAM,sBAAsB,CAAC;AAC7F,OAAO,KAAK,EACV,YAAY,EACZ,0BAA0B,EAM1B,qBAAqB,EACrB,UAAU,EACX,MAAM,SAAS,CAAC;AAEjB,mEAAmE;AACnE,eAAO,MAAM,6BAA6B,QAAa,CAAC;AAExD,+EAA+E;AAC/E,eAAO,MAAM,0CAA0C,QAAS,CAAC;AAEjE;;;;GAIG;AACH,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAE9C,0FAA0F;AAC1F,eAAO,MAAM,8BAA8B,QAA4B,CAAC;AAWxE;;;;;;;;GAQG;AACH,eAAO,MAAM,iCAAiC,IAAI,CAAC;AACnD,eAAO,MAAM,iCAAiC,MAAM,CAAC;AAIrD,MAAM,WAAW,YAAY;IAC3B,2DAA2D;IAC3D,IAAI,EAAE,MAAM,CAAC;IACb,0EAA0E;IAC1E,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;IAC3B,kFAAkF;IAClF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,MAAM,MAAM,GAAG,SAAS,CAAC;IAClD,0EAA0E;IAC1E,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,gFAAgF;IAChF,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC,YAAY,CAAC,CAAC;IACzD,UAAU,CAAC,EAAE,6BAA6B,CAAC,YAAY,CAAC,CAAC;IACzD,yEAAyE;IACzE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAC,CAAC,CAAC;IAC3F,6EAA6E;IAC7E,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IACtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,OAAO,GAAG,mBAAmB,CAAC;IACtC;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,IAAI,CAAC;IAC5B;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9B;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;OAQG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,2GAA2G;IAC3G,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,qBAAqB,GAC7B,eAAe,GACf,SAAS,GACT,YAAY,GACZ,WAAW,GACX,YAAY,CAAC;AAEjB,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC,yEAAyE;IACzE,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,qBAAqB,CAAC;IACjC,0EAA0E;IAC1E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,MAAM;IACrB,2DAA2D;IAC3D,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,+BAA+B;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,wEAAwE;IACxE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,IAAI,CAAC;IACtB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B;;;;OAIG;IACH,MAAM,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,CAAC;IAC/D,+EAA+E;IAC/E,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B;;;OAGG;IACH,aAAa,EAAE,CAAC,IAAI,EAAE;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,GAAG,EAAE,MAAM,EAAE,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAC3C;;;OAGG;IACH,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;IACpC;;;;;OAKG;IACH,WAAW,EAAE,MAAM,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC9C,uDAAuD;IACvD,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,2DAA2D;IAC3D,eAAe,EAAE,CAAC,IAAI,EAAE;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,0BAA0B,CAAA;KAAC,KAAK,IAAI,CAAC;IAC5F;;;;;OAKG;IACH,WAAW,EAAE,CAAC,IAAI,EAAE;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAC,KAAK,IAAI,CAAC;IAChD;;;;;;;;OAQG;IACH,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,0CAA0C;IAC1C,aAAa,EAAE,MAAM,UAAU,CAAC;IAChC;;;;;;OAMG;IACH,cAAc,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,MAAM,IAAI,CAAC;IACrD;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CAC/B;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,YAAY,GAAI,QAAQ,YAAY,KAAG,MA4+DnD,CAAC"}
|
package/dist/client.js
CHANGED
|
@@ -12,7 +12,7 @@ import { SYNC_SCHEMA_VERSION } from "./storage/schema";
|
|
|
12
12
|
import { createSyncStore } from "./storage/store";
|
|
13
13
|
import { CURSORS_TABLE, KNOWN_STREAMS_TABLE } from "./storage/types";
|
|
14
14
|
import { wipeLocalData } from "./storage/wipe";
|
|
15
|
-
import { bootstrapStream } from "./sync/bootstrap";
|
|
15
|
+
import { bootstrapStream, bootstrapStreamsBatch } from "./sync/bootstrap";
|
|
16
16
|
import { getAllCursors } from "./sync/cursor";
|
|
17
17
|
import { applyDelta } from "./sync/deltaApplier";
|
|
18
18
|
import { repairMarkedEntities } from "./sync/entityRepair";
|
|
@@ -24,8 +24,22 @@ import { hydrateWindowEntities } from "./sync/windowHydrate";
|
|
|
24
24
|
export const DEFAULT_RECONCILE_INTERVAL_MS = 5 * 60_000;
|
|
25
25
|
/** Minimum interval between seq-jump-triggered reconciles per stream (30s). */
|
|
26
26
|
export const DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS = 30_000;
|
|
27
|
+
/**
|
|
28
|
+
* Streams snapshot-paged at once by reconcile, subscribe catch-up, force resync and
|
|
29
|
+
* wipe re-bootstrap. Every stream costs at least one `GET /sync/snapshot` round trip,
|
|
30
|
+
* so paging them one at a time made cold start scale with collections × scopes.
|
|
31
|
+
*/
|
|
32
|
+
export const DEFAULT_SNAPSHOT_CONCURRENCY = 6;
|
|
27
33
|
/** E5: default local tombstone retention window (90 days), matching the server's (C7). */
|
|
28
34
|
export const DEFAULT_TOMBSTONE_RETENTION_MS = 90 * 24 * 60 * 60 * 1_000;
|
|
35
|
+
/**
|
|
36
|
+
* Entities one `POST /sync/snapshot/batch` response may carry, shared across its streams.
|
|
37
|
+
* Bounds both the response size (about 2 MB of typical rows) and the work per round.
|
|
38
|
+
*/
|
|
39
|
+
const SNAPSHOT_BATCH_LIMIT = 5_000;
|
|
40
|
+
/** The server has no `POST /sync/snapshot/batch` route (404). */
|
|
41
|
+
class SnapshotBatchUnsupportedError extends Error {
|
|
42
|
+
}
|
|
29
43
|
/**
|
|
30
44
|
* `start()` auth-resolution retry defaults. A host app only calls `start()` once it
|
|
31
45
|
* already believes the user is authenticated (e.g. right after its own login flow
|
|
@@ -135,6 +149,14 @@ export const createSyncDb = (config) => {
|
|
|
135
149
|
/** The reconcile pass every concurrent trigger shares; see `reconcile()`. */
|
|
136
150
|
let inFlightReconcile;
|
|
137
151
|
const lastSeqJumpReconcileAt = new Map();
|
|
152
|
+
/**
|
|
153
|
+
* The latest snapshot pass per stream (single-flight). Two passes paging the same
|
|
154
|
+
* stream at once double the requests, and a purge in one (force resync, retention
|
|
155
|
+
* gap, leave) lets the other write its high cursor back and mark the stream
|
|
156
|
+
* bootstrapped over rows it never paged. Tagged with generation + user like
|
|
157
|
+
* `inFlightReconcile`, so a pass from a superseded lifecycle is never joined.
|
|
158
|
+
*/
|
|
159
|
+
const streamPasses = new Map();
|
|
138
160
|
const statusListeners = new Set();
|
|
139
161
|
// A4: auth-pause state. `authPaused` gates reconcile/replay/timer triggers
|
|
140
162
|
// (INV-2 — nothing else may unpause except a same-user auth change).
|
|
@@ -142,6 +164,9 @@ export const createSyncDb = (config) => {
|
|
|
142
164
|
// refresh has already run for the CURRENT pause episode.
|
|
143
165
|
let authPaused = false;
|
|
144
166
|
let refreshAttempted = false;
|
|
167
|
+
// Set once the server answers 404 to the snapshot batch route; per-stream paging is
|
|
168
|
+
// used for the rest of this client's life.
|
|
169
|
+
let snapshotBatchUnsupported = false;
|
|
145
170
|
// E1: lifecycle serialization. `generation` bumps on every start()/stop() so
|
|
146
171
|
// an in-flight operation that resumes after an await can detect it has been
|
|
147
172
|
// superseded (a rapid stop()-then-start() for a new user) and abort instead
|
|
@@ -189,6 +214,103 @@ export const createSyncDb = (config) => {
|
|
|
189
214
|
await pending;
|
|
190
215
|
}
|
|
191
216
|
};
|
|
217
|
+
/**
|
|
218
|
+
* Run a snapshot pass over `streams`, single-flight per stream. Each stream is either
|
|
219
|
+
* joined to its latest pass or owned by this one; `run` receives the owned streams once
|
|
220
|
+
* every pass it queued behind has settled.
|
|
221
|
+
*
|
|
222
|
+
* - `join`: share the stream's latest pass when one exists (reconcile — any pass to
|
|
223
|
+
* the head will do).
|
|
224
|
+
* - `fresh`: the pass must start after this call (subscribe catch-up — a pass that
|
|
225
|
+
* began before the room join may have missed a write). Joins a queued pass that has
|
|
226
|
+
* not started yet, otherwise queues behind the running one.
|
|
227
|
+
* - `exclusive`: always queue behind whatever is in flight (force resync and wipe
|
|
228
|
+
* re-bootstrap purge first, which must never overlap another pass's paging).
|
|
229
|
+
*/
|
|
230
|
+
const runStreamsPass = ({ mode, run, streams, }) => {
|
|
231
|
+
const joined = [];
|
|
232
|
+
const owned = [];
|
|
233
|
+
const previous = [];
|
|
234
|
+
for (const stream of streams) {
|
|
235
|
+
const existing = streamPasses.get(stream);
|
|
236
|
+
const current = existing && existing.generation === generation && existing.userId === currentUserId
|
|
237
|
+
? existing
|
|
238
|
+
: undefined;
|
|
239
|
+
if (current && (mode === "join" || (mode === "fresh" && !current.started))) {
|
|
240
|
+
joined.push(current.promise);
|
|
241
|
+
continue;
|
|
242
|
+
}
|
|
243
|
+
owned.push(stream);
|
|
244
|
+
if (existing) {
|
|
245
|
+
previous.push(existing.promise);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
if (owned.length === 0) {
|
|
249
|
+
return Promise.all(joined).then(() => undefined);
|
|
250
|
+
}
|
|
251
|
+
const entry = { generation, promise: Promise.resolve(), started: false, userId: currentUserId };
|
|
252
|
+
entry.promise = (async () => {
|
|
253
|
+
// The previous passes' failures belong to their own callers.
|
|
254
|
+
await Promise.all(previous.map((pass) => pass.catch(() => undefined)));
|
|
255
|
+
// A stop()/start() or user switch while this pass was queued supersedes it:
|
|
256
|
+
// paging now would write into a store that belongs to a newer lifecycle.
|
|
257
|
+
if (entry.generation !== generation || entry.userId !== currentUserId) {
|
|
258
|
+
return;
|
|
259
|
+
}
|
|
260
|
+
entry.started = true;
|
|
261
|
+
await run(owned);
|
|
262
|
+
})().finally(() => {
|
|
263
|
+
for (const stream of owned) {
|
|
264
|
+
if (streamPasses.get(stream) === entry) {
|
|
265
|
+
streamPasses.delete(stream);
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
});
|
|
269
|
+
for (const stream of owned) {
|
|
270
|
+
streamPasses.set(stream, entry);
|
|
271
|
+
}
|
|
272
|
+
return Promise.all([entry.promise, ...joined]).then(() => undefined);
|
|
273
|
+
};
|
|
274
|
+
/** {@link runStreamsPass} for a single stream. */
|
|
275
|
+
const runStreamPass = ({ mode, run, stream, }) => runStreamsPass({ mode, run: () => run(), streams: [stream] });
|
|
276
|
+
/**
|
|
277
|
+
* Run `worker` over `items` with at most `snapshotConcurrency` in flight. Stops taking
|
|
278
|
+
* new items once `isSuperseded()` or any worker throws, waits for the in-flight ones,
|
|
279
|
+
* then rethrows — `AuthRequiredError` first, so callers still auth-pause on a 401 that
|
|
280
|
+
* raced another failure.
|
|
281
|
+
*/
|
|
282
|
+
const runSnapshotPool = async ({ isSuperseded, items, worker, }) => {
|
|
283
|
+
const limit = Math.max(1, Math.floor(config.snapshotConcurrency ?? DEFAULT_SNAPSHOT_CONCURRENCY));
|
|
284
|
+
let next = 0;
|
|
285
|
+
let authError;
|
|
286
|
+
let firstError;
|
|
287
|
+
let failed = false;
|
|
288
|
+
const runWorker = async () => {
|
|
289
|
+
while (next < items.length && !failed && !isSuperseded()) {
|
|
290
|
+
const item = items[next];
|
|
291
|
+
next += 1;
|
|
292
|
+
try {
|
|
293
|
+
await worker(item);
|
|
294
|
+
}
|
|
295
|
+
catch (error) {
|
|
296
|
+
failed = true;
|
|
297
|
+
if (error instanceof AuthRequiredError) {
|
|
298
|
+
authError ??= error;
|
|
299
|
+
}
|
|
300
|
+
else if (firstError === undefined) {
|
|
301
|
+
firstError = error;
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
};
|
|
306
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, runWorker));
|
|
307
|
+
if (authError) {
|
|
308
|
+
throw authError;
|
|
309
|
+
}
|
|
310
|
+
if (failed) {
|
|
311
|
+
throw firstError;
|
|
312
|
+
}
|
|
313
|
+
};
|
|
192
314
|
const notifyStatusChange = () => {
|
|
193
315
|
for (const listener of statusListeners) {
|
|
194
316
|
listener();
|
|
@@ -405,6 +527,72 @@ export const createSyncDb = (config) => {
|
|
|
405
527
|
}
|
|
406
528
|
return serverSet;
|
|
407
529
|
};
|
|
530
|
+
/**
|
|
531
|
+
* Page `streams` to their heads as one single-flight pass (see {@link runStreamsPass}).
|
|
532
|
+
* One `POST /sync/snapshot/batch` round trip per round covers every owned stream;
|
|
533
|
+
* against a server without that route, falls back to `GET /sync/snapshot` per stream,
|
|
534
|
+
* `snapshotConcurrency` streams at a time.
|
|
535
|
+
*/
|
|
536
|
+
const pageStreams = async ({ streams, mode, isSuperseded, purgeFirst = false, }) => {
|
|
537
|
+
const channel = httpChannel;
|
|
538
|
+
if (!channel || streams.length === 0) {
|
|
539
|
+
return;
|
|
540
|
+
}
|
|
541
|
+
const collections = new Map(streams.map(({ stream, collection }) => [stream, collection]));
|
|
542
|
+
await runStreamsPass({
|
|
543
|
+
mode,
|
|
544
|
+
run: async (owned) => {
|
|
545
|
+
const ownedStreams = owned.map((stream) => ({
|
|
546
|
+
collection: collections.get(stream),
|
|
547
|
+
stream,
|
|
548
|
+
}));
|
|
549
|
+
if (purgeFirst) {
|
|
550
|
+
for (const { stream, collection } of ownedStreams) {
|
|
551
|
+
store.purgeStream({ stream });
|
|
552
|
+
store.addKnownStream({ collection, stream });
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
const fetchSnapshotBatch = channel.fetchSnapshotBatch;
|
|
556
|
+
if (fetchSnapshotBatch && !snapshotBatchUnsupported) {
|
|
557
|
+
try {
|
|
558
|
+
await bootstrapStreamsBatch({
|
|
559
|
+
fetchSnapshotBatch: async (request) => {
|
|
560
|
+
const result = await fetchSnapshotBatch(request);
|
|
561
|
+
if (result.type === "unsupported") {
|
|
562
|
+
throw new SnapshotBatchUnsupportedError();
|
|
563
|
+
}
|
|
564
|
+
return result;
|
|
565
|
+
},
|
|
566
|
+
isSuperseded,
|
|
567
|
+
limit: SNAPSHOT_BATCH_LIMIT,
|
|
568
|
+
store,
|
|
569
|
+
streams: ownedStreams,
|
|
570
|
+
});
|
|
571
|
+
return;
|
|
572
|
+
}
|
|
573
|
+
catch (error) {
|
|
574
|
+
if (!(error instanceof SnapshotBatchUnsupportedError)) {
|
|
575
|
+
throw error;
|
|
576
|
+
}
|
|
577
|
+
// Every round before the 404 is already written, so per-stream paging
|
|
578
|
+
// resumes from the store's cursors without losing anything.
|
|
579
|
+
snapshotBatchUnsupported = true;
|
|
580
|
+
debugLog?.record({
|
|
581
|
+
direction: "system",
|
|
582
|
+
label: "snapshot batch unsupported by server; paging per stream",
|
|
583
|
+
type: "reconcile",
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
await runSnapshotPool({
|
|
588
|
+
isSuperseded,
|
|
589
|
+
items: ownedStreams,
|
|
590
|
+
worker: ({ stream, collection }) => bootstrapStream({ channel, collection, isSuperseded, store, stream }),
|
|
591
|
+
});
|
|
592
|
+
},
|
|
593
|
+
streams: streams.map(({ stream }) => stream),
|
|
594
|
+
});
|
|
595
|
+
};
|
|
408
596
|
const runReconcile = async () => {
|
|
409
597
|
// Simulated offline severs all network activity, including the HTTP
|
|
410
598
|
// channel; an auth pause stands down every network trigger (INV-2) until
|
|
@@ -437,14 +625,15 @@ export const createSyncDb = (config) => {
|
|
|
437
625
|
// lifecycle/user while awaiting discovery.
|
|
438
626
|
return;
|
|
439
627
|
}
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
628
|
+
await pageStreams({
|
|
629
|
+
isSuperseded,
|
|
630
|
+
mode: "join",
|
|
631
|
+
streams: [...streams]
|
|
632
|
+
.filter(([stream, collection]) => !skipSnapshotPaging({ collection, stream }))
|
|
633
|
+
.map(([stream, collection]) => ({ collection, stream })),
|
|
634
|
+
});
|
|
635
|
+
if (isSuperseded()) {
|
|
636
|
+
return;
|
|
448
637
|
}
|
|
449
638
|
// E5: client-side compaction runs only after a successful per-stream reconcile —
|
|
450
639
|
// reconcile just proved the local store caught up with the server via real network
|
|
@@ -510,6 +699,63 @@ export const createSyncDb = (config) => {
|
|
|
510
699
|
inFlightReconcile = { generation, promise, userId: currentUserId };
|
|
511
700
|
return promise;
|
|
512
701
|
};
|
|
702
|
+
// Streams confirmed by `sync:subscribed` and not yet paged, tagged with the lifecycle
|
|
703
|
+
// and user that confirmed them. One confirmation arrives per collection; those landing
|
|
704
|
+
// while a catch-up request is in flight are paged together in the next one.
|
|
705
|
+
const pendingCatchUp = new Map();
|
|
706
|
+
let isCatchUpDraining = false;
|
|
707
|
+
const drainCatchUp = async () => {
|
|
708
|
+
if (isCatchUpDraining) {
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
isCatchUpDraining = true;
|
|
712
|
+
addSyncing(1);
|
|
713
|
+
try {
|
|
714
|
+
while (pendingCatchUp.size > 0) {
|
|
715
|
+
const myGeneration = generation;
|
|
716
|
+
const myUserId = currentUserId;
|
|
717
|
+
const isSuperseded = () => generation !== myGeneration || currentUserId !== myUserId;
|
|
718
|
+
const streams = [];
|
|
719
|
+
for (const [stream, pending] of pendingCatchUp) {
|
|
720
|
+
if (pending.generation === myGeneration && pending.userId === myUserId) {
|
|
721
|
+
streams.push({ collection: pending.collection, stream });
|
|
722
|
+
}
|
|
723
|
+
}
|
|
724
|
+
pendingCatchUp.clear();
|
|
725
|
+
if (!httpChannel || simulatedOffline || authPaused) {
|
|
726
|
+
return;
|
|
727
|
+
}
|
|
728
|
+
if (streams.length === 0) {
|
|
729
|
+
continue;
|
|
730
|
+
}
|
|
731
|
+
await pageStreams({ isSuperseded, mode: "fresh", streams });
|
|
732
|
+
if (isSuperseded()) {
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
debugLog?.record({
|
|
736
|
+
detail: { streams: streams.map(({ stream }) => stream) },
|
|
737
|
+
direction: "system",
|
|
738
|
+
label: `subscribe catch-up: ${streams.map(({ stream }) => stream).join(", ")}`,
|
|
739
|
+
type: "reconcile",
|
|
740
|
+
});
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
catch (error) {
|
|
744
|
+
if (error instanceof AuthRequiredError) {
|
|
745
|
+
setAuthPaused(true);
|
|
746
|
+
return;
|
|
747
|
+
}
|
|
748
|
+
throw error;
|
|
749
|
+
}
|
|
750
|
+
finally {
|
|
751
|
+
isCatchUpDraining = false;
|
|
752
|
+
addSyncing(-1);
|
|
753
|
+
// Confirmations for a newer lifecycle can queue while a superseded pass unwinds.
|
|
754
|
+
if (pendingCatchUp.size > 0) {
|
|
755
|
+
void drainCatchUp().catch(warn("subscribe catch-up failed"));
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
};
|
|
513
759
|
/**
|
|
514
760
|
* Catch up the streams the server just confirmed this socket joined
|
|
515
761
|
* (`sync:subscribed`).
|
|
@@ -526,8 +772,9 @@ export const createSyncDb = (config) => {
|
|
|
526
772
|
* confirmed stream from its cursor here makes snapshot + delta coverage contiguous.
|
|
527
773
|
* It deliberately does NOT go through `reconcile()`: that call coalesces onto a pass
|
|
528
774
|
* which may have started before the join (exactly the pass that could have missed the
|
|
529
|
-
* write).
|
|
530
|
-
*
|
|
775
|
+
* write). For the same reason a confirmation arriving while a catch-up request is in
|
|
776
|
+
* flight waits for the next request rather than joining that one. Paging is
|
|
777
|
+
* idempotent and cursor-guarded, so running it beside an in-flight reconcile is safe.
|
|
531
778
|
*/
|
|
532
779
|
const catchUpSubscribedStreams = async ({ collection, mode, streams, }) => {
|
|
533
780
|
if (!httpChannel || simulatedOffline || authPaused || streams.length === 0) {
|
|
@@ -539,38 +786,22 @@ export const createSyncDb = (config) => {
|
|
|
539
786
|
if (mode === "window" || isWindowCollection(collection)) {
|
|
540
787
|
return;
|
|
541
788
|
}
|
|
542
|
-
const
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
addSyncing(1);
|
|
546
|
-
try {
|
|
547
|
-
for (const stream of streams) {
|
|
548
|
-
if (isSuperseded()) {
|
|
549
|
-
return;
|
|
550
|
-
}
|
|
551
|
-
if (skipSnapshotPaging({ collection, mode, stream })) {
|
|
552
|
-
continue;
|
|
553
|
-
}
|
|
554
|
-
store.addKnownStream({ collection, stream });
|
|
555
|
-
await bootstrapStream({ channel: httpChannel, collection, store, stream });
|
|
789
|
+
for (const stream of streams) {
|
|
790
|
+
if (skipSnapshotPaging({ collection, mode, stream })) {
|
|
791
|
+
continue;
|
|
556
792
|
}
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
if (error instanceof AuthRequiredError) {
|
|
566
|
-
setAuthPaused(true);
|
|
567
|
-
return;
|
|
793
|
+
store.addKnownStream({ collection, stream });
|
|
794
|
+
// A queued pass that has not started yet already starts after this join.
|
|
795
|
+
const queued = streamPasses.get(stream);
|
|
796
|
+
if (queued &&
|
|
797
|
+
!queued.started &&
|
|
798
|
+
queued.generation === generation &&
|
|
799
|
+
queued.userId === currentUserId) {
|
|
800
|
+
continue;
|
|
568
801
|
}
|
|
569
|
-
|
|
570
|
-
}
|
|
571
|
-
finally {
|
|
572
|
-
addSyncing(-1);
|
|
802
|
+
pendingCatchUp.set(stream, { collection, generation, userId: currentUserId });
|
|
573
803
|
}
|
|
804
|
+
await drainCatchUp();
|
|
574
805
|
};
|
|
575
806
|
/**
|
|
576
807
|
* C2 migration: deployed clients hold legacy `snapshot:{collection}` pseudo-cursors.
|
|
@@ -686,19 +917,30 @@ export const createSyncDb = (config) => {
|
|
|
686
917
|
for (const collection of new Set(streamInfos.map((info) => info.collection))) {
|
|
687
918
|
purged += store.purgeUnknownStreamEntities({ collection });
|
|
688
919
|
}
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
}
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
920
|
+
const channel = httpChannel;
|
|
921
|
+
await runSnapshotPool({
|
|
922
|
+
isSuperseded,
|
|
923
|
+
// Query collections have no snapshot to rebuild from; their retained windows
|
|
924
|
+
// are refetched below instead.
|
|
925
|
+
items: streamInfos.filter(({ collection }) => !isQueryCollection(collection)),
|
|
926
|
+
worker: async ({ stream, collection }) => {
|
|
927
|
+
await runStreamPass({
|
|
928
|
+
mode: "exclusive",
|
|
929
|
+
run: async () => {
|
|
930
|
+
if (isSuperseded()) {
|
|
931
|
+
return;
|
|
932
|
+
}
|
|
933
|
+
purged += store.purgeStream({ stream });
|
|
934
|
+
store.addKnownStream({ collection, stream });
|
|
935
|
+
await bootstrapStream({ channel, collection, isSuperseded, store, stream });
|
|
936
|
+
},
|
|
937
|
+
stream,
|
|
938
|
+
});
|
|
939
|
+
await waitForLifecycleIdle();
|
|
940
|
+
},
|
|
941
|
+
});
|
|
942
|
+
if (isSuperseded()) {
|
|
943
|
+
return { ok: false, purged, reason: "superseded", repaired, streams: streamInfos.length };
|
|
702
944
|
}
|
|
703
945
|
repaired = await repairAllMarkedEntities();
|
|
704
946
|
await queryWindows.refetchRetained();
|
|
@@ -940,9 +1182,15 @@ export const createSyncDb = (config) => {
|
|
|
940
1182
|
// INV-2 auth-pause case (returns undefined on a 401) — skip bootstrap then.
|
|
941
1183
|
const streams = await syncStreams();
|
|
942
1184
|
if (streams) {
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
1185
|
+
await pageStreams({
|
|
1186
|
+
isSuperseded: () => false,
|
|
1187
|
+
mode: "exclusive",
|
|
1188
|
+
// A pass that was paging when the wipe ran can write its cursor and
|
|
1189
|
+
// `bootstrapped` mark into the wiped store before this one starts. Purge
|
|
1190
|
+
// again so this pass pages from 0 instead of resuming from that cursor.
|
|
1191
|
+
purgeFirst: true,
|
|
1192
|
+
streams: [...streams].map(([stream, collection]) => ({ collection, stream })),
|
|
1193
|
+
});
|
|
946
1194
|
}
|
|
947
1195
|
}
|
|
948
1196
|
};
|