@terreno/syncdb 57.14.0 → 57.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/README.md +6 -5
  2. package/dist/client.d.ts.map +1 -1
  3. package/dist/client.js +200 -80
  4. package/dist/client.js.map +1 -1
  5. package/dist/client.test.js +86 -0
  6. package/dist/client.test.js.map +1 -1
  7. package/dist/persisters/encryptedIndexedDbPersister.d.ts +4 -3
  8. package/dist/persisters/encryptedIndexedDbPersister.d.ts.map +1 -1
  9. package/dist/persisters/encryptedIndexedDbPersister.js +21 -16
  10. package/dist/persisters/encryptedIndexedDbPersister.js.map +1 -1
  11. package/dist/persisters/encryptedIndexedDbPersister.test.js +40 -1
  12. package/dist/persisters/encryptedIndexedDbPersister.test.js.map +1 -1
  13. package/dist/sync/bootstrap.d.ts +34 -3
  14. package/dist/sync/bootstrap.d.ts.map +1 -1
  15. package/dist/sync/bootstrap.js +224 -71
  16. package/dist/sync/bootstrap.js.map +1 -1
  17. package/dist/sync/bootstrap.test.js +210 -2
  18. package/dist/sync/bootstrap.test.js.map +1 -1
  19. package/dist/sync/httpChannel.d.ts +14 -1
  20. package/dist/sync/httpChannel.d.ts.map +1 -1
  21. package/dist/sync/httpChannel.js +34 -0
  22. package/dist/sync/httpChannel.js.map +1 -1
  23. package/dist/sync/httpChannel.test.js +54 -0
  24. package/dist/sync/httpChannel.test.js.map +1 -1
  25. package/dist/types.d.ts +14 -0
  26. package/dist/types.d.ts.map +1 -1
  27. package/package.json +1 -1
  28. package/src/client.test.ts +111 -1
  29. package/src/client.ts +232 -86
  30. package/src/persisters/encryptedIndexedDbPersister.test.ts +44 -1
  31. package/src/persisters/encryptedIndexedDbPersister.ts +24 -17
  32. package/src/sync/bootstrap.test.ts +269 -3
  33. package/src/sync/bootstrap.ts +336 -82
  34. package/src/sync/httpChannel.test.ts +67 -0
  35. package/src/sync/httpChannel.ts +50 -0
  36. package/src/types.ts +12 -0
package/README.md CHANGED
@@ -25,7 +25,7 @@ Local-first data layer for Terreno apps. A TinyBase `MergeableStore` (encrypted
25
25
  │ sync:mutate ─────────────────────────────►│ │
26
26
  │ sync:ack / sync:nack ◄────────────────────┘ │
27
27
  │ sync:delta ◄──────────────────────────────────────────────┘
28
- │ GET /sync/snapshot (bootstrap + catch-up, HTTP)
28
+ │ POST /sync/snapshot/batch (bootstrap + catch-up, HTTP)
29
29
  └── POST /sync/mutate (fallback while the socket is down)
30
30
  ```
31
31
 
@@ -102,7 +102,7 @@ const todoRouter = modelRouter("/todos", Todo, {
102
102
 
103
103
  new TerrenoApp({userModel: User})
104
104
  .register(todoRouter)
105
- .register(new SyncApp()) // GET /sync/snapshot, POST /sync/mutate, GET /sync/key
105
+ .register(new SyncApp()) // POST /sync/snapshot/batch, GET /sync/snapshot, POST /sync/mutate, GET /sync/key
106
106
  .register(new RealtimeApp()) // Socket.io server; installs sync:subscribe/sync:mutate handlers
107
107
  .start();
108
108
  ```
@@ -168,7 +168,7 @@ const client = createSyncDb({
168
168
  seqJumpReconcileMinIntervalMs?, // seq-jump reconcile rate limit (default 30s)
169
169
  batchSize?, // max mutations per batched drain send (default 50; server caps at 100)
170
170
  haltQueueOnConflict?, // conflict policy — see "Conflict handling modes" below (default false)
171
- snapshotConcurrency?, // streams snapshot-paged at once (default 6; 1 = one at a time)
171
+ snapshotConcurrency?, // per-stream GET /sync/snapshot passes at once (default 6; 1 = one at a time)
172
172
  onDecryptFailure?, // override the default wipe+re-bootstrap on undecryptable data (web)
173
173
  tombstoneRetentionMs?, // client-side tombstone compaction window (default 90 days; 0 disables)
174
174
  onAuthRequired?, // fires once per auth-pause episode — prompt for re-login
@@ -323,6 +323,7 @@ Sequencing guarantees: validation failures never consume a seq (the claim happen
323
323
  | Endpoint | Purpose | Auth |
324
324
  |---|---|---|
325
325
  | `GET /sync/snapshot?collection=&cursor=&limit=` | Bootstrap + catch-up. Returns `{entities: [{id, data, seq, deleted}], cursor, hasMore}`. `cursor=0` = full snapshot (legacy docs without `_syncSeq` arrive in the first page with seq 0). Default page 100, max 100. | Model `list` permissions + server-enforced scope filter |
326
+ | `POST /sync/snapshot/batch` | Bootstrap + catch-up for many streams in one round trip. Body `{streams: [{stream, cursor?, legacyCursor?}], limit?}` (max 500 streams; `limit` is the entity budget for the whole response, default 5,000, max 10,000). Returns `{pages}`, one `GET /sync/snapshot`-shaped page per stream in request order. | Same as `GET /sync/snapshot`, per stream; any denied stream fails the request |
326
327
  | `POST /sync/mutate` | HTTP fallback for outbox replay (same handler as the socket channel). Body: `{mutationId, collection, operation, id?, data?, baseVersion?}`. Returns `{ack}` or `{nack}` with status 409 (conflict), 403 (unauthorized), 422 (validation), 429 (rate_limited, carries `retryAfterMs`), 500 (error). | modelRouter create/update/delete write path |
327
328
  | `POST /sync/mutate/batch` | HTTP fallback for batched outbox replay. Body: `{mutations: SyncMutateRequest[]}` (max 100; intra-batch duplicate `mutationId`s rejected up front). Returns `{results: ({type:"ack",ack}\|{type:"nack",nack})[]}` — applied strictly in array order, stopping at the first non-ack (a shorter `results` array than the request means everything after it was never attempted). | modelRouter create/update/delete write path, per mutation |
328
329
  | `GET /sync/key` | Caller's per-user key material for the server key provider (32 random bytes, base64; created on first call). | Own key only |
@@ -450,9 +451,9 @@ This writes `store/syncDbSdk.ts` with `SYNC_COLLECTIONS`, entity types, and hook
450
451
  - **Multi-tab web**: two tabs of the same user share one IndexedDB blob; concurrent persister saves are last-writer-wins at the blob level and can drop the other tab's queued outbox rows. Single-writer coordination (Web Locks) is not implemented yet — avoid relying on offline writes from multiple simultaneous tabs.
451
452
  - **`Model.bulkWrite` is unavailable on synced models**: it skips Mongoose middleware, so it can never stamp seqs. `registerSync` replaces the static with one that throws — bulk updates to a synced collection have to loop per document.
452
453
  - **Native plaintext by design**: no SQLCipher; the OS sandbox is deemed sufficient.
453
- - **Whole-store persistence**: each save serializes and (on web) encrypts the full store — cost scales with store size, not change size. Bound it by scoping which collections sync; saves are debounced.
454
+ - **Whole-store persistence**: each save serializes and (on web) encrypts the full store — cost scales with store size, not change size. Bound it by scoping which collections sync; saves are debounced, and a burst of transactions costs one serialization per write, not one per transaction.
454
455
  - **`realtime` + `sync` coexistence**: a model may enable both (distinct events, `sync` vs `sync:delta`, so clients never double-apply), at the cost of double emission work. `modelRouter` `realtime` is **deprecated and will be removed in Terreno 58**; use `sync` only.
455
- - **Cold start scales with stream count**: every stream (one collection in one scope) costs at least one `GET /sync/snapshot` round trip, even when empty. Streams page side by side (`snapshotConcurrency`, default 6), so cold start is about `streams / 6` round trips. Sync only the collections the client reads. See [Use syncdb correctly](https://github.com/TerrenoLabs/terreno/blob/master/docs/how-to/use-syncdb-correctly.md) for this and the write-path pitfalls apps must handle.
456
+ - **Cold start scales with row count, not stream count**: streams page together through `POST /sync/snapshot/batch`, one round trip per round of up to 5,000 rows. Against a server without that route, every stream costs at least one `GET /sync/snapshot` round trip (`snapshotConcurrency` at a time, default 6). Sync only the collections the client reads. See [Use syncdb correctly](https://github.com/TerrenoLabs/terreno/blob/master/docs/how-to/use-syncdb-correctly.md) for this and the write-path pitfalls apps must handle.
456
457
  - **Seq counter write amplification**: every synced write does an atomic `$inc` on a per-stream counter doc. Acceptable at current scale; Redis-based counters are the documented upgrade path.
457
458
 
458
459
  ## Documentation
@@ -1 +1 @@
1
- {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,gBAAgB,CAAC;AAEhD,OAAO,EAAkB,KAAK,YAAY,EAAE,KAAK,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AAG9F,OAAO,EAAmC,KAAK,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAGjF,OAAO,KAAK,EAAC,6BAA6B,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AACxF,OAAO,EAAqB,KAAK,YAAY,EAAC,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAkB,KAAK,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAOhE,OAAO,EAAuC,KAAK,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAG1F,OAAO,KAAK,EAA8C,aAAa,EAAC,MAAM,kBAAkB,CAAC;AACjG,OAAO,EAAC,KAAK,2BAA2B,EAAwB,MAAM,sBAAsB,CAAC;AAC7F,OAAO,KAAK,EACV,YAAY,EACZ,0BAA0B,EAM1B,qBAAqB,EACrB,UAAU,EACX,MAAM,SAAS,CAAC;AAEjB,mEAAmE;AACnE,eAAO,MAAM,6BAA6B,QAAa,CAAC;AAExD,+EAA+E;AAC/E,eAAO,MAAM,0CAA0C,QAAS,CAAC;AAEjE;;;;GAIG;AACH,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAE9C,0FAA0F;AAC1F,eAAO,MAAM,8BAA8B,QAA4B,CAAC;AAExE;;;;;;;;GAQG;AACH,eAAO,MAAM,iCAAiC,IAAI,CAAC;AACnD,eAAO,MAAM,iCAAiC,MAAM,CAAC;AAIrD,MAAM,WAAW,YAAY;IAC3B,2DAA2D;IAC3D,IAAI,EAAE,MAAM,CAAC;IACb,0EAA0E;IAC1E,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;IAC3B,kFAAkF;IAClF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,MAAM,MAAM,GAAG,SAAS,CAAC;IAClD,0EAA0E;IAC1E,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,gFAAgF;IAChF,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC,YAAY,CAAC,CAAC;IACzD,UAAU,CAAC,EAAE,6BAA6B,CAAC,YAAY,CAAC,CAAC;IACzD,yEAAyE;IACzE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAC,CAAC,CAAC;IAC3F,6EAA6E;IAC7E,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IACtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,OAAO,GAAG,mBAAmB,CAAC;IACtC;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,IAAI,CAAC;IAC5B;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9B;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;OAQG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,2GAA2G;IAC3G,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,qBAAqB,GAC7B,eAAe,GACf,SAAS,GACT,YAAY,GACZ,WAAW,GACX,YAAY,CAAC;AAEjB,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC,yEAAyE;IACzE,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,qBAAqB,CAAC;IACjC,0EAA0E;IAC1E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,MAAM;IACrB,2DAA2D;IAC3D,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,+BAA+B;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,wEAAwE;IACxE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,IAAI,CAAC;IACtB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B;;;;OAIG;IACH,MAAM,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,CAAC;IAC/D,+EAA+E;IAC/E,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B;;;OAGG;IACH,aAAa,EAAE,CAAC,IAAI,EAAE;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,GAAG,EAAE,MAAM,EAAE,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAC3C;;;OAGG;IACH,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;IACpC;;;;;OAKG;IACH,WAAW,EAAE,MAAM,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC9C,uDAAuD;IACvD,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,2DAA2D;IAC3D,eAAe,EAAE,CAAC,IAAI,EAAE;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,0BAA0B,CAAA;KAAC,KAAK,IAAI,CAAC;IAC5F;;;;;OAKG;IACH,WAAW,EAAE,CAAC,IAAI,EAAE;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAC,KAAK,IAAI,CAAC;IAChD;;;;;;;;OAQG;IACH,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,0CAA0C;IAC1C,aAAa,EAAE,MAAM,UAAU,CAAC;IAChC;;;;;;OAMG;IACH,cAAc,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,MAAM,IAAI,CAAC;IACrD;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CAC/B;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,YAAY,GAAI,QAAQ,YAAY,KAAG,MAm2DnD,CAAC"}
1
+ {"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,WAAW,EAAC,MAAM,gBAAgB,CAAC;AAEhD,OAAO,EAAkB,KAAK,YAAY,EAAE,KAAK,mBAAmB,EAAC,MAAM,kBAAkB,CAAC;AAG9F,OAAO,EAAmC,KAAK,MAAM,EAAC,MAAM,oBAAoB,CAAC;AAGjF,OAAO,KAAK,EAAC,6BAA6B,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AACxF,OAAO,EAAqB,KAAK,YAAY,EAAC,MAAM,sBAAsB,CAAC;AAE3E,OAAO,EAAkB,KAAK,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAOhE,OAAO,EAAuC,KAAK,WAAW,EAAC,MAAM,oBAAoB,CAAC;AAG1F,OAAO,KAAK,EAA8C,aAAa,EAAC,MAAM,kBAAkB,CAAC;AACjG,OAAO,EAAC,KAAK,2BAA2B,EAAwB,MAAM,sBAAsB,CAAC;AAC7F,OAAO,KAAK,EACV,YAAY,EACZ,0BAA0B,EAM1B,qBAAqB,EACrB,UAAU,EACX,MAAM,SAAS,CAAC;AAEjB,mEAAmE;AACnE,eAAO,MAAM,6BAA6B,QAAa,CAAC;AAExD,+EAA+E;AAC/E,eAAO,MAAM,0CAA0C,QAAS,CAAC;AAEjE;;;;GAIG;AACH,eAAO,MAAM,4BAA4B,IAAI,CAAC;AAE9C,0FAA0F;AAC1F,eAAO,MAAM,8BAA8B,QAA4B,CAAC;AAWxE;;;;;;;;GAQG;AACH,eAAO,MAAM,iCAAiC,IAAI,CAAC;AACnD,eAAO,MAAM,iCAAiC,MAAM,CAAC;AAIrD,MAAM,WAAW,YAAY;IAC3B,2DAA2D;IAC3D,IAAI,EAAE,MAAM,CAAC;IACb,0EAA0E;IAC1E,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,YAAY,EAAE,YAAY,CAAC;IAC3B,kFAAkF;IAClF,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iFAAiF;IACjF,SAAS,CAAC,EAAE,aAAa,CAAC;IAC1B;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,MAAM,MAAM,GAAG,SAAS,CAAC;IAClD,0EAA0E;IAC1E,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,gBAAgB,CAAC;IACpC,gFAAgF;IAChF,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;;;OAKG;IACH,UAAU,CAAC,EAAE,6BAA6B,CAAC,YAAY,CAAC,CAAC;IACzD,UAAU,CAAC,EAAE,6BAA6B,CAAC,YAAY,CAAC,CAAC;IACzD,yEAAyE;IACzE,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,GAAG;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KAAC,CAAC,CAAC;IAC3F,6EAA6E;IAC7E,6BAA6B,CAAC,EAAE,MAAM,CAAC;IACvC,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,MAAM,CAAC;IACnB,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,MAAM,CAAC;IACtB;;;;;OAKG;IACH,KAAK,CAAC,EAAE,OAAO,GAAG,mBAAmB,CAAC;IACtC;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,IAAI,CAAC;IAC5B;;;;;OAKG;IACH,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,iFAAiF;IACjF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;OAIG;IACH,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,MAAM,IAAI,CAAC;IAC9B;;;;;;;OAOG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B;;;;;;;;OAQG;IACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,2GAA2G;IAC3G,qBAAqB,CAAC,EAAE,MAAM,CAAC;CAChC;AAED;;;;;;;;GAQG;AACH,MAAM,MAAM,qBAAqB,GAC7B,eAAe,GACf,SAAS,GACT,YAAY,GACZ,WAAW,GACX,YAAY,CAAC;AAEjB,oDAAoD;AACpD,MAAM,WAAW,iBAAiB;IAChC,yEAAyE;IACzE,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,0CAA0C;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,qBAAqB,CAAC;IACjC,0EAA0E;IAC1E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,uCAAuC;IACvC,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC/B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,MAAM;IACrB,2DAA2D;IAC3D,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,+BAA+B;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;;;OAKG;IACH,KAAK,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,wEAAwE;IACxE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1B;;;;;;OAMG;IACH,SAAS,EAAE,MAAM,IAAI,CAAC;IACtB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9B;;;;OAIG;IACH,MAAM,EAAE,CAAC,IAAI,EAAE,UAAU,KAAK;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAC,CAAC;IAC/D,+EAA+E;IAC/E,SAAS,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC/B;;;OAGG;IACH,aAAa,EAAE,CAAC,IAAI,EAAE;QACpB,UAAU,EAAE,MAAM,CAAC;QACnB,GAAG,EAAE,MAAM,EAAE,CAAC;QACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,KAAK,OAAO,CAAC,2BAA2B,CAAC,CAAC;IAC3C;;;OAGG;IACH,QAAQ,CAAC,YAAY,EAAE,YAAY,CAAC;IACpC;;;;;OAKG;IACH,WAAW,EAAE,MAAM,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC9C,uDAAuD;IACvD,YAAY,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAClC,2DAA2D;IAC3D,eAAe,EAAE,CAAC,IAAI,EAAE;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,0BAA0B,CAAA;KAAC,KAAK,IAAI,CAAC;IAC5F;;;;;OAKG;IACH,WAAW,EAAE,CAAC,IAAI,EAAE;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAC,KAAK,IAAI,CAAC;IAChD;;;;;;;;OAQG;IACH,OAAO,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAC7B,0CAA0C;IAC1C,aAAa,EAAE,MAAM,UAAU,CAAC;IAChC;;;;;;OAMG;IACH,cAAc,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,MAAM,IAAI,CAAC;IACrD;;;;OAIG;IACH,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAC;CAC/B;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,YAAY,GAAI,QAAQ,YAAY,KAAG,MA4+DnD,CAAC"}
package/dist/client.js CHANGED
@@ -12,7 +12,7 @@ import { SYNC_SCHEMA_VERSION } from "./storage/schema";
12
12
  import { createSyncStore } from "./storage/store";
13
13
  import { CURSORS_TABLE, KNOWN_STREAMS_TABLE } from "./storage/types";
14
14
  import { wipeLocalData } from "./storage/wipe";
15
- import { bootstrapStream } from "./sync/bootstrap";
15
+ import { bootstrapStream, bootstrapStreamsBatch } from "./sync/bootstrap";
16
16
  import { getAllCursors } from "./sync/cursor";
17
17
  import { applyDelta } from "./sync/deltaApplier";
18
18
  import { repairMarkedEntities } from "./sync/entityRepair";
@@ -32,6 +32,14 @@ export const DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS = 30_000;
32
32
  export const DEFAULT_SNAPSHOT_CONCURRENCY = 6;
33
33
  /** E5: default local tombstone retention window (90 days), matching the server's (C7). */
34
34
  export const DEFAULT_TOMBSTONE_RETENTION_MS = 90 * 24 * 60 * 60 * 1_000;
35
+ /**
36
+ * Entities one `POST /sync/snapshot/batch` response may carry, shared across its streams.
37
+ * Bounds both the response size (about 2 MB of typical rows) and the work per round.
38
+ */
39
+ const SNAPSHOT_BATCH_LIMIT = 5_000;
40
+ /** The server has no `POST /sync/snapshot/batch` route (404). */
41
+ class SnapshotBatchUnsupportedError extends Error {
42
+ }
35
43
  /**
36
44
  * `start()` auth-resolution retry defaults. A host app only calls `start()` once it
37
45
  * already believes the user is authenticated (e.g. right after its own login flow
@@ -156,6 +164,9 @@ export const createSyncDb = (config) => {
156
164
  // refresh has already run for the CURRENT pause episode.
157
165
  let authPaused = false;
158
166
  let refreshAttempted = false;
167
+ // Set once the server answers 404 to the snapshot batch route; per-stream paging is
168
+ // used for the rest of this client's life.
169
+ let snapshotBatchUnsupported = false;
159
170
  // E1: lifecycle serialization. `generation` bumps on every start()/stop() so
160
171
  // an in-flight operation that resumes after an await can detect it has been
161
172
  // superseded (a rapid stop()-then-start() for a new user) and abort instead
@@ -204,7 +215,9 @@ export const createSyncDb = (config) => {
204
215
  }
205
216
  };
206
217
  /**
207
- * Run a snapshot pass for one stream, single-flight per stream.
218
+ * Run a snapshot pass over `streams`, single-flight per stream. Each stream is either
219
+ * joined to its latest pass or owned by this one; `run` receives the owned streams once
220
+ * every pass it queued behind has settled.
208
221
  *
209
222
  * - `join`: share the stream's latest pass when one exists (reconcile — any pass to
210
223
  * the head will do).
@@ -214,36 +227,52 @@ export const createSyncDb = (config) => {
214
227
  * - `exclusive`: always queue behind whatever is in flight (force resync and wipe
215
228
  * re-bootstrap purge first, which must never overlap another pass's paging).
216
229
  */
217
- const runStreamPass = ({ mode, run, stream, }) => {
218
- const existing = streamPasses.get(stream);
219
- const current = existing && existing.generation === generation && existing.userId === currentUserId
220
- ? existing
221
- : undefined;
222
- if (current && (mode === "join" || (mode === "fresh" && !current.started))) {
223
- return current.promise;
230
+ const runStreamsPass = ({ mode, run, streams, }) => {
231
+ const joined = [];
232
+ const owned = [];
233
+ const previous = [];
234
+ for (const stream of streams) {
235
+ const existing = streamPasses.get(stream);
236
+ const current = existing && existing.generation === generation && existing.userId === currentUserId
237
+ ? existing
238
+ : undefined;
239
+ if (current && (mode === "join" || (mode === "fresh" && !current.started))) {
240
+ joined.push(current.promise);
241
+ continue;
242
+ }
243
+ owned.push(stream);
244
+ if (existing) {
245
+ previous.push(existing.promise);
246
+ }
247
+ }
248
+ if (owned.length === 0) {
249
+ return Promise.all(joined).then(() => undefined);
224
250
  }
225
251
  const entry = { generation, promise: Promise.resolve(), started: false, userId: currentUserId };
226
- const previous = existing?.promise;
227
252
  entry.promise = (async () => {
228
- if (previous) {
229
- // The previous pass's failure belongs to its own caller.
230
- await previous.catch(() => undefined);
231
- }
253
+ // The previous passes' failures belong to their own callers.
254
+ await Promise.all(previous.map((pass) => pass.catch(() => undefined)));
232
255
  // A stop()/start() or user switch while this pass was queued supersedes it:
233
256
  // paging now would write into a store that belongs to a newer lifecycle.
234
257
  if (entry.generation !== generation || entry.userId !== currentUserId) {
235
258
  return;
236
259
  }
237
260
  entry.started = true;
238
- await run();
261
+ await run(owned);
239
262
  })().finally(() => {
240
- if (streamPasses.get(stream) === entry) {
241
- streamPasses.delete(stream);
263
+ for (const stream of owned) {
264
+ if (streamPasses.get(stream) === entry) {
265
+ streamPasses.delete(stream);
266
+ }
242
267
  }
243
268
  });
244
- streamPasses.set(stream, entry);
245
- return entry.promise;
269
+ for (const stream of owned) {
270
+ streamPasses.set(stream, entry);
271
+ }
272
+ return Promise.all([entry.promise, ...joined]).then(() => undefined);
246
273
  };
274
+ /** {@link runStreamsPass} for a single stream. */
275
+ const runStreamPass = ({ mode, run, stream, }) => runStreamsPass({ mode, run: () => run(), streams: [stream] });
247
276
  /**
248
277
  * Run `worker` over `items` with at most `snapshotConcurrency` in flight. Stops taking
249
278
  * new items once `isSuperseded()` or any worker throws, waits for the in-flight ones,
@@ -498,6 +527,72 @@ export const createSyncDb = (config) => {
498
527
  }
499
528
  return serverSet;
500
529
  };
530
+ /**
531
+ * Page `streams` to their heads as one single-flight pass (see {@link runStreamsPass}).
532
+ * One `POST /sync/snapshot/batch` round trip per round covers every owned stream;
533
+ * against a server without that route, falls back to `GET /sync/snapshot` per stream,
534
+ * `snapshotConcurrency` streams at a time.
535
+ */
536
+ const pageStreams = async ({ streams, mode, isSuperseded, purgeFirst = false, }) => {
537
+ const channel = httpChannel;
538
+ if (!channel || streams.length === 0) {
539
+ return;
540
+ }
541
+ const collections = new Map(streams.map(({ stream, collection }) => [stream, collection]));
542
+ await runStreamsPass({
543
+ mode,
544
+ run: async (owned) => {
545
+ const ownedStreams = owned.map((stream) => ({
546
+ collection: collections.get(stream),
547
+ stream,
548
+ }));
549
+ if (purgeFirst) {
550
+ for (const { stream, collection } of ownedStreams) {
551
+ store.purgeStream({ stream });
552
+ store.addKnownStream({ collection, stream });
553
+ }
554
+ }
555
+ const fetchSnapshotBatch = channel.fetchSnapshotBatch;
556
+ if (fetchSnapshotBatch && !snapshotBatchUnsupported) {
557
+ try {
558
+ await bootstrapStreamsBatch({
559
+ fetchSnapshotBatch: async (request) => {
560
+ const result = await fetchSnapshotBatch(request);
561
+ if (result.type === "unsupported") {
562
+ throw new SnapshotBatchUnsupportedError();
563
+ }
564
+ return result;
565
+ },
566
+ isSuperseded,
567
+ limit: SNAPSHOT_BATCH_LIMIT,
568
+ store,
569
+ streams: ownedStreams,
570
+ });
571
+ return;
572
+ }
573
+ catch (error) {
574
+ if (!(error instanceof SnapshotBatchUnsupportedError)) {
575
+ throw error;
576
+ }
577
+ // Every round before the 404 is already written, so per-stream paging
578
+ // resumes from the store's cursors without losing anything.
579
+ snapshotBatchUnsupported = true;
580
+ debugLog?.record({
581
+ direction: "system",
582
+ label: "snapshot batch unsupported by server; paging per stream",
583
+ type: "reconcile",
584
+ });
585
+ }
586
+ }
587
+ await runSnapshotPool({
588
+ isSuperseded,
589
+ items: ownedStreams,
590
+ worker: ({ stream, collection }) => bootstrapStream({ channel, collection, isSuperseded, store, stream }),
591
+ });
592
+ },
593
+ streams: streams.map(({ stream }) => stream),
594
+ });
595
+ };
501
596
  const runReconcile = async () => {
502
597
  // Simulated offline severs all network activity, including the HTTP
503
598
  // channel; an auth pause stands down every network trigger (INV-2) until
@@ -530,15 +625,12 @@ export const createSyncDb = (config) => {
530
625
  // lifecycle/user while awaiting discovery.
531
626
  return;
532
627
  }
533
- const channel = httpChannel;
534
- await runSnapshotPool({
628
+ await pageStreams({
535
629
  isSuperseded,
536
- items: [...streams].filter(([stream, collection]) => !skipSnapshotPaging({ collection, stream })),
537
- worker: ([stream, collection]) => runStreamPass({
538
- mode: "join",
539
- run: () => bootstrapStream({ channel, collection, store, stream }),
540
- stream,
541
- }),
630
+ mode: "join",
631
+ streams: [...streams]
632
+ .filter(([stream, collection]) => !skipSnapshotPaging({ collection, stream }))
633
+ .map(([stream, collection]) => ({ collection, stream })),
542
634
  });
543
635
  if (isSuperseded()) {
544
636
  return;
@@ -607,6 +699,63 @@ export const createSyncDb = (config) => {
607
699
  inFlightReconcile = { generation, promise, userId: currentUserId };
608
700
  return promise;
609
701
  };
702
+ // Streams confirmed by `sync:subscribed` and not yet paged, tagged with the lifecycle
703
+ // and user that confirmed them. One confirmation arrives per collection; those landing
704
+ // while a catch-up request is in flight are paged together in the next one.
705
+ const pendingCatchUp = new Map();
706
+ let isCatchUpDraining = false;
707
+ const drainCatchUp = async () => {
708
+ if (isCatchUpDraining) {
709
+ return;
710
+ }
711
+ isCatchUpDraining = true;
712
+ addSyncing(1);
713
+ try {
714
+ while (pendingCatchUp.size > 0) {
715
+ const myGeneration = generation;
716
+ const myUserId = currentUserId;
717
+ const isSuperseded = () => generation !== myGeneration || currentUserId !== myUserId;
718
+ const streams = [];
719
+ for (const [stream, pending] of pendingCatchUp) {
720
+ if (pending.generation === myGeneration && pending.userId === myUserId) {
721
+ streams.push({ collection: pending.collection, stream });
722
+ }
723
+ }
724
+ pendingCatchUp.clear();
725
+ if (!httpChannel || simulatedOffline || authPaused) {
726
+ return;
727
+ }
728
+ if (streams.length === 0) {
729
+ continue;
730
+ }
731
+ await pageStreams({ isSuperseded, mode: "fresh", streams });
732
+ if (isSuperseded()) {
733
+ return;
734
+ }
735
+ debugLog?.record({
736
+ detail: { streams: streams.map(({ stream }) => stream) },
737
+ direction: "system",
738
+ label: `subscribe catch-up: ${streams.map(({ stream }) => stream).join(", ")}`,
739
+ type: "reconcile",
740
+ });
741
+ }
742
+ }
743
+ catch (error) {
744
+ if (error instanceof AuthRequiredError) {
745
+ setAuthPaused(true);
746
+ return;
747
+ }
748
+ throw error;
749
+ }
750
+ finally {
751
+ isCatchUpDraining = false;
752
+ addSyncing(-1);
753
+ // Confirmations for a newer lifecycle can queue while a superseded pass unwinds.
754
+ if (pendingCatchUp.size > 0) {
755
+ void drainCatchUp().catch(warn("subscribe catch-up failed"));
756
+ }
757
+ }
758
+ };
610
759
  /**
611
760
  * Catch up the streams the server just confirmed this socket joined
612
761
  * (`sync:subscribed`).
@@ -623,8 +772,9 @@ export const createSyncDb = (config) => {
623
772
  * confirmed stream from its cursor here makes snapshot + delta coverage contiguous.
624
773
  * It deliberately does NOT go through `reconcile()`: that call coalesces onto a pass
625
774
  * which may have started before the join (exactly the pass that could have missed the
626
- * write). Per-stream paging is idempotent and cursor-guarded, so running it beside an
627
- * in-flight reconcile is safe.
775
+ * write). For the same reason a confirmation arriving while a catch-up request is in
776
+ * flight waits for the next request rather than joining that one. Paging is
777
+ * idempotent and cursor-guarded, so running it beside an in-flight reconcile is safe.
628
778
  */
629
779
  const catchUpSubscribedStreams = async ({ collection, mode, streams, }) => {
630
780
  if (!httpChannel || simulatedOffline || authPaused || streams.length === 0) {
@@ -636,44 +786,22 @@ export const createSyncDb = (config) => {
636
786
  if (mode === "window" || isWindowCollection(collection)) {
637
787
  return;
638
788
  }
639
- const myGeneration = generation;
640
- const myUserId = currentUserId;
641
- const isSuperseded = () => generation !== myGeneration || currentUserId !== myUserId;
642
- addSyncing(1);
643
- try {
644
- const channel = httpChannel;
645
- await runSnapshotPool({
646
- isSuperseded,
647
- items: streams.filter((stream) => !skipSnapshotPaging({ collection, mode, stream })),
648
- worker: (stream) => {
649
- store.addKnownStream({ collection, stream });
650
- return runStreamPass({
651
- mode: "fresh",
652
- run: () => bootstrapStream({ channel, collection, store, stream }),
653
- stream,
654
- });
655
- },
656
- });
657
- if (isSuperseded()) {
658
- return;
789
+ for (const stream of streams) {
790
+ if (skipSnapshotPaging({ collection, mode, stream })) {
791
+ continue;
659
792
  }
660
- debugLog?.record({
661
- detail: { collection, streams },
662
- direction: "system",
663
- label: `subscribe catch-up: ${streams.join(", ")}`,
664
- type: "reconcile",
665
- });
666
- }
667
- catch (error) {
668
- if (error instanceof AuthRequiredError) {
669
- setAuthPaused(true);
670
- return;
793
+ store.addKnownStream({ collection, stream });
794
+ // A queued pass that has not started yet already starts after this join.
795
+ const queued = streamPasses.get(stream);
796
+ if (queued &&
797
+ !queued.started &&
798
+ queued.generation === generation &&
799
+ queued.userId === currentUserId) {
800
+ continue;
671
801
  }
672
- throw error;
673
- }
674
- finally {
675
- addSyncing(-1);
802
+ pendingCatchUp.set(stream, { collection, generation, userId: currentUserId });
676
803
  }
804
+ await drainCatchUp();
677
805
  };
678
806
  /**
679
807
  * C2 migration: deployed clients hold legacy `snapshot:{collection}` pseudo-cursors.
@@ -804,7 +932,7 @@ export const createSyncDb = (config) => {
804
932
  }
805
933
  purged += store.purgeStream({ stream });
806
934
  store.addKnownStream({ collection, stream });
807
- await bootstrapStream({ channel, collection, store, stream });
935
+ await bootstrapStream({ channel, collection, isSuperseded, store, stream });
808
936
  },
809
937
  stream,
810
938
  });
@@ -1054,22 +1182,14 @@ export const createSyncDb = (config) => {
1054
1182
  // INV-2 auth-pause case (returns undefined on a 401) — skip bootstrap then.
1055
1183
  const streams = await syncStreams();
1056
1184
  if (streams) {
1057
- const channel = httpChannel;
1058
- await runSnapshotPool({
1185
+ await pageStreams({
1059
1186
  isSuperseded: () => false,
1060
- items: [...streams],
1061
- worker: ([stream, collection]) => runStreamPass({
1062
- mode: "exclusive",
1063
- // A pass that was paging when the wipe ran can write its cursor and
1064
- // `bootstrapped` mark into the wiped store before this one starts. Purge
1065
- // again so this pass pages from 0 instead of resuming from that cursor.
1066
- run: async () => {
1067
- store.purgeStream({ stream });
1068
- store.addKnownStream({ collection, stream });
1069
- await bootstrapStream({ channel, collection, store, stream });
1070
- },
1071
- stream,
1072
- }),
1187
+ mode: "exclusive",
1188
+ // A pass that was paging when the wipe ran can write its cursor and
1189
+ // `bootstrapped` mark into the wiped store before this one starts. Purge
1190
+ // again so this pass pages from 0 instead of resuming from that cursor.
1191
+ purgeFirst: true,
1192
+ streams: [...streams].map(([stream, collection]) => ({ collection, stream })),
1073
1193
  });
1074
1194
  }
1075
1195
  }