@terreno/syncdb 57.13.0 → 57.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/client.d.ts +12 -0
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +161 -33
- package/dist/client.js.map +1 -1
- package/dist/client.test.js +169 -0
- package/dist/client.test.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/client.test.ts +203 -0
- package/src/client.ts +195 -33
- package/src/index.ts +1 -0
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EACL,YAAY,EACZ,6BAA6B,EAC7B,0CAA0C,EAC1C,iCAAiC,EACjC,iCAAiC,EACjC,8BAA8B,EAC9B,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,EAC1B,KAAK,UAAU,EACf,KAAK,MAAM,EACX,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAIlB,YAAY,EACV,YAAY,EACZ,0BAA0B,EAC1B,cAAc,EACd,YAAY,EACZ,OAAO,EACP,oBAAoB,EACpB,YAAY,EACZ,SAAS,EACT,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,EACrB,iBAAiB,EACjB,qBAAqB,EACrB,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,oBAAoB,EACpB,UAAU,EACV,cAAc,EACd,cAAc,GACf,MAAM,SAAS,CAAC;AAMjB,YAAY,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAClD,OAAO,EACL,wBAAwB,EACxB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,WAAW,GACjB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAC,aAAa,EAAE,QAAQ,EAAC,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAC,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,cAAc,EAAC,MAAM,eAAe,CAAC;AASlF,OAAO,EAAC,aAAa,EAAC,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAC,kBAAkB,EAAE,KAAK,MAAM,EAAC,MAAM,oBAAoB,CAAC;AACnE,YAAY,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAC,YAAY,EAAE,KAAK,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAI7C,OAAO,EACL,KAAK,wBAAwB,EAC7B,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,KAAK,yBAAyB,EAC9B,2BAA2B,GAC5B,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EACzB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAItB,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,qBAAqB,EACrB,2BAA2B,GAC5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAC,WAAW,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAI9D,OAAO,EAAC,6BAA6B,EAAC,MAAM,sCAAsC,CAAC;AACnF,OAAO,EAAC,iCAAiC,EAAC,MAAM,0CAA0C,CAAC;AAC3F,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAC,6BAA6B,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AAIxF,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,KAAK,SAAS,EACd,KAAK,qBAAqB,EAC1B,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,gBAAgB,EAChB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAC,qBAAqB,EAAE,KAAK,qBAAqB,EAAC,MAAM,wBAAwB,CAAC;AACzF,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,eAAe,GACrB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAC,YAAY,EAAC,MAAM,SAAS,CAAC;AAM1C,YAAY,EACV,kBAAkB,EAClB,cAAc,EACd,kBAAkB,EAClB,YAAY,EACZ,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,GACf,MAAM,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EACL,YAAY,EACZ,6BAA6B,EAC7B,0CAA0C,EAC1C,4BAA4B,EAC5B,iCAAiC,EACjC,iCAAiC,EACjC,8BAA8B,EAC9B,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,EAC1B,KAAK,UAAU,EACf,KAAK,MAAM,EACX,KAAK,YAAY,GAClB,MAAM,UAAU,CAAC;AAIlB,YAAY,EACV,YAAY,EACZ,0BAA0B,EAC1B,cAAc,EACd,YAAY,EACZ,OAAO,EACP,oBAAoB,EACpB,YAAY,EACZ,SAAS,EACT,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,EACvB,qBAAqB,EACrB,iBAAiB,EACjB,qBAAqB,EACrB,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,oBAAoB,EACpB,UAAU,EACV,cAAc,EACd,cAAc,GACf,MAAM,SAAS,CAAC;AAMjB,YAAY,EAAC,SAAS,EAAC,MAAM,oBAAoB,CAAC;AAClD,OAAO,EACL,wBAAwB,EACxB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,iBAAiB,EACtB,KAAK,WAAW,GACjB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAC,aAAa,EAAE,QAAQ,EAAC,MAAM,cAAc,CAAC;AAC1D,OAAO,EAAC,YAAY,EAAE,KAAK,WAAW,EAAE,KAAK,cAAc,EAAC,MAAM,eAAe,CAAC;AASlF,OAAO,EAAC,aAAa,EAAC,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAC,kBAAkB,EAAE,KAAK,MAAM,EAAC,MAAM,oBAAoB,CAAC;AACnE,YAAY,EAAC,SAAS,EAAC,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAC,YAAY,EAAE,KAAK,UAAU,EAAC,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAI7C,OAAO,EACL,KAAK,wBAAwB,EAC7B,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EACL,KAAK,yBAAyB,EAC9B,2BAA2B,GAC5B,MAAM,8BAA8B,CAAC;AACtC,YAAY,EACV,oBAAoB,EACpB,0BAA0B,EAC1B,yBAAyB,EACzB,yBAAyB,EACzB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAItB,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,qBAAqB,EACrB,2BAA2B,GAC5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAC,WAAW,EAAE,YAAY,EAAC,MAAM,gBAAgB,CAAC;AAI9D,OAAO,EAAC,6BAA6B,EAAC,MAAM,sCAAsC,CAAC;AACnF,OAAO,EAAC,iCAAiC,EAAC,MAAM,0CAA0C,CAAC;AAC3F,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAC,6BAA6B,EAAE,gBAAgB,EAAC,MAAM,oBAAoB,CAAC;AAIxF,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,KAAK,SAAS,EACd,KAAK,qBAAqB,EAC1B,KAAK,WAAW,EAChB,KAAK,iBAAiB,EACtB,gBAAgB,EAChB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAC,qBAAqB,EAAE,KAAK,qBAAqB,EAAC,MAAM,wBAAwB,CAAC;AACzF,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EACvB,KAAK,aAAa,EAClB,KAAK,eAAe,GACrB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EAAC,YAAY,EAAC,MAAM,SAAS,CAAC;AAM1C,YAAY,EACV,kBAAkB,EAClB,cAAc,EACd,kBAAkB,EAClB,YAAY,EACZ,mBAAmB,EACnB,oBAAoB,EACpB,iBAAiB,EACjB,cAAc,GACf,MAAM,kBAAkB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -23,7 +23,7 @@
|
|
|
23
23
|
* `createFakeTransport`, kept out of production bundles.
|
|
24
24
|
*/
|
|
25
25
|
// --- Client ---------------------------------------------------------------
|
|
26
|
-
export { createSyncDb, DEFAULT_RECONCILE_INTERVAL_MS, DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS, DEFAULT_START_AUTH_RETRY_ATTEMPTS, DEFAULT_START_AUTH_RETRY_DELAY_MS, DEFAULT_TOMBSTONE_RETENTION_MS, } from "./client";
|
|
26
|
+
export { createSyncDb, DEFAULT_RECONCILE_INTERVAL_MS, DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS, DEFAULT_SNAPSHOT_CONCURRENCY, DEFAULT_START_AUTH_RETRY_ATTEMPTS, DEFAULT_START_AUTH_RETRY_DELAY_MS, DEFAULT_TOMBSTONE_RETENTION_MS, } from "./client";
|
|
27
27
|
export { DEFAULT_WINDOW_PAGE_SIZE, } from "./query/queryWindows";
|
|
28
28
|
export { compileWhere } from "./query/where";
|
|
29
29
|
// --- Local store and outbox (read surface) -------------------------------
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,6EAA6E;AAE7E,OAAO,EACL,YAAY,EACZ,6BAA6B,EAC7B,0CAA0C,EAC1C,iCAAiC,EACjC,iCAAiC,EACjC,8BAA8B,GAM/B,MAAM,UAAU,CAAC;AAiClB,OAAO,EACL,wBAAwB,GAKzB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,YAAY,EAAwC,MAAM,eAAe,CAAC;AAElF,4EAA4E;AAC5E,sEAAsE;AACtE,gFAAgF;AAChF,8EAA8E;AAC9E,kFAAkF;AAClF,gDAAgD;AAEhD,OAAO,EAAC,aAAa,EAAC,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAC,kBAAkB,EAAc,MAAM,oBAAoB,CAAC;AAEnE,OAAO,EAAC,YAAY,EAAkB,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAE7C,6EAA6E;AAE7E,OAAO,EAEL,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,2BAA2B,GAC5B,MAAM,8BAA8B,CAAC;AAUtC,4EAA4E;AAE5E,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,qBAAqB,EACrB,2BAA2B,GAC5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,uBAAuB,CAAC;AAG/B,6EAA6E;AAE7E,OAAO,EAAC,6BAA6B,EAAC,MAAM,sCAAsC,CAAC;AACnF,OAAO,EAAC,iCAAiC,EAAC,MAAM,0CAA0C,CAAC;AAC3F,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,8BAA8B,CAAC;AAGtC,4EAA4E;AAE5E,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EAKjB,gBAAgB,EAChB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAC,qBAAqB,EAA6B,MAAM,wBAAwB,CAAC;AACzF,OAAO,EACL,kBAAkB,EAClB,2BAA2B,GAK5B,MAAM,kBAAkB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,6EAA6E;AAE7E,OAAO,EACL,YAAY,EACZ,6BAA6B,EAC7B,0CAA0C,EAC1C,4BAA4B,EAC5B,iCAAiC,EACjC,iCAAiC,EACjC,8BAA8B,GAM/B,MAAM,UAAU,CAAC;AAiClB,OAAO,EACL,wBAAwB,GAKzB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAC,YAAY,EAAwC,MAAM,eAAe,CAAC;AAElF,4EAA4E;AAC5E,sEAAsE;AACtE,gFAAgF;AAChF,8EAA8E;AAC9E,kFAAkF;AAClF,gDAAgD;AAEhD,OAAO,EAAC,aAAa,EAAC,MAAM,uBAAuB,CAAC;AACpD,OAAO,EAAC,kBAAkB,EAAc,MAAM,oBAAoB,CAAC;AAEnE,OAAO,EAAC,YAAY,EAAkB,MAAM,iBAAiB,CAAC;AAC9D,OAAO,EAAC,aAAa,EAAC,MAAM,gBAAgB,CAAC;AAE7C,6EAA6E;AAE7E,OAAO,EAEL,iBAAiB,EACjB,6BAA6B,GAC9B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAEL,2BAA2B,GAC5B,MAAM,8BAA8B,CAAC;AAUtC,4EAA4E;AAE5E,OAAO,EACL,wBAAwB,EACxB,iBAAiB,EACjB,qBAAqB,EACrB,2BAA2B,GAC5B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,aAAa,EAAC,MAAM,wBAAwB,CAAC;AACrD,OAAO,EACL,sBAAsB,EACtB,sBAAsB,EACtB,uBAAuB,EACvB,yBAAyB,GAC1B,MAAM,uBAAuB,CAAC;AAG/B,6EAA6E;AAE7E,OAAO,EAAC,6BAA6B,EAAC,MAAM,sCAAsC,CAAC;AACnF,OAAO,EAAC,iCAAiC,EAAC,MAAM,0CAA0C,CAAC;AAC3F,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,8BAA8B,CAAC;AAGtC,4EAA4E;AAE5E,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EAKjB,gBAAgB,EAChB,yBAAyB,GAC1B,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAC,qBAAqB,EAA6B,MAAM,wBAAwB,CAAC;AACzF,OAAO,EACL,kBAAkB,EAClB,2BAA2B,GAK5B,MAAM,kBAAkB,CAAC"}
|
package/package.json
CHANGED
package/src/client.test.ts
CHANGED
|
@@ -3305,3 +3305,206 @@ describe("createSyncDb", () => {
|
|
|
3305
3305
|
});
|
|
3306
3306
|
});
|
|
3307
3307
|
});
|
|
3308
|
+
|
|
3309
|
+
describe("snapshot paging concurrency", () => {
|
|
3310
|
+
/**
|
|
3311
|
+
* Wrap the harness channel so every snapshot page waits on a gate the test releases,
|
|
3312
|
+
* and record how many pages are in flight overall and per stream.
|
|
3313
|
+
*/
|
|
3314
|
+
const trackSnapshotPages = (
|
|
3315
|
+
harness: Harness
|
|
3316
|
+
): {
|
|
3317
|
+
fetched: string[];
|
|
3318
|
+
maxInFlight: () => number;
|
|
3319
|
+
maxInFlightPerStream: () => number;
|
|
3320
|
+
pending: () => number;
|
|
3321
|
+
releaseAll: () => void;
|
|
3322
|
+
} => {
|
|
3323
|
+
const fetched: string[] = [];
|
|
3324
|
+
const inFlight = new Map<string, number>();
|
|
3325
|
+
const gates: (() => void)[] = [];
|
|
3326
|
+
let total = 0;
|
|
3327
|
+
let maxTotal = 0;
|
|
3328
|
+
let maxPerStream = 0;
|
|
3329
|
+
const fetchSnapshotPage = harness.http.channel.fetchSnapshotPage;
|
|
3330
|
+
harness.http.channel.fetchSnapshotPage = async (args) => {
|
|
3331
|
+
fetched.push(args.stream);
|
|
3332
|
+
total += 1;
|
|
3333
|
+
maxTotal = Math.max(maxTotal, total);
|
|
3334
|
+
const streamCount = (inFlight.get(args.stream) ?? 0) + 1;
|
|
3335
|
+
inFlight.set(args.stream, streamCount);
|
|
3336
|
+
maxPerStream = Math.max(maxPerStream, streamCount);
|
|
3337
|
+
await new Promise<void>((resolve) => gates.push(resolve));
|
|
3338
|
+
total -= 1;
|
|
3339
|
+
inFlight.set(args.stream, (inFlight.get(args.stream) ?? 1) - 1);
|
|
3340
|
+
return fetchSnapshotPage(args);
|
|
3341
|
+
};
|
|
3342
|
+
return {
|
|
3343
|
+
fetched,
|
|
3344
|
+
maxInFlight: () => maxTotal,
|
|
3345
|
+
maxInFlightPerStream: () => maxPerStream,
|
|
3346
|
+
pending: () => gates.length,
|
|
3347
|
+
releaseAll: () => {
|
|
3348
|
+
for (const release of gates.splice(0)) {
|
|
3349
|
+
release();
|
|
3350
|
+
}
|
|
3351
|
+
},
|
|
3352
|
+
};
|
|
3353
|
+
};
|
|
3354
|
+
|
|
3355
|
+
/** Release gated pages until `done` settles. */
|
|
3356
|
+
const drain = async <T>(
|
|
3357
|
+
pages: ReturnType<typeof trackSnapshotPages>,
|
|
3358
|
+
done: Promise<T>
|
|
3359
|
+
): Promise<T> => {
|
|
3360
|
+
let settled = false;
|
|
3361
|
+
const result = done.finally(() => {
|
|
3362
|
+
settled = true;
|
|
3363
|
+
});
|
|
3364
|
+
for (let attempt = 0; attempt < 100 && !settled; attempt += 1) {
|
|
3365
|
+
pages.releaseAll();
|
|
3366
|
+
await flush();
|
|
3367
|
+
}
|
|
3368
|
+
return result;
|
|
3369
|
+
};
|
|
3370
|
+
|
|
3371
|
+
const manyStreams = (count: number): SyncStreamInfo[] =>
|
|
3372
|
+
Array.from({length: count}, (_, index) => ({
|
|
3373
|
+
collection: "todos",
|
|
3374
|
+
stream: `todos|tenant:w${index}`,
|
|
3375
|
+
}));
|
|
3376
|
+
|
|
3377
|
+
const isBootstrapped = (client: ReturnType<typeof createSyncDb>, stream: string): boolean =>
|
|
3378
|
+
client.store.raw.getCell(CURSORS_TABLE, stream, "bootstrapped") === true;
|
|
3379
|
+
|
|
3380
|
+
it("pages streams side by side, up to snapshotConcurrency at once", async () => {
|
|
3381
|
+
const harness = makeHarness({snapshotConcurrency: 3});
|
|
3382
|
+
harness.http.state.streams = manyStreams(10);
|
|
3383
|
+
const pages = trackSnapshotPages(harness);
|
|
3384
|
+
const client = createSyncDb(harness.config);
|
|
3385
|
+
await client.start();
|
|
3386
|
+
|
|
3387
|
+
await waitUntil(() => pages.pending() === 3);
|
|
3388
|
+
await flush();
|
|
3389
|
+
expect(pages.pending()).toBe(3);
|
|
3390
|
+
await drain(pages, client.reconcile());
|
|
3391
|
+
|
|
3392
|
+
expect(pages.maxInFlight()).toBe(3);
|
|
3393
|
+
for (const {stream} of harness.http.state.streams) {
|
|
3394
|
+
expect(isBootstrapped(client, stream)).toBe(true);
|
|
3395
|
+
}
|
|
3396
|
+
await client.stop();
|
|
3397
|
+
});
|
|
3398
|
+
|
|
3399
|
+
it("pages one stream at a time when snapshotConcurrency is 1", async () => {
|
|
3400
|
+
const harness = makeHarness({snapshotConcurrency: 1});
|
|
3401
|
+
harness.http.state.streams = manyStreams(4);
|
|
3402
|
+
const pages = trackSnapshotPages(harness);
|
|
3403
|
+
const client = createSyncDb(harness.config);
|
|
3404
|
+
await client.start();
|
|
3405
|
+
await drain(pages, client.reconcile());
|
|
3406
|
+
|
|
3407
|
+
expect(pages.maxInFlight()).toBe(1);
|
|
3408
|
+
expect(pages.fetched).toHaveLength(4);
|
|
3409
|
+
await client.stop();
|
|
3410
|
+
});
|
|
3411
|
+
|
|
3412
|
+
it("queues a subscribe catch-up behind an in-flight pass on the same stream", async () => {
|
|
3413
|
+
const harness = makeHarness();
|
|
3414
|
+
const pages = trackSnapshotPages(harness);
|
|
3415
|
+
const client = createSyncDb(harness.config);
|
|
3416
|
+
await client.start();
|
|
3417
|
+
await waitUntil(() => pages.pending() === 1);
|
|
3418
|
+
|
|
3419
|
+
harness.transport.confirmSubscribed({collection: "todos", streams: [DEFAULT_STREAM]});
|
|
3420
|
+
harness.transport.confirmSubscribed({collection: "todos", streams: [DEFAULT_STREAM]});
|
|
3421
|
+
await flush();
|
|
3422
|
+
// The catch-up must not page beside the reconcile pass that began before the join.
|
|
3423
|
+
expect(pages.pending()).toBe(1);
|
|
3424
|
+
await drain(pages, client.reconcile());
|
|
3425
|
+
await waitUntil(() => pages.fetched.length === 2);
|
|
3426
|
+
pages.releaseAll();
|
|
3427
|
+
await flush();
|
|
3428
|
+
|
|
3429
|
+
// One startup pass, then one fresh pass shared by both confirmations.
|
|
3430
|
+
expect(pages.fetched).toEqual([DEFAULT_STREAM, DEFAULT_STREAM]);
|
|
3431
|
+
expect(pages.maxInFlightPerStream()).toBe(1);
|
|
3432
|
+
await client.stop();
|
|
3433
|
+
});
|
|
3434
|
+
|
|
3435
|
+
it("drops a queued catch-up when the client stops before it starts", async () => {
|
|
3436
|
+
const harness = makeHarness();
|
|
3437
|
+
const pages = trackSnapshotPages(harness);
|
|
3438
|
+
const client = createSyncDb(harness.config);
|
|
3439
|
+
await client.start();
|
|
3440
|
+
await waitUntil(() => pages.pending() === 1);
|
|
3441
|
+
harness.transport.confirmSubscribed({collection: "todos", streams: [DEFAULT_STREAM]});
|
|
3442
|
+
await flush();
|
|
3443
|
+
|
|
3444
|
+
const stopped = client.stop();
|
|
3445
|
+
for (let attempt = 0; attempt < 10; attempt += 1) {
|
|
3446
|
+
pages.releaseAll();
|
|
3447
|
+
await flush();
|
|
3448
|
+
}
|
|
3449
|
+
await stopped;
|
|
3450
|
+
|
|
3451
|
+
// Only the startup pass paged; the catch-up queued behind it belonged to the
|
|
3452
|
+
// stopped lifecycle.
|
|
3453
|
+
expect(pages.fetched).toEqual([DEFAULT_STREAM]);
|
|
3454
|
+
});
|
|
3455
|
+
|
|
3456
|
+
it("forceResync waits for an in-flight pass before purging the stream", async () => {
|
|
3457
|
+
const harness = makeHarness();
|
|
3458
|
+
harness.http.state.pages[DEFAULT_STREAM] = {
|
|
3459
|
+
cursor: 3,
|
|
3460
|
+
entities: [{data: {title: "kept"}, deleted: false, id: "t1", seq: 3}],
|
|
3461
|
+
frontierSeq: 3,
|
|
3462
|
+
hasMore: false,
|
|
3463
|
+
oldestRetainedSeq: 0,
|
|
3464
|
+
stream: DEFAULT_STREAM,
|
|
3465
|
+
};
|
|
3466
|
+
const pages = trackSnapshotPages(harness);
|
|
3467
|
+
const client = createSyncDb(harness.config);
|
|
3468
|
+
await client.start();
|
|
3469
|
+
await waitUntil(() => pages.pending() === 1);
|
|
3470
|
+
|
|
3471
|
+
const resync = client.forceResync();
|
|
3472
|
+
await flush();
|
|
3473
|
+
await flush();
|
|
3474
|
+
// The purge-and-repage must not start while the startup pass is still paging.
|
|
3475
|
+
expect(pages.pending()).toBe(1);
|
|
3476
|
+
const result = await drain(pages, resync);
|
|
3477
|
+
|
|
3478
|
+
assert.equal(result.ok, true);
|
|
3479
|
+
expect(pages.maxInFlightPerStream()).toBe(1);
|
|
3480
|
+
expect(client.store.getEntity({collection: "todos", id: "t1"})?.data).toEqual({
|
|
3481
|
+
title: "kept",
|
|
3482
|
+
});
|
|
3483
|
+
expect(isBootstrapped(client, DEFAULT_STREAM)).toBe(true);
|
|
3484
|
+
await client.stop();
|
|
3485
|
+
});
|
|
3486
|
+
|
|
3487
|
+
it("stops taking new streams after one fails", async () => {
|
|
3488
|
+
const harness = makeHarness({snapshotConcurrency: 1});
|
|
3489
|
+
harness.http.state.streams = manyStreams(3);
|
|
3490
|
+
const failing = "todos|tenant:w1";
|
|
3491
|
+
const fetched: string[] = [];
|
|
3492
|
+
const fetchSnapshotPage = harness.http.channel.fetchSnapshotPage;
|
|
3493
|
+
harness.http.channel.fetchSnapshotPage = async (args) => {
|
|
3494
|
+
fetched.push(args.stream);
|
|
3495
|
+
if (args.stream === failing) {
|
|
3496
|
+
throw new Error("snapshot failed");
|
|
3497
|
+
}
|
|
3498
|
+
return fetchSnapshotPage(args);
|
|
3499
|
+
};
|
|
3500
|
+
const client = createSyncDb(harness.config);
|
|
3501
|
+
|
|
3502
|
+
// start() fires a reconcile of its own; this call shares that pass.
|
|
3503
|
+
await client.start();
|
|
3504
|
+
await expect(client.reconcile()).rejects.toThrow("snapshot failed");
|
|
3505
|
+
|
|
3506
|
+
expect(fetched).toEqual(["todos|tenant:w0", failing]);
|
|
3507
|
+
expect(isBootstrapped(client, "todos|tenant:w2")).toBe(false);
|
|
3508
|
+
await client.stop();
|
|
3509
|
+
});
|
|
3510
|
+
});
|
package/src/client.ts
CHANGED
|
@@ -43,6 +43,13 @@ export const DEFAULT_RECONCILE_INTERVAL_MS = 5 * 60_000;
|
|
|
43
43
|
/** Minimum interval between seq-jump-triggered reconciles per stream (30s). */
|
|
44
44
|
export const DEFAULT_SEQ_JUMP_RECONCILE_MIN_INTERVAL_MS = 30_000;
|
|
45
45
|
|
|
46
|
+
/**
|
|
47
|
+
* Streams snapshot-paged at once by reconcile, subscribe catch-up, force resync and
|
|
48
|
+
* wipe re-bootstrap. Every stream costs at least one `GET /sync/snapshot` round trip,
|
|
49
|
+
* so paging them one at a time made cold start scale with collections × scopes.
|
|
50
|
+
*/
|
|
51
|
+
export const DEFAULT_SNAPSHOT_CONCURRENCY = 6;
|
|
52
|
+
|
|
46
53
|
/** E5: default local tombstone retention window (90 days), matching the server's (C7). */
|
|
47
54
|
export const DEFAULT_TOMBSTONE_RETENTION_MS = 90 * 24 * 60 * 60 * 1_000;
|
|
48
55
|
|
|
@@ -136,6 +143,12 @@ export interface SyncDbConfig {
|
|
|
136
143
|
wipeOnSignOut?: boolean;
|
|
137
144
|
/** Max mutations per batched drain send (B3, default 50; server caps at 100). */
|
|
138
145
|
batchSize?: number;
|
|
146
|
+
/**
|
|
147
|
+
* Max streams snapshot-paged at once (default {@link DEFAULT_SNAPSHOT_CONCURRENCY}).
|
|
148
|
+
* Each stream still pages its own pages in order; this bounds how many streams page
|
|
149
|
+
* side by side. `1` restores one-stream-at-a-time paging.
|
|
150
|
+
*/
|
|
151
|
+
snapshotConcurrency?: number;
|
|
139
152
|
/**
|
|
140
153
|
* B4: when true, a conflict halts the ENTIRE drain instead of just blocking
|
|
141
154
|
* its entity. Default false (per-entity blocking — see README "Conflict
|
|
@@ -427,6 +440,17 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
427
440
|
| {generation: number; promise: Promise<void>; userId: string | undefined}
|
|
428
441
|
| undefined;
|
|
429
442
|
const lastSeqJumpReconcileAt = new Map<string, number>();
|
|
443
|
+
/**
|
|
444
|
+
* The latest snapshot pass per stream (single-flight). Two passes paging the same
|
|
445
|
+
* stream at once double the requests, and a purge in one (force resync, retention
|
|
446
|
+
* gap, leave) lets the other write its high cursor back and mark the stream
|
|
447
|
+
* bootstrapped over rows it never paged. Tagged with generation + user like
|
|
448
|
+
* `inFlightReconcile`, so a pass from a superseded lifecycle is never joined.
|
|
449
|
+
*/
|
|
450
|
+
const streamPasses = new Map<
|
|
451
|
+
string,
|
|
452
|
+
{generation: number; promise: Promise<void>; started: boolean; userId: string | undefined}
|
|
453
|
+
>();
|
|
430
454
|
const statusListeners = new Set<() => void>();
|
|
431
455
|
// A4: auth-pause state. `authPaused` gates reconcile/replay/timer triggers
|
|
432
456
|
// (INV-2 — nothing else may unpause except a same-user auth change).
|
|
@@ -487,6 +511,105 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
487
511
|
}
|
|
488
512
|
};
|
|
489
513
|
|
|
514
|
+
/**
|
|
515
|
+
* Run a snapshot pass for one stream, single-flight per stream.
|
|
516
|
+
*
|
|
517
|
+
* - `join`: share the stream's latest pass when one exists (reconcile — any pass to
|
|
518
|
+
* the head will do).
|
|
519
|
+
* - `fresh`: the pass must start after this call (subscribe catch-up — a pass that
|
|
520
|
+
* began before the room join may have missed a write). Joins a queued pass that has
|
|
521
|
+
* not started yet, otherwise queues behind the running one.
|
|
522
|
+
* - `exclusive`: always queue behind whatever is in flight (force resync and wipe
|
|
523
|
+
* re-bootstrap purge first, which must never overlap another pass's paging).
|
|
524
|
+
*/
|
|
525
|
+
const runStreamPass = ({
|
|
526
|
+
mode,
|
|
527
|
+
run,
|
|
528
|
+
stream,
|
|
529
|
+
}: {
|
|
530
|
+
mode: "join" | "fresh" | "exclusive";
|
|
531
|
+
run: () => Promise<void>;
|
|
532
|
+
stream: string;
|
|
533
|
+
}): Promise<void> => {
|
|
534
|
+
const existing = streamPasses.get(stream);
|
|
535
|
+
const current =
|
|
536
|
+
existing && existing.generation === generation && existing.userId === currentUserId
|
|
537
|
+
? existing
|
|
538
|
+
: undefined;
|
|
539
|
+
if (current && (mode === "join" || (mode === "fresh" && !current.started))) {
|
|
540
|
+
return current.promise;
|
|
541
|
+
}
|
|
542
|
+
const entry = {generation, promise: Promise.resolve(), started: false, userId: currentUserId};
|
|
543
|
+
const previous = existing?.promise;
|
|
544
|
+
entry.promise = (async (): Promise<void> => {
|
|
545
|
+
if (previous) {
|
|
546
|
+
// The previous pass's failure belongs to its own caller.
|
|
547
|
+
await previous.catch(() => undefined);
|
|
548
|
+
}
|
|
549
|
+
// A stop()/start() or user switch while this pass was queued supersedes it:
|
|
550
|
+
// paging now would write into a store that belongs to a newer lifecycle.
|
|
551
|
+
if (entry.generation !== generation || entry.userId !== currentUserId) {
|
|
552
|
+
return;
|
|
553
|
+
}
|
|
554
|
+
entry.started = true;
|
|
555
|
+
await run();
|
|
556
|
+
})().finally(() => {
|
|
557
|
+
if (streamPasses.get(stream) === entry) {
|
|
558
|
+
streamPasses.delete(stream);
|
|
559
|
+
}
|
|
560
|
+
});
|
|
561
|
+
streamPasses.set(stream, entry);
|
|
562
|
+
return entry.promise;
|
|
563
|
+
};
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* Run `worker` over `items` with at most `snapshotConcurrency` in flight. Stops taking
|
|
567
|
+
* new items once `isSuperseded()` or any worker throws, waits for the in-flight ones,
|
|
568
|
+
* then rethrows — `AuthRequiredError` first, so callers still auth-pause on a 401 that
|
|
569
|
+
* raced another failure.
|
|
570
|
+
*/
|
|
571
|
+
const runSnapshotPool = async <T>({
|
|
572
|
+
isSuperseded,
|
|
573
|
+
items,
|
|
574
|
+
worker,
|
|
575
|
+
}: {
|
|
576
|
+
isSuperseded: () => boolean;
|
|
577
|
+
items: T[];
|
|
578
|
+
worker: (item: T) => Promise<void>;
|
|
579
|
+
}): Promise<void> => {
|
|
580
|
+
const limit = Math.max(
|
|
581
|
+
1,
|
|
582
|
+
Math.floor(config.snapshotConcurrency ?? DEFAULT_SNAPSHOT_CONCURRENCY)
|
|
583
|
+
);
|
|
584
|
+
let next = 0;
|
|
585
|
+
let authError: AuthRequiredError | undefined;
|
|
586
|
+
let firstError: unknown;
|
|
587
|
+
let failed = false;
|
|
588
|
+
const runWorker = async (): Promise<void> => {
|
|
589
|
+
while (next < items.length && !failed && !isSuperseded()) {
|
|
590
|
+
const item = items[next] as T;
|
|
591
|
+
next += 1;
|
|
592
|
+
try {
|
|
593
|
+
await worker(item);
|
|
594
|
+
} catch (error) {
|
|
595
|
+
failed = true;
|
|
596
|
+
if (error instanceof AuthRequiredError) {
|
|
597
|
+
authError ??= error;
|
|
598
|
+
} else if (firstError === undefined) {
|
|
599
|
+
firstError = error;
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
};
|
|
604
|
+
await Promise.all(Array.from({length: Math.min(limit, items.length)}, runWorker));
|
|
605
|
+
if (authError) {
|
|
606
|
+
throw authError;
|
|
607
|
+
}
|
|
608
|
+
if (failed) {
|
|
609
|
+
throw firstError;
|
|
610
|
+
}
|
|
611
|
+
};
|
|
612
|
+
|
|
490
613
|
const notifyStatusChange = (): void => {
|
|
491
614
|
for (const listener of statusListeners) {
|
|
492
615
|
listener();
|
|
@@ -759,14 +882,21 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
759
882
|
// lifecycle/user while awaiting discovery.
|
|
760
883
|
return;
|
|
761
884
|
}
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
885
|
+
const channel = httpChannel;
|
|
886
|
+
await runSnapshotPool({
|
|
887
|
+
isSuperseded,
|
|
888
|
+
items: [...streams].filter(
|
|
889
|
+
([stream, collection]) => !skipSnapshotPaging({collection, stream})
|
|
890
|
+
),
|
|
891
|
+
worker: ([stream, collection]) =>
|
|
892
|
+
runStreamPass({
|
|
893
|
+
mode: "join",
|
|
894
|
+
run: () => bootstrapStream({channel, collection, store, stream}),
|
|
895
|
+
stream,
|
|
896
|
+
}),
|
|
897
|
+
});
|
|
898
|
+
if (isSuperseded()) {
|
|
899
|
+
return;
|
|
770
900
|
}
|
|
771
901
|
// E5: client-side compaction runs only after a successful per-stream reconcile —
|
|
772
902
|
// reconcile just proved the local store caught up with the server via real network
|
|
@@ -874,15 +1004,21 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
874
1004
|
const isSuperseded = (): boolean => generation !== myGeneration || currentUserId !== myUserId;
|
|
875
1005
|
addSyncing(1);
|
|
876
1006
|
try {
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
}
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
1007
|
+
const channel = httpChannel;
|
|
1008
|
+
await runSnapshotPool({
|
|
1009
|
+
isSuperseded,
|
|
1010
|
+
items: streams.filter((stream) => !skipSnapshotPaging({collection, mode, stream})),
|
|
1011
|
+
worker: (stream) => {
|
|
1012
|
+
store.addKnownStream({collection, stream});
|
|
1013
|
+
return runStreamPass({
|
|
1014
|
+
mode: "fresh",
|
|
1015
|
+
run: () => bootstrapStream({channel, collection, store, stream}),
|
|
1016
|
+
stream,
|
|
1017
|
+
});
|
|
1018
|
+
},
|
|
1019
|
+
});
|
|
1020
|
+
if (isSuperseded()) {
|
|
1021
|
+
return;
|
|
886
1022
|
}
|
|
887
1023
|
debugLog?.record({
|
|
888
1024
|
detail: {collection, streams},
|
|
@@ -1017,19 +1153,30 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
1017
1153
|
for (const collection of new Set(streamInfos.map((info) => info.collection))) {
|
|
1018
1154
|
purged += store.purgeUnknownStreamEntities({collection});
|
|
1019
1155
|
}
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
}
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1156
|
+
const channel = httpChannel;
|
|
1157
|
+
await runSnapshotPool({
|
|
1158
|
+
isSuperseded,
|
|
1159
|
+
// Query collections have no snapshot to rebuild from; their retained windows
|
|
1160
|
+
// are refetched below instead.
|
|
1161
|
+
items: streamInfos.filter(({collection}) => !isQueryCollection(collection)),
|
|
1162
|
+
worker: async ({stream, collection}) => {
|
|
1163
|
+
await runStreamPass({
|
|
1164
|
+
mode: "exclusive",
|
|
1165
|
+
run: async () => {
|
|
1166
|
+
if (isSuperseded()) {
|
|
1167
|
+
return;
|
|
1168
|
+
}
|
|
1169
|
+
purged += store.purgeStream({stream});
|
|
1170
|
+
store.addKnownStream({collection, stream});
|
|
1171
|
+
await bootstrapStream({channel, collection, store, stream});
|
|
1172
|
+
},
|
|
1173
|
+
stream,
|
|
1174
|
+
});
|
|
1175
|
+
await waitForLifecycleIdle();
|
|
1176
|
+
},
|
|
1177
|
+
});
|
|
1178
|
+
if (isSuperseded()) {
|
|
1179
|
+
return {ok: false, purged, reason: "superseded", repaired, streams: streamInfos.length};
|
|
1033
1180
|
}
|
|
1034
1181
|
repaired = await repairAllMarkedEntities();
|
|
1035
1182
|
await queryWindows.refetchRetained();
|
|
@@ -1278,9 +1425,24 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
1278
1425
|
// INV-2 auth-pause case (returns undefined on a 401) — skip bootstrap then.
|
|
1279
1426
|
const streams = await syncStreams();
|
|
1280
1427
|
if (streams) {
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1428
|
+
const channel = httpChannel;
|
|
1429
|
+
await runSnapshotPool({
|
|
1430
|
+
isSuperseded: () => false,
|
|
1431
|
+
items: [...streams],
|
|
1432
|
+
worker: ([stream, collection]) =>
|
|
1433
|
+
runStreamPass({
|
|
1434
|
+
mode: "exclusive",
|
|
1435
|
+
// A pass that was paging when the wipe ran can write its cursor and
|
|
1436
|
+
// `bootstrapped` mark into the wiped store before this one starts. Purge
|
|
1437
|
+
// again so this pass pages from 0 instead of resuming from that cursor.
|
|
1438
|
+
run: async () => {
|
|
1439
|
+
store.purgeStream({stream});
|
|
1440
|
+
store.addKnownStream({collection, stream});
|
|
1441
|
+
await bootstrapStream({channel, collection, store, stream});
|
|
1442
|
+
},
|
|
1443
|
+
stream,
|
|
1444
|
+
}),
|
|
1445
|
+
});
|
|
1284
1446
|
}
|
|
1285
1447
|
}
|
|
1286
1448
|
};
|
package/src/index.ts
CHANGED