@terreno/syncdb 57.12.0 → 57.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/client.d.ts +12 -0
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +161 -33
- package/dist/client.js.map +1 -1
- package/dist/client.test.js +169 -0
- package/dist/client.test.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/client.test.ts +203 -0
- package/src/client.ts +195 -33
- package/src/index.ts +1 -0
package/dist/client.test.js
CHANGED
|
@@ -2878,4 +2878,173 @@ describe("createSyncDb", () => {
|
|
|
2878
2878
|
});
|
|
2879
2879
|
});
|
|
2880
2880
|
});
|
|
2881
|
+
describe("snapshot paging concurrency", () => {
|
|
2882
|
+
/**
|
|
2883
|
+
* Wrap the harness channel so every snapshot page waits on a gate the test releases,
|
|
2884
|
+
* and record how many pages are in flight overall and per stream.
|
|
2885
|
+
*/
|
|
2886
|
+
const trackSnapshotPages = (harness) => {
|
|
2887
|
+
const fetched = [];
|
|
2888
|
+
const inFlight = new Map();
|
|
2889
|
+
const gates = [];
|
|
2890
|
+
let total = 0;
|
|
2891
|
+
let maxTotal = 0;
|
|
2892
|
+
let maxPerStream = 0;
|
|
2893
|
+
const fetchSnapshotPage = harness.http.channel.fetchSnapshotPage;
|
|
2894
|
+
harness.http.channel.fetchSnapshotPage = async (args) => {
|
|
2895
|
+
fetched.push(args.stream);
|
|
2896
|
+
total += 1;
|
|
2897
|
+
maxTotal = Math.max(maxTotal, total);
|
|
2898
|
+
const streamCount = (inFlight.get(args.stream) ?? 0) + 1;
|
|
2899
|
+
inFlight.set(args.stream, streamCount);
|
|
2900
|
+
maxPerStream = Math.max(maxPerStream, streamCount);
|
|
2901
|
+
await new Promise((resolve) => gates.push(resolve));
|
|
2902
|
+
total -= 1;
|
|
2903
|
+
inFlight.set(args.stream, (inFlight.get(args.stream) ?? 1) - 1);
|
|
2904
|
+
return fetchSnapshotPage(args);
|
|
2905
|
+
};
|
|
2906
|
+
return {
|
|
2907
|
+
fetched,
|
|
2908
|
+
maxInFlight: () => maxTotal,
|
|
2909
|
+
maxInFlightPerStream: () => maxPerStream,
|
|
2910
|
+
pending: () => gates.length,
|
|
2911
|
+
releaseAll: () => {
|
|
2912
|
+
for (const release of gates.splice(0)) {
|
|
2913
|
+
release();
|
|
2914
|
+
}
|
|
2915
|
+
},
|
|
2916
|
+
};
|
|
2917
|
+
};
|
|
2918
|
+
/** Release gated pages until `done` settles. */
|
|
2919
|
+
const drain = async (pages, done) => {
|
|
2920
|
+
let settled = false;
|
|
2921
|
+
const result = done.finally(() => {
|
|
2922
|
+
settled = true;
|
|
2923
|
+
});
|
|
2924
|
+
for (let attempt = 0; attempt < 100 && !settled; attempt += 1) {
|
|
2925
|
+
pages.releaseAll();
|
|
2926
|
+
await flush();
|
|
2927
|
+
}
|
|
2928
|
+
return result;
|
|
2929
|
+
};
|
|
2930
|
+
const manyStreams = (count) => Array.from({ length: count }, (_, index) => ({
|
|
2931
|
+
collection: "todos",
|
|
2932
|
+
stream: `todos|tenant:w${index}`,
|
|
2933
|
+
}));
|
|
2934
|
+
const isBootstrapped = (client, stream) => client.store.raw.getCell(CURSORS_TABLE, stream, "bootstrapped") === true;
|
|
2935
|
+
it("pages streams side by side, up to snapshotConcurrency at once", async () => {
|
|
2936
|
+
const harness = makeHarness({ snapshotConcurrency: 3 });
|
|
2937
|
+
harness.http.state.streams = manyStreams(10);
|
|
2938
|
+
const pages = trackSnapshotPages(harness);
|
|
2939
|
+
const client = createSyncDb(harness.config);
|
|
2940
|
+
await client.start();
|
|
2941
|
+
await waitUntil(() => pages.pending() === 3);
|
|
2942
|
+
await flush();
|
|
2943
|
+
expect(pages.pending()).toBe(3);
|
|
2944
|
+
await drain(pages, client.reconcile());
|
|
2945
|
+
expect(pages.maxInFlight()).toBe(3);
|
|
2946
|
+
for (const { stream } of harness.http.state.streams) {
|
|
2947
|
+
expect(isBootstrapped(client, stream)).toBe(true);
|
|
2948
|
+
}
|
|
2949
|
+
await client.stop();
|
|
2950
|
+
});
|
|
2951
|
+
it("pages one stream at a time when snapshotConcurrency is 1", async () => {
|
|
2952
|
+
const harness = makeHarness({ snapshotConcurrency: 1 });
|
|
2953
|
+
harness.http.state.streams = manyStreams(4);
|
|
2954
|
+
const pages = trackSnapshotPages(harness);
|
|
2955
|
+
const client = createSyncDb(harness.config);
|
|
2956
|
+
await client.start();
|
|
2957
|
+
await drain(pages, client.reconcile());
|
|
2958
|
+
expect(pages.maxInFlight()).toBe(1);
|
|
2959
|
+
expect(pages.fetched).toHaveLength(4);
|
|
2960
|
+
await client.stop();
|
|
2961
|
+
});
|
|
2962
|
+
it("queues a subscribe catch-up behind an in-flight pass on the same stream", async () => {
|
|
2963
|
+
const harness = makeHarness();
|
|
2964
|
+
const pages = trackSnapshotPages(harness);
|
|
2965
|
+
const client = createSyncDb(harness.config);
|
|
2966
|
+
await client.start();
|
|
2967
|
+
await waitUntil(() => pages.pending() === 1);
|
|
2968
|
+
harness.transport.confirmSubscribed({ collection: "todos", streams: [DEFAULT_STREAM] });
|
|
2969
|
+
harness.transport.confirmSubscribed({ collection: "todos", streams: [DEFAULT_STREAM] });
|
|
2970
|
+
await flush();
|
|
2971
|
+
// The catch-up must not page beside the reconcile pass that began before the join.
|
|
2972
|
+
expect(pages.pending()).toBe(1);
|
|
2973
|
+
await drain(pages, client.reconcile());
|
|
2974
|
+
await waitUntil(() => pages.fetched.length === 2);
|
|
2975
|
+
pages.releaseAll();
|
|
2976
|
+
await flush();
|
|
2977
|
+
// One startup pass, then one fresh pass shared by both confirmations.
|
|
2978
|
+
expect(pages.fetched).toEqual([DEFAULT_STREAM, DEFAULT_STREAM]);
|
|
2979
|
+
expect(pages.maxInFlightPerStream()).toBe(1);
|
|
2980
|
+
await client.stop();
|
|
2981
|
+
});
|
|
2982
|
+
it("drops a queued catch-up when the client stops before it starts", async () => {
|
|
2983
|
+
const harness = makeHarness();
|
|
2984
|
+
const pages = trackSnapshotPages(harness);
|
|
2985
|
+
const client = createSyncDb(harness.config);
|
|
2986
|
+
await client.start();
|
|
2987
|
+
await waitUntil(() => pages.pending() === 1);
|
|
2988
|
+
harness.transport.confirmSubscribed({ collection: "todos", streams: [DEFAULT_STREAM] });
|
|
2989
|
+
await flush();
|
|
2990
|
+
const stopped = client.stop();
|
|
2991
|
+
for (let attempt = 0; attempt < 10; attempt += 1) {
|
|
2992
|
+
pages.releaseAll();
|
|
2993
|
+
await flush();
|
|
2994
|
+
}
|
|
2995
|
+
await stopped;
|
|
2996
|
+
// Only the startup pass paged; the catch-up queued behind it belonged to the
|
|
2997
|
+
// stopped lifecycle.
|
|
2998
|
+
expect(pages.fetched).toEqual([DEFAULT_STREAM]);
|
|
2999
|
+
});
|
|
3000
|
+
it("forceResync waits for an in-flight pass before purging the stream", async () => {
|
|
3001
|
+
const harness = makeHarness();
|
|
3002
|
+
harness.http.state.pages[DEFAULT_STREAM] = {
|
|
3003
|
+
cursor: 3,
|
|
3004
|
+
entities: [{ data: { title: "kept" }, deleted: false, id: "t1", seq: 3 }],
|
|
3005
|
+
frontierSeq: 3,
|
|
3006
|
+
hasMore: false,
|
|
3007
|
+
oldestRetainedSeq: 0,
|
|
3008
|
+
stream: DEFAULT_STREAM,
|
|
3009
|
+
};
|
|
3010
|
+
const pages = trackSnapshotPages(harness);
|
|
3011
|
+
const client = createSyncDb(harness.config);
|
|
3012
|
+
await client.start();
|
|
3013
|
+
await waitUntil(() => pages.pending() === 1);
|
|
3014
|
+
const resync = client.forceResync();
|
|
3015
|
+
await flush();
|
|
3016
|
+
await flush();
|
|
3017
|
+
// The purge-and-repage must not start while the startup pass is still paging.
|
|
3018
|
+
expect(pages.pending()).toBe(1);
|
|
3019
|
+
const result = await drain(pages, resync);
|
|
3020
|
+
assert.equal(result.ok, true);
|
|
3021
|
+
expect(pages.maxInFlightPerStream()).toBe(1);
|
|
3022
|
+
expect(client.store.getEntity({ collection: "todos", id: "t1" })?.data).toEqual({
|
|
3023
|
+
title: "kept",
|
|
3024
|
+
});
|
|
3025
|
+
expect(isBootstrapped(client, DEFAULT_STREAM)).toBe(true);
|
|
3026
|
+
await client.stop();
|
|
3027
|
+
});
|
|
3028
|
+
it("stops taking new streams after one fails", async () => {
|
|
3029
|
+
const harness = makeHarness({ snapshotConcurrency: 1 });
|
|
3030
|
+
harness.http.state.streams = manyStreams(3);
|
|
3031
|
+
const failing = "todos|tenant:w1";
|
|
3032
|
+
const fetched = [];
|
|
3033
|
+
const fetchSnapshotPage = harness.http.channel.fetchSnapshotPage;
|
|
3034
|
+
harness.http.channel.fetchSnapshotPage = async (args) => {
|
|
3035
|
+
fetched.push(args.stream);
|
|
3036
|
+
if (args.stream === failing) {
|
|
3037
|
+
throw new Error("snapshot failed");
|
|
3038
|
+
}
|
|
3039
|
+
return fetchSnapshotPage(args);
|
|
3040
|
+
};
|
|
3041
|
+
const client = createSyncDb(harness.config);
|
|
3042
|
+
// start() fires a reconcile of its own; this call shares that pass.
|
|
3043
|
+
await client.start();
|
|
3044
|
+
await expect(client.reconcile()).rejects.toThrow("snapshot failed");
|
|
3045
|
+
expect(fetched).toEqual(["todos|tenant:w0", failing]);
|
|
3046
|
+
expect(isBootstrapped(client, "todos|tenant:w2")).toBe(false);
|
|
3047
|
+
await client.stop();
|
|
3048
|
+
});
|
|
3049
|
+
});
|
|
2881
3050
|
//# sourceMappingURL=client.test.js.map
|