@terreno/syncdb 57.16.2 → 57.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/auth/betterAuthAdapter.d.ts +15 -4
- package/dist/auth/betterAuthAdapter.d.ts.map +1 -1
- package/dist/auth/betterAuthAdapter.js +130 -12
- package/dist/auth/betterAuthAdapter.js.map +1 -1
- package/dist/auth/betterAuthAdapter.test.js +321 -0
- package/dist/auth/betterAuthAdapter.test.js.map +1 -1
- package/dist/client.d.ts +6 -0
- package/dist/client.d.ts.map +1 -1
- package/dist/client.js +1 -0
- package/dist/client.js.map +1 -1
- package/dist/client.test.js +226 -0
- package/dist/client.test.js.map +1 -1
- package/dist/sync/replayCoordinator.test.js +262 -1
- package/dist/sync/replayCoordinator.test.js.map +1 -1
- package/package.json +1 -1
- package/src/auth/betterAuthAdapter.test.ts +384 -0
- package/src/auth/betterAuthAdapter.ts +155 -14
- package/src/client.test.ts +256 -0
- package/src/client.ts +7 -0
- package/src/sync/replayCoordinator.test.ts +313 -1
package/src/client.test.ts
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import "fake-indexeddb/auto";
|
|
2
2
|
|
|
3
3
|
import {describe, expect, it} from "bun:test";
|
|
4
|
+
import {createServer} from "node:http";
|
|
5
|
+
import type {AddressInfo} from "node:net";
|
|
4
6
|
import {assert} from "chai";
|
|
7
|
+
import {Server} from "socket.io";
|
|
5
8
|
|
|
9
|
+
import {betterAuthAdapter} from "./auth/betterAuthAdapter";
|
|
6
10
|
import {createSyncDb, type SyncDbConfig} from "./client";
|
|
7
11
|
import {createLocalKeyProvider, DEFAULT_KEY_CACHE_DB_NAME} from "./crypto/keyProviders";
|
|
8
12
|
import {getConflict} from "./mutations/conflicts";
|
|
@@ -254,6 +258,39 @@ describe("createSyncDb", () => {
|
|
|
254
258
|
});
|
|
255
259
|
});
|
|
256
260
|
|
|
261
|
+
it("opens the default socket with the configured socketTransports", async () => {
|
|
262
|
+
const httpServer = createServer();
|
|
263
|
+
const io = new Server(httpServer);
|
|
264
|
+
const firstTransports: string[] = [];
|
|
265
|
+
io.on("connection", (socket) => {
|
|
266
|
+
firstTransports.push(socket.conn.transport.name);
|
|
267
|
+
});
|
|
268
|
+
await new Promise<void>((resolve) => httpServer.listen(0, "127.0.0.1", resolve));
|
|
269
|
+
const {port} = httpServer.address() as AddressInfo;
|
|
270
|
+
|
|
271
|
+
const harness = makeHarness();
|
|
272
|
+
const client = createSyncDb({
|
|
273
|
+
...harness.config,
|
|
274
|
+
baseUrl: `http://127.0.0.1:${port}`,
|
|
275
|
+
socketTransports: ["websocket"],
|
|
276
|
+
transport: undefined,
|
|
277
|
+
});
|
|
278
|
+
try {
|
|
279
|
+
await client.start();
|
|
280
|
+
for (let attempt = 0; attempt < 50 && firstTransports.length === 0; attempt++) {
|
|
281
|
+
await new Promise((resolve) => setTimeout(resolve, 20));
|
|
282
|
+
}
|
|
283
|
+
expect(firstTransports).toEqual(["websocket"]);
|
|
284
|
+
} finally {
|
|
285
|
+
await client.stop();
|
|
286
|
+
io.disconnectSockets(true);
|
|
287
|
+
httpServer.closeAllConnections();
|
|
288
|
+
await new Promise<void>((resolve) => {
|
|
289
|
+
io.close(() => resolve());
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
});
|
|
293
|
+
|
|
257
294
|
it("start() rejects without an authenticated user", async () => {
|
|
258
295
|
// startAuthRetryAttempts: 1 disables retrying so this stays instant — the
|
|
259
296
|
// no-retry-attempts contract is tested here; retry behavior is tested below.
|
|
@@ -2636,6 +2673,46 @@ describe("createSyncDb", () => {
|
|
|
2636
2673
|
});
|
|
2637
2674
|
});
|
|
2638
2675
|
|
|
2676
|
+
describe("auth change while the session read fails (betterAuthAdapter)", () => {
|
|
2677
|
+
it("keeps the user and mutate() working when the device goes offline before the auth-change handler reads the session", async () => {
|
|
2678
|
+
// start() subscribes to the session atom, which hands over the current session at
|
|
2679
|
+
// once; the handler that emission queues runs after start() resolves. When the
|
|
2680
|
+
// network drops in between, its get-session read fails. That must not count as a
|
|
2681
|
+
// sign-out.
|
|
2682
|
+
const listeners = new Set<(value: unknown) => void>();
|
|
2683
|
+
// Only start()'s own read gets through; the network is gone for every later one.
|
|
2684
|
+
let sessionReads = 0;
|
|
2685
|
+
const session = {data: {session: {token: "tok"}, user: {id: "u1"}}};
|
|
2686
|
+
const harness = makeHarness({
|
|
2687
|
+
authProvider: betterAuthAdapter({
|
|
2688
|
+
getSession: async () => {
|
|
2689
|
+
sessionReads += 1;
|
|
2690
|
+
if (sessionReads > 1) {
|
|
2691
|
+
throw new TypeError("Failed to fetch");
|
|
2692
|
+
}
|
|
2693
|
+
return session;
|
|
2694
|
+
},
|
|
2695
|
+
useSession: {
|
|
2696
|
+
subscribe: (listener) => {
|
|
2697
|
+
listeners.add(listener);
|
|
2698
|
+
listener(session);
|
|
2699
|
+
return () => listeners.delete(listener);
|
|
2700
|
+
},
|
|
2701
|
+
},
|
|
2702
|
+
}),
|
|
2703
|
+
});
|
|
2704
|
+
const client = createSyncDb(harness.config);
|
|
2705
|
+
await client.start();
|
|
2706
|
+
await waitUntil(() => sessionReads > 1);
|
|
2707
|
+
await flush();
|
|
2708
|
+
|
|
2709
|
+
expect(client.getSyncStatus().paused).toBeUndefined();
|
|
2710
|
+
const {id} = client.mutate({collection: "todos", data: {title: "x"}, operation: "create"});
|
|
2711
|
+
expect(id).toBeTruthy();
|
|
2712
|
+
await client.stop();
|
|
2713
|
+
});
|
|
2714
|
+
});
|
|
2715
|
+
|
|
2639
2716
|
describe("signOut() (A4)", () => {
|
|
2640
2717
|
it("serializes against a concurrent auth change and leaves no destroyed persister behind", async () => {
|
|
2641
2718
|
const harness = makeHarness();
|
|
@@ -3618,3 +3695,182 @@ describe("snapshot paging concurrency", () => {
|
|
|
3618
3695
|
await client.stop();
|
|
3619
3696
|
});
|
|
3620
3697
|
});
|
|
3698
|
+
|
|
3699
|
+
describe("debug event stream: reconcile decisions", () => {
|
|
3700
|
+
/** Labels of every recorded reconcile event, in order. */
|
|
3701
|
+
const reconcileLabels = (client: ReturnType<typeof createSyncDb>): string[] =>
|
|
3702
|
+
(client.debug?.getEvents() ?? [])
|
|
3703
|
+
.filter((event) => event.type === "reconcile")
|
|
3704
|
+
.map((event) => event.label);
|
|
3705
|
+
|
|
3706
|
+
it("records a skip for a stream whose collection is not configured and a join for the rest", async () => {
|
|
3707
|
+
const harness = makeHarness({debug: true});
|
|
3708
|
+
const unconfigured = "projects|tenant:org1";
|
|
3709
|
+
harness.http.state.streams = [
|
|
3710
|
+
{collection: "todos", stream: DEFAULT_STREAM},
|
|
3711
|
+
{collection: "projects", stream: unconfigured},
|
|
3712
|
+
];
|
|
3713
|
+
const client = createSyncDb(harness.config);
|
|
3714
|
+
await client.start();
|
|
3715
|
+
await flush();
|
|
3716
|
+
|
|
3717
|
+
const skipped = client.debug
|
|
3718
|
+
?.getEvents()
|
|
3719
|
+
.find((event) => event.label.startsWith("stream skipped"));
|
|
3720
|
+
expect(skipped).toMatchObject({
|
|
3721
|
+
detail: {collection: "projects", stream: unconfigured},
|
|
3722
|
+
direction: "system",
|
|
3723
|
+
label: `stream skipped (unconfigured collection): ${unconfigured}`,
|
|
3724
|
+
type: "reconcile",
|
|
3725
|
+
});
|
|
3726
|
+
expect(reconcileLabels(client)).toContain(`stream join: ${DEFAULT_STREAM}`);
|
|
3727
|
+
await client.stop();
|
|
3728
|
+
});
|
|
3729
|
+
|
|
3730
|
+
it("records a leave with the number of entities it purged", async () => {
|
|
3731
|
+
const harness = makeHarness({debug: true});
|
|
3732
|
+
const leaving = "todos|tenant:org1";
|
|
3733
|
+
harness.http.state.streams = [
|
|
3734
|
+
{collection: "todos", stream: DEFAULT_STREAM},
|
|
3735
|
+
{collection: "todos", stream: leaving},
|
|
3736
|
+
];
|
|
3737
|
+
harness.http.state.pages[leaving] = {
|
|
3738
|
+
cursor: 2,
|
|
3739
|
+
entities: [
|
|
3740
|
+
{data: {title: "org doc"}, deleted: false, id: "og1", seq: 1},
|
|
3741
|
+
{data: {title: "org doc 2"}, deleted: false, id: "og2", seq: 2},
|
|
3742
|
+
],
|
|
3743
|
+
frontierSeq: 2,
|
|
3744
|
+
hasMore: false,
|
|
3745
|
+
oldestRetainedSeq: 0,
|
|
3746
|
+
stream: leaving,
|
|
3747
|
+
};
|
|
3748
|
+
const client = createSyncDb(harness.config);
|
|
3749
|
+
await client.start();
|
|
3750
|
+
await flush();
|
|
3751
|
+
|
|
3752
|
+
harness.http.state.streams = [{collection: "todos", stream: DEFAULT_STREAM}];
|
|
3753
|
+
await client.reconcile();
|
|
3754
|
+
|
|
3755
|
+
const leave = client.debug?.getEvents().find((event) => event.label.startsWith("stream leave"));
|
|
3756
|
+
expect(leave).toMatchObject({
|
|
3757
|
+
detail: {purged: 2, stream: leaving},
|
|
3758
|
+
label: `stream leave: purged ${leaving} (2 entities)`,
|
|
3759
|
+
});
|
|
3760
|
+
await client.stop();
|
|
3761
|
+
});
|
|
3762
|
+
|
|
3763
|
+
it("records how many legacy snapshot cursors it migrated", async () => {
|
|
3764
|
+
const harness = makeHarness({collections: ["todos", "notes"], debug: true});
|
|
3765
|
+
const client = createSyncDb(harness.config);
|
|
3766
|
+
client.store.raw.setRow(CURSORS_TABLE, "snapshot:todos", {seq: 42, updatedAt: "old"});
|
|
3767
|
+
client.store.raw.setRow(CURSORS_TABLE, "snapshot:notes", {seq: 7, updatedAt: "old"});
|
|
3768
|
+
|
|
3769
|
+
await client.start();
|
|
3770
|
+
await flush();
|
|
3771
|
+
|
|
3772
|
+
const migration = client.debug?.getEvents().find((event) => event.label.startsWith("migrated"));
|
|
3773
|
+
expect(migration).toMatchObject({
|
|
3774
|
+
detail: {migrated: 2},
|
|
3775
|
+
label: "migrated 2 legacy snapshot cursors",
|
|
3776
|
+
});
|
|
3777
|
+
await client.stop();
|
|
3778
|
+
});
|
|
3779
|
+
|
|
3780
|
+
it("records tombstone compaction only when a reconcile actually removes rows", async () => {
|
|
3781
|
+
const retentionMs = 90 * 24 * 60 * 60 * 1_000;
|
|
3782
|
+
const harness = makeHarness({debug: true, tombstoneRetentionMs: retentionMs});
|
|
3783
|
+
const client = createSyncDb(harness.config);
|
|
3784
|
+
await client.start();
|
|
3785
|
+
await flush();
|
|
3786
|
+
harness.transport.deliverDelta({
|
|
3787
|
+
collection: "todos",
|
|
3788
|
+
deleted: true,
|
|
3789
|
+
id: "old-tombstone",
|
|
3790
|
+
method: "delete",
|
|
3791
|
+
seq: 1,
|
|
3792
|
+
stream: DEFAULT_STREAM,
|
|
3793
|
+
});
|
|
3794
|
+
|
|
3795
|
+
await client.reconcile();
|
|
3796
|
+
expect(reconcileLabels(client).some((label) => label.startsWith("compacted"))).toBe(false);
|
|
3797
|
+
|
|
3798
|
+
harness.clock.value += retentionMs + 1_000;
|
|
3799
|
+
await client.reconcile();
|
|
3800
|
+
|
|
3801
|
+
const compaction = client.debug
|
|
3802
|
+
?.getEvents()
|
|
3803
|
+
.find((event) => event.label.startsWith("compacted"));
|
|
3804
|
+
expect(compaction).toMatchObject({detail: {removed: 1}, label: "compacted 1 tombstone(s)"});
|
|
3805
|
+
await client.stop();
|
|
3806
|
+
});
|
|
3807
|
+
|
|
3808
|
+
it("records the fallback to per-stream paging when the server lacks snapshot batch", async () => {
|
|
3809
|
+
const harness = makeHarness({debug: true});
|
|
3810
|
+
harness.http.channel.fetchSnapshotBatch = async () => ({type: "unsupported"});
|
|
3811
|
+
const client = createSyncDb(harness.config);
|
|
3812
|
+
await client.start();
|
|
3813
|
+
await flush();
|
|
3814
|
+
|
|
3815
|
+
expect(reconcileLabels(client)).toContain(
|
|
3816
|
+
"snapshot batch unsupported by server; paging per stream"
|
|
3817
|
+
);
|
|
3818
|
+
// The per-stream fallback still bootstrapped the stream.
|
|
3819
|
+
expect(harness.http.state.fetchCount).toBeGreaterThan(0);
|
|
3820
|
+
await client.stop();
|
|
3821
|
+
});
|
|
3822
|
+
|
|
3823
|
+
it("brackets forceResync with start and end events that carry the purge totals", async () => {
|
|
3824
|
+
const harness = makeHarness({debug: true});
|
|
3825
|
+
harness.http.state.pages[DEFAULT_STREAM] = {
|
|
3826
|
+
cursor: 1,
|
|
3827
|
+
entities: [{data: {title: "stale"}, deleted: false, id: "local-1", seq: 1}],
|
|
3828
|
+
frontierSeq: 1,
|
|
3829
|
+
hasMore: false,
|
|
3830
|
+
oldestRetainedSeq: 0,
|
|
3831
|
+
stream: DEFAULT_STREAM,
|
|
3832
|
+
};
|
|
3833
|
+
const client = createSyncDb(harness.config);
|
|
3834
|
+
await client.start();
|
|
3835
|
+
await flush();
|
|
3836
|
+
harness.http.state.pages[DEFAULT_STREAM] = emptyPage(DEFAULT_STREAM, 5);
|
|
3837
|
+
|
|
3838
|
+
const result = await client.forceResync();
|
|
3839
|
+
|
|
3840
|
+
assert.equal(result.ok, true);
|
|
3841
|
+
const events = client.debug?.getEvents() ?? [];
|
|
3842
|
+
const start = events.find((event) => event.label === "force resync start");
|
|
3843
|
+
const end = events.find((event) => event.label.startsWith("force resync end"));
|
|
3844
|
+
expect(start).toMatchObject({phase: "start", type: "reconcile"});
|
|
3845
|
+
expect(end).toMatchObject({
|
|
3846
|
+
detail: {purged: result.purged, repaired: result.repaired, streams: result.streams},
|
|
3847
|
+
label: `force resync end (${result.streams} streams, ${result.purged} purged, ${result.repaired} repaired)`,
|
|
3848
|
+
phase: "end",
|
|
3849
|
+
});
|
|
3850
|
+
// The injected clock never advanced, so the measured duration is exact.
|
|
3851
|
+
expect(end?.durationMs).toBe(0);
|
|
3852
|
+
expect(start?.id ?? Number.POSITIVE_INFINITY).toBeLessThan(end?.id ?? 0);
|
|
3853
|
+
await client.stop();
|
|
3854
|
+
});
|
|
3855
|
+
|
|
3856
|
+
it("records the streams a subscribe confirmation caught up", async () => {
|
|
3857
|
+
const harness = makeHarness({debug: true});
|
|
3858
|
+
const client = createSyncDb(harness.config);
|
|
3859
|
+
await client.start();
|
|
3860
|
+
await flush();
|
|
3861
|
+
|
|
3862
|
+
harness.transport.confirmSubscribed({collection: "todos", streams: ["todos|owner:new"]});
|
|
3863
|
+
await waitUntil(() =>
|
|
3864
|
+
reconcileLabels(client).some((label) => label.startsWith("subscribe catch-up"))
|
|
3865
|
+
);
|
|
3866
|
+
|
|
3867
|
+
const catchUp = client.debug
|
|
3868
|
+
?.getEvents()
|
|
3869
|
+
.find((event) => event.label.startsWith("subscribe catch-up"));
|
|
3870
|
+
expect(catchUp).toMatchObject({
|
|
3871
|
+
detail: {streams: ["todos|owner:new"]},
|
|
3872
|
+
label: "subscribe catch-up: todos|owner:new",
|
|
3873
|
+
});
|
|
3874
|
+
await client.stop();
|
|
3875
|
+
});
|
|
3876
|
+
});
|
package/src/client.ts
CHANGED
|
@@ -86,6 +86,12 @@ export interface SyncDbConfig {
|
|
|
86
86
|
baseUrl?: string;
|
|
87
87
|
/** Transport override (tests inject a fake; default is the socket transport). */
|
|
88
88
|
transport?: SyncTransport;
|
|
89
|
+
/**
|
|
90
|
+
* Socket.io transports for the default socket transport, in order. Defaults to polling then
|
|
91
|
+
* websocket. Pass `["websocket"]` for hosts without session affinity, such as Vercel
|
|
92
|
+
* Functions, where a polling request can land on a different instance than its session.
|
|
93
|
+
*/
|
|
94
|
+
socketTransports?: string[];
|
|
89
95
|
/**
|
|
90
96
|
* Optional selected organization id for `X-Organization-Id` on HTTP sync
|
|
91
97
|
* calls and the `organizationId` field on socket mutate payloads. Read at
|
|
@@ -357,6 +363,7 @@ export const createSyncDb = (config: SyncDbConfig): SyncDb => {
|
|
|
357
363
|
// biome-ignore lint/style/noNonNullAssertion: guarded above — no transport implies baseUrl.
|
|
358
364
|
baseUrl: config.baseUrl!,
|
|
359
365
|
organizationIdProvider: config.organizationIdProvider,
|
|
366
|
+
transports: config.socketTransports,
|
|
360
367
|
});
|
|
361
368
|
const httpChannel =
|
|
362
369
|
config.httpChannel ??
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {describe, expect, it} from "bun:test";
|
|
2
2
|
|
|
3
|
+
import {createSyncDebugLog, type SyncDebugEvent, type SyncDebugLog} from "../debug/debugLog";
|
|
3
4
|
import {getConflict} from "../mutations/conflicts";
|
|
4
5
|
import {createOutbox, type Outbox} from "../mutations/outbox";
|
|
5
6
|
import {createSyncStore, type SyncStore} from "../storage/store";
|
|
@@ -26,7 +27,8 @@ interface Harness {
|
|
|
26
27
|
clock: {value: number};
|
|
27
28
|
}
|
|
28
29
|
|
|
29
|
-
|
|
30
|
+
/** Single-send harness (no batch transport), plus optional coordinator overrides. */
|
|
31
|
+
const makeHarness = (overrides: Partial<CreateReplayCoordinatorArgs> = {}): Harness => {
|
|
30
32
|
const store = createSyncStore({collections: ["notes", "todos"]});
|
|
31
33
|
const outbox = createOutbox({store});
|
|
32
34
|
const transport = createFakeTransport();
|
|
@@ -36,6 +38,7 @@ const makeHarness = (): Harness => {
|
|
|
36
38
|
outbox,
|
|
37
39
|
sendMutation: transport.sendMutation,
|
|
38
40
|
store,
|
|
41
|
+
...overrides,
|
|
39
42
|
});
|
|
40
43
|
return {clock, coordinator, outbox, store, transport};
|
|
41
44
|
};
|
|
@@ -1795,4 +1798,313 @@ describe("createReplayCoordinator", () => {
|
|
|
1795
1798
|
expect(harness.outbox.getMutation({mutationId: "m2"})?.status).toBe("acked");
|
|
1796
1799
|
});
|
|
1797
1800
|
});
|
|
1801
|
+
describe("debug event stream", () => {
|
|
1802
|
+
const DEBUG_CLOCK = "2026-01-01T00:00:00.000Z";
|
|
1803
|
+
|
|
1804
|
+
/** Fixed random source: every jittered backoff is half its cap. */
|
|
1805
|
+
const halfRandom = (): number => 0.5;
|
|
1806
|
+
|
|
1807
|
+
const makeDebugLog = (): SyncDebugLog => createSyncDebugLog({clock: () => DEBUG_CLOCK});
|
|
1808
|
+
|
|
1809
|
+
const eventsOfType = (debug: SyncDebugLog, type: SyncDebugEvent["type"]): SyncDebugEvent[] =>
|
|
1810
|
+
debug.getEvents().filter((event) => event.type === type);
|
|
1811
|
+
|
|
1812
|
+
it("a validation nack records a failed event with the parsed mutation body and the reason", async () => {
|
|
1813
|
+
const debug = makeDebugLog();
|
|
1814
|
+
const harness = makeHarness({debug});
|
|
1815
|
+
enqueue(harness, {data: {title: "draft"}, entityId: "t1", mutationId: "m1"});
|
|
1816
|
+
harness.transport.respondWithNack({code: "validation", message: "title too short"});
|
|
1817
|
+
|
|
1818
|
+
await harness.coordinator.replay({userId: USER});
|
|
1819
|
+
|
|
1820
|
+
const [failed] = eventsOfType(debug, "failed");
|
|
1821
|
+
expect(failed).toMatchObject({
|
|
1822
|
+
collection: "todos",
|
|
1823
|
+
detail: {data: {title: "draft"}, reason: "validation"},
|
|
1824
|
+
direction: "inbound",
|
|
1825
|
+
entityId: "t1",
|
|
1826
|
+
label: "failed todos/t1 (validation)",
|
|
1827
|
+
mutationId: "m1",
|
|
1828
|
+
ok: false,
|
|
1829
|
+
operation: "create",
|
|
1830
|
+
});
|
|
1831
|
+
expect(eventsOfType(debug, "failed")).toHaveLength(1);
|
|
1832
|
+
});
|
|
1833
|
+
|
|
1834
|
+
it("records the raw stored args when they are not valid JSON", async () => {
|
|
1835
|
+
const debug = makeDebugLog();
|
|
1836
|
+
const harness = makeHarness({debug});
|
|
1837
|
+
harness.store.upsertEntity({
|
|
1838
|
+
collection: "todos",
|
|
1839
|
+
data: null,
|
|
1840
|
+
id: "t1",
|
|
1841
|
+
pendingMutationId: "m1",
|
|
1842
|
+
});
|
|
1843
|
+
harness.outbox.enqueue({
|
|
1844
|
+
args: {},
|
|
1845
|
+
collection: "todos",
|
|
1846
|
+
entityId: "t1",
|
|
1847
|
+
mutationId: "m1",
|
|
1848
|
+
operation: "create",
|
|
1849
|
+
userId: USER,
|
|
1850
|
+
});
|
|
1851
|
+
harness.store.raw.setCell("_outbox", "m1", "args", "{not json");
|
|
1852
|
+
harness.transport.respondWithNack({code: "validation"});
|
|
1853
|
+
|
|
1854
|
+
await harness.coordinator.replay({userId: USER});
|
|
1855
|
+
|
|
1856
|
+
// The debug stream shows exactly what was stored, so a corrupt row is
|
|
1857
|
+
// visible to whoever is inspecting the failure.
|
|
1858
|
+
expect(eventsOfType(debug, "send")[0]?.detail?.data).toBe("{not json");
|
|
1859
|
+
expect(eventsOfType(debug, "failed")[0]?.detail?.data).toBe("{not json");
|
|
1860
|
+
});
|
|
1861
|
+
|
|
1862
|
+
it("an unauthorized nack records a nack event that marks the drain as paused", async () => {
|
|
1863
|
+
const debug = makeDebugLog();
|
|
1864
|
+
const harness = makeHarness({debug});
|
|
1865
|
+
enqueue(harness, {entityId: "t1", mutationId: "m1"});
|
|
1866
|
+
harness.transport.respondWithNack({code: "unauthorized"});
|
|
1867
|
+
|
|
1868
|
+
const result = await harness.coordinator.replay({userId: USER});
|
|
1869
|
+
|
|
1870
|
+
expect(result).toEqual({paused: "auth"});
|
|
1871
|
+
const nacks = eventsOfType(debug, "nack");
|
|
1872
|
+
expect(nacks).toHaveLength(1);
|
|
1873
|
+
expect(nacks[0]).toMatchObject({
|
|
1874
|
+
detail: {code: "unauthorized", data: {title: "t1"}, paused: true},
|
|
1875
|
+
direction: "inbound",
|
|
1876
|
+
label: "nack todos/t1 (unauthorized)",
|
|
1877
|
+
mutationId: "m1",
|
|
1878
|
+
ok: false,
|
|
1879
|
+
});
|
|
1880
|
+
expect(nacks[0]?.detail).not.toHaveProperty("source");
|
|
1881
|
+
});
|
|
1882
|
+
|
|
1883
|
+
it("an AuthRequiredError from the send channel records an unauthorized nack sourced to HTTP", async () => {
|
|
1884
|
+
const debug = makeDebugLog();
|
|
1885
|
+
const harness = makeHarness({debug});
|
|
1886
|
+
enqueue(harness, {entityId: "t1", mutationId: "m1"});
|
|
1887
|
+
harness.transport.respondWith(() => {
|
|
1888
|
+
throw new AuthRequiredError();
|
|
1889
|
+
});
|
|
1890
|
+
|
|
1891
|
+
const result = await harness.coordinator.replay({userId: USER});
|
|
1892
|
+
|
|
1893
|
+
expect(result).toEqual({paused: "auth"});
|
|
1894
|
+
expect(eventsOfType(debug, "nack")).toEqual([
|
|
1895
|
+
expect.objectContaining({
|
|
1896
|
+
detail: {code: "unauthorized", data: {title: "t1"}, paused: true, source: "http"},
|
|
1897
|
+
label: "nack todos/t1 (unauthorized)",
|
|
1898
|
+
mutationId: "m1",
|
|
1899
|
+
ok: false,
|
|
1900
|
+
}),
|
|
1901
|
+
]);
|
|
1902
|
+
});
|
|
1903
|
+
|
|
1904
|
+
it("an error nack records a retry event with the attempt number and the backoff it armed", async () => {
|
|
1905
|
+
const debug = makeDebugLog();
|
|
1906
|
+
const harness = makeHarness({debug, random: halfRandom});
|
|
1907
|
+
enqueue(harness, {entityId: "t1", mutationId: "m1"});
|
|
1908
|
+
harness.transport.respondWithNack({code: "error"});
|
|
1909
|
+
|
|
1910
|
+
await harness.coordinator.replay({userId: USER});
|
|
1911
|
+
|
|
1912
|
+
const backoffMs = ERROR_NACK_BASE_BACKOFF_MS / 2;
|
|
1913
|
+
expect(eventsOfType(debug, "retry")).toEqual([
|
|
1914
|
+
expect.objectContaining({
|
|
1915
|
+
detail: {attempt: 1, backoffMs, data: {title: "t1"}, reason: "error"},
|
|
1916
|
+
direction: "system",
|
|
1917
|
+
entityId: "t1",
|
|
1918
|
+
label: `retry todos/t1 (error, ${backoffMs}ms)`,
|
|
1919
|
+
mutationId: "m1",
|
|
1920
|
+
operation: "create",
|
|
1921
|
+
}),
|
|
1922
|
+
]);
|
|
1923
|
+
expect(eventsOfType(debug, "failed")).toHaveLength(0);
|
|
1924
|
+
});
|
|
1925
|
+
|
|
1926
|
+
it("a transport failure records a retry event whose attempt count grows on each failure", async () => {
|
|
1927
|
+
const debug = makeDebugLog();
|
|
1928
|
+
const harness = makeHarness({debug, random: halfRandom});
|
|
1929
|
+
enqueue(harness, {entityId: "t1", mutationId: "m1"});
|
|
1930
|
+
harness.transport.setDefaultResponder(() => {
|
|
1931
|
+
throw new Error("timeout");
|
|
1932
|
+
});
|
|
1933
|
+
|
|
1934
|
+
await harness.coordinator.replay({userId: USER});
|
|
1935
|
+
harness.clock.value += MAX_BACKOFF_MS;
|
|
1936
|
+
await harness.coordinator.replay({userId: USER});
|
|
1937
|
+
|
|
1938
|
+
const retries = eventsOfType(debug, "retry");
|
|
1939
|
+
expect(retries.map((event) => event.detail)).toEqual([
|
|
1940
|
+
{attempt: 1, backoffMs: 500, data: {title: "t1"}, reason: "transport"},
|
|
1941
|
+
{attempt: 2, backoffMs: 1_000, data: {title: "t1"}, reason: "transport"},
|
|
1942
|
+
]);
|
|
1943
|
+
expect(retries.map((event) => event.label)).toEqual([
|
|
1944
|
+
"retry todos/t1 (transport, 500ms)",
|
|
1945
|
+
"retry todos/t1 (transport, 1000ms)",
|
|
1946
|
+
]);
|
|
1947
|
+
});
|
|
1948
|
+
|
|
1949
|
+
it("a rate_limited nack records the server's nack, then a retry floored at retryAfterMs", async () => {
|
|
1950
|
+
const debug = makeDebugLog();
|
|
1951
|
+
const harness = makeHarness({debug, random: halfRandom});
|
|
1952
|
+
enqueue(harness, {entityId: "t1", mutationId: "m1"});
|
|
1953
|
+
harness.transport.respondWithNack({
|
|
1954
|
+
code: "rate_limited",
|
|
1955
|
+
message: "slow down",
|
|
1956
|
+
retryAfterMs: 10_000,
|
|
1957
|
+
});
|
|
1958
|
+
|
|
1959
|
+
await harness.coordinator.replay({userId: USER});
|
|
1960
|
+
|
|
1961
|
+
const outcomes = debug
|
|
1962
|
+
.getEvents()
|
|
1963
|
+
.filter((event) => event.type === "nack" || event.type === "retry");
|
|
1964
|
+
expect(outcomes).toEqual([
|
|
1965
|
+
expect.objectContaining({
|
|
1966
|
+
detail: {
|
|
1967
|
+
code: "rate_limited",
|
|
1968
|
+
data: {title: "t1"},
|
|
1969
|
+
message: "slow down",
|
|
1970
|
+
retryAfterMs: 10_000,
|
|
1971
|
+
},
|
|
1972
|
+
direction: "inbound",
|
|
1973
|
+
label: "nack todos/t1 (rate_limited)",
|
|
1974
|
+
ok: false,
|
|
1975
|
+
type: "nack",
|
|
1976
|
+
}),
|
|
1977
|
+
expect.objectContaining({
|
|
1978
|
+
detail: {attempt: 1, backoffMs: 10_000, data: {title: "t1"}, reason: "rate_limited"},
|
|
1979
|
+
label: "retry todos/t1 (rate_limited, 10000ms)",
|
|
1980
|
+
type: "retry",
|
|
1981
|
+
}),
|
|
1982
|
+
]);
|
|
1983
|
+
});
|
|
1984
|
+
|
|
1985
|
+
it("a batched chunk records one send event per mutation, flagged as part of a batch", async () => {
|
|
1986
|
+
const debug = makeDebugLog();
|
|
1987
|
+
const harness = makeBatchHarness({debug});
|
|
1988
|
+
enqueue(harness, {entityId: "e1", mutationId: "m1"});
|
|
1989
|
+
enqueue(harness, {data: {title: "second"}, entityId: "e2", mutationId: "m2"});
|
|
1990
|
+
|
|
1991
|
+
await harness.coordinator.replay({userId: USER});
|
|
1992
|
+
|
|
1993
|
+
expect(harness.transport.sentBatches).toEqual([["m1", "m2"]]);
|
|
1994
|
+
expect(eventsOfType(debug, "send")).toEqual([
|
|
1995
|
+
expect.objectContaining({
|
|
1996
|
+
detail: {attempt: 1, batch: true, data: {title: "e1"}},
|
|
1997
|
+
direction: "outbound",
|
|
1998
|
+
label: "send create todos/e1",
|
|
1999
|
+
mutationId: "m1",
|
|
2000
|
+
}),
|
|
2001
|
+
expect.objectContaining({
|
|
2002
|
+
detail: {attempt: 1, batch: true, data: {title: "second"}},
|
|
2003
|
+
label: "send create todos/e2",
|
|
2004
|
+
mutationId: "m2",
|
|
2005
|
+
}),
|
|
2006
|
+
]);
|
|
2007
|
+
expect(eventsOfType(debug, "ack").map((event) => event.mutationId)).toEqual(["m1", "m2"]);
|
|
2008
|
+
});
|
|
2009
|
+
|
|
2010
|
+
it("a batch transport failure records a transport retry for every mutation in the chunk", async () => {
|
|
2011
|
+
const debug = makeDebugLog();
|
|
2012
|
+
const harness = makeBatchHarness({debug, random: halfRandom});
|
|
2013
|
+
enqueue(harness, {entityId: "e1", mutationId: "m1"});
|
|
2014
|
+
enqueue(harness, {entityId: "e2", mutationId: "m2"});
|
|
2015
|
+
harness.transport.setBatchResponder(() => {
|
|
2016
|
+
throw new Error("socket disconnected mid-batch");
|
|
2017
|
+
});
|
|
2018
|
+
|
|
2019
|
+
await harness.coordinator.replay({userId: USER});
|
|
2020
|
+
|
|
2021
|
+
expect(
|
|
2022
|
+
eventsOfType(debug, "retry").map((event) => [event.mutationId, event.detail?.reason])
|
|
2023
|
+
).toEqual([
|
|
2024
|
+
["m1", "transport"],
|
|
2025
|
+
["m2", "transport"],
|
|
2026
|
+
]);
|
|
2027
|
+
});
|
|
2028
|
+
});
|
|
2029
|
+
|
|
2030
|
+
describe("batch edge cases", () => {
|
|
2031
|
+
it("a mutation with unparseable args still joins the chunk alongside other entities", async () => {
|
|
2032
|
+
const harness = makeBatchHarness();
|
|
2033
|
+
enqueue(harness, {entityId: "e1", mutationId: "m1"});
|
|
2034
|
+
enqueue(harness, {entityId: "e2", mutationId: "m2"});
|
|
2035
|
+
harness.store.raw.setCell("_outbox", "m2", "args", "{not json");
|
|
2036
|
+
|
|
2037
|
+
await harness.coordinator.replay({userId: USER});
|
|
2038
|
+
|
|
2039
|
+
// The reference-blocking scan cannot inspect corrupt args, so it treats
|
|
2040
|
+
// them as referencing nothing rather than holding the mutation back.
|
|
2041
|
+
expect(harness.transport.sentBatches).toEqual([["m1", "m2"]]);
|
|
2042
|
+
expect(harness.outbox.getMutation({mutationId: "m2"})?.status).toBe("acked");
|
|
2043
|
+
});
|
|
2044
|
+
|
|
2045
|
+
it("ignores a batch response that resolves after reset()", async () => {
|
|
2046
|
+
const harness = makeBatchHarness();
|
|
2047
|
+
enqueue(harness, {entityId: "e1", mutationId: "m1"});
|
|
2048
|
+
enqueue(harness, {entityId: "e2", mutationId: "m2"});
|
|
2049
|
+
let release: (() => void) | undefined;
|
|
2050
|
+
harness.transport.respondBatchWith(
|
|
2051
|
+
(request) =>
|
|
2052
|
+
new Promise((resolve) => {
|
|
2053
|
+
release = (): void =>
|
|
2054
|
+
resolve({
|
|
2055
|
+
results: request.mutations.map((mutation, index) => ({
|
|
2056
|
+
ack: {id: mutation.id ?? "", mutationId: mutation.mutationId, seq: index + 1},
|
|
2057
|
+
type: "ack" as const,
|
|
2058
|
+
})),
|
|
2059
|
+
type: "results",
|
|
2060
|
+
});
|
|
2061
|
+
})
|
|
2062
|
+
);
|
|
2063
|
+
|
|
2064
|
+
const replay = harness.coordinator.replay({userId: USER});
|
|
2065
|
+
await Promise.resolve();
|
|
2066
|
+
expect(release).toBeDefined();
|
|
2067
|
+
harness.coordinator.reset();
|
|
2068
|
+
release?.();
|
|
2069
|
+
|
|
2070
|
+
expect(await replay).toEqual({});
|
|
2071
|
+
// The stale acks never land: neither mutation is finalized.
|
|
2072
|
+
expect(harness.outbox.getMutation({mutationId: "m1"})?.status).toBe("inFlight");
|
|
2073
|
+
expect(harness.outbox.getMutation({mutationId: "m2"})?.status).toBe("inFlight");
|
|
2074
|
+
});
|
|
2075
|
+
|
|
2076
|
+
it("ignores a batch transport failure that rejects after reset()", async () => {
|
|
2077
|
+
const harness = makeBatchHarness();
|
|
2078
|
+
enqueue(harness, {entityId: "e1", mutationId: "m1"});
|
|
2079
|
+
enqueue(harness, {entityId: "e2", mutationId: "m2"});
|
|
2080
|
+
let fail: (() => void) | undefined;
|
|
2081
|
+
harness.transport.respondBatchWith(
|
|
2082
|
+
() =>
|
|
2083
|
+
new Promise((_resolve, reject) => {
|
|
2084
|
+
fail = (): void => reject(new Error("socket closed"));
|
|
2085
|
+
})
|
|
2086
|
+
);
|
|
2087
|
+
|
|
2088
|
+
const replay = harness.coordinator.replay({userId: USER});
|
|
2089
|
+
await Promise.resolve();
|
|
2090
|
+
expect(fail).toBeDefined();
|
|
2091
|
+
harness.coordinator.reset();
|
|
2092
|
+
fail?.();
|
|
2093
|
+
|
|
2094
|
+
expect(await replay).toEqual({});
|
|
2095
|
+
// No backoff is armed for the old lifecycle's failure: the rows are left
|
|
2096
|
+
// exactly as the reset found them.
|
|
2097
|
+
expect(harness.outbox.getMutation({mutationId: "m1"})?.status).toBe("inFlight");
|
|
2098
|
+
expect(harness.outbox.getMutation({mutationId: "m1"})?.errorNackCount).toBe(0);
|
|
2099
|
+
});
|
|
2100
|
+
|
|
2101
|
+
it("sends nothing while the host has gated draining off", async () => {
|
|
2102
|
+
const harness = makeHarness({shouldDrain: () => false});
|
|
2103
|
+
enqueue(harness, {entityId: "t1", mutationId: "m1"});
|
|
2104
|
+
|
|
2105
|
+
expect(await harness.coordinator.replay({userId: USER})).toEqual({});
|
|
2106
|
+
expect(harness.transport.sentMutations).toHaveLength(0);
|
|
2107
|
+
expect(harness.outbox.getMutation({mutationId: "m1"})?.status).toBe("queued");
|
|
2108
|
+
});
|
|
2109
|
+
});
|
|
1798
2110
|
});
|