@niwa-labs/host 0.0.0-stage → 0.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +2 -2
- package/package.json +16 -4
- package/src/access-actions.ts +193 -0
- package/src/access-control.ts +333 -0
- package/src/call-operations.ts +176 -0
- package/src/cockpit-catalog.ts +99 -0
- package/src/describe.ts +55 -0
- package/src/doors.ts +272 -0
- package/src/host-types.ts +43 -0
- package/src/host.ts +3346 -0
- package/src/index.ts +10 -0
- package/src/plugin-install.ts +249 -0
- package/src/state.ts +261 -0
- package/src/stores.ts +131 -0
- package/test/abandon.test.ts +58 -0
- package/test/action-refusal-restart.test.ts +46 -0
- package/test/action-step.test.ts +363 -0
- package/test/action-wait-restart.test.ts +96 -0
- package/test/builtin-actions.test.ts +21 -0
- package/test/cancel-steps.test.ts +150 -0
- package/test/catch-up.test.ts +378 -0
- package/test/causes.test.ts +138 -0
- package/test/client.test.ts +190 -0
- package/test/cockpit-catalog.test.ts +118 -0
- package/test/crash-between-steps.test.ts +88 -0
- package/test/declared-values.test.ts +175 -0
- package/test/deep-nested-pause.test.ts +147 -0
- package/test/fixtures/action-step-host.ts +166 -0
- package/test/fixtures/action-wait-restart-host.ts +62 -0
- package/test/fixtures/catch-up-host.ts +91 -0
- package/test/fixtures/causes-host.ts +78 -0
- package/test/fixtures/client-host.ts +52 -0
- package/test/fixtures/crash-between-steps-host.ts +71 -0
- package/test/fixtures/declared-values-host.ts +110 -0
- package/test/fixtures/intercept-order-host.ts +112 -0
- package/test/fixtures/intercept-step-host.ts +106 -0
- package/test/fixtures/live-frames-host.ts +81 -0
- package/test/fixtures/long-wait-host.ts +78 -0
- package/test/fixtures/nested-wait-host.ts +120 -0
- package/test/fixtures/no-repeat-crash-host.ts +79 -0
- package/test/fixtures/plugin-replace-host.ts +176 -0
- package/test/fixtures/record-refusal-crash.ts +50 -0
- package/test/fixtures/recovery-contracts-host.ts +79 -0
- package/test/fixtures/repeated-write-host.ts +39 -0
- package/test/fixtures/replay-mismatch-host.ts +66 -0
- package/test/fixtures/single-writer-host.ts +113 -0
- package/test/fixtures/state-host.ts +118 -0
- package/test/fixtures/wait-restart-host.ts +82 -0
- package/test/guards.test.ts +448 -0
- package/test/handler-emit.test.ts +116 -0
- package/test/host.test.ts +114 -0
- package/test/intercept-adjust.test.ts +318 -0
- package/test/intercept-order.test.ts +276 -0
- package/test/intercept-step.test.ts +144 -0
- package/test/large-outcome.test.ts +60 -0
- package/test/live-frames.test.ts +150 -0
- package/test/live-ready.test.ts +61 -0
- package/test/long-wait.test.ts +168 -0
- package/test/nested-outcome.test.ts +75 -0
- package/test/nested-wait.test.ts +111 -0
- package/test/no-repeat-crash.test.ts +132 -0
- package/test/offer-replace.test.ts +39 -0
- package/test/plugin-async-setup.test.ts +22 -0
- package/test/plugin-copies.test.ts +87 -0
- package/test/plugin-emit.test.ts +43 -0
- package/test/plugin-install-failure.test.ts +121 -0
- package/test/plugin-install-hash.test.ts +57 -0
- package/test/plugin-needs.test.ts +80 -0
- package/test/plugin-replace.test.ts +225 -0
- package/test/plugin-uninstall.test.ts +70 -0
- package/test/plugin-version-copies.test.ts +53 -0
- package/test/plugin-version.test.ts +180 -0
- package/test/read-filter.test.ts +136 -0
- package/test/read-query.test.ts +92 -0
- package/test/record-admission.test.ts +294 -0
- package/test/recovery-contracts.test.ts +107 -0
- package/test/repeated-write.test.ts +119 -0
- package/test/replay-mismatch.test.ts +79 -0
- package/test/replay.test.ts +135 -0
- package/test/single-writer.test.ts +267 -0
- package/test/state.test.ts +246 -0
- package/test/step-by-name.test.ts +122 -0
- package/test/stores.test.ts +102 -0
- package/test/stream.test.ts +154 -0
- package/test/subscriber-stream.test.ts +71 -0
- package/test/subscribers.test.ts +101 -0
- package/test/subscriptions.test.ts +632 -0
- package/test/trace-call.test.ts +124 -0
- package/test/trigger-filter.test.ts +42 -0
- package/test/use-many.test.ts +151 -0
- package/test/wait-query.test.ts +117 -0
- package/test/wait-restart.test.ts +187 -0
- package/test/wire-read-access.test.ts +85 -0
- package/tsconfig.json +6 -0
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
// Replay of a recorded key by a different call: the invocation pauses, nothing is written twice.
|
|
2
|
+
//
|
|
3
|
+
// The host runs in this process. The handler of one invocation runs twice: the first run records
|
|
4
|
+
// something at the key "k" and fails, the owner retries, and the second run calls something else
|
|
5
|
+
// at the same key. The recorded fact is not the answer to that call, so the invocation pauses with
|
|
6
|
+
// a replay mismatch instead of returning the old value or crashing the host.
|
|
7
|
+
|
|
8
|
+
import { expect, test } from "bun:test";
|
|
9
|
+
|
|
10
|
+
import { action, record, seq } from "@niwa-labs/core";
|
|
11
|
+
import { createNiwa } from "@niwa-labs/host";
|
|
12
|
+
import { plugin } from "@niwa-labs/plugin";
|
|
13
|
+
import type { Handler, StoredRecord } from "@niwa-labs/core";
|
|
14
|
+
import { eventually, tempHome } from "@niwa-labs/testing";
|
|
15
|
+
import { z } from "zod";
|
|
16
|
+
|
|
17
|
+
const Start = record("replay.start", z.object({}));
|
|
18
|
+
const Note = record("replay.note", z.object({ n: z.number() }));
|
|
19
|
+
|
|
20
|
+
const STATE_TYPE = "niwa.invocation.state";
|
|
21
|
+
|
|
22
|
+
/** Two actions a step may call: the same key is called with another input, or as the other one. */
|
|
23
|
+
const First = action("replay.first", {
|
|
24
|
+
description: "does nothing",
|
|
25
|
+
input: z.object({ text: z.string() }),
|
|
26
|
+
output: z.object({}),
|
|
27
|
+
recovery: "repeat",
|
|
28
|
+
effect: "write",
|
|
29
|
+
run: async () => ({}),
|
|
30
|
+
});
|
|
31
|
+
const Second = action("replay.second", {
|
|
32
|
+
description: "does nothing",
|
|
33
|
+
input: z.object({ text: z.string() }),
|
|
34
|
+
output: z.object({}),
|
|
35
|
+
recovery: "repeat",
|
|
36
|
+
effect: "write",
|
|
37
|
+
run: async () => ({}),
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
/** Run a handler twice (the owner retries after the first run fails); return the last state. */
|
|
41
|
+
async function lastStateAfterRetry(
|
|
42
|
+
handler: (run: number) => Handler<{}>,
|
|
43
|
+
): Promise<{ state: string; reason?: string; seen: StoredRecord[] }> {
|
|
44
|
+
const home = await tempHome();
|
|
45
|
+
const niwa = createNiwa({ home: home.path });
|
|
46
|
+
let runs = 0;
|
|
47
|
+
niwa.use(
|
|
48
|
+
plugin("replay", (api) => {
|
|
49
|
+
api.declare(Start, Note);
|
|
50
|
+
api.action(First);
|
|
51
|
+
api.action(Second);
|
|
52
|
+
api.on(Start, async (data, step, rec) => {
|
|
53
|
+
runs += 1;
|
|
54
|
+
await handler(runs)(data, step, rec);
|
|
55
|
+
});
|
|
56
|
+
}),
|
|
57
|
+
);
|
|
58
|
+
await niwa.serve();
|
|
59
|
+
const seen: StoredRecord[] = [];
|
|
60
|
+
const watching = (async () => {
|
|
61
|
+
for await (const rec of niwa.records.since()) seen.push(rec);
|
|
62
|
+
})();
|
|
63
|
+
const states = () =>
|
|
64
|
+
seen
|
|
65
|
+
.filter((rec) => rec.type === STATE_TYPE)
|
|
66
|
+
.map((rec) => rec.data as { invocation: string; state: string; reason?: string });
|
|
67
|
+
try {
|
|
68
|
+
await niwa.emit(Start, {});
|
|
69
|
+
await eventually(() => states().at(-1)?.state === "paused", 10_000, "the first run pauses");
|
|
70
|
+
const invocation = states().at(-1)?.invocation ?? "";
|
|
71
|
+
await niwa.call("niwa.invocation.resolve" as never, { invocation, choice: "retry" } as never);
|
|
72
|
+
await eventually(
|
|
73
|
+
() => states().filter((s) => s.state === "paused").length === 2,
|
|
74
|
+
10_000,
|
|
75
|
+
"the second run pauses",
|
|
76
|
+
);
|
|
77
|
+
return { state: states().at(-1)?.state ?? "", reason: states().at(-1)?.reason, seen };
|
|
78
|
+
} finally {
|
|
79
|
+
await niwa.close();
|
|
80
|
+
await watching.catch(() => {});
|
|
81
|
+
await home.cleanup();
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
test("a clock key read by a random call pauses the invocation", async () => {
|
|
86
|
+
const result = await lastStateAfterRetry((run) => async (_data, step) => {
|
|
87
|
+
if (run === 1) {
|
|
88
|
+
await step.now({ key: "k" });
|
|
89
|
+
throw new Error("first run stops here");
|
|
90
|
+
}
|
|
91
|
+
await step.random({ key: "k" });
|
|
92
|
+
});
|
|
93
|
+
expect(result.reason).toBe("replay mismatch at step k");
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
test("an emit key written again with other content pauses the invocation", async () => {
|
|
97
|
+
const result = await lastStateAfterRetry((run) => async (_data, step) => {
|
|
98
|
+
if (run === 1) {
|
|
99
|
+
await step.emit(Note, { n: 1 }, { key: "k" });
|
|
100
|
+
throw new Error("first run stops here");
|
|
101
|
+
}
|
|
102
|
+
await step.emit(Note, { n: 2 }, { key: "k" });
|
|
103
|
+
});
|
|
104
|
+
expect(result.reason).toBe("replay mismatch at emit k");
|
|
105
|
+
const notes = result.seen.filter((rec) => rec.type === Note.name);
|
|
106
|
+
expect(notes.map((rec) => rec.data)).toEqual([{ n: 1 }]);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("a step called with another input says which call it was and that the input changed", async () => {
|
|
110
|
+
const result = await lastStateAfterRetry((run) => async (_data, step) => {
|
|
111
|
+
if (run === 1) {
|
|
112
|
+
await step(First, { text: "the old prompt" }, { key: "k" });
|
|
113
|
+
throw new Error("first run stops here");
|
|
114
|
+
}
|
|
115
|
+
await step(First, { text: "the new prompt" }, { key: "k" });
|
|
116
|
+
});
|
|
117
|
+
expect(result.state).toBe("paused");
|
|
118
|
+
expect(result.reason).toBe(
|
|
119
|
+
"replay mismatch at step k: the call to replay.first was recorded with another input than the code gives it now",
|
|
120
|
+
);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test("a step that is another action under the same key says what was recorded and what is called now", async () => {
|
|
124
|
+
const result = await lastStateAfterRetry((run) => async (_data, step) => {
|
|
125
|
+
if (run === 1) {
|
|
126
|
+
await step(First, { text: "same" }, { key: "k" });
|
|
127
|
+
throw new Error("first run stops here");
|
|
128
|
+
}
|
|
129
|
+
await step(Second, { text: "same" }, { key: "k" });
|
|
130
|
+
});
|
|
131
|
+
expect(result.state).toBe("paused");
|
|
132
|
+
expect(result.reason).toBe(
|
|
133
|
+
"replay mismatch at step k: it was recorded as a call to replay.first, and the code now calls replay.second",
|
|
134
|
+
);
|
|
135
|
+
});
|
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
// One writer.
|
|
2
|
+
//
|
|
3
|
+
// The scenario runs against real host processes (fixtures/single-writer-host.ts):
|
|
4
|
+
//
|
|
5
|
+
// 1. spawn host A; it serves the home
|
|
6
|
+
// 2. spawn host B on the same home — B is refused with WriterBusy, and its output says so
|
|
7
|
+
// 3. kill A with SIGKILL — a real crash, no clean shutdown
|
|
8
|
+
// 4. spawn B again — the dead holder is taken over, and B serves
|
|
9
|
+
// 5. the journal is the same one: a record written through B is readable
|
|
10
|
+
//
|
|
11
|
+
// The stale holder is the second half of the check:
|
|
12
|
+
//
|
|
13
|
+
// 6. host A parks inside a step, on a gate that the test opens later
|
|
14
|
+
// 7. host B starts with "takeover": it takes the journal from the living A
|
|
15
|
+
// 8. the gate opens — A tries to write the outcome of its step and the journal refuses
|
|
16
|
+
// it: A has lost ownership. A says so and stops
|
|
17
|
+
// 9. B's journal holds no record written by a handler after the take-over
|
|
18
|
+
//
|
|
19
|
+
// Only the public API is used: the entries of the packages and "@niwa-labs/testing".
|
|
20
|
+
|
|
21
|
+
import { expect, test } from "bun:test";
|
|
22
|
+
import { fileURLToPath } from "node:url";
|
|
23
|
+
|
|
24
|
+
import type { StoredRecord } from "@niwa-labs/core";
|
|
25
|
+
import { connect } from "@niwa-labs/client";
|
|
26
|
+
import { seq } from "@niwa-labs/core";
|
|
27
|
+
import { counter, eventually, gate, spawnHost, tempHome, connectAsOwner } from "@niwa-labs/testing";
|
|
28
|
+
|
|
29
|
+
import { Probe, Start } from "./fixtures/single-writer-host.ts";
|
|
30
|
+
|
|
31
|
+
const HOST_SCRIPT = fileURLToPath(new URL("./fixtures/single-writer-host.ts", import.meta.url));
|
|
32
|
+
|
|
33
|
+
/** From the beginning of the journal. */
|
|
34
|
+
const FROM_START = seq(0);
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Read the journal through the public subscription.
|
|
38
|
+
*
|
|
39
|
+
* A subscription never ends by itself, so the read stops itself: it waits for the journal to
|
|
40
|
+
* settle and then closes the subscription.
|
|
41
|
+
*/
|
|
42
|
+
async function readJournal(
|
|
43
|
+
client: Awaited<ReturnType<typeof connect>>,
|
|
44
|
+
settleMs = 500,
|
|
45
|
+
): Promise<StoredRecord[]> {
|
|
46
|
+
const records: StoredRecord[] = [];
|
|
47
|
+
const reading = (async () => {
|
|
48
|
+
for await (const record of client.records.since()) {
|
|
49
|
+
records.push(record);
|
|
50
|
+
}
|
|
51
|
+
})().catch(() => {});
|
|
52
|
+
await Bun.sleep(settleMs);
|
|
53
|
+
await client.close().catch(() => {});
|
|
54
|
+
await reading;
|
|
55
|
+
return records;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* True for a record written by one plugin's handler: the actor names the handler that
|
|
60
|
+
* wrote it, and a handler id is the plugin's name and the handler's own name
|
|
61
|
+
*.
|
|
62
|
+
*/
|
|
63
|
+
function writtenByHandler(record: StoredRecord, pluginName = "single-writer"): boolean {
|
|
64
|
+
return record.actor.kind === "handler" && record.actor.handler.startsWith(`${pluginName}/`);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** The highest journal position in these records. */
|
|
68
|
+
function highestSeq(records: StoredRecord[]): number {
|
|
69
|
+
return records.reduce((highest, record) => Math.max(highest, record.seq), 0);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
test("one writer: a second host is refused while the first lives, and it takes the journal over after the first dies", async () => {
|
|
73
|
+
const home = await tempHome();
|
|
74
|
+
const homePath = home.path;
|
|
75
|
+
let first: ReturnType<typeof spawnHost> | undefined;
|
|
76
|
+
let refused: ReturnType<typeof spawnHost> | undefined;
|
|
77
|
+
let second: ReturnType<typeof spawnHost> | undefined;
|
|
78
|
+
let client: Awaited<ReturnType<typeof connect>> | undefined;
|
|
79
|
+
|
|
80
|
+
try {
|
|
81
|
+
// 1. The first host serves the home.
|
|
82
|
+
first = spawnHost(HOST_SCRIPT, homePath);
|
|
83
|
+
await first.ready();
|
|
84
|
+
|
|
85
|
+
// 2. A second host on the same home is refused while the first lives. The
|
|
86
|
+
// refusal is a fact about the second host, so it is read from its output.
|
|
87
|
+
refused = spawnHost(HOST_SCRIPT, homePath);
|
|
88
|
+
await eventually(
|
|
89
|
+
() => refused?.output().includes("WriterBusy") === true,
|
|
90
|
+
10_000,
|
|
91
|
+
"the second host should be refused with WriterBusy",
|
|
92
|
+
);
|
|
93
|
+
expect(refused.output()).toContain("WriterBusy");
|
|
94
|
+
await refused.kill();
|
|
95
|
+
refused = undefined;
|
|
96
|
+
|
|
97
|
+
// 3. The first host dies without a clean shutdown: SIGKILL, no goodbye.
|
|
98
|
+
await first.kill();
|
|
99
|
+
first = undefined;
|
|
100
|
+
|
|
101
|
+
// 4. The second host takes the journal over: the holder is dead, so the
|
|
102
|
+
// journal may be taken.
|
|
103
|
+
second = spawnHost(HOST_SCRIPT, homePath);
|
|
104
|
+
await second.ready();
|
|
105
|
+
|
|
106
|
+
// 5. The journal is the same one and it works: a record written through
|
|
107
|
+
// the second host comes back with its data.
|
|
108
|
+
client = await connectAsOwner(homePath);
|
|
109
|
+
const written = await client.emit(Start, { n: 1 });
|
|
110
|
+
expect(written.type).toBe(Start.name);
|
|
111
|
+
|
|
112
|
+
const records = await readJournal(client);
|
|
113
|
+
expect(records.some((record) => record.id === written.id)).toBe(true);
|
|
114
|
+
} finally {
|
|
115
|
+
await client?.close().catch(() => {});
|
|
116
|
+
await first?.kill().catch(() => {});
|
|
117
|
+
await refused?.kill().catch(() => {});
|
|
118
|
+
await second?.kill().catch(() => {});
|
|
119
|
+
await home.cleanup();
|
|
120
|
+
}
|
|
121
|
+
}, 30_000);
|
|
122
|
+
|
|
123
|
+
test("one writer: a stale holder that writes learns it lost the journal, writes nothing and stops", async () => {
|
|
124
|
+
const home = await tempHome();
|
|
125
|
+
const homePath = home.path;
|
|
126
|
+
let stale: ReturnType<typeof spawnHost> | undefined;
|
|
127
|
+
let taker: ReturnType<typeof spawnHost> | undefined;
|
|
128
|
+
let client: Awaited<ReturnType<typeof connect>> | undefined;
|
|
129
|
+
|
|
130
|
+
try {
|
|
131
|
+
// 1. The first host serves the home, and parks inside a step: the gate is
|
|
132
|
+
// not open, so the invocation waits in the middle of the step.
|
|
133
|
+
stale = spawnHost(HOST_SCRIPT, homePath, ["block", "single-writer"]);
|
|
134
|
+
await stale.ready();
|
|
135
|
+
|
|
136
|
+
client = await connectAsOwner(homePath);
|
|
137
|
+
await client.emit(Start, { n: 1 });
|
|
138
|
+
await eventually(
|
|
139
|
+
() => counter(homePath, "block").count() === 1,
|
|
140
|
+
10_000,
|
|
141
|
+
"the first host should be parked inside the step",
|
|
142
|
+
);
|
|
143
|
+
|
|
144
|
+
// Where the journal stands before the take-over. The step intent of the
|
|
145
|
+
// parked invocation is the last thing a handler wrote.
|
|
146
|
+
const before = await readJournal(await connectAsOwner(homePath));
|
|
147
|
+
const handlerRecords = before.filter((record) => writtenByHandler(record, "single-writer"));
|
|
148
|
+
expect(handlerRecords.length).toBeGreaterThan(0);
|
|
149
|
+
const position = highestSeq(handlerRecords);
|
|
150
|
+
|
|
151
|
+
// 2. The second host takes the journal over from the living holder. The
|
|
152
|
+
// take-over is explicit: the holder did not ask for it, so it learns it
|
|
153
|
+
// lost the journal from its next write.
|
|
154
|
+
//
|
|
155
|
+
// The socket of the stale host is still there, so `ready()` would be
|
|
156
|
+
// answered by the stale host. The take-over is therefore observed where
|
|
157
|
+
// it is real: in the journal.
|
|
158
|
+
taker = spawnHost(HOST_SCRIPT, homePath, ["takeover", "block2", "single-writer-taker"]);
|
|
159
|
+
|
|
160
|
+
// The take-over is done when the socket belongs to the new holder. The
|
|
161
|
+
// socket of the stale host is still there, so `ready()` would be answered
|
|
162
|
+
// by the stale host. The new holder is told apart by what it declares:
|
|
163
|
+
// its plugin has its own name, and the description says which host is
|
|
164
|
+
// listening.
|
|
165
|
+
await eventually(
|
|
166
|
+
async () => {
|
|
167
|
+
// The connection itself may fail while the take-over is under way (the file is removed
|
|
168
|
+
// and bound again), so it is part of the attempt.
|
|
169
|
+
let probe: Awaited<ReturnType<typeof connect>> | undefined;
|
|
170
|
+
try {
|
|
171
|
+
probe = await connectAsOwner(homePath);
|
|
172
|
+
const described = await probe.describe();
|
|
173
|
+
return described.actions.some((action) => action.name === "single-writer-taker-finish");
|
|
174
|
+
} catch {
|
|
175
|
+
return false;
|
|
176
|
+
} finally {
|
|
177
|
+
await probe?.close().catch(() => {});
|
|
178
|
+
}
|
|
179
|
+
},
|
|
180
|
+
15_000,
|
|
181
|
+
"the second host should be the one listening",
|
|
182
|
+
);
|
|
183
|
+
|
|
184
|
+
// 3. The gate opens: the parked invocation of the stale host tries to write
|
|
185
|
+
// the outcome of its step, and the journal refuses it. The stale host says
|
|
186
|
+
// so and stops — it is not the writer any more, and it writes nothing.
|
|
187
|
+
gate(homePath, "block").open();
|
|
188
|
+
await eventually(
|
|
189
|
+
() => stale?.output().includes("LostOwnership") === true,
|
|
190
|
+
15_000,
|
|
191
|
+
"the stale holder should learn it lost the journal",
|
|
192
|
+
);
|
|
193
|
+
expect(stale.output()).toContain("LostOwnership");
|
|
194
|
+
|
|
195
|
+
// The stale holder does not go on: its next step never ran, because the
|
|
196
|
+
// write of this one was refused.
|
|
197
|
+
expect(counter(homePath, "single-writer-finish").count()).toBe(0);
|
|
198
|
+
|
|
199
|
+
// 4. The journal of the new holder holds no record written by the stale
|
|
200
|
+
// holder after the take-over: the write of the old holder was refused,
|
|
201
|
+
// and nothing was written in its place.
|
|
202
|
+
//
|
|
203
|
+
// Only the records of the stale holder's own plugin are counted: the new
|
|
204
|
+
// holder has its own plugin and writes its own records.
|
|
205
|
+
const after = await readJournal(await connectAsOwner(homePath));
|
|
206
|
+
const staleRecords = after.filter((record) => writtenByHandler(record, "single-writer"));
|
|
207
|
+
const afterTakeover = staleRecords.filter((record) => record.seq > position);
|
|
208
|
+
expect(afterTakeover).toHaveLength(0);
|
|
209
|
+
// The records of the stale holder from before the take-over are still there.
|
|
210
|
+
expect(staleRecords.length).toBe(handlerRecords.length);
|
|
211
|
+
|
|
212
|
+
// 5. The new holder is the writer now: it writes, and it may write.
|
|
213
|
+
const takerClient = await connectAsOwner(homePath);
|
|
214
|
+
const probe = await takerClient.emit(Probe, { n: 2 });
|
|
215
|
+
expect(probe.type).toBe(Probe.name);
|
|
216
|
+
await takerClient.close().catch(() => {});
|
|
217
|
+
} finally {
|
|
218
|
+
await client?.close().catch(() => {});
|
|
219
|
+
await stale?.kill().catch(() => {});
|
|
220
|
+
await taker?.kill().catch(() => {});
|
|
221
|
+
await home.cleanup();
|
|
222
|
+
}
|
|
223
|
+
}, 30_000);
|
|
224
|
+
|
|
225
|
+
test("one writer: a stale host that is stopped leaves the socket of the new holder alone", async () => {
|
|
226
|
+
const home = await tempHome();
|
|
227
|
+
let stale: ReturnType<typeof spawnHost> | undefined;
|
|
228
|
+
let taker: ReturnType<typeof spawnHost> | undefined;
|
|
229
|
+
|
|
230
|
+
try {
|
|
231
|
+
stale = spawnHost(HOST_SCRIPT, home.path);
|
|
232
|
+
await stale.ready();
|
|
233
|
+
taker = spawnHost(HOST_SCRIPT, home.path, ["takeover", "block2", "single-writer-taker"]);
|
|
234
|
+
await eventually(
|
|
235
|
+
async () => {
|
|
236
|
+
let probe: Awaited<ReturnType<typeof connect>> | undefined;
|
|
237
|
+
try {
|
|
238
|
+
probe = await connectAsOwner(home.path);
|
|
239
|
+
return (await probe.describe()).actions.some((action) => action.name === "single-writer-taker-finish");
|
|
240
|
+
} catch {
|
|
241
|
+
return false;
|
|
242
|
+
} finally {
|
|
243
|
+
await probe?.close().catch(() => {});
|
|
244
|
+
}
|
|
245
|
+
},
|
|
246
|
+
15_000,
|
|
247
|
+
"the second host should be the one listening",
|
|
248
|
+
);
|
|
249
|
+
|
|
250
|
+
// The stale host shuts down cleanly. It did not write yet, so it has not noticed the
|
|
251
|
+
// take-over: stopping its listener must not remove the file the new holder listens on.
|
|
252
|
+
await stale.stop();
|
|
253
|
+
stale = undefined;
|
|
254
|
+
|
|
255
|
+
const client = await connectAsOwner(home.path);
|
|
256
|
+
try {
|
|
257
|
+
const described = await client.describe();
|
|
258
|
+
expect(described.actions.some((action) => action.name === "single-writer-taker-finish")).toBe(true);
|
|
259
|
+
} finally {
|
|
260
|
+
await client.close().catch(() => {});
|
|
261
|
+
}
|
|
262
|
+
} finally {
|
|
263
|
+
await stale?.kill().catch(() => {});
|
|
264
|
+
await taker?.kill().catch(() => {});
|
|
265
|
+
await home.cleanup();
|
|
266
|
+
}
|
|
267
|
+
}, 30_000);
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
// State.
|
|
2
|
+
//
|
|
3
|
+
// The scenario runs against a real host process (fixtures/state-host.ts):
|
|
4
|
+
//
|
|
5
|
+
// 1. spawn the host, connect a client, emit many Add and Bump records; the
|
|
6
|
+
// two handlers of the same counter wait on the gate "g2", so every one of
|
|
7
|
+
// their invocations is in flight at the same time
|
|
8
|
+
// 2. open the gate: all the invocations update the same counter at once, and
|
|
9
|
+
// every update must be in it — no update is lost
|
|
10
|
+
// 3. emit Fill { items } and Take { tag }; the handler of Take takes the
|
|
11
|
+
// batch and blocks in the gated step "g"
|
|
12
|
+
// 4. kill the host with SIGKILL — a real crash, after the take and before the
|
|
13
|
+
// next step
|
|
14
|
+
// 5. spawn the host again, open the gate "g": the invocation of Take is
|
|
15
|
+
// replayed and returns the same batch
|
|
16
|
+
// 6. the batch is taken once and not twice, and the invocation ends as done
|
|
17
|
+
//
|
|
18
|
+
// Only the public API is used: the entries of the packages and "@niwa-labs/testing".
|
|
19
|
+
|
|
20
|
+
import { expect, test } from "bun:test";
|
|
21
|
+
import { fileURLToPath } from "node:url";
|
|
22
|
+
|
|
23
|
+
import { connect } from "@niwa-labs/client";
|
|
24
|
+
import { seq } from "@niwa-labs/core";
|
|
25
|
+
import type { StoredRecord } from "@niwa-labs/core";
|
|
26
|
+
import { counter, eventually, gate, spawnHost, tempHome, connectAsOwner } from "@niwa-labs/testing";
|
|
27
|
+
import { z } from "zod";
|
|
28
|
+
|
|
29
|
+
import { Add, Bump, Fill, Seen, Take } from "./fixtures/state-host.ts";
|
|
30
|
+
|
|
31
|
+
const HOST_SCRIPT = fileURLToPath(new URL("./fixtures/state-host.ts", import.meta.url));
|
|
32
|
+
|
|
33
|
+
/** From the beginning of the journal. */
|
|
34
|
+
const FROM_START = seq(0);
|
|
35
|
+
|
|
36
|
+
/** The type the core writes for every change of an invocation's state. */
|
|
37
|
+
const STATE_TYPE = "niwa.invocation.state";
|
|
38
|
+
|
|
39
|
+
/** The type the core writes for every change of a plugin's state. */
|
|
40
|
+
const CHANGED_TYPE = "niwa.state.changed";
|
|
41
|
+
|
|
42
|
+
/** The data of a CORE.invocationState record: { invocation, state, reason? }. */
|
|
43
|
+
const InvocationState = z.object({
|
|
44
|
+
invocation: z.string(),
|
|
45
|
+
state: z.string(),
|
|
46
|
+
reason: z.string().optional(),
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
/** The data of a state change: { plugin, name, op, value }. */
|
|
50
|
+
const Change = z.object({
|
|
51
|
+
plugin: z.string(),
|
|
52
|
+
name: z.string(),
|
|
53
|
+
op: z.string(),
|
|
54
|
+
value: z.unknown(),
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* The current value of a state key, as the journal says it.
|
|
59
|
+
*
|
|
60
|
+
* Every change is a record, so the value a key holds is the value of the last
|
|
61
|
+
* change of that key. This is what the host's own projection holds too: the
|
|
62
|
+
* value is folded from the journal, never kept anywhere else.
|
|
63
|
+
*/
|
|
64
|
+
function currentOf(records: StoredRecord[], name: string): unknown {
|
|
65
|
+
let value: unknown;
|
|
66
|
+
for (const record of records) {
|
|
67
|
+
if (record.type !== CHANGED_TYPE) continue;
|
|
68
|
+
const parsed = Change.safeParse(record.data);
|
|
69
|
+
if (!parsed.success) continue;
|
|
70
|
+
if (parsed.data.name !== name) continue;
|
|
71
|
+
value = parsed.data.value;
|
|
72
|
+
}
|
|
73
|
+
return value;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** The changes of a state key, in the order the journal holds them. */
|
|
77
|
+
function changesOf(records: StoredRecord[], name: string): Array<z.infer<typeof Change>> {
|
|
78
|
+
const changes: Array<z.infer<typeof Change>> = [];
|
|
79
|
+
for (const record of records) {
|
|
80
|
+
if (record.type !== CHANGED_TYPE) continue;
|
|
81
|
+
const parsed = Change.safeParse(record.data);
|
|
82
|
+
if (!parsed.success) continue;
|
|
83
|
+
if (parsed.data.name !== name) continue;
|
|
84
|
+
changes.push(parsed.data);
|
|
85
|
+
}
|
|
86
|
+
return changes;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** True for a record "niwa.invocation.state" whose state is "done". */
|
|
90
|
+
function isDoneState(record: StoredRecord): boolean {
|
|
91
|
+
if (record.type !== STATE_TYPE) return false;
|
|
92
|
+
const parsed = InvocationState.safeParse(record.data);
|
|
93
|
+
return parsed.success && parsed.data.state === "done";
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** The batch a Seen record carries. */
|
|
97
|
+
function seenItems(record: StoredRecord): string[] | undefined {
|
|
98
|
+
if (record.type !== Seen.name) return undefined;
|
|
99
|
+
const parsed = Seen.schema.safeParse(record.data);
|
|
100
|
+
return parsed.success ? parsed.data.items : undefined;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
test("state: concurrent updates are not lost, and a batch taken before a crash is taken again after the restart", async () => {
|
|
104
|
+
const home = await tempHome();
|
|
105
|
+
const homePath = home.path;
|
|
106
|
+
const clients: Array<{ close(): Promise<void> }> = [];
|
|
107
|
+
let first: ReturnType<typeof spawnHost> | undefined;
|
|
108
|
+
let second: ReturnType<typeof spawnHost> | undefined;
|
|
109
|
+
|
|
110
|
+
try {
|
|
111
|
+
// 1. The first host, and many records that wait for the same gate.
|
|
112
|
+
first = spawnHost(HOST_SCRIPT, homePath);
|
|
113
|
+
await first.ready();
|
|
114
|
+
|
|
115
|
+
const client = await connectAsOwner(homePath);
|
|
116
|
+
const watcher = await connectAsOwner(homePath);
|
|
117
|
+
clients.push(client, watcher);
|
|
118
|
+
const seen: StoredRecord[] = [];
|
|
119
|
+
const watch = (async () => {
|
|
120
|
+
for await (const record of watcher.records.since()) {
|
|
121
|
+
seen.push(record);
|
|
122
|
+
}
|
|
123
|
+
})().catch(() => undefined);
|
|
124
|
+
|
|
125
|
+
// Two handlers, the same counter. Every handler waits on the gate first, so
|
|
126
|
+
// all of them are in flight at the same time when the gate opens.
|
|
127
|
+
const adds = 10;
|
|
128
|
+
const bumps = 10;
|
|
129
|
+
for (let i = 0; i < adds; i += 1) {
|
|
130
|
+
await client.emit(Add, { n: 1 });
|
|
131
|
+
}
|
|
132
|
+
for (let i = 0; i < bumps; i += 1) {
|
|
133
|
+
await client.emit(Bump, { n: 1 });
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Every invocation reached the gated step: they are all in flight.
|
|
137
|
+
await eventually(
|
|
138
|
+
() => counter(homePath, "tick").count() === adds + bumps,
|
|
139
|
+
15_000,
|
|
140
|
+
"every handler should have reached the gated step",
|
|
141
|
+
);
|
|
142
|
+
// No update has happened yet, so the journal holds no change of the counter:
|
|
143
|
+
// it still holds its initial value.
|
|
144
|
+
expect(changesOf(seen, "counter").length).toBe(0);
|
|
145
|
+
|
|
146
|
+
// 2. The gate opens: all the invocations update the same counter at once.
|
|
147
|
+
gate(homePath, "g2").open();
|
|
148
|
+
|
|
149
|
+
await eventually(
|
|
150
|
+
() => currentOf(seen, "counter") === adds + bumps,
|
|
151
|
+
15_000,
|
|
152
|
+
"the counter should hold every update",
|
|
153
|
+
);
|
|
154
|
+
|
|
155
|
+
// 2. No update is lost: the counter holds every one of them.
|
|
156
|
+
expect(currentOf(seen, "counter")).toBe(adds + bumps);
|
|
157
|
+
const counterChanges = changesOf(seen, "counter");
|
|
158
|
+
expect(counterChanges.length).toBe(adds + bumps);
|
|
159
|
+
expect(counterChanges.every((change) => change.plugin === "state")).toBe(true);
|
|
160
|
+
|
|
161
|
+
// 3. The batch is filled and then taken.
|
|
162
|
+
const items = ["a", "b", "c"];
|
|
163
|
+
await client.emit(Fill, { items });
|
|
164
|
+
await client.emit(Take, { tag: "take-1" });
|
|
165
|
+
|
|
166
|
+
// The handler of Take took the batch and is now blocked in the gated step.
|
|
167
|
+
await eventually(
|
|
168
|
+
() => counter(homePath, "hold").count() === 1,
|
|
169
|
+
15_000,
|
|
170
|
+
"the take handler should have reached the gated step",
|
|
171
|
+
);
|
|
172
|
+
|
|
173
|
+
// The batch is gone from the state: the take reset it in one change.
|
|
174
|
+
await eventually(
|
|
175
|
+
() => changesOf(seen, "batch").some((change) => change.op === "take"),
|
|
176
|
+
10_000,
|
|
177
|
+
"the batch should have been taken",
|
|
178
|
+
);
|
|
179
|
+
expect(currentOf(seen, "batch")).toEqual([]);
|
|
180
|
+
|
|
181
|
+
// 4. A real crash: SIGKILL, after the take and before the next step.
|
|
182
|
+
await first.kill();
|
|
183
|
+
first = undefined;
|
|
184
|
+
await client.close().catch(() => {});
|
|
185
|
+
await watcher.close().catch(() => {});
|
|
186
|
+
await watch;
|
|
187
|
+
|
|
188
|
+
// 5. The second host takes the unfinished invocation up again. The gate is
|
|
189
|
+
// opened right away, before waiting for the socket: the resumed step
|
|
190
|
+
// must not block the host from accepting connections.
|
|
191
|
+
second = spawnHost(HOST_SCRIPT, homePath);
|
|
192
|
+
gate(homePath, "g").open();
|
|
193
|
+
await second.ready();
|
|
194
|
+
|
|
195
|
+
const observer = await connectAsOwner(homePath);
|
|
196
|
+
clients.push(observer);
|
|
197
|
+
const history: StoredRecord[] = [];
|
|
198
|
+
const observe = (async () => {
|
|
199
|
+
for await (const record of observer.records.since()) {
|
|
200
|
+
history.push(record);
|
|
201
|
+
}
|
|
202
|
+
})().catch(() => undefined);
|
|
203
|
+
|
|
204
|
+
// The replayed invocation finished and wrote down the batch it took.
|
|
205
|
+
await eventually(
|
|
206
|
+
() => history.some((record) => seenItems(record) !== undefined),
|
|
207
|
+
15_000,
|
|
208
|
+
"the replayed take should have written down the batch",
|
|
209
|
+
);
|
|
210
|
+
|
|
211
|
+
// 6. The same batch is returned after the restart.
|
|
212
|
+
const taken = history.find((record) => seenItems(record) !== undefined);
|
|
213
|
+
expect(seenItems(taken as StoredRecord)).toEqual(items);
|
|
214
|
+
|
|
215
|
+
// The batch was taken once and not twice: the state holds one take.
|
|
216
|
+
const takes = changesOf(history, "batch").filter((change) => change.op === "take");
|
|
217
|
+
expect(takes.length).toBe(1);
|
|
218
|
+
|
|
219
|
+
// The batch was read once, and the replay returned that record, not today's empty batch.
|
|
220
|
+
const batchReads = history.filter((record) => {
|
|
221
|
+
if (record.type !== "niwa.state.read") return false;
|
|
222
|
+
const parsed = Change.safeParse(record.data);
|
|
223
|
+
return parsed.success && parsed.data.name === "batch" && parsed.data.op === "take";
|
|
224
|
+
});
|
|
225
|
+
expect(batchReads.length).toBe(1);
|
|
226
|
+
expect(batchReads[0]?.data).toMatchObject({ value: items });
|
|
227
|
+
|
|
228
|
+
// The invocation of Take ended as done.
|
|
229
|
+
await eventually(
|
|
230
|
+
() => history.some(isDoneState),
|
|
231
|
+
15_000,
|
|
232
|
+
"the invocation of the take should have ended as done",
|
|
233
|
+
);
|
|
234
|
+
|
|
235
|
+
// The counter is untouched by the restart: its value is a projection of the
|
|
236
|
+
// journal, and the journal still holds every update.
|
|
237
|
+
expect(currentOf(history, "counter")).toBe(adds + bumps);
|
|
238
|
+
} finally {
|
|
239
|
+
for (const client of clients) {
|
|
240
|
+
await client.close().catch(() => {});
|
|
241
|
+
}
|
|
242
|
+
await first?.kill().catch(() => {});
|
|
243
|
+
await second?.kill().catch(() => {});
|
|
244
|
+
await home.cleanup();
|
|
245
|
+
}
|
|
246
|
+
}, 60_000);
|