@botiverse/k-carrier 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +24 -0
- package/README.md +62 -0
- package/core/src/artifact/README.md +1 -0
- package/core/src/artifact/collectStream.ts +49 -0
- package/core/src/artifact/download.ts +300 -0
- package/core/src/artifact/errors.ts +24 -0
- package/core/src/artifact/partialPath.ts +18 -0
- package/core/src/artifact/source.ts +59 -0
- package/core/src/artifact/staticManifestSource.ts +185 -0
- package/core/src/artifact/swap.ts +21 -0
- package/core/src/clock.ts +22 -0
- package/core/src/converge/lifecycle.ts +114 -0
- package/core/src/converge/predicates.ts +57 -0
- package/core/src/converge/report.ts +38 -0
- package/core/src/createUpgrader.ts +300 -0
- package/core/src/drive/README.md +1 -0
- package/core/src/index.ts +17 -0
- package/core/src/invariants.ts +243 -0
- package/core/src/lifecycle/hostAdapter.ts +77 -0
- package/core/src/liveness.ts +80 -0
- package/core/src/platform/README.md +1 -0
- package/core/src/platform/index.ts +11 -0
- package/core/src/platform/ops.ts +53 -0
- package/core/src/platform/posix.ts +99 -0
- package/core/src/platform/windows.ts +73 -0
- package/core/src/policy/README.md +1 -0
- package/core/src/progress.ts +54 -0
- package/core/src/provenance/journal.ts +216 -0
- package/core/src/status/report.ts +76 -0
- package/core/src/status/reportStore.ts +67 -0
- package/core/src/txn/effects.ts +40 -0
- package/core/src/txn/engine.ts +275 -0
- package/core/src/txn/fileEffects.ts +129 -0
- package/core/src/txn/hostCallBudget.ts +28 -0
- package/core/src/txn/lock.ts +124 -0
- package/core/src/txn/state.ts +77 -0
- package/core/src/txn/transitions.ts +49 -0
- package/core/src/upgrade/outcome.ts +53 -0
- package/core/src/upgrade/retire.ts +44 -0
- package/core/src/upgrader.ts +124 -0
- package/docs/design-v1.md +246 -0
- package/docs/harness-design.md +170 -0
- package/docs/integration.md +326 -0
- package/docs/prior-art.md +150 -0
- package/docs/test-plan.md +100 -0
- package/harness/README.md +20 -0
- package/harness/src/adapter/probeChecks.ts +93 -0
- package/harness/src/adapter/releaseKnob.ts +65 -0
- package/harness/src/adapter/serviceChecks.ts +260 -0
- package/harness/src/artifact/checks.ts +232 -0
- package/harness/src/artifact/downloadHoles.ts +300 -0
- package/harness/src/artifact/downloadHolesMutations.ts +170 -0
- package/harness/src/artifact/m1.ts +204 -0
- package/harness/src/artifact/m1Resume.ts +141 -0
- package/harness/src/artifact/m3.ts +296 -0
- package/harness/src/artifact/m3Hosts.ts +241 -0
- package/harness/src/artifact/m4.ts +201 -0
- package/harness/src/artifact/m5.ts +239 -0
- package/harness/src/artifact/m6.ts +274 -0
- package/harness/src/artifact/m6Drive.ts +296 -0
- package/harness/src/artifact/m6DriveMutations.ts +70 -0
- package/harness/src/artifact/m6Mutations.ts +136 -0
- package/harness/src/artifact/m6Status.ts +224 -0
- package/harness/src/artifact-factory/checks.ts +68 -0
- package/harness/src/artifact-factory/demo.ts +70 -0
- package/harness/src/artifact-factory/factory.ts +134 -0
- package/harness/src/artifact-factory/run.ts +107 -0
- package/harness/src/blackbox.ts +284 -0
- package/harness/src/cli.ts +281 -0
- package/harness/src/crash/enumerate.ts +66 -0
- package/harness/src/crash/runner.ts +162 -0
- package/harness/src/examples/checks.ts +286 -0
- package/harness/src/fake-host/checks.ts +188 -0
- package/harness/src/fake-host/daemon-entry.ts +39 -0
- package/harness/src/fake-host/daemon.ts +159 -0
- package/harness/src/fake-host/inproc.ts +256 -0
- package/harness/src/fake-server/manifest.ts +68 -0
- package/harness/src/fake-server/server.ts +205 -0
- package/harness/src/fake-server/store.ts +208 -0
- package/harness/src/fixtures/minimal-adapter.ts +108 -0
- package/harness/src/fixtures/service-adapter.ts +187 -0
- package/harness/src/index.ts +11 -0
- package/harness/src/receipt.ts +83 -0
- package/harness/src/runner.ts +268 -0
- package/harness/src/scenario/processScan.ts +55 -0
- package/harness/src/scenario/sandbox.ts +259 -0
- package/harness/src/scenario/virtualClock.ts +56 -0
- package/harness/src/selfverify/selfVerify.ts +136 -0
- package/harness/src/sim/checks.ts +46 -0
- package/harness/src/sim/corpus.ts +36 -0
- package/harness/src/sim/effectRuntime.ts +113 -0
- package/harness/src/sim/error.ts +14 -0
- package/harness/src/sim/prng.ts +29 -0
- package/harness/src/sim/record.ts +76 -0
- package/harness/src/sim/run.ts +148 -0
- package/harness/src/sim/scheduler.ts +72 -0
- package/harness/src/sim/world.ts +284 -0
- package/harness/src/target.ts +86 -0
- package/harness/src/targetCheck.ts +47 -0
- package/harness/src/teeth/artifact.ts +215 -0
- package/harness/src/teeth/artifactDownload.ts +136 -0
- package/harness/src/teeth/artifactFactory.ts +41 -0
- package/harness/src/teeth/artifactM5.ts +68 -0
- package/harness/src/teeth/artifactM6.ts +264 -0
- package/harness/src/teeth/blackbox.ts +22 -0
- package/harness/src/teeth/checks.ts +278 -0
- package/harness/src/teeth/examples.ts +57 -0
- package/harness/src/teeth/fakeHost.ts +120 -0
- package/harness/src/teeth/index.ts +16 -0
- package/harness/src/teeth/m0.ts +135 -0
- package/harness/src/teeth/registry.ts +267 -0
- package/harness/src/teeth/selfCheck.ts +45 -0
- package/harness/src/teeth/sim.ts +61 -0
- package/package.json +43 -0
|
@@ -0,0 +1,259 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* scenario sandbox (harness-design §1.3) — one scenario, one sandbox.
|
|
3
|
+
*
|
|
4
|
+
* Each sandbox owns:
|
|
5
|
+
* - a fresh temp stateDir (mkdtemp) — install dir, stateDir, and the
|
|
6
|
+
* fake-server's release store all live INSIDE it, so teardown =
|
|
7
|
+
* delete the sandbox (harness-design §1.77 "沙箱边界即清场边界");
|
|
8
|
+
* - an independently allocated localhost port, unique among live
|
|
9
|
+
* sandboxes, so parallel scenarios never collide;
|
|
10
|
+
* - a marker file carrying the sandbox id, so the future fake-host
|
|
11
|
+
* daemon's teardown half can pgrep by marker ("按沙箱标记 pgrep 复核
|
|
12
|
+
* 零残留" — verify-dead; that half lands with the daemon, this is the
|
|
13
|
+
* skeleton it plugs into).
|
|
14
|
+
*
|
|
15
|
+
* teardown() currently deletes the sandbox tree; the process-tree
|
|
16
|
+
* verify-dead step is a documented no-op stub until fake-host daemon
|
|
17
|
+
* exists (the process-tree verify-dead half lands once the fake-host daemon is in place).
|
|
18
|
+
*/
|
|
19
|
+
import { promises as fs } from "node:fs";
|
|
20
|
+
import * as os from "node:os";
|
|
21
|
+
import * as path from "node:path";
|
|
22
|
+
import { createServer } from "node:net";
|
|
23
|
+
import { spawn } from "node:child_process";
|
|
24
|
+
import assert from "node:assert/strict";
|
|
25
|
+
import { processAlive } from "../fake-host/daemon.ts";
|
|
26
|
+
import { findPidsByMarkerToken } from "./processScan.ts";
|
|
27
|
+
import { type ToothContext } from "../teeth/registry.ts";
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The sandbox marker: processes spawned inside a sandbox carry the literal
|
|
31
|
+
* token `K_SANDBOX_MARKER=<sandbox id>` on TWO channels at once —
|
|
32
|
+
*
|
|
33
|
+
* env (`envMarker()`) observable via `ps e` on POSIX, even for
|
|
34
|
+
* orphans whose parent died;
|
|
35
|
+
* argv (`argvMarker()`) observable in the command line everywhere:
|
|
36
|
+
* `ps` shows argv on POSIX, and Windows exposes
|
|
37
|
+
* `Win32_Process.CommandLine` — its ONLY public
|
|
38
|
+
* out-of-band channel (other processes' env
|
|
39
|
+
* blocks are not readable without NT internals).
|
|
40
|
+
*
|
|
41
|
+
* One token, two carriers: the scanner greps for the same literal string on
|
|
42
|
+
* every platform, only the enumeration command forks (harness-design §1.77:
|
|
43
|
+
* "按沙箱标记 pgrep 复核零残留").
|
|
44
|
+
*/
|
|
45
|
+
export const MARKER_ENV = "K_SANDBOX_MARKER";
|
|
46
|
+
|
|
47
|
+
export interface Sandbox {
|
|
48
|
+
/** Unique temp dir (mkdtemp); the scenario's entire world. */
|
|
49
|
+
readonly dir: string;
|
|
50
|
+
/** Port reserved for this sandbox's fake-server. */
|
|
51
|
+
readonly port: number;
|
|
52
|
+
/** Sandbox marker file content (the pgrep-by-marker key). */
|
|
53
|
+
readonly id: string;
|
|
54
|
+
/** The env var a spawned process needs to be claimable by this sandbox. */
|
|
55
|
+
envMarker(): Record<string, string>;
|
|
56
|
+
/** The argv token making the marker command-line-visible (Windows' only
|
|
57
|
+
* public out-of-band channel; harmless trailing arg elsewhere). */
|
|
58
|
+
argvMarker(): string[];
|
|
59
|
+
/** Delete the sandbox tree; idempotent. */
|
|
60
|
+
teardown(): Promise<void>;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface SandboxOptions {
|
|
64
|
+
/** Prefix for the mkdtemp dir name. Default: "scenario". */
|
|
65
|
+
prefix?: string;
|
|
66
|
+
/** Parent dir for the sandbox. Default: os.tmpdir(). */
|
|
67
|
+
baseDir?: string;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Ports currently reserved by live sandboxes (distinctness guarantee). */
|
|
71
|
+
const reservedPorts = new Set<number>();
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Allocate a free localhost port, unique among live sandboxes.
|
|
75
|
+
* Binds 127.0.0.1:0, reads the OS-assigned port, closes, reserves it.
|
|
76
|
+
* (The probe-close-bind window is the standard tradeoff; the reservation
|
|
77
|
+
* set removes the practical parallel-collision case.)
|
|
78
|
+
*/
|
|
79
|
+
export async function allocatePort(): Promise<number> {
|
|
80
|
+
for (;;) {
|
|
81
|
+
const port = await probeFreePort();
|
|
82
|
+
if (reservedPorts.has(port)) continue; // recently handed out; avoid reuse
|
|
83
|
+
reservedPorts.add(port);
|
|
84
|
+
return port;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function probeFreePort(): Promise<number> {
|
|
89
|
+
return new Promise((resolve, reject) => {
|
|
90
|
+
const s = createServer();
|
|
91
|
+
s.once("error", reject);
|
|
92
|
+
s.listen(0, "127.0.0.1", () => {
|
|
93
|
+
const addr = s.address();
|
|
94
|
+
s.close(() => {
|
|
95
|
+
if (addr && typeof addr === "object") resolve(addr.port);
|
|
96
|
+
else reject(new Error("allocatePort: no port assigned"));
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export async function createSandbox(opts: SandboxOptions = {}): Promise<Sandbox> {
|
|
103
|
+
const baseDir = opts.baseDir ?? os.tmpdir();
|
|
104
|
+
const prefix = opts.prefix ?? "scenario";
|
|
105
|
+
const dir = await fs.mkdtemp(path.join(baseDir, `${SANDBOX_DIR_PREFIX}${prefix}-`));
|
|
106
|
+
const id = path.basename(dir);
|
|
107
|
+
const port = await allocatePort();
|
|
108
|
+
// Marker file: future fake-host daemon teardown pgrep's by this id to
|
|
109
|
+
// verify zero residual processes (harness-design §1.77).
|
|
110
|
+
await fs.writeFile(path.join(dir, ".k-sandbox-marker"), `${id}\n`);
|
|
111
|
+
|
|
112
|
+
let tornDown = false;
|
|
113
|
+
const teardown = async (): Promise<void> => {
|
|
114
|
+
if (tornDown) return;
|
|
115
|
+
await verifyProcessTreeDead(dir, id); // throws on residuals; dir kept as evidence
|
|
116
|
+
await fs.rm(dir, { recursive: true, force: true });
|
|
117
|
+
reservedPorts.delete(port);
|
|
118
|
+
tornDown = true;
|
|
119
|
+
};
|
|
120
|
+
|
|
121
|
+
return {
|
|
122
|
+
dir,
|
|
123
|
+
port,
|
|
124
|
+
id,
|
|
125
|
+
envMarker: () => ({ [MARKER_ENV]: id }),
|
|
126
|
+
argvMarker: () => [`${MARKER_ENV}=${id}`],
|
|
127
|
+
teardown,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Prefix every sandbox directory name carries (see createSandbox). */
|
|
132
|
+
const SANDBOX_DIR_PREFIX = "k-harness-";
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* The marker a process must carry to be found by this sandbox's teardown.
|
|
136
|
+
*
|
|
137
|
+
* Teeth often derive a NESTED context (e.g. `<sandbox>/respawn` per host
|
|
138
|
+
* shape). Taking basename() of that nested dir yields "respawn", which no
|
|
139
|
+
* teardown scan will ever match -- so the scan returns zero, the tooth reports
|
|
140
|
+
* a clean teardown, and the leaked process is still running. The zero means
|
|
141
|
+
* "the query matched nothing", not "nothing leaked". (Found 08-05 by looking
|
|
142
|
+
* for the surviving pid instead of trusting the count.)
|
|
143
|
+
*
|
|
144
|
+
* So the marker is always resolved back to the SANDBOX's own id, however deep
|
|
145
|
+
* the caller's context is nested.
|
|
146
|
+
*/
|
|
147
|
+
export function sandboxMarkerFor(dir: string): string {
|
|
148
|
+
for (let cur = path.resolve(dir); ; cur = path.dirname(cur)) {
|
|
149
|
+
const base = path.basename(cur);
|
|
150
|
+
if (base.startsWith(SANDBOX_DIR_PREFIX)) return base;
|
|
151
|
+
if (path.dirname(cur) === cur) return path.basename(path.resolve(dir));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export { findPidsByMarkerToken };
|
|
156
|
+
|
|
157
|
+
/** Typed teardown failure: residual processes survived the kill+verify. */
|
|
158
|
+
export class VerifyDeadError extends Error {
|
|
159
|
+
readonly code = "SANDBOX_VERIFY_DEAD";
|
|
160
|
+
readonly survivors: number[];
|
|
161
|
+
|
|
162
|
+
constructor(survivors: number[]) {
|
|
163
|
+
super(`SANDBOX_VERIFY_DEAD: ${survivors.length} process(es) still alive after teardown kill: ${survivors.join(", ")}`);
|
|
164
|
+
this.name = "VerifyDeadError";
|
|
165
|
+
this.survivors = survivors;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const KILL_GRACE_MS = 2000;
|
|
170
|
+
const POLL_MS = 10;
|
|
171
|
+
|
|
172
|
+
function sleep(ms: number): Promise<void> {
|
|
173
|
+
return new Promise((r) => {
|
|
174
|
+
setTimeout(r, ms);
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Process-tree verify-dead — the teardown half that proves "发了 kill"≠
|
|
180
|
+
* "死了" (zombie `__service` was a real production lesson): scan for every
|
|
181
|
+
* process carrying this sandbox's marker, SIGKILL them, then make the OS
|
|
182
|
+
* confirm each is gone. Any survivor raises a typed VerifyDeadError and
|
|
183
|
+
* the sandbox dir is kept as evidence (the teardown did NOT succeed).
|
|
184
|
+
*/
|
|
185
|
+
export async function verifyProcessTreeDead(_sandboxDir: string, markerId: string): Promise<void> {
|
|
186
|
+
const alive = findPidsByMarkerToken(MARKER_ENV, markerId);
|
|
187
|
+
if (alive.length === 0) return;
|
|
188
|
+
for (const pid of alive) {
|
|
189
|
+
try {
|
|
190
|
+
process.kill(pid, "SIGKILL");
|
|
191
|
+
} catch {
|
|
192
|
+
// already gone between the scan and the signal
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
const survivors: number[] = [];
|
|
196
|
+
const deadline = Date.now() + KILL_GRACE_MS;
|
|
197
|
+
for (const pid of alive) {
|
|
198
|
+
while (processAlive(pid)) {
|
|
199
|
+
if (Date.now() > deadline) {
|
|
200
|
+
survivors.push(pid);
|
|
201
|
+
break;
|
|
202
|
+
}
|
|
203
|
+
await sleep(POLL_MS);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
if (survivors.length > 0) throw new VerifyDeadError(survivors);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* The sandbox verify-dead tooth check (registered as
|
|
211
|
+
* scenario.sandbox-verify-dead): a process carrying the sandbox marker
|
|
212
|
+
* must be SIGKILLed and OS-confirmed gone by teardown ("发了 kill"≠"死
|
|
213
|
+
* 了"). `skipTeardownKill` simulates the must-red mutation (teardown's
|
|
214
|
+
* kill/verify is a no-op) for known-red driving.
|
|
215
|
+
*/
|
|
216
|
+
export async function checkSandboxVerifyDead(
|
|
217
|
+
ctx: ToothContext,
|
|
218
|
+
opts: { skipTeardownKill?: boolean } = {},
|
|
219
|
+
): Promise<void> {
|
|
220
|
+
const sb = await createSandbox({ prefix: "verify-dead", baseDir: ctx.sandboxDir });
|
|
221
|
+
// Marker on both channels: env (POSIX ps e) + argv (the command-line token
|
|
222
|
+
// Windows' CIM scan needs; node ignores args after the -e script).
|
|
223
|
+
const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", ...sb.argvMarker()], {
|
|
224
|
+
env: { ...process.env, ...sb.envMarker() },
|
|
225
|
+
stdio: "ignore", // no pipes: a failed assertion must not degrade into a hang
|
|
226
|
+
});
|
|
227
|
+
try {
|
|
228
|
+
await waitForMarker(sb.id);
|
|
229
|
+
if (opts.skipTeardownKill) {
|
|
230
|
+
// mutation: teardown's kill/verify never ran — the marker process is
|
|
231
|
+
// still alive, and the zero-residual assertion must go RED
|
|
232
|
+
assert.deepEqual(
|
|
233
|
+
findPidsByMarkerToken(MARKER_ENV, sb.id),
|
|
234
|
+
[],
|
|
235
|
+
"after teardown no marker processes may remain (kill step skipped => RED)",
|
|
236
|
+
);
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
await sb.teardown();
|
|
240
|
+
assert.ok(!processAlive(child.pid ?? -1), "teardown must kill the marker process");
|
|
241
|
+
assert.deepEqual(
|
|
242
|
+
findPidsByMarkerToken(MARKER_ENV, sb.id),
|
|
243
|
+
[],
|
|
244
|
+
"no marker processes may remain after teardown",
|
|
245
|
+
);
|
|
246
|
+
await assert.rejects(fs.access(sb.dir), "teardown must remove the sandbox dir");
|
|
247
|
+
} finally {
|
|
248
|
+
if (processAlive(child.pid ?? -1)) child.kill("SIGKILL");
|
|
249
|
+
await sb.teardown().catch(() => {});
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async function waitForMarker(markerId: string): Promise<void> {
|
|
254
|
+
const deadline = Date.now() + 2000;
|
|
255
|
+
while (findPidsByMarkerToken(MARKER_ENV, markerId).length === 0) {
|
|
256
|
+
if (Date.now() > deadline) throw new Error(`marker process ${markerId} never became visible`);
|
|
257
|
+
await sleep(20);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VirtualClock — deterministic Clock for scenarios (harness-design §1.3).
|
|
3
|
+
* Time moves only via advance(); timers fire in due-time order (FIFO among
|
|
4
|
+
* equals), and timers scheduled by fired callbacks within the advanced
|
|
5
|
+
* window fire in the same advance() call. No real sleeping anywhere.
|
|
6
|
+
*/
|
|
7
|
+
import type { Clock } from "../../../core/src/clock.ts";
|
|
8
|
+
|
|
9
|
+
interface PendingTimer {
|
|
10
|
+
dueMs: number;
|
|
11
|
+
seq: number;
|
|
12
|
+
fn: () => void;
|
|
13
|
+
cancelled: boolean;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export class VirtualClock implements Clock {
|
|
17
|
+
private currentMs = 0;
|
|
18
|
+
private seq = 0;
|
|
19
|
+
private timers: PendingTimer[] = [];
|
|
20
|
+
|
|
21
|
+
nowMs(): number {
|
|
22
|
+
return this.currentMs;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
after(ms: number, fn: () => void): () => void {
|
|
26
|
+
const t: PendingTimer = {
|
|
27
|
+
dueMs: this.currentMs + Math.max(0, ms),
|
|
28
|
+
seq: this.seq++,
|
|
29
|
+
fn,
|
|
30
|
+
cancelled: false,
|
|
31
|
+
};
|
|
32
|
+
this.timers.push(t);
|
|
33
|
+
return () => {
|
|
34
|
+
t.cancelled = true;
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Advance virtual time, firing due timers in (dueMs, seq) order. */
|
|
39
|
+
advance(ms: number): void {
|
|
40
|
+
const target = this.currentMs + Math.max(0, ms);
|
|
41
|
+
for (;;) {
|
|
42
|
+
const due = this.timers
|
|
43
|
+
.filter((t) => !t.cancelled && t.dueMs <= target)
|
|
44
|
+
.toSorted((a, b) => a.dueMs - b.dueMs || a.seq - b.seq)[0];
|
|
45
|
+
if (!due) break;
|
|
46
|
+
this.timers = this.timers.filter((t) => t !== due);
|
|
47
|
+
this.currentMs = due.dueMs;
|
|
48
|
+
due.fn(); // may schedule more timers within the window
|
|
49
|
+
}
|
|
50
|
+
this.currentMs = target;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
pendingCount(): number {
|
|
54
|
+
return this.timers.filter((t) => !t.cancelled).length;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Harness self-verification — M0's exit gate (harness-design §1.6).
|
|
3
|
+
*
|
|
4
|
+
* The harness must prove it can judge before it is allowed to judge anything.
|
|
5
|
+
* Three samples, all required:
|
|
6
|
+
*
|
|
7
|
+
* known-green a correct world must pass (no false alarms)
|
|
8
|
+
* known-red a seeded defect must be caught (it detects at all)
|
|
9
|
+
* adversarial a world that satisfies the CHECKS while violating the
|
|
10
|
+
* real oracle must STILL be caught (it detects what the
|
|
11
|
+
* author did not think to check)
|
|
12
|
+
*
|
|
13
|
+
* The third is the one that matters. Anyone can write a checker that passes
|
|
14
|
+
* its own examples; the question is whether it survives something built
|
|
15
|
+
* specifically to slip through it. Escapes found in the wild get added here
|
|
16
|
+
* permanently, so the suite only ever gets harder.
|
|
17
|
+
*/
|
|
18
|
+
import {
|
|
19
|
+
checkInvariants,
|
|
20
|
+
workloadPreserved,
|
|
21
|
+
type Invariant,
|
|
22
|
+
type WorldSnapshot,
|
|
23
|
+
} from "../../../core/src/invariants.ts";
|
|
24
|
+
|
|
25
|
+
export type SampleVerdict = "caught" | "passed";
|
|
26
|
+
|
|
27
|
+
export interface SelfVerifyOutcome {
|
|
28
|
+
knownGreen: SampleVerdict;
|
|
29
|
+
knownRed: SampleVerdict;
|
|
30
|
+
adversarial: SampleVerdict;
|
|
31
|
+
/** True only when all three behaved as required. */
|
|
32
|
+
qualified: boolean;
|
|
33
|
+
detail: string[];
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function healthyWorld(): WorldSnapshot {
|
|
37
|
+
return {
|
|
38
|
+
phase: "promoted",
|
|
39
|
+
slots: { stable: "2.0.0", experiment: null },
|
|
40
|
+
liveProcesses: [{ slot: "stable", pid: 7, startId: "s-7", version: "2.0.0" }],
|
|
41
|
+
journalIntents: ["staged", "handing-over", "running-experiment", "readback", "promoted"],
|
|
42
|
+
workloadDigest: "ledger:42:abc",
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** A seeded defect the checks are explicitly designed to catch. */
|
|
47
|
+
function defectiveWorld(): WorldSnapshot {
|
|
48
|
+
return {
|
|
49
|
+
...healthyWorld(),
|
|
50
|
+
// promote completed but the experiment slot was never cleared
|
|
51
|
+
slots: { stable: "2.0.0", experiment: "2.0.0" },
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* The adversarial sample: a world engineered to satisfy every structural
|
|
57
|
+
* check while violating the real oracle.
|
|
58
|
+
*
|
|
59
|
+
* Here: the upgrade reports success, invariants about slots/journal/phase all
|
|
60
|
+
* hold, and exactly one process is live — but that process is the OLD binary
|
|
61
|
+
* (the new version never actually took over). Every "shape" check passes;
|
|
62
|
+
* only comparing the live process against the version that was promoted
|
|
63
|
+
* exposes it. This is the live-PID false-success family.
|
|
64
|
+
*/
|
|
65
|
+
function adversarialWorld(): WorldSnapshot {
|
|
66
|
+
return {
|
|
67
|
+
phase: "promoted",
|
|
68
|
+
slots: { stable: "2.0.0", experiment: null },
|
|
69
|
+
// EVERY structural check passes: one live process, its version matches
|
|
70
|
+
// the stable slot, journal complete, no leftover experiment slot.
|
|
71
|
+
liveProcesses: [{ slot: "stable", pid: 7, startId: "s-BEFORE", version: "2.0.0" }],
|
|
72
|
+
journalIntents: ["staged", "handing-over", "running-experiment", "readback", "promoted"],
|
|
73
|
+
workloadDigest: "ledger:42:abc",
|
|
74
|
+
// ...but this is the SAME incarnation that was live before the upgrade:
|
|
75
|
+
// the service never restarted; it merely reports the new version string.
|
|
76
|
+
priorIncarnationStartId: "s-BEFORE",
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* The oracle the adversarial sample violates: a promote means the service was
|
|
82
|
+
* actually restarted onto the new bytes. A version STRING can be reported by
|
|
83
|
+
* the old incarnation; only a fresh incarnation identity proves the restart.
|
|
84
|
+
* Registered as a first-class invariant so this escape can never reopen.
|
|
85
|
+
*/
|
|
86
|
+
export const promotedVersionIsLive: Invariant = {
|
|
87
|
+
id: "k.promoted-runs-a-fresh-incarnation",
|
|
88
|
+
description: "after promote, the live incarnation is not the one that preceded the upgrade",
|
|
89
|
+
// Conditional: we can only trust incarnation identity if the host's probe
|
|
90
|
+
// reports from the live process rather than from a file or cache.
|
|
91
|
+
assumes: ["probe-from-live-process"],
|
|
92
|
+
check: (s) => {
|
|
93
|
+
if (s.phase !== "promoted" || s.priorIncarnationStartId === undefined) return null;
|
|
94
|
+
const stale = s.liveProcesses.filter((p) => p.startId === s.priorIncarnationStartId);
|
|
95
|
+
return stale.length > 0
|
|
96
|
+
? `promoted ${s.slots.stable} but the live incarnation (startId ${s.priorIncarnationStartId}) is the pre-upgrade one — it reports the new version without having restarted`
|
|
97
|
+
: null;
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
export function runSelfVerification(invariants: readonly Invariant[]): SelfVerifyOutcome {
|
|
102
|
+
const detail: string[] = [];
|
|
103
|
+
|
|
104
|
+
const greenViolations = checkInvariants(healthyWorld(), invariants);
|
|
105
|
+
const knownGreen: SampleVerdict = greenViolations.length === 0 ? "passed" : "caught";
|
|
106
|
+
if (knownGreen === "caught") {
|
|
107
|
+
detail.push(`known-green FALSE ALARM: ${greenViolations.map((v) => v.invariantId).join(", ")}`);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
const redViolations = checkInvariants(defectiveWorld(), invariants);
|
|
111
|
+
const knownRed: SampleVerdict = redViolations.length > 0 ? "caught" : "passed";
|
|
112
|
+
if (knownRed === "passed") detail.push("known-red ESCAPED: seeded defect went undetected");
|
|
113
|
+
|
|
114
|
+
const advViolations = checkInvariants(adversarialWorld(), invariants);
|
|
115
|
+
const adversarial: SampleVerdict = advViolations.length > 0 ? "caught" : "passed";
|
|
116
|
+
if (adversarial === "passed") {
|
|
117
|
+
detail.push(
|
|
118
|
+
"adversarial ESCAPED: a world that satisfies every structural check while running the OLD binary was judged healthy",
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// The workload oracle must also survive a faked-preservation attempt.
|
|
123
|
+
const fakedPreservation = workloadPreserved(healthyWorld(), {
|
|
124
|
+
...healthyWorld(),
|
|
125
|
+
workloadDigest: "ledger:0:reset",
|
|
126
|
+
});
|
|
127
|
+
if (fakedPreservation === null) detail.push("workload oracle ESCAPED: digest change went unnoticed");
|
|
128
|
+
|
|
129
|
+
const qualified =
|
|
130
|
+
knownGreen === "passed" &&
|
|
131
|
+
knownRed === "caught" &&
|
|
132
|
+
adversarial === "caught" &&
|
|
133
|
+
fakedPreservation !== null;
|
|
134
|
+
|
|
135
|
+
return { knownGreen, knownRed, adversarial, qualified, detail };
|
|
136
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { SMOKE_SEEDS } from "./corpus.ts";
|
|
3
|
+
import { replayBytes, runSimulation, runSimulationBatch } from "./run.ts";
|
|
4
|
+
|
|
5
|
+
export async function checkSeedReplayIdentical(): Promise<void> {
|
|
6
|
+
const seed = 0xdecafbad;
|
|
7
|
+
const first = await runSimulation(seed);
|
|
8
|
+
const second = await runSimulation(seed);
|
|
9
|
+
assert.equal(replayBytes(first), replayBytes(second), "the same seed must replay byte-for-byte");
|
|
10
|
+
|
|
11
|
+
const other = await runSimulation(seed + 1);
|
|
12
|
+
assert.notEqual(
|
|
13
|
+
first.transcriptSha256,
|
|
14
|
+
other.transcriptSha256,
|
|
15
|
+
"different seeds must control a different observable schedule",
|
|
16
|
+
);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function checkSmokeSeeds(): Promise<void> {
|
|
20
|
+
const batch = await runSimulationBatch(SMOKE_SEEDS);
|
|
21
|
+
const failures = batch.results
|
|
22
|
+
.filter((result) => result.status === "fail")
|
|
23
|
+
.map((result) => `seed ${result.seed}: ${result.failure} (${result.replay})`);
|
|
24
|
+
assert.deepEqual(failures, [], failures.join("\n"));
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export async function checkFaultSurfaceCovered(): Promise<void> {
|
|
28
|
+
const batch = await runSimulationBatch(SMOKE_SEEDS);
|
|
29
|
+
for (const decision of [
|
|
30
|
+
"delay",
|
|
31
|
+
"crash-before",
|
|
32
|
+
"crash-after",
|
|
33
|
+
"fail-before",
|
|
34
|
+
"partial-write",
|
|
35
|
+
"reorder-volatile",
|
|
36
|
+
] as const) {
|
|
37
|
+
assert.ok(
|
|
38
|
+
batch.coverage[decision] > 0,
|
|
39
|
+
`fixed smoke corpus never exercised ${decision}; add a seed before claiming that surface`,
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
assert.ok(
|
|
43
|
+
batch.results.some((result) => result.restarts > 0),
|
|
44
|
+
"fixed smoke corpus must exercise recovery, not only clean upgrades",
|
|
45
|
+
);
|
|
46
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/** Fixed, reviewable PR corpus. Nightly runs use a much larger range. */
|
|
2
|
+
export const SMOKE_SEEDS: readonly number[] = [
|
|
3
|
+
1,
|
|
4
|
+
2,
|
|
5
|
+
3,
|
|
6
|
+
5,
|
|
7
|
+
8,
|
|
8
|
+
13,
|
|
9
|
+
21,
|
|
10
|
+
34,
|
|
11
|
+
55,
|
|
12
|
+
89,
|
|
13
|
+
144,
|
|
14
|
+
233,
|
|
15
|
+
377,
|
|
16
|
+
610,
|
|
17
|
+
987,
|
|
18
|
+
1597,
|
|
19
|
+
2584,
|
|
20
|
+
4181,
|
|
21
|
+
6765,
|
|
22
|
+
10946,
|
|
23
|
+
0x12345678,
|
|
24
|
+
0x6a09e667,
|
|
25
|
+
0x9e3779b9,
|
|
26
|
+
0xffffffff,
|
|
27
|
+
];
|
|
28
|
+
|
|
29
|
+
export function sequentialSeeds(startSeed: number, count: number): number[] {
|
|
30
|
+
if (!Number.isSafeInteger(count) || count <= 0) {
|
|
31
|
+
throw new RangeError(`seed count must be a positive safe integer, got ${count}`);
|
|
32
|
+
}
|
|
33
|
+
if (count > 1_000_000) throw new RangeError(`seed count ${count} exceeds the 1000000 safety cap`);
|
|
34
|
+
const start = startSeed >>> 0;
|
|
35
|
+
return Array.from({ length: count }, (_unused, index) => (start + index) >>> 0);
|
|
36
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import type { Clock } from "../../../core/src/clock.ts";
|
|
2
|
+
import { SimulationError } from "./error.ts";
|
|
3
|
+
import {
|
|
4
|
+
FaultScheduler,
|
|
5
|
+
type EffectKind,
|
|
6
|
+
type FaultCoverage,
|
|
7
|
+
} from "./scheduler.ts";
|
|
8
|
+
|
|
9
|
+
interface Timer {
|
|
10
|
+
id: number;
|
|
11
|
+
dueMs: number;
|
|
12
|
+
fn: () => void;
|
|
13
|
+
canceled: boolean;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface VolatileFaultHooks {
|
|
17
|
+
partial?: () => void;
|
|
18
|
+
reorder?: () => void;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Seeded scheduling and virtual time, separated from the simulated machine. */
|
|
22
|
+
export class EffectRuntime {
|
|
23
|
+
readonly trace: string[] = [];
|
|
24
|
+
readonly clock: Clock;
|
|
25
|
+
private readonly scheduler: FaultScheduler;
|
|
26
|
+
private now = 1_000;
|
|
27
|
+
private nextTimerId = 1;
|
|
28
|
+
private readonly timers: Timer[] = [];
|
|
29
|
+
private effectNumber = 0;
|
|
30
|
+
private readonly assertInvariants: (name: string) => void;
|
|
31
|
+
|
|
32
|
+
constructor(seed: number, faults: boolean, assertInvariants: (name: string) => void) {
|
|
33
|
+
this.scheduler = new FaultScheduler(seed, faults);
|
|
34
|
+
this.assertInvariants = assertInvariants;
|
|
35
|
+
this.clock = {
|
|
36
|
+
nowMs: () => this.now,
|
|
37
|
+
after: (ms, fn) => {
|
|
38
|
+
const timer: Timer = {
|
|
39
|
+
id: this.nextTimerId++,
|
|
40
|
+
dueMs: this.now + ms,
|
|
41
|
+
fn,
|
|
42
|
+
canceled: false,
|
|
43
|
+
};
|
|
44
|
+
this.timers.push(timer);
|
|
45
|
+
return () => {
|
|
46
|
+
timer.canceled = true;
|
|
47
|
+
};
|
|
48
|
+
},
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
get coverage(): FaultCoverage {
|
|
53
|
+
return { ...this.scheduler.coverage };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async effect<T>(
|
|
57
|
+
name: string,
|
|
58
|
+
kind: EffectKind,
|
|
59
|
+
apply: () => T,
|
|
60
|
+
volatile: VolatileFaultHooks = {},
|
|
61
|
+
): Promise<T> {
|
|
62
|
+
const decision = this.scheduler.decide(kind);
|
|
63
|
+
this.effectNumber += 1;
|
|
64
|
+
this.trace.push(`${this.effectNumber}:${name}:${decision}`);
|
|
65
|
+
|
|
66
|
+
if (decision === "delay") this.advance(this.scheduler.delayMs());
|
|
67
|
+
if (decision === "partial-write") {
|
|
68
|
+
volatile.partial?.();
|
|
69
|
+
this.assertInvariants(name);
|
|
70
|
+
throw new SimulationError("crash", name, `SIM_CRASH_PARTIAL: ${name}`);
|
|
71
|
+
}
|
|
72
|
+
if (decision === "reorder-volatile") {
|
|
73
|
+
volatile.reorder?.();
|
|
74
|
+
this.assertInvariants(name);
|
|
75
|
+
throw new SimulationError(
|
|
76
|
+
"effect-failure",
|
|
77
|
+
name,
|
|
78
|
+
`SIM_EFFECT_FAIL: ${name}: volatile journal fragments reordered before fsync`,
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
if (decision === "crash-before") {
|
|
82
|
+
this.assertInvariants(name);
|
|
83
|
+
throw new SimulationError("crash", name, `SIM_CRASH_BEFORE: ${name}`);
|
|
84
|
+
}
|
|
85
|
+
if (decision === "fail-before") {
|
|
86
|
+
this.assertInvariants(name);
|
|
87
|
+
throw new SimulationError("effect-failure", name, `SIM_EFFECT_FAIL: ${name}: injected failure`);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const result = apply();
|
|
91
|
+
this.assertInvariants(name);
|
|
92
|
+
if (decision === "crash-after") {
|
|
93
|
+
throw new SimulationError("crash", name, `SIM_CRASH_AFTER: ${name}`);
|
|
94
|
+
}
|
|
95
|
+
return result;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
reboot(reason: string, volatileTail: string): void {
|
|
99
|
+
this.trace.push(`reboot:${reason}:drop-tail=${volatileTail}`);
|
|
100
|
+
for (const timer of this.timers) timer.canceled = true;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
private advance(ms: number): void {
|
|
104
|
+
this.now += ms;
|
|
105
|
+
const due = this.timers
|
|
106
|
+
.filter((timer) => !timer.canceled && timer.dueMs <= this.now)
|
|
107
|
+
.toSorted((left, right) => left.dueMs - right.dueMs || left.id - right.id);
|
|
108
|
+
for (const timer of due) {
|
|
109
|
+
timer.canceled = true;
|
|
110
|
+
timer.fn();
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export type SimulationErrorKind = "crash" | "effect-failure" | "invariant";
|
|
2
|
+
|
|
3
|
+
/** One typed failure family keeps the runner's recovery policy explicit. */
|
|
4
|
+
export class SimulationError extends Error {
|
|
5
|
+
readonly kind: SimulationErrorKind;
|
|
6
|
+
readonly effectName: string;
|
|
7
|
+
|
|
8
|
+
constructor(kind: SimulationErrorKind, effectName: string, message: string) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = "SimulationError";
|
|
11
|
+
this.kind = kind;
|
|
12
|
+
this.effectName = effectName;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Small, specified 32-bit PRNG for deterministic simulation.
|
|
3
|
+
*
|
|
4
|
+
* Mulberry32 is deliberately implemented here instead of depending on a
|
|
5
|
+
* package: the integer operations below are the replay format. A seed means
|
|
6
|
+
* the same fault schedule on every OS and every future invocation.
|
|
7
|
+
*/
|
|
8
|
+
export class SeededPrng {
|
|
9
|
+
private state: number;
|
|
10
|
+
|
|
11
|
+
constructor(seed: number) {
|
|
12
|
+
this.state = seed >>> 0;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
nextUint32(): number {
|
|
16
|
+
this.state = (this.state + 0x6d2b79f5) >>> 0;
|
|
17
|
+
let value = this.state;
|
|
18
|
+
value = Math.imul(value ^ (value >>> 15), value | 1);
|
|
19
|
+
value ^= value + Math.imul(value ^ (value >>> 7), value | 61);
|
|
20
|
+
return (value ^ (value >>> 14)) >>> 0;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
below(exclusiveUpperBound: number): number {
|
|
24
|
+
if (!Number.isSafeInteger(exclusiveUpperBound) || exclusiveUpperBound <= 0) {
|
|
25
|
+
throw new RangeError(`PRNG bound must be a positive safe integer, got ${exclusiveUpperBound}`);
|
|
26
|
+
}
|
|
27
|
+
return this.nextUint32() % exclusiveUpperBound;
|
|
28
|
+
}
|
|
29
|
+
}
|