@botiverse/k-carrier 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +24 -0
- package/README.md +62 -0
- package/core/src/artifact/README.md +1 -0
- package/core/src/artifact/collectStream.ts +49 -0
- package/core/src/artifact/download.ts +300 -0
- package/core/src/artifact/errors.ts +24 -0
- package/core/src/artifact/partialPath.ts +18 -0
- package/core/src/artifact/source.ts +59 -0
- package/core/src/artifact/staticManifestSource.ts +185 -0
- package/core/src/artifact/swap.ts +21 -0
- package/core/src/clock.ts +22 -0
- package/core/src/converge/lifecycle.ts +114 -0
- package/core/src/converge/predicates.ts +57 -0
- package/core/src/converge/report.ts +38 -0
- package/core/src/createUpgrader.ts +300 -0
- package/core/src/drive/README.md +1 -0
- package/core/src/index.ts +17 -0
- package/core/src/invariants.ts +243 -0
- package/core/src/lifecycle/hostAdapter.ts +77 -0
- package/core/src/liveness.ts +80 -0
- package/core/src/platform/README.md +1 -0
- package/core/src/platform/index.ts +11 -0
- package/core/src/platform/ops.ts +53 -0
- package/core/src/platform/posix.ts +99 -0
- package/core/src/platform/windows.ts +73 -0
- package/core/src/policy/README.md +1 -0
- package/core/src/progress.ts +54 -0
- package/core/src/provenance/journal.ts +216 -0
- package/core/src/status/report.ts +76 -0
- package/core/src/status/reportStore.ts +67 -0
- package/core/src/txn/effects.ts +40 -0
- package/core/src/txn/engine.ts +275 -0
- package/core/src/txn/fileEffects.ts +129 -0
- package/core/src/txn/hostCallBudget.ts +28 -0
- package/core/src/txn/lock.ts +124 -0
- package/core/src/txn/state.ts +77 -0
- package/core/src/txn/transitions.ts +49 -0
- package/core/src/upgrade/outcome.ts +53 -0
- package/core/src/upgrade/retire.ts +44 -0
- package/core/src/upgrader.ts +124 -0
- package/docs/design-v1.md +246 -0
- package/docs/harness-design.md +170 -0
- package/docs/integration.md +326 -0
- package/docs/prior-art.md +150 -0
- package/docs/test-plan.md +100 -0
- package/harness/README.md +20 -0
- package/harness/src/adapter/probeChecks.ts +93 -0
- package/harness/src/adapter/releaseKnob.ts +65 -0
- package/harness/src/adapter/serviceChecks.ts +260 -0
- package/harness/src/artifact/checks.ts +232 -0
- package/harness/src/artifact/downloadHoles.ts +300 -0
- package/harness/src/artifact/downloadHolesMutations.ts +170 -0
- package/harness/src/artifact/m1.ts +204 -0
- package/harness/src/artifact/m1Resume.ts +141 -0
- package/harness/src/artifact/m3.ts +296 -0
- package/harness/src/artifact/m3Hosts.ts +241 -0
- package/harness/src/artifact/m4.ts +201 -0
- package/harness/src/artifact/m5.ts +239 -0
- package/harness/src/artifact/m6.ts +274 -0
- package/harness/src/artifact/m6Drive.ts +296 -0
- package/harness/src/artifact/m6DriveMutations.ts +70 -0
- package/harness/src/artifact/m6Mutations.ts +136 -0
- package/harness/src/artifact/m6Status.ts +224 -0
- package/harness/src/artifact-factory/checks.ts +68 -0
- package/harness/src/artifact-factory/demo.ts +70 -0
- package/harness/src/artifact-factory/factory.ts +134 -0
- package/harness/src/artifact-factory/run.ts +107 -0
- package/harness/src/blackbox.ts +284 -0
- package/harness/src/cli.ts +281 -0
- package/harness/src/crash/enumerate.ts +66 -0
- package/harness/src/crash/runner.ts +162 -0
- package/harness/src/examples/checks.ts +286 -0
- package/harness/src/fake-host/checks.ts +188 -0
- package/harness/src/fake-host/daemon-entry.ts +39 -0
- package/harness/src/fake-host/daemon.ts +159 -0
- package/harness/src/fake-host/inproc.ts +256 -0
- package/harness/src/fake-server/manifest.ts +68 -0
- package/harness/src/fake-server/server.ts +205 -0
- package/harness/src/fake-server/store.ts +208 -0
- package/harness/src/fixtures/minimal-adapter.ts +108 -0
- package/harness/src/fixtures/service-adapter.ts +187 -0
- package/harness/src/index.ts +11 -0
- package/harness/src/receipt.ts +83 -0
- package/harness/src/runner.ts +268 -0
- package/harness/src/scenario/processScan.ts +55 -0
- package/harness/src/scenario/sandbox.ts +259 -0
- package/harness/src/scenario/virtualClock.ts +56 -0
- package/harness/src/selfverify/selfVerify.ts +136 -0
- package/harness/src/sim/checks.ts +46 -0
- package/harness/src/sim/corpus.ts +36 -0
- package/harness/src/sim/effectRuntime.ts +113 -0
- package/harness/src/sim/error.ts +14 -0
- package/harness/src/sim/prng.ts +29 -0
- package/harness/src/sim/record.ts +76 -0
- package/harness/src/sim/run.ts +148 -0
- package/harness/src/sim/scheduler.ts +72 -0
- package/harness/src/sim/world.ts +284 -0
- package/harness/src/target.ts +86 -0
- package/harness/src/targetCheck.ts +47 -0
- package/harness/src/teeth/artifact.ts +215 -0
- package/harness/src/teeth/artifactDownload.ts +136 -0
- package/harness/src/teeth/artifactFactory.ts +41 -0
- package/harness/src/teeth/artifactM5.ts +68 -0
- package/harness/src/teeth/artifactM6.ts +264 -0
- package/harness/src/teeth/blackbox.ts +22 -0
- package/harness/src/teeth/checks.ts +278 -0
- package/harness/src/teeth/examples.ts +57 -0
- package/harness/src/teeth/fakeHost.ts +120 -0
- package/harness/src/teeth/index.ts +16 -0
- package/harness/src/teeth/m0.ts +135 -0
- package/harness/src/teeth/registry.ts +267 -0
- package/harness/src/teeth/selfCheck.ts +45 -0
- package/harness/src/teeth/sim.ts +61 -0
- package/package.json +43 -0
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Report store — the LAST convergence report, persisted across restarts
|
|
3
|
+
* (M6, L5; a machine that DID converge must not report
|
|
4
|
+
* NOT_OBSERVED after a restart, AND an unreadable report must not be
|
|
5
|
+
* reported as "never observed" — the provenance three-state lesson applies
|
|
6
|
+
* here too: "I cannot read it" is not "it never happened").
|
|
7
|
+
*
|
|
8
|
+
* Three states, mechanically distinct:
|
|
9
|
+
* - genesis: no report file — the machine never promoted. NOT_OBSERVED.
|
|
10
|
+
* - observed: a valid, version-stamped report from the last promote.
|
|
11
|
+
* - unreadable: the file exists but cannot be read (EACCES/EISDIR/EIO) or
|
|
12
|
+
* is corrupt. The machine DID observe something; its record is hidden,
|
|
13
|
+
* not absent.
|
|
14
|
+
*
|
|
15
|
+
* persistReport is durable BEFORE visible (write tmp, fsync, then rename —
|
|
16
|
+
* same promise as the provenance append): a rename alone guarantees
|
|
17
|
+
* atomic visibility, not that the bytes survived a power cut.
|
|
18
|
+
*/
|
|
19
|
+
import { promises as fs } from "node:fs";
|
|
20
|
+
import * as path from "node:path";
|
|
21
|
+
import type { ConvergenceReport } from "../converge/predicates.ts";
|
|
22
|
+
import { platformOpsFor } from "../platform/index.ts";
|
|
23
|
+
|
|
24
|
+
const REPORT_FILE = "report.json";
|
|
25
|
+
|
|
26
|
+
export type ReportRead =
|
|
27
|
+
| { kind: "genesis" }
|
|
28
|
+
| { kind: "observed"; report: ConvergenceReport }
|
|
29
|
+
| { kind: "unreadable"; reason: string };
|
|
30
|
+
|
|
31
|
+
export async function persistReport(stateDir: string, report: ConvergenceReport): Promise<void> {
|
|
32
|
+
await fs.mkdir(stateDir, { recursive: true });
|
|
33
|
+
const target = path.join(stateDir, REPORT_FILE);
|
|
34
|
+
const tmp = `${target}.tmp`;
|
|
35
|
+
const fh = await fs.open(tmp, "w");
|
|
36
|
+
try {
|
|
37
|
+
await fh.writeFile(JSON.stringify(report));
|
|
38
|
+
await fh.sync(); // durable BEFORE the rename makes it visible
|
|
39
|
+
} finally {
|
|
40
|
+
await fh.close();
|
|
41
|
+
}
|
|
42
|
+
await platformOpsFor().renamePath(tmp, target);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export async function loadLastReport(stateDir: string): Promise<ReportRead> {
|
|
46
|
+
let text: string;
|
|
47
|
+
try {
|
|
48
|
+
text = await fs.readFile(path.join(stateDir, REPORT_FILE), "utf8");
|
|
49
|
+
} catch (err) {
|
|
50
|
+
const code = (err as NodeJS.ErrnoException).code;
|
|
51
|
+
if (code === "ENOENT") return { kind: "genesis" };
|
|
52
|
+
return {
|
|
53
|
+
kind: "unreadable",
|
|
54
|
+
reason: `cannot read ${REPORT_FILE} (${code ?? (err as Error).message}); unreadable is NOT never-observed`,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
try {
|
|
58
|
+
const parsed = JSON.parse(text) as ConvergenceReport;
|
|
59
|
+
if (typeof parsed.version !== "string") throw new Error("report carries no version stamp");
|
|
60
|
+
if (!parsed.binaryAtTarget || typeof parsed.binaryAtTarget.passed !== "boolean") {
|
|
61
|
+
throw new Error("report carries no binaryAtTarget");
|
|
62
|
+
}
|
|
63
|
+
return { kind: "observed", report: parsed };
|
|
64
|
+
} catch (err) {
|
|
65
|
+
return { kind: "unreadable", reason: `corrupt ${REPORT_FILE}: ${(err as Error).message}` };
|
|
66
|
+
}
|
|
67
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Effects — the txn engine's ONLY doorway to the world (sim-first
|
|
3
|
+
* constraint, harness-design §1.45).
|
|
4
|
+
*
|
|
5
|
+
* The engine is a pure state machine: no direct IO, time, or randomness.
|
|
6
|
+
* Everything durable goes through these interfaces. Production wires them
|
|
7
|
+
* to the real filesystem/platform adapters; the simulator wires them to an
|
|
8
|
+
* in-memory disk with seeded fault injection. This is a product-grade
|
|
9
|
+
* abstraction (platform adapters differ in fsync/swap semantics anyway),
|
|
10
|
+
* not a test seam.
|
|
11
|
+
*
|
|
12
|
+
* Durability contract: appendAndSync resolves only after the entry is
|
|
13
|
+
* durable (fsync'd). The engine writes intent BEFORE acting (WAL); recovery
|
|
14
|
+
* trusts the journal, never directory listings or guesses.
|
|
15
|
+
*/
|
|
16
|
+
import type { JournalEntry } from "./state.ts";
|
|
17
|
+
import type { Slot } from "../lifecycle/hostAdapter.ts";
|
|
18
|
+
|
|
19
|
+
export interface JournalStore {
|
|
20
|
+
/** Append one entry durably. Resolves only after fsync. */
|
|
21
|
+
appendAndSync(entry: JournalEntry): Promise<void>;
|
|
22
|
+
/** All entries in append order (crash-truncated tails already dropped). */
|
|
23
|
+
readAll(): Promise<JournalEntry[]>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface SlotStore {
|
|
27
|
+
/** Populate the experiment slot with a verified artifact (atomic: visible fully or not at all). */
|
|
28
|
+
stageExperiment(artifact: { version: string; bytesRef: string }): Promise<void>;
|
|
29
|
+
/** Which version each slot holds; null = empty. */
|
|
30
|
+
slotVersions(): Promise<Record<Slot, string | null>>;
|
|
31
|
+
/** experiment -> stable, old stable dropped. Atomic. */
|
|
32
|
+
promoteExperiment(): Promise<void>;
|
|
33
|
+
/** Drop the experiment slot. Atomic, idempotent. */
|
|
34
|
+
clearExperiment(): Promise<void>;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface TxnEffects {
|
|
38
|
+
journal: JournalStore;
|
|
39
|
+
slots: SlotStore;
|
|
40
|
+
}
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Upgrade transaction engine — pure state machine over TxnEffects +
|
|
3
|
+
* HostAdapter (sim-first: zero direct IO/time/randomness; see effects.ts).
|
|
4
|
+
*
|
|
5
|
+
* Invariants (each is a harness tooth):
|
|
6
|
+
* - WAL: intent is journaled+fsync'd BEFORE the action it names.
|
|
7
|
+
* - Crash anywhere -> recover() lands on stable-running or completes the
|
|
8
|
+
* transition, decided by journal replay. Never dual-run, never bricked.
|
|
9
|
+
* - Promote only after the caller-supplied predicate evaluation passed.
|
|
10
|
+
* - Rollback is always available until promote; its reason is journaled.
|
|
11
|
+
*/
|
|
12
|
+
import type { HostAdapter, ProcessEvidence } from "../lifecycle/hostAdapter.ts";
|
|
13
|
+
import type { TxnEffects } from "./effects.ts";
|
|
14
|
+
import type { JournalEntry, TxnPhase } from "./state.ts";
|
|
15
|
+
import { STATE_FORMAT_VERSION } from "./state.ts";
|
|
16
|
+
import type { Clock } from "../clock.ts";
|
|
17
|
+
import { HostCallTimeout, DEFAULT_HOST_CALL_BUDGET_MS } from "./hostCallBudget.ts";
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
export interface EngineDeps {
|
|
21
|
+
effects: TxnEffects;
|
|
22
|
+
host: HostAdapter;
|
|
23
|
+
clock: Clock;
|
|
24
|
+
/**
|
|
25
|
+
* Evaluate convergence for the freshly started experiment process.
|
|
26
|
+
* Returns null when converged; otherwise a human-readable refusal that
|
|
27
|
+
* becomes the rollback reason. (Full ConvergenceReport wiring lands with
|
|
28
|
+
* converge/; the engine only cares about pass/fail + reason.)
|
|
29
|
+
*/
|
|
30
|
+
evaluatePredicates: (evidence: ProcessEvidence, targetVersion: string) => Promise<string | null>;
|
|
31
|
+
/**
|
|
32
|
+
* Budget for a single host call (quiesce/stop/start/probe).
|
|
33
|
+
*
|
|
34
|
+
* A host that HANGS is worse than one that crashes: nothing is journaled,
|
|
35
|
+
* the lock stays held by a process that is still alive (so stale-lock
|
|
36
|
+
* takeover does not apply), and every later attempt queues behind it
|
|
37
|
+
* forever. That is the "wedged half-way" failure, and it is the one an
|
|
38
|
+
* updater is least able to explain afterwards.
|
|
39
|
+
*
|
|
40
|
+
* Default 120s: long enough for a real service to drain sessions on a busy
|
|
41
|
+
* machine, short enough that a wedge is reported the same day it happens.
|
|
42
|
+
*/
|
|
43
|
+
hostCallBudgetMs?: number;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export type EngineOutcome =
|
|
47
|
+
| { result: "promoted"; version: string }
|
|
48
|
+
| { result: "rolled-back"; reason: string }
|
|
49
|
+
| { result: "up-to-date" };
|
|
50
|
+
|
|
51
|
+
export class UpgradeEngine {
|
|
52
|
+
private readonly deps: EngineDeps;
|
|
53
|
+
private seq = 0;
|
|
54
|
+
|
|
55
|
+
constructor(deps: EngineDeps) {
|
|
56
|
+
this.deps = deps;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
private async journal(intent: TxnPhase, detail: Record<string, string> = {}): Promise<void> {
|
|
60
|
+
const entry: JournalEntry = {
|
|
61
|
+
seq: this.seq++,
|
|
62
|
+
timestampMs: this.deps.clock.nowMs(),
|
|
63
|
+
intent,
|
|
64
|
+
detail: { ...detail, formatVersion: String(STATE_FORMAT_VERSION) },
|
|
65
|
+
};
|
|
66
|
+
await this.deps.effects.journal.appendAndSync(entry);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Replay the journal and finish or undo whatever was in flight.
|
|
71
|
+
* Must be called before upgrade() on every process start.
|
|
72
|
+
*/
|
|
73
|
+
async recover(): Promise<void> {
|
|
74
|
+
const entries = await this.deps.effects.journal.readAll();
|
|
75
|
+
const last = entries.at(-1);
|
|
76
|
+
this.seq = (last?.seq ?? -1) + 1;
|
|
77
|
+
if (!last) return; // fresh install, stable running
|
|
78
|
+
|
|
79
|
+
const version = (await this.deps.effects.slots.slotVersions()).experiment;
|
|
80
|
+
switch (last.intent) {
|
|
81
|
+
case "idle":
|
|
82
|
+
return;
|
|
83
|
+
case "promoted":
|
|
84
|
+
// WAL redo: the intent is durable but its action may not have run
|
|
85
|
+
// (crash in the after-journal window). Completing it is idempotent —
|
|
86
|
+
// promoteExperiment on an already-promoted world is a no-op because
|
|
87
|
+
// the experiment slot is empty.
|
|
88
|
+
await this.deps.effects.slots.promoteExperiment();
|
|
89
|
+
// resume is part of the terminal action too. A crash after promote()
|
|
90
|
+
// but before resume() used to leave the service alive and its hosted
|
|
91
|
+
// work permanently parked; DST found this exact effect boundary.
|
|
92
|
+
await this.deps.host.resume();
|
|
93
|
+
return;
|
|
94
|
+
case "rolled-back":
|
|
95
|
+
// The terminal journal entry is WAL intent, not proof that the host
|
|
96
|
+
// restore ran. Redo the whole idempotent rollback action: a crash
|
|
97
|
+
// immediately after journaling `rolled-back` may still have the
|
|
98
|
+
// experiment process live and workloads parked.
|
|
99
|
+
await this.deps.host.stop("experiment");
|
|
100
|
+
await this.deps.host.start("stable");
|
|
101
|
+
await this.deps.host.resume();
|
|
102
|
+
await this.deps.effects.slots.clearExperiment();
|
|
103
|
+
return;
|
|
104
|
+
case "staged":
|
|
105
|
+
// Download completed but handover never started: cheap undo.
|
|
106
|
+
await this.rollbackTo("crash before handover", { skipHostRestart: true });
|
|
107
|
+
return;
|
|
108
|
+
case "handing-over":
|
|
109
|
+
case "running-experiment":
|
|
110
|
+
case "readback": {
|
|
111
|
+
// We may have died with the experiment (partially) live -- or the
|
|
112
|
+
// handover succeeded and killed the process that was driving it.
|
|
113
|
+
// EVIDENCE decides, never a "this restart was planned" flag: a flag is
|
|
114
|
+
// a claim the crash path could make just as easily.
|
|
115
|
+
if (version !== null && (await this.handoverSucceeded(last, version))) {
|
|
116
|
+
await this.finishHandover(version);
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
// Fail closed: stop whatever runs, restore stable, resume workloads.
|
|
120
|
+
await this.rollbackTo(`crash during ${last.intent}` + (version ? ` (experiment ${version})` : ""));
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
default: {
|
|
124
|
+
// Unknown intent => journal written by a NEWER core. Fail closed.
|
|
125
|
+
throw new Error(
|
|
126
|
+
`journal intent ${JSON.stringify(last.intent)} is not understood by this core (state format newer than binary); refusing to act`,
|
|
127
|
+
);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** Full transactional upgrade to targetVersion via a pre-verified artifact. */
|
|
133
|
+
async upgrade(target: { version: string; bytesRef: string }): Promise<EngineOutcome> {
|
|
134
|
+
const versions = await this.deps.effects.slots.slotVersions();
|
|
135
|
+
if (versions.stable === target.version) return { result: "up-to-date" };
|
|
136
|
+
|
|
137
|
+
await this.journal("staged", { version: target.version });
|
|
138
|
+
await this.deps.effects.slots.stageExperiment(target);
|
|
139
|
+
|
|
140
|
+
// Who is handing over. Recorded BEFORE the handover because on some hosts
|
|
141
|
+
// this very process does not survive it: a service that is replaced by
|
|
142
|
+
// exiting (its supervisor respawns it from the new bytes) dies here on the
|
|
143
|
+
// SUCCESS path, and the successor -- not us -- finishes the transaction.
|
|
144
|
+
// Without this identity the successor cannot tell "the handover worked"
|
|
145
|
+
// from "we crashed mid-handover", because both leave the same journal.
|
|
146
|
+
const priorStartId = await this.probeStartId();
|
|
147
|
+
await this.journal("handing-over", {
|
|
148
|
+
version: target.version,
|
|
149
|
+
...(priorStartId === null ? {} : { priorStartId }),
|
|
150
|
+
});
|
|
151
|
+
await this.withBudget("quiesce", () => this.deps.host.quiesce());
|
|
152
|
+
await this.withBudget("stop", () => this.deps.host.stop("stable"));
|
|
153
|
+
await this.withBudget("start", () => this.deps.host.start("experiment"));
|
|
154
|
+
|
|
155
|
+
await this.journal("running-experiment", { version: target.version });
|
|
156
|
+
let evidence: ProcessEvidence;
|
|
157
|
+
try {
|
|
158
|
+
evidence = await this.withBudget("healthProbe", () => this.deps.host.healthProbe());
|
|
159
|
+
} catch (err) {
|
|
160
|
+
// A probe that FAILED is information: the host answered "not healthy",
|
|
161
|
+
// so rolling back (which calls stop/start/resume) is sound. A probe that
|
|
162
|
+
// WEDGED is not information -- we do not know what the host is doing, and
|
|
163
|
+
// issuing more host calls to a host that never answered the last one is
|
|
164
|
+
// how a stuck upgrade becomes two live incarnations. Let it out; the
|
|
165
|
+
// journal keeps the in-flight phase and the next start resolves it from
|
|
166
|
+
// evidence.
|
|
167
|
+
if (err instanceof HostCallTimeout) throw err;
|
|
168
|
+
return this.rollbackOutcome(`experiment probe failed: ${(err as Error).message}`);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
await this.journal("readback", { version: target.version });
|
|
172
|
+
const refusal = await this.deps.evaluatePredicates(evidence, target.version);
|
|
173
|
+
if (refusal !== null) {
|
|
174
|
+
return this.rollbackOutcome(`predicates refused: ${refusal}`);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
await this.journal("promoted", { version: target.version });
|
|
178
|
+
await this.deps.effects.slots.promoteExperiment();
|
|
179
|
+
await this.deps.host.resume();
|
|
180
|
+
return { result: "promoted", version: target.version };
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/**
|
|
184
|
+
* Did the handover actually happen? True only when a live process reports
|
|
185
|
+
* the EXPERIMENT version from a DIFFERENT incarnation than the one that
|
|
186
|
+
* journaled the handover. Same incarnation answering => nothing was
|
|
187
|
+
* replaced, whatever it claims about its version.
|
|
188
|
+
*/
|
|
189
|
+
private async handoverSucceeded(last: JournalEntry, experimentVersion: string): Promise<boolean> {
|
|
190
|
+
const priorStartId = last.detail.priorStartId;
|
|
191
|
+
// No recorded identity (older journal, or a host whose probe was
|
|
192
|
+
// unavailable) => we cannot prove a successor exists => fail closed.
|
|
193
|
+
if (priorStartId === undefined) return false;
|
|
194
|
+
let evidence: ProcessEvidence;
|
|
195
|
+
try {
|
|
196
|
+
evidence = await this.withBudget("healthProbe", () => this.deps.host.healthProbe());
|
|
197
|
+
} catch {
|
|
198
|
+
return false; // nothing alive to vouch for the handover
|
|
199
|
+
}
|
|
200
|
+
return evidence.version === experimentVersion && evidence.startId !== priorStartId;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/** Finish a transaction whose handover outlived the process that began it. */
|
|
204
|
+
private async finishHandover(experimentVersion: string): Promise<void> {
|
|
205
|
+
let evidence: ProcessEvidence;
|
|
206
|
+
try {
|
|
207
|
+
evidence = await this.withBudget("healthProbe", () => this.deps.host.healthProbe());
|
|
208
|
+
} catch (err) {
|
|
209
|
+
if (err instanceof HostCallTimeout) throw err; // same rule: do not act on a wedged host
|
|
210
|
+
await this.rollbackTo(`successor probe failed: ${(err as Error).message}`);
|
|
211
|
+
return;
|
|
212
|
+
}
|
|
213
|
+
await this.journal("readback", { version: experimentVersion });
|
|
214
|
+
const refusal = await this.deps.evaluatePredicates(evidence, experimentVersion);
|
|
215
|
+
if (refusal !== null) {
|
|
216
|
+
await this.rollbackTo(`predicates refused after handover: ${refusal}`);
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
await this.journal("promoted", { version: experimentVersion });
|
|
220
|
+
await this.deps.effects.slots.promoteExperiment();
|
|
221
|
+
await this.deps.host.resume();
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Bound a host call. On expiry we do NOT pretend the call failed cleanly:
|
|
226
|
+
* a pending promise cannot be cancelled in JS, so the host may still be
|
|
227
|
+
* mid-operation. The transaction gives up instead of issuing further host
|
|
228
|
+
* calls it cannot reason about, and the journal records why. Recovery is
|
|
229
|
+
* the next process start, which probes for evidence and decides from facts.
|
|
230
|
+
*/
|
|
231
|
+
private async withBudget<T>(label: string, call: () => Promise<T>): Promise<T> {
|
|
232
|
+
const budgetMs = this.deps.hostCallBudgetMs ?? DEFAULT_HOST_CALL_BUDGET_MS;
|
|
233
|
+
let cancel: (() => void) | undefined;
|
|
234
|
+
try {
|
|
235
|
+
return await Promise.race([
|
|
236
|
+
call(),
|
|
237
|
+
new Promise<never>((_resolve, reject) => {
|
|
238
|
+
cancel = this.deps.clock.after(budgetMs, () => {
|
|
239
|
+
reject(new HostCallTimeout(label, budgetMs));
|
|
240
|
+
});
|
|
241
|
+
}),
|
|
242
|
+
]);
|
|
243
|
+
} finally {
|
|
244
|
+
cancel?.();
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Best-effort identity of the live process; null when nothing answers. */
|
|
249
|
+
private async probeStartId(): Promise<string | null> {
|
|
250
|
+
try {
|
|
251
|
+
// Budgeted like every other host call: found by the wedged-host test,
|
|
252
|
+
// which hung HERE -- an unbounded probe before the handover is the same
|
|
253
|
+
// trap one line earlier.
|
|
254
|
+
return (await this.withBudget("healthProbe", () => this.deps.host.healthProbe())).startId;
|
|
255
|
+
} catch {
|
|
256
|
+
return null;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
private async rollbackOutcome(reason: string): Promise<EngineOutcome> {
|
|
261
|
+
await this.rollbackTo(reason);
|
|
262
|
+
return { result: "rolled-back", reason };
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
private async rollbackTo(reason: string, opts: { skipHostRestart?: boolean } = {}): Promise<void> {
|
|
266
|
+
await this.journal("rolled-back", { reason });
|
|
267
|
+
if (!opts.skipHostRestart) {
|
|
268
|
+
// Stop whatever may be running (either slot), restore stable, resume.
|
|
269
|
+
await this.deps.host.stop("experiment");
|
|
270
|
+
await this.deps.host.start("stable");
|
|
271
|
+
await this.deps.host.resume();
|
|
272
|
+
}
|
|
273
|
+
await this.deps.effects.slots.clearExperiment();
|
|
274
|
+
}
|
|
275
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Filesystem-backed TxnEffects: the real journal and slot store.
|
|
3
|
+
*
|
|
4
|
+
* Layout under stateDir:
|
|
5
|
+
* journal.jsonl append-only, one intent per line, fsync'd before return
|
|
6
|
+
* slots/stable/ the trusted version's bytes
|
|
7
|
+
* slots/experiment/ the version on trial
|
|
8
|
+
* slots/<slot>/VERSION the version string held by that slot
|
|
9
|
+
*
|
|
10
|
+
* Durability is the whole point of this file: appendAndSync must not resolve
|
|
11
|
+
* until the entry survives a power cut, or the WAL guarantee the engine
|
|
12
|
+
* depends on is a fiction.
|
|
13
|
+
*/
|
|
14
|
+
import { promises as fs } from "node:fs";
|
|
15
|
+
import path from "node:path";
|
|
16
|
+
import type { JournalEntry } from "./state.ts";
|
|
17
|
+
import type { JournalStore, SlotStore, TxnEffects } from "./effects.ts";
|
|
18
|
+
import type { Slot } from "../lifecycle/hostAdapter.ts";
|
|
19
|
+
import { platformOpsFor } from "../platform/index.ts";
|
|
20
|
+
|
|
21
|
+
const JOURNAL = "journal.jsonl";
|
|
22
|
+
const VERSION_FILE = "VERSION";
|
|
23
|
+
const ARTIFACT_FILE = "artifact.bin";
|
|
24
|
+
|
|
25
|
+
function slotDir(stateDir: string, slot: Slot): string {
|
|
26
|
+
return path.join(stateDir, "slots", slot);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function fileJournalStore(stateDir: string): JournalStore {
|
|
30
|
+
const journalPath = path.join(stateDir, JOURNAL);
|
|
31
|
+
return {
|
|
32
|
+
async appendAndSync(entry: JournalEntry): Promise<void> {
|
|
33
|
+
await fs.mkdir(stateDir, { recursive: true });
|
|
34
|
+
const fh = await fs.open(journalPath, "a");
|
|
35
|
+
try {
|
|
36
|
+
await fh.writeFile(`${JSON.stringify(entry)}\n`);
|
|
37
|
+
await fh.sync(); // durable BEFORE we return: the WAL promise
|
|
38
|
+
} finally {
|
|
39
|
+
await fh.close();
|
|
40
|
+
}
|
|
41
|
+
},
|
|
42
|
+
async readAll(): Promise<JournalEntry[]> {
|
|
43
|
+
let text: string;
|
|
44
|
+
try {
|
|
45
|
+
text = await fs.readFile(journalPath, "utf8");
|
|
46
|
+
} catch {
|
|
47
|
+
return [];
|
|
48
|
+
}
|
|
49
|
+
const out: JournalEntry[] = [];
|
|
50
|
+
for (const line of text.split("\n")) {
|
|
51
|
+
if (!line.trim()) continue;
|
|
52
|
+
try {
|
|
53
|
+
out.push(JSON.parse(line) as JournalEntry);
|
|
54
|
+
} catch {
|
|
55
|
+
// A torn final line is expected after a crash mid-append: the entry
|
|
56
|
+
// never completed, so it never happened. Earlier lines stand.
|
|
57
|
+
break;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
return out;
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function fileSlotStore(stateDir: string): SlotStore {
|
|
66
|
+
async function readVersion(slot: Slot): Promise<string | null> {
|
|
67
|
+
try {
|
|
68
|
+
return (await fs.readFile(path.join(slotDir(stateDir, slot), VERSION_FILE), "utf8")).trim();
|
|
69
|
+
} catch {
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return {
|
|
75
|
+
async stageExperiment(artifact: { version: string; bytesRef: string }): Promise<void> {
|
|
76
|
+
const dir = slotDir(stateDir, "experiment");
|
|
77
|
+
const staging = `${dir}.staging`;
|
|
78
|
+
await fs.rm(staging, { recursive: true, force: true });
|
|
79
|
+
await fs.mkdir(staging, { recursive: true });
|
|
80
|
+
// bytesRef is a path to the verified bytes the caller downloaded.
|
|
81
|
+
await fs.copyFile(artifact.bytesRef, path.join(staging, ARTIFACT_FILE));
|
|
82
|
+
await fs.writeFile(path.join(staging, VERSION_FILE), artifact.version);
|
|
83
|
+
// Publish the slot atomically: a half-written experiment must never be
|
|
84
|
+
// visible as a stageable slot.
|
|
85
|
+
await fs.rm(dir, { recursive: true, force: true });
|
|
86
|
+
await platformOpsFor().renamePath(staging, dir);
|
|
87
|
+
},
|
|
88
|
+
|
|
89
|
+
async slotVersions(): Promise<Record<Slot, string | null>> {
|
|
90
|
+
return { stable: await readVersion("stable"), experiment: await readVersion("experiment") };
|
|
91
|
+
},
|
|
92
|
+
|
|
93
|
+
async promoteExperiment(): Promise<void> {
|
|
94
|
+
const experiment = slotDir(stateDir, "experiment");
|
|
95
|
+
const stable = slotDir(stateDir, "stable");
|
|
96
|
+
if ((await readVersion("experiment")) === null) return; // idempotent redo
|
|
97
|
+
await fs.rm(`${stable}.old`, { recursive: true, force: true });
|
|
98
|
+
await platformOpsFor().renamePath(stable, `${stable}.old`).catch(() => {});
|
|
99
|
+
await platformOpsFor().renamePath(experiment, stable);
|
|
100
|
+
await fs.rm(`${stable}.old`, { recursive: true, force: true });
|
|
101
|
+
},
|
|
102
|
+
|
|
103
|
+
async clearExperiment(): Promise<void> {
|
|
104
|
+
await fs.rm(slotDir(stateDir, "experiment"), { recursive: true, force: true });
|
|
105
|
+
},
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function fileEffects(stateDir: string): TxnEffects {
|
|
110
|
+
return { journal: fileJournalStore(stateDir), slots: fileSlotStore(stateDir) };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** Path of the executable a slot holds, for hosts that need to launch it. */
|
|
114
|
+
export function slotArtifactPath(stateDir: string, slot: Slot): string {
|
|
115
|
+
return path.join(slotDir(stateDir, slot), ARTIFACT_FILE);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Write verified bytes into the slot layout and mark them executable. */
|
|
119
|
+
export async function materializeArtifact(
|
|
120
|
+
stateDir: string,
|
|
121
|
+
bytes: Uint8Array,
|
|
122
|
+
): Promise<string> {
|
|
123
|
+
const tmpDir = path.join(stateDir, "incoming");
|
|
124
|
+
await fs.mkdir(tmpDir, { recursive: true });
|
|
125
|
+
const target = path.join(tmpDir, ARTIFACT_FILE);
|
|
126
|
+
await platformOpsFor().swapExecutable(target, bytes);
|
|
127
|
+
await platformOpsFor().makeExecutable(target);
|
|
128
|
+
return target;
|
|
129
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A bound on how long one host call may take.
|
|
3
|
+
*
|
|
4
|
+
* A host that HANGS is worse than one that crashes: nothing is journaled, the
|
|
5
|
+
* upgrade lock stays held by a process that is still ALIVE (so stale-lock
|
|
6
|
+
* takeover never fires), and every later attempt queues behind it forever.
|
|
7
|
+
* That is the "wedged half-way" failure, and it is the one an updater is least
|
|
8
|
+
* able to explain afterwards.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Default budget for a single host call. Long enough for a real service to
|
|
13
|
+
* drain sessions on a busy machine, short enough that a wedge is reported the
|
|
14
|
+
* same day it happens. Adopters override it via EngineDeps.hostCallBudgetMs.
|
|
15
|
+
*/
|
|
16
|
+
export const DEFAULT_HOST_CALL_BUDGET_MS = 120_000;
|
|
17
|
+
|
|
18
|
+
/** A host call exceeded its budget: the host is wedged, not merely failing. */
|
|
19
|
+
export class HostCallTimeout extends Error {
|
|
20
|
+
readonly call: string;
|
|
21
|
+
readonly budgetMs: number;
|
|
22
|
+
constructor(call: string, budgetMs: number) {
|
|
23
|
+
super(`host ${call}() did not return within ${budgetMs}ms — treating the host as wedged`);
|
|
24
|
+
this.name = "HostCallTimeout";
|
|
25
|
+
this.call = call;
|
|
26
|
+
this.budgetMs = budgetMs;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Upgrade lock — one transaction at a time per service identity.
|
|
3
|
+
*
|
|
4
|
+
* Every entrypoint constructs the same Upgrader (daemon loop, `self upgrade`,
|
|
5
|
+
* install script, remote drive), so two of them CAN fire at once. Without a
|
|
6
|
+
* lock the semantics of "what happened" are undefined: two transactions would
|
|
7
|
+
* interleave over one journal and one pair of slots.
|
|
8
|
+
*
|
|
9
|
+
* Deliberately simple, matching the declared scope: the lock covers ONE
|
|
10
|
+
* service identity (one stateDir). It is not a cross-instance orchestrator —
|
|
11
|
+
* running several instances means several stateDirs, each with its own lock.
|
|
12
|
+
*
|
|
13
|
+
* Crash safety: the lock file records the holder's pid and start time, so a
|
|
14
|
+
* lock left behind by a killed process is detected as stale rather than
|
|
15
|
+
* wedging the install forever. "Stale" means the OS no longer knows that pid;
|
|
16
|
+
* we never time out a lock whose holder is demonstrably alive.
|
|
17
|
+
*/
|
|
18
|
+
import { promises as fs } from "node:fs";
|
|
19
|
+
import path from "node:path";
|
|
20
|
+
import { platformOpsFor } from "../platform/index.ts";
|
|
21
|
+
|
|
22
|
+
/** Bounded retries: an unclaimable lock is a typed failure, never a hang. */
|
|
23
|
+
const MAX_ATTEMPTS = 50;
|
|
24
|
+
|
|
25
|
+
export class UpgradeLockError extends Error {
|
|
26
|
+
readonly code = "UPGRADE_IN_PROGRESS";
|
|
27
|
+
|
|
28
|
+
constructor(holderPid: number) {
|
|
29
|
+
super(
|
|
30
|
+
`[UPGRADE_IN_PROGRESS] another upgrade is running (pid ${holderPid}); ` +
|
|
31
|
+
`refusing to start a second transaction over the same state`,
|
|
32
|
+
);
|
|
33
|
+
this.name = "UpgradeLockError";
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
interface LockRecord {
|
|
38
|
+
pid: number;
|
|
39
|
+
acquiredAtMs: number;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface UpgradeLock {
|
|
43
|
+
release(): Promise<void>;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Acquire the single-transaction lock, or throw UpgradeLockError.
|
|
48
|
+
* `nowMs` is injected rather than read from Date so simulation stays
|
|
49
|
+
* deterministic (clock seam discipline).
|
|
50
|
+
*/
|
|
51
|
+
export async function acquireUpgradeLock(stateDir: string, nowMs: number): Promise<UpgradeLock> {
|
|
52
|
+
const lockPath = path.join(stateDir, "upgrade.lock");
|
|
53
|
+
await fs.mkdir(stateDir, { recursive: true });
|
|
54
|
+
const ops = platformOpsFor();
|
|
55
|
+
|
|
56
|
+
const record: LockRecord = { pid: process.pid, acquiredAtMs: nowMs };
|
|
57
|
+
let attempts = 0;
|
|
58
|
+
for (;;) {
|
|
59
|
+
try {
|
|
60
|
+
// wx: fails if the file exists — the atomic "claim it" primitive.
|
|
61
|
+
const fh = await fs.open(lockPath, "wx");
|
|
62
|
+
try {
|
|
63
|
+
await fh.writeFile(JSON.stringify(record));
|
|
64
|
+
} finally {
|
|
65
|
+
await fh.close();
|
|
66
|
+
}
|
|
67
|
+
return {
|
|
68
|
+
async release() {
|
|
69
|
+
await fs.rm(lockPath, { force: true });
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
} catch (err) {
|
|
73
|
+
if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
|
|
74
|
+
attempts += 1;
|
|
75
|
+
if (attempts > MAX_ATTEMPTS) {
|
|
76
|
+
// Never spin forever: a lock we can neither claim nor clear is a
|
|
77
|
+
// typed failure, not a hang. (Hangs hide bugs; failures report them.)
|
|
78
|
+
throw new Error(
|
|
79
|
+
`[UPGRADE_LOCK_UNRESOLVABLE] could not acquire or clear ${lockPath} after ${MAX_ATTEMPTS} attempts`,
|
|
80
|
+
{ cause: err },
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
const holder = await readHolder(lockPath);
|
|
84
|
+
if (holder === "vanished") continue; // gone between open and read: retry
|
|
85
|
+
// A non-positive pid is never a real holder — and must NEVER reach
|
|
86
|
+
// process.kill, where pid<=0 addresses process GROUPS or every process.
|
|
87
|
+
if (holder !== "unreadable" && holder.pid > 0 && ops.isProcessAlive(holder.pid)) {
|
|
88
|
+
throw new UpgradeLockError(holder.pid);
|
|
89
|
+
}
|
|
90
|
+
// Holder is gone: its transaction died mid-flight. Recovery (journal
|
|
91
|
+
// replay) will decide what to do with the state; clear the lock and
|
|
92
|
+
// take it. Removing a specific stale file is safe to race — whoever
|
|
93
|
+
// wins the next `wx` owns the lock.
|
|
94
|
+
await fs.rm(lockPath, { force: true });
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* "vanished" = the file disappeared; retrying the claim is correct.
|
|
101
|
+
* "unreadable" = present but not a lock record; no holder can be proven
|
|
102
|
+
* alive, so it must not block forever.
|
|
103
|
+
*/
|
|
104
|
+
async function readHolder(lockPath: string): Promise<LockRecord | "vanished" | "unreadable"> {
|
|
105
|
+
let text: string;
|
|
106
|
+
try {
|
|
107
|
+
text = await fs.readFile(lockPath, "utf8");
|
|
108
|
+
} catch {
|
|
109
|
+
return "vanished";
|
|
110
|
+
}
|
|
111
|
+
try {
|
|
112
|
+
const parsed: unknown = JSON.parse(text);
|
|
113
|
+
if (
|
|
114
|
+
typeof parsed === "object" &&
|
|
115
|
+
parsed !== null &&
|
|
116
|
+
typeof (parsed as LockRecord).pid === "number"
|
|
117
|
+
) {
|
|
118
|
+
return parsed as LockRecord;
|
|
119
|
+
}
|
|
120
|
+
return "unreadable";
|
|
121
|
+
} catch {
|
|
122
|
+
return "unreadable";
|
|
123
|
+
}
|
|
124
|
+
}
|