@botiverse/k-carrier 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. package/LICENSE +202 -0
  2. package/NOTICE +24 -0
  3. package/README.md +62 -0
  4. package/core/src/artifact/README.md +1 -0
  5. package/core/src/artifact/collectStream.ts +49 -0
  6. package/core/src/artifact/download.ts +300 -0
  7. package/core/src/artifact/errors.ts +24 -0
  8. package/core/src/artifact/partialPath.ts +18 -0
  9. package/core/src/artifact/source.ts +59 -0
  10. package/core/src/artifact/staticManifestSource.ts +185 -0
  11. package/core/src/artifact/swap.ts +21 -0
  12. package/core/src/clock.ts +22 -0
  13. package/core/src/converge/lifecycle.ts +114 -0
  14. package/core/src/converge/predicates.ts +57 -0
  15. package/core/src/converge/report.ts +38 -0
  16. package/core/src/createUpgrader.ts +300 -0
  17. package/core/src/drive/README.md +1 -0
  18. package/core/src/index.ts +17 -0
  19. package/core/src/invariants.ts +243 -0
  20. package/core/src/lifecycle/hostAdapter.ts +77 -0
  21. package/core/src/liveness.ts +80 -0
  22. package/core/src/platform/README.md +1 -0
  23. package/core/src/platform/index.ts +11 -0
  24. package/core/src/platform/ops.ts +53 -0
  25. package/core/src/platform/posix.ts +99 -0
  26. package/core/src/platform/windows.ts +73 -0
  27. package/core/src/policy/README.md +1 -0
  28. package/core/src/progress.ts +54 -0
  29. package/core/src/provenance/journal.ts +216 -0
  30. package/core/src/status/report.ts +76 -0
  31. package/core/src/status/reportStore.ts +67 -0
  32. package/core/src/txn/effects.ts +40 -0
  33. package/core/src/txn/engine.ts +275 -0
  34. package/core/src/txn/fileEffects.ts +129 -0
  35. package/core/src/txn/hostCallBudget.ts +28 -0
  36. package/core/src/txn/lock.ts +124 -0
  37. package/core/src/txn/state.ts +77 -0
  38. package/core/src/txn/transitions.ts +49 -0
  39. package/core/src/upgrade/outcome.ts +53 -0
  40. package/core/src/upgrade/retire.ts +44 -0
  41. package/core/src/upgrader.ts +124 -0
  42. package/docs/design-v1.md +246 -0
  43. package/docs/harness-design.md +170 -0
  44. package/docs/integration.md +326 -0
  45. package/docs/prior-art.md +150 -0
  46. package/docs/test-plan.md +100 -0
  47. package/harness/README.md +20 -0
  48. package/harness/src/adapter/probeChecks.ts +93 -0
  49. package/harness/src/adapter/releaseKnob.ts +65 -0
  50. package/harness/src/adapter/serviceChecks.ts +260 -0
  51. package/harness/src/artifact/checks.ts +232 -0
  52. package/harness/src/artifact/downloadHoles.ts +300 -0
  53. package/harness/src/artifact/downloadHolesMutations.ts +170 -0
  54. package/harness/src/artifact/m1.ts +204 -0
  55. package/harness/src/artifact/m1Resume.ts +141 -0
  56. package/harness/src/artifact/m3.ts +296 -0
  57. package/harness/src/artifact/m3Hosts.ts +241 -0
  58. package/harness/src/artifact/m4.ts +201 -0
  59. package/harness/src/artifact/m5.ts +239 -0
  60. package/harness/src/artifact/m6.ts +274 -0
  61. package/harness/src/artifact/m6Drive.ts +296 -0
  62. package/harness/src/artifact/m6DriveMutations.ts +70 -0
  63. package/harness/src/artifact/m6Mutations.ts +136 -0
  64. package/harness/src/artifact/m6Status.ts +224 -0
  65. package/harness/src/artifact-factory/checks.ts +68 -0
  66. package/harness/src/artifact-factory/demo.ts +70 -0
  67. package/harness/src/artifact-factory/factory.ts +134 -0
  68. package/harness/src/artifact-factory/run.ts +107 -0
  69. package/harness/src/blackbox.ts +284 -0
  70. package/harness/src/cli.ts +281 -0
  71. package/harness/src/crash/enumerate.ts +66 -0
  72. package/harness/src/crash/runner.ts +162 -0
  73. package/harness/src/examples/checks.ts +286 -0
  74. package/harness/src/fake-host/checks.ts +188 -0
  75. package/harness/src/fake-host/daemon-entry.ts +39 -0
  76. package/harness/src/fake-host/daemon.ts +159 -0
  77. package/harness/src/fake-host/inproc.ts +256 -0
  78. package/harness/src/fake-server/manifest.ts +68 -0
  79. package/harness/src/fake-server/server.ts +205 -0
  80. package/harness/src/fake-server/store.ts +208 -0
  81. package/harness/src/fixtures/minimal-adapter.ts +108 -0
  82. package/harness/src/fixtures/service-adapter.ts +187 -0
  83. package/harness/src/index.ts +11 -0
  84. package/harness/src/receipt.ts +83 -0
  85. package/harness/src/runner.ts +268 -0
  86. package/harness/src/scenario/processScan.ts +55 -0
  87. package/harness/src/scenario/sandbox.ts +259 -0
  88. package/harness/src/scenario/virtualClock.ts +56 -0
  89. package/harness/src/selfverify/selfVerify.ts +136 -0
  90. package/harness/src/sim/checks.ts +46 -0
  91. package/harness/src/sim/corpus.ts +36 -0
  92. package/harness/src/sim/effectRuntime.ts +113 -0
  93. package/harness/src/sim/error.ts +14 -0
  94. package/harness/src/sim/prng.ts +29 -0
  95. package/harness/src/sim/record.ts +76 -0
  96. package/harness/src/sim/run.ts +148 -0
  97. package/harness/src/sim/scheduler.ts +72 -0
  98. package/harness/src/sim/world.ts +284 -0
  99. package/harness/src/target.ts +86 -0
  100. package/harness/src/targetCheck.ts +47 -0
  101. package/harness/src/teeth/artifact.ts +215 -0
  102. package/harness/src/teeth/artifactDownload.ts +136 -0
  103. package/harness/src/teeth/artifactFactory.ts +41 -0
  104. package/harness/src/teeth/artifactM5.ts +68 -0
  105. package/harness/src/teeth/artifactM6.ts +264 -0
  106. package/harness/src/teeth/blackbox.ts +22 -0
  107. package/harness/src/teeth/checks.ts +278 -0
  108. package/harness/src/teeth/examples.ts +57 -0
  109. package/harness/src/teeth/fakeHost.ts +120 -0
  110. package/harness/src/teeth/index.ts +16 -0
  111. package/harness/src/teeth/m0.ts +135 -0
  112. package/harness/src/teeth/registry.ts +267 -0
  113. package/harness/src/teeth/selfCheck.ts +45 -0
  114. package/harness/src/teeth/sim.ts +61 -0
  115. package/package.json +43 -0
@@ -0,0 +1,67 @@
1
+ /**
2
+ * Report store — the LAST convergence report, persisted across restarts
3
+ * (M6, L5; a machine that DID converge must not report
4
+ * NOT_OBSERVED after a restart, AND an unreadable report must not be
5
+ * reported as "never observed" — the provenance three-state lesson applies
6
+ * here too: "I cannot read it" is not "it never happened").
7
+ *
8
+ * Three states, mechanically distinct:
9
+ * - genesis: no report file — the machine never promoted. NOT_OBSERVED.
10
+ * - observed: a valid, version-stamped report from the last promote.
11
+ * - unreadable: the file exists but cannot be read (EACCES/EISDIR/EIO) or
12
+ * is corrupt. The machine DID observe something; its record is hidden,
13
+ * not absent.
14
+ *
15
+ * persistReport is durable BEFORE visible (write tmp, fsync, then rename —
16
+ * same promise as the provenance append): a rename alone guarantees
17
+ * atomic visibility, not that the bytes survived a power cut.
18
+ */
19
+ import { promises as fs } from "node:fs";
20
+ import * as path from "node:path";
21
+ import type { ConvergenceReport } from "../converge/predicates.ts";
22
+ import { platformOpsFor } from "../platform/index.ts";
23
+
24
+ const REPORT_FILE = "report.json";
25
+
26
+ export type ReportRead =
27
+ | { kind: "genesis" }
28
+ | { kind: "observed"; report: ConvergenceReport }
29
+ | { kind: "unreadable"; reason: string };
30
+
31
+ export async function persistReport(stateDir: string, report: ConvergenceReport): Promise<void> {
32
+ await fs.mkdir(stateDir, { recursive: true });
33
+ const target = path.join(stateDir, REPORT_FILE);
34
+ const tmp = `${target}.tmp`;
35
+ const fh = await fs.open(tmp, "w");
36
+ try {
37
+ await fh.writeFile(JSON.stringify(report));
38
+ await fh.sync(); // durable BEFORE the rename makes it visible
39
+ } finally {
40
+ await fh.close();
41
+ }
42
+ await platformOpsFor().renamePath(tmp, target);
43
+ }
44
+
45
+ export async function loadLastReport(stateDir: string): Promise<ReportRead> {
46
+ let text: string;
47
+ try {
48
+ text = await fs.readFile(path.join(stateDir, REPORT_FILE), "utf8");
49
+ } catch (err) {
50
+ const code = (err as NodeJS.ErrnoException).code;
51
+ if (code === "ENOENT") return { kind: "genesis" };
52
+ return {
53
+ kind: "unreadable",
54
+ reason: `cannot read ${REPORT_FILE} (${code ?? (err as Error).message}); unreadable is NOT never-observed`,
55
+ };
56
+ }
57
+ try {
58
+ const parsed = JSON.parse(text) as ConvergenceReport;
59
+ if (typeof parsed.version !== "string") throw new Error("report carries no version stamp");
60
+ if (!parsed.binaryAtTarget || typeof parsed.binaryAtTarget.passed !== "boolean") {
61
+ throw new Error("report carries no binaryAtTarget");
62
+ }
63
+ return { kind: "observed", report: parsed };
64
+ } catch (err) {
65
+ return { kind: "unreadable", reason: `corrupt ${REPORT_FILE}: ${(err as Error).message}` };
66
+ }
67
+ }
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Effects — the txn engine's ONLY doorway to the world (sim-first
3
+ * constraint, harness-design §1.45).
4
+ *
5
+ * The engine is a pure state machine: no direct IO, time, or randomness.
6
+ * Everything durable goes through these interfaces. Production wires them
7
+ * to the real filesystem/platform adapters; the simulator wires them to an
8
+ * in-memory disk with seeded fault injection. This is a product-grade
9
+ * abstraction (platform adapters differ in fsync/swap semantics anyway),
10
+ * not a test seam.
11
+ *
12
+ * Durability contract: appendAndSync resolves only after the entry is
13
+ * durable (fsync'd). The engine writes intent BEFORE acting (WAL); recovery
14
+ * trusts the journal, never directory listings or guesses.
15
+ */
16
+ import type { JournalEntry } from "./state.ts";
17
+ import type { Slot } from "../lifecycle/hostAdapter.ts";
18
+
19
+ export interface JournalStore {
20
+ /** Append one entry durably. Resolves only after fsync. */
21
+ appendAndSync(entry: JournalEntry): Promise<void>;
22
+ /** All entries in append order (crash-truncated tails already dropped). */
23
+ readAll(): Promise<JournalEntry[]>;
24
+ }
25
+
26
+ export interface SlotStore {
27
+ /** Populate the experiment slot with a verified artifact (atomic: visible fully or not at all). */
28
+ stageExperiment(artifact: { version: string; bytesRef: string }): Promise<void>;
29
+ /** Which version each slot holds; null = empty. */
30
+ slotVersions(): Promise<Record<Slot, string | null>>;
31
+ /** experiment -> stable, old stable dropped. Atomic. */
32
+ promoteExperiment(): Promise<void>;
33
+ /** Drop the experiment slot. Atomic, idempotent. */
34
+ clearExperiment(): Promise<void>;
35
+ }
36
+
37
+ export interface TxnEffects {
38
+ journal: JournalStore;
39
+ slots: SlotStore;
40
+ }
@@ -0,0 +1,275 @@
1
+ /**
2
+ * Upgrade transaction engine — pure state machine over TxnEffects +
3
+ * HostAdapter (sim-first: zero direct IO/time/randomness; see effects.ts).
4
+ *
5
+ * Invariants (each is a harness tooth):
6
+ * - WAL: intent is journaled+fsync'd BEFORE the action it names.
7
+ * - Crash anywhere -> recover() lands on stable-running or completes the
8
+ * transition, decided by journal replay. Never dual-run, never bricked.
9
+ * - Promote only after the caller-supplied predicate evaluation passed.
10
+ * - Rollback is always available until promote; its reason is journaled.
11
+ */
12
+ import type { HostAdapter, ProcessEvidence } from "../lifecycle/hostAdapter.ts";
13
+ import type { TxnEffects } from "./effects.ts";
14
+ import type { JournalEntry, TxnPhase } from "./state.ts";
15
+ import { STATE_FORMAT_VERSION } from "./state.ts";
16
+ import type { Clock } from "../clock.ts";
17
+ import { HostCallTimeout, DEFAULT_HOST_CALL_BUDGET_MS } from "./hostCallBudget.ts";
18
+
19
+
20
+ export interface EngineDeps {
21
+ effects: TxnEffects;
22
+ host: HostAdapter;
23
+ clock: Clock;
24
+ /**
25
+ * Evaluate convergence for the freshly started experiment process.
26
+ * Returns null when converged; otherwise a human-readable refusal that
27
+ * becomes the rollback reason. (Full ConvergenceReport wiring lands with
28
+ * converge/; the engine only cares about pass/fail + reason.)
29
+ */
30
+ evaluatePredicates: (evidence: ProcessEvidence, targetVersion: string) => Promise<string | null>;
31
+ /**
32
+ * Budget for a single host call (quiesce/stop/start/probe).
33
+ *
34
+ * A host that HANGS is worse than one that crashes: nothing is journaled,
35
+ * the lock stays held by a process that is still alive (so stale-lock
36
+ * takeover does not apply), and every later attempt queues behind it
37
+ * forever. That is the "wedged half-way" failure, and it is the one an
38
+ * updater is least able to explain afterwards.
39
+ *
40
+ * Default 120s: long enough for a real service to drain sessions on a busy
41
+ * machine, short enough that a wedge is reported the same day it happens.
42
+ */
43
+ hostCallBudgetMs?: number;
44
+ }
45
+
46
+ export type EngineOutcome =
47
+ | { result: "promoted"; version: string }
48
+ | { result: "rolled-back"; reason: string }
49
+ | { result: "up-to-date" };
50
+
51
+ export class UpgradeEngine {
52
+ private readonly deps: EngineDeps;
53
+ private seq = 0;
54
+
55
+ constructor(deps: EngineDeps) {
56
+ this.deps = deps;
57
+ }
58
+
59
+ private async journal(intent: TxnPhase, detail: Record<string, string> = {}): Promise<void> {
60
+ const entry: JournalEntry = {
61
+ seq: this.seq++,
62
+ timestampMs: this.deps.clock.nowMs(),
63
+ intent,
64
+ detail: { ...detail, formatVersion: String(STATE_FORMAT_VERSION) },
65
+ };
66
+ await this.deps.effects.journal.appendAndSync(entry);
67
+ }
68
+
69
+ /**
70
+ * Replay the journal and finish or undo whatever was in flight.
71
+ * Must be called before upgrade() on every process start.
72
+ */
73
+ async recover(): Promise<void> {
74
+ const entries = await this.deps.effects.journal.readAll();
75
+ const last = entries.at(-1);
76
+ this.seq = (last?.seq ?? -1) + 1;
77
+ if (!last) return; // fresh install, stable running
78
+
79
+ const version = (await this.deps.effects.slots.slotVersions()).experiment;
80
+ switch (last.intent) {
81
+ case "idle":
82
+ return;
83
+ case "promoted":
84
+ // WAL redo: the intent is durable but its action may not have run
85
+ // (crash in the after-journal window). Completing it is idempotent —
86
+ // promoteExperiment on an already-promoted world is a no-op because
87
+ // the experiment slot is empty.
88
+ await this.deps.effects.slots.promoteExperiment();
89
+ // resume is part of the terminal action too. A crash after promote()
90
+ // but before resume() used to leave the service alive and its hosted
91
+ // work permanently parked; DST found this exact effect boundary.
92
+ await this.deps.host.resume();
93
+ return;
94
+ case "rolled-back":
95
+ // The terminal journal entry is WAL intent, not proof that the host
96
+ // restore ran. Redo the whole idempotent rollback action: a crash
97
+ // immediately after journaling `rolled-back` may still have the
98
+ // experiment process live and workloads parked.
99
+ await this.deps.host.stop("experiment");
100
+ await this.deps.host.start("stable");
101
+ await this.deps.host.resume();
102
+ await this.deps.effects.slots.clearExperiment();
103
+ return;
104
+ case "staged":
105
+ // Download completed but handover never started: cheap undo.
106
+ await this.rollbackTo("crash before handover", { skipHostRestart: true });
107
+ return;
108
+ case "handing-over":
109
+ case "running-experiment":
110
+ case "readback": {
111
+ // We may have died with the experiment (partially) live -- or the
112
+ // handover succeeded and killed the process that was driving it.
113
+ // EVIDENCE decides, never a "this restart was planned" flag: a flag is
114
+ // a claim the crash path could make just as easily.
115
+ if (version !== null && (await this.handoverSucceeded(last, version))) {
116
+ await this.finishHandover(version);
117
+ return;
118
+ }
119
+ // Fail closed: stop whatever runs, restore stable, resume workloads.
120
+ await this.rollbackTo(`crash during ${last.intent}` + (version ? ` (experiment ${version})` : ""));
121
+ return;
122
+ }
123
+ default: {
124
+ // Unknown intent => journal written by a NEWER core. Fail closed.
125
+ throw new Error(
126
+ `journal intent ${JSON.stringify(last.intent)} is not understood by this core (state format newer than binary); refusing to act`,
127
+ );
128
+ }
129
+ }
130
+ }
131
+
132
+ /** Full transactional upgrade to targetVersion via a pre-verified artifact. */
133
+ async upgrade(target: { version: string; bytesRef: string }): Promise<EngineOutcome> {
134
+ const versions = await this.deps.effects.slots.slotVersions();
135
+ if (versions.stable === target.version) return { result: "up-to-date" };
136
+
137
+ await this.journal("staged", { version: target.version });
138
+ await this.deps.effects.slots.stageExperiment(target);
139
+
140
+ // Who is handing over. Recorded BEFORE the handover because on some hosts
141
+ // this very process does not survive it: a service that is replaced by
142
+ // exiting (its supervisor respawns it from the new bytes) dies here on the
143
+ // SUCCESS path, and the successor -- not us -- finishes the transaction.
144
+ // Without this identity the successor cannot tell "the handover worked"
145
+ // from "we crashed mid-handover", because both leave the same journal.
146
+ const priorStartId = await this.probeStartId();
147
+ await this.journal("handing-over", {
148
+ version: target.version,
149
+ ...(priorStartId === null ? {} : { priorStartId }),
150
+ });
151
+ await this.withBudget("quiesce", () => this.deps.host.quiesce());
152
+ await this.withBudget("stop", () => this.deps.host.stop("stable"));
153
+ await this.withBudget("start", () => this.deps.host.start("experiment"));
154
+
155
+ await this.journal("running-experiment", { version: target.version });
156
+ let evidence: ProcessEvidence;
157
+ try {
158
+ evidence = await this.withBudget("healthProbe", () => this.deps.host.healthProbe());
159
+ } catch (err) {
160
+ // A probe that FAILED is information: the host answered "not healthy",
161
+ // so rolling back (which calls stop/start/resume) is sound. A probe that
162
+ // WEDGED is not information -- we do not know what the host is doing, and
163
+ // issuing more host calls to a host that never answered the last one is
164
+ // how a stuck upgrade becomes two live incarnations. Let it out; the
165
+ // journal keeps the in-flight phase and the next start resolves it from
166
+ // evidence.
167
+ if (err instanceof HostCallTimeout) throw err;
168
+ return this.rollbackOutcome(`experiment probe failed: ${(err as Error).message}`);
169
+ }
170
+
171
+ await this.journal("readback", { version: target.version });
172
+ const refusal = await this.deps.evaluatePredicates(evidence, target.version);
173
+ if (refusal !== null) {
174
+ return this.rollbackOutcome(`predicates refused: ${refusal}`);
175
+ }
176
+
177
+ await this.journal("promoted", { version: target.version });
178
+ await this.deps.effects.slots.promoteExperiment();
179
+ await this.deps.host.resume();
180
+ return { result: "promoted", version: target.version };
181
+ }
182
+
183
+ /**
184
+ * Did the handover actually happen? True only when a live process reports
185
+ * the EXPERIMENT version from a DIFFERENT incarnation than the one that
186
+ * journaled the handover. Same incarnation answering => nothing was
187
+ * replaced, whatever it claims about its version.
188
+ */
189
+ private async handoverSucceeded(last: JournalEntry, experimentVersion: string): Promise<boolean> {
190
+ const priorStartId = last.detail.priorStartId;
191
+ // No recorded identity (older journal, or a host whose probe was
192
+ // unavailable) => we cannot prove a successor exists => fail closed.
193
+ if (priorStartId === undefined) return false;
194
+ let evidence: ProcessEvidence;
195
+ try {
196
+ evidence = await this.withBudget("healthProbe", () => this.deps.host.healthProbe());
197
+ } catch {
198
+ return false; // nothing alive to vouch for the handover
199
+ }
200
+ return evidence.version === experimentVersion && evidence.startId !== priorStartId;
201
+ }
202
+
203
+ /** Finish a transaction whose handover outlived the process that began it. */
204
+ private async finishHandover(experimentVersion: string): Promise<void> {
205
+ let evidence: ProcessEvidence;
206
+ try {
207
+ evidence = await this.withBudget("healthProbe", () => this.deps.host.healthProbe());
208
+ } catch (err) {
209
+ if (err instanceof HostCallTimeout) throw err; // same rule: do not act on a wedged host
210
+ await this.rollbackTo(`successor probe failed: ${(err as Error).message}`);
211
+ return;
212
+ }
213
+ await this.journal("readback", { version: experimentVersion });
214
+ const refusal = await this.deps.evaluatePredicates(evidence, experimentVersion);
215
+ if (refusal !== null) {
216
+ await this.rollbackTo(`predicates refused after handover: ${refusal}`);
217
+ return;
218
+ }
219
+ await this.journal("promoted", { version: experimentVersion });
220
+ await this.deps.effects.slots.promoteExperiment();
221
+ await this.deps.host.resume();
222
+ }
223
+
224
+ /**
225
+ * Bound a host call. On expiry we do NOT pretend the call failed cleanly:
226
+ * a pending promise cannot be cancelled in JS, so the host may still be
227
+ * mid-operation. The transaction gives up instead of issuing further host
228
+ * calls it cannot reason about, and the journal records why. Recovery is
229
+ * the next process start, which probes for evidence and decides from facts.
230
+ */
231
+ private async withBudget<T>(label: string, call: () => Promise<T>): Promise<T> {
232
+ const budgetMs = this.deps.hostCallBudgetMs ?? DEFAULT_HOST_CALL_BUDGET_MS;
233
+ let cancel: (() => void) | undefined;
234
+ try {
235
+ return await Promise.race([
236
+ call(),
237
+ new Promise<never>((_resolve, reject) => {
238
+ cancel = this.deps.clock.after(budgetMs, () => {
239
+ reject(new HostCallTimeout(label, budgetMs));
240
+ });
241
+ }),
242
+ ]);
243
+ } finally {
244
+ cancel?.();
245
+ }
246
+ }
247
+
248
+ /** Best-effort identity of the live process; null when nothing answers. */
249
+ private async probeStartId(): Promise<string | null> {
250
+ try {
251
+ // Budgeted like every other host call: found by the wedged-host test,
252
+ // which hung HERE -- an unbounded probe before the handover is the same
253
+ // trap one line earlier.
254
+ return (await this.withBudget("healthProbe", () => this.deps.host.healthProbe())).startId;
255
+ } catch {
256
+ return null;
257
+ }
258
+ }
259
+
260
+ private async rollbackOutcome(reason: string): Promise<EngineOutcome> {
261
+ await this.rollbackTo(reason);
262
+ return { result: "rolled-back", reason };
263
+ }
264
+
265
+ private async rollbackTo(reason: string, opts: { skipHostRestart?: boolean } = {}): Promise<void> {
266
+ await this.journal("rolled-back", { reason });
267
+ if (!opts.skipHostRestart) {
268
+ // Stop whatever may be running (either slot), restore stable, resume.
269
+ await this.deps.host.stop("experiment");
270
+ await this.deps.host.start("stable");
271
+ await this.deps.host.resume();
272
+ }
273
+ await this.deps.effects.slots.clearExperiment();
274
+ }
275
+ }
@@ -0,0 +1,129 @@
1
+ /**
2
+ * Filesystem-backed TxnEffects: the real journal and slot store.
3
+ *
4
+ * Layout under stateDir:
5
+ * journal.jsonl append-only, one intent per line, fsync'd before return
6
+ * slots/stable/ the trusted version's bytes
7
+ * slots/experiment/ the version on trial
8
+ * slots/<slot>/VERSION the version string held by that slot
9
+ *
10
+ * Durability is the whole point of this file: appendAndSync must not resolve
11
+ * until the entry survives a power cut, or the WAL guarantee the engine
12
+ * depends on is a fiction.
13
+ */
14
+ import { promises as fs } from "node:fs";
15
+ import path from "node:path";
16
+ import type { JournalEntry } from "./state.ts";
17
+ import type { JournalStore, SlotStore, TxnEffects } from "./effects.ts";
18
+ import type { Slot } from "../lifecycle/hostAdapter.ts";
19
+ import { platformOpsFor } from "../platform/index.ts";
20
+
21
+ const JOURNAL = "journal.jsonl";
22
+ const VERSION_FILE = "VERSION";
23
+ const ARTIFACT_FILE = "artifact.bin";
24
+
25
+ function slotDir(stateDir: string, slot: Slot): string {
26
+ return path.join(stateDir, "slots", slot);
27
+ }
28
+
29
+ export function fileJournalStore(stateDir: string): JournalStore {
30
+ const journalPath = path.join(stateDir, JOURNAL);
31
+ return {
32
+ async appendAndSync(entry: JournalEntry): Promise<void> {
33
+ await fs.mkdir(stateDir, { recursive: true });
34
+ const fh = await fs.open(journalPath, "a");
35
+ try {
36
+ await fh.writeFile(`${JSON.stringify(entry)}\n`);
37
+ await fh.sync(); // durable BEFORE we return: the WAL promise
38
+ } finally {
39
+ await fh.close();
40
+ }
41
+ },
42
+ async readAll(): Promise<JournalEntry[]> {
43
+ let text: string;
44
+ try {
45
+ text = await fs.readFile(journalPath, "utf8");
46
+ } catch {
47
+ return [];
48
+ }
49
+ const out: JournalEntry[] = [];
50
+ for (const line of text.split("\n")) {
51
+ if (!line.trim()) continue;
52
+ try {
53
+ out.push(JSON.parse(line) as JournalEntry);
54
+ } catch {
55
+ // A torn final line is expected after a crash mid-append: the entry
56
+ // never completed, so it never happened. Earlier lines stand.
57
+ break;
58
+ }
59
+ }
60
+ return out;
61
+ },
62
+ };
63
+ }
64
+
65
+ export function fileSlotStore(stateDir: string): SlotStore {
66
+ async function readVersion(slot: Slot): Promise<string | null> {
67
+ try {
68
+ return (await fs.readFile(path.join(slotDir(stateDir, slot), VERSION_FILE), "utf8")).trim();
69
+ } catch {
70
+ return null;
71
+ }
72
+ }
73
+
74
+ return {
75
+ async stageExperiment(artifact: { version: string; bytesRef: string }): Promise<void> {
76
+ const dir = slotDir(stateDir, "experiment");
77
+ const staging = `${dir}.staging`;
78
+ await fs.rm(staging, { recursive: true, force: true });
79
+ await fs.mkdir(staging, { recursive: true });
80
+ // bytesRef is a path to the verified bytes the caller downloaded.
81
+ await fs.copyFile(artifact.bytesRef, path.join(staging, ARTIFACT_FILE));
82
+ await fs.writeFile(path.join(staging, VERSION_FILE), artifact.version);
83
+ // Publish the slot atomically: a half-written experiment must never be
84
+ // visible as a stageable slot.
85
+ await fs.rm(dir, { recursive: true, force: true });
86
+ await platformOpsFor().renamePath(staging, dir);
87
+ },
88
+
89
+ async slotVersions(): Promise<Record<Slot, string | null>> {
90
+ return { stable: await readVersion("stable"), experiment: await readVersion("experiment") };
91
+ },
92
+
93
+ async promoteExperiment(): Promise<void> {
94
+ const experiment = slotDir(stateDir, "experiment");
95
+ const stable = slotDir(stateDir, "stable");
96
+ if ((await readVersion("experiment")) === null) return; // idempotent redo
97
+ await fs.rm(`${stable}.old`, { recursive: true, force: true });
98
+ await platformOpsFor().renamePath(stable, `${stable}.old`).catch(() => {});
99
+ await platformOpsFor().renamePath(experiment, stable);
100
+ await fs.rm(`${stable}.old`, { recursive: true, force: true });
101
+ },
102
+
103
+ async clearExperiment(): Promise<void> {
104
+ await fs.rm(slotDir(stateDir, "experiment"), { recursive: true, force: true });
105
+ },
106
+ };
107
+ }
108
+
109
+ export function fileEffects(stateDir: string): TxnEffects {
110
+ return { journal: fileJournalStore(stateDir), slots: fileSlotStore(stateDir) };
111
+ }
112
+
113
+ /** Path of the executable a slot holds, for hosts that need to launch it. */
114
+ export function slotArtifactPath(stateDir: string, slot: Slot): string {
115
+ return path.join(slotDir(stateDir, slot), ARTIFACT_FILE);
116
+ }
117
+
118
+ /** Write verified bytes into the slot layout and mark them executable. */
119
+ export async function materializeArtifact(
120
+ stateDir: string,
121
+ bytes: Uint8Array,
122
+ ): Promise<string> {
123
+ const tmpDir = path.join(stateDir, "incoming");
124
+ await fs.mkdir(tmpDir, { recursive: true });
125
+ const target = path.join(tmpDir, ARTIFACT_FILE);
126
+ await platformOpsFor().swapExecutable(target, bytes);
127
+ await platformOpsFor().makeExecutable(target);
128
+ return target;
129
+ }
@@ -0,0 +1,28 @@
1
+ /**
2
+ * A bound on how long one host call may take.
3
+ *
4
+ * A host that HANGS is worse than one that crashes: nothing is journaled, the
5
+ * upgrade lock stays held by a process that is still ALIVE (so stale-lock
6
+ * takeover never fires), and every later attempt queues behind it forever.
7
+ * That is the "wedged half-way" failure, and it is the one an updater is least
8
+ * able to explain afterwards.
9
+ */
10
+
11
+ /**
12
+ * Default budget for a single host call. Long enough for a real service to
13
+ * drain sessions on a busy machine, short enough that a wedge is reported the
14
+ * same day it happens. Adopters override it via EngineDeps.hostCallBudgetMs.
15
+ */
16
+ export const DEFAULT_HOST_CALL_BUDGET_MS = 120_000;
17
+
18
+ /** A host call exceeded its budget: the host is wedged, not merely failing. */
19
+ export class HostCallTimeout extends Error {
20
+ readonly call: string;
21
+ readonly budgetMs: number;
22
+ constructor(call: string, budgetMs: number) {
23
+ super(`host ${call}() did not return within ${budgetMs}ms — treating the host as wedged`);
24
+ this.name = "HostCallTimeout";
25
+ this.call = call;
26
+ this.budgetMs = budgetMs;
27
+ }
28
+ }
@@ -0,0 +1,124 @@
1
+ /**
2
+ * Upgrade lock — one transaction at a time per service identity.
3
+ *
4
+ * Every entrypoint constructs the same Upgrader (daemon loop, `self upgrade`,
5
+ * install script, remote drive), so two of them CAN fire at once. Without a
6
+ * lock the semantics of "what happened" are undefined: two transactions would
7
+ * interleave over one journal and one pair of slots.
8
+ *
9
+ * Deliberately simple, matching the declared scope: the lock covers ONE
10
+ * service identity (one stateDir). It is not a cross-instance orchestrator —
11
+ * running several instances means several stateDirs, each with its own lock.
12
+ *
13
+ * Crash safety: the lock file records the holder's pid and start time, so a
14
+ * lock left behind by a killed process is detected as stale rather than
15
+ * wedging the install forever. "Stale" means the OS no longer knows that pid;
16
+ * we never time out a lock whose holder is demonstrably alive.
17
+ */
18
+ import { promises as fs } from "node:fs";
19
+ import path from "node:path";
20
+ import { platformOpsFor } from "../platform/index.ts";
21
+
22
+ /** Bounded retries: an unclaimable lock is a typed failure, never a hang. */
23
+ const MAX_ATTEMPTS = 50;
24
+
25
+ export class UpgradeLockError extends Error {
26
+ readonly code = "UPGRADE_IN_PROGRESS";
27
+
28
+ constructor(holderPid: number) {
29
+ super(
30
+ `[UPGRADE_IN_PROGRESS] another upgrade is running (pid ${holderPid}); ` +
31
+ `refusing to start a second transaction over the same state`,
32
+ );
33
+ this.name = "UpgradeLockError";
34
+ }
35
+ }
36
+
37
+ interface LockRecord {
38
+ pid: number;
39
+ acquiredAtMs: number;
40
+ }
41
+
42
+ export interface UpgradeLock {
43
+ release(): Promise<void>;
44
+ }
45
+
46
+ /**
47
+ * Acquire the single-transaction lock, or throw UpgradeLockError.
48
+ * `nowMs` is injected rather than read from Date so simulation stays
49
+ * deterministic (clock seam discipline).
50
+ */
51
+ export async function acquireUpgradeLock(stateDir: string, nowMs: number): Promise<UpgradeLock> {
52
+ const lockPath = path.join(stateDir, "upgrade.lock");
53
+ await fs.mkdir(stateDir, { recursive: true });
54
+ const ops = platformOpsFor();
55
+
56
+ const record: LockRecord = { pid: process.pid, acquiredAtMs: nowMs };
57
+ let attempts = 0;
58
+ for (;;) {
59
+ try {
60
+ // wx: fails if the file exists — the atomic "claim it" primitive.
61
+ const fh = await fs.open(lockPath, "wx");
62
+ try {
63
+ await fh.writeFile(JSON.stringify(record));
64
+ } finally {
65
+ await fh.close();
66
+ }
67
+ return {
68
+ async release() {
69
+ await fs.rm(lockPath, { force: true });
70
+ },
71
+ };
72
+ } catch (err) {
73
+ if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
74
+ attempts += 1;
75
+ if (attempts > MAX_ATTEMPTS) {
76
+ // Never spin forever: a lock we can neither claim nor clear is a
77
+ // typed failure, not a hang. (Hangs hide bugs; failures report them.)
78
+ throw new Error(
79
+ `[UPGRADE_LOCK_UNRESOLVABLE] could not acquire or clear ${lockPath} after ${MAX_ATTEMPTS} attempts`,
80
+ { cause: err },
81
+ );
82
+ }
83
+ const holder = await readHolder(lockPath);
84
+ if (holder === "vanished") continue; // gone between open and read: retry
85
+ // A non-positive pid is never a real holder — and must NEVER reach
86
+ // process.kill, where pid<=0 addresses process GROUPS or every process.
87
+ if (holder !== "unreadable" && holder.pid > 0 && ops.isProcessAlive(holder.pid)) {
88
+ throw new UpgradeLockError(holder.pid);
89
+ }
90
+ // Holder is gone: its transaction died mid-flight. Recovery (journal
91
+ // replay) will decide what to do with the state; clear the lock and
92
+ // take it. Removing a specific stale file is safe to race — whoever
93
+ // wins the next `wx` owns the lock.
94
+ await fs.rm(lockPath, { force: true });
95
+ }
96
+ }
97
+ }
98
+
99
+ /**
100
+ * "vanished" = the file disappeared; retrying the claim is correct.
101
+ * "unreadable" = present but not a lock record; no holder can be proven
102
+ * alive, so it must not block forever.
103
+ */
104
+ async function readHolder(lockPath: string): Promise<LockRecord | "vanished" | "unreadable"> {
105
+ let text: string;
106
+ try {
107
+ text = await fs.readFile(lockPath, "utf8");
108
+ } catch {
109
+ return "vanished";
110
+ }
111
+ try {
112
+ const parsed: unknown = JSON.parse(text);
113
+ if (
114
+ typeof parsed === "object" &&
115
+ parsed !== null &&
116
+ typeof (parsed as LockRecord).pid === "number"
117
+ ) {
118
+ return parsed as LockRecord;
119
+ }
120
+ return "unreadable";
121
+ } catch {
122
+ return "unreadable";
123
+ }
124
+ }