@botiverse/k-carrier 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +5 -2
- package/README.md +72 -26
- package/core/src/artifact/download.ts +23 -1
- package/core/src/artifact/gzip.ts +26 -0
- package/core/src/artifact/source.ts +2 -0
- package/core/src/{createUpgrader.ts → createRunner.ts} +15 -14
- package/core/src/index.ts +12 -3
- package/core/src/launcher/launch.ts +15 -0
- package/core/src/launcher/supervise.ts +170 -0
- package/core/src/lifecycle/commandHost.ts +111 -0
- package/core/src/lifecycle/hostAdapter.ts +28 -16
- package/core/src/operation.ts +47 -25
- package/core/src/operationLifecycle.ts +2 -7
- package/core/src/platform/ops.ts +7 -0
- package/core/src/platform/posix.ts +26 -6
- package/core/src/platform/windows.ts +8 -2
- package/core/src/protocol/runner.ts +81 -0
- package/core/src/provenance/journal.ts +1 -1
- package/core/src/quarantine.ts +167 -0
- package/core/src/runner/cli.ts +27 -0
- package/core/src/runner/execute.ts +68 -0
- package/core/src/txn/engine.ts +41 -85
- package/core/src/txn/fileEffects.ts +15 -1
- package/core/src/txn/hostCallBudget.ts +4 -1
- package/core/src/txn/hostCallUncertain.ts +2 -0
- package/core/src/txn/lock.ts +81 -37
- package/core/src/txn/state.ts +1 -1
- package/core/src/upgrade/drive.ts +31 -2
- package/core/src/upgrade/outcome.ts +1 -1
- package/core/src/upgrade/recover.ts +22 -1
- package/core/src/upgrade/retire.ts +1 -1
- package/core/src/upgrader.ts +7 -8
- package/docs/design.md +173 -0
- package/docs/guide.md +196 -0
- package/docs/harness-design.md +75 -170
- package/docs/integration.md +221 -354
- package/docs/prior-art/design-influences.md +26 -0
- package/docs/prior-art/external-runner-research.md +49 -0
- package/docs/reference.md +209 -0
- package/docs/test-plan.md +89 -92
- package/harness/src/adapter/releaseKnob.ts +1 -1
- package/harness/src/adapter/serviceChecks.ts +5 -5
- package/harness/src/artifact/m1.ts +8 -8
- package/harness/src/artifact/m1Resume.ts +2 -2
- package/harness/src/artifact/m3.ts +25 -104
- package/harness/src/artifact/m3Hosts.ts +9 -61
- package/harness/src/artifact/m4.ts +3 -3
- package/harness/src/artifact/m5.ts +5 -5
- package/harness/src/artifact/m6.ts +6 -6
- package/harness/src/artifact/m6Status.ts +1 -1
- package/harness/src/examples/checks.ts +10 -13
- package/harness/src/fixtures/cliToolSource.ts +166 -0
- package/harness/src/fixtures/externalCrashAdapter.ts +19 -0
- package/harness/src/fixtures/managedHost.ts +100 -0
- package/harness/src/fixtures/serviceSource.ts +181 -0
- package/harness/src/fixtures/supervisedAdapter.ts +57 -0
- package/harness/src/scenario/processScan.ts +3 -1
- package/harness/src/scenario/sandbox.ts +2 -2
- package/harness/src/teeth/artifact.ts +3 -3
- package/harness/src/teeth/examples.ts +1 -1
- package/package.json +5 -3
- package/docs/design-v1.md +0 -246
- package/docs/prior-art.md +0 -150
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
/** Internal byte-replacement fixture driven by the harness.
|
|
2
|
+
* Version/behavior placeholders are stamped by ArtifactFactory. The self-upgrade
|
|
3
|
+
* command exercises core mechanisms; it is not a product integration API.
|
|
4
|
+
* K_CORE_UPGRADER supplies the file URL of core/src/createRunner.ts.
|
|
5
|
+
*/
|
|
6
|
+
export const CLI_TOOL_SOURCE = `#!/usr/bin/env node
|
|
7
|
+
// Internal swap-tool fixture built by ArtifactFactory.
|
|
8
|
+
// self upgrade runs through core's Upgrader; --probe is the headless
|
|
9
|
+
// start check the swap-profile host uses to verify new bytes.
|
|
10
|
+
"use strict";
|
|
11
|
+
const VERSION = "__K_VERSION__";
|
|
12
|
+
const BEHAVIOR = "__K_BEHAVIOR__";
|
|
13
|
+
const fs = require("node:fs");
|
|
14
|
+
const path = require("node:path");
|
|
15
|
+
const { spawnSync } = require("node:child_process");
|
|
16
|
+
const RELEASE_BASE = process.env.K_RELEASE_BASE;
|
|
17
|
+
const STATE_DIR = process.env.K_STATE_DIR ?? path.join(path.dirname(process.argv[1]), "state");
|
|
18
|
+
const CORE_UPGRADER = process.env.K_CORE_UPGRADER;
|
|
19
|
+
const args = process.argv.slice(2);
|
|
20
|
+
const startId = process.pid + "-" + process.hrtime.bigint().toString(36);
|
|
21
|
+
// Synchronous writes: process.exit() can truncate buffered pipe writes,
|
|
22
|
+
// and the black-box assertions read stdout as evidence.
|
|
23
|
+
|
|
24
|
+
if (args[0] === "--probe") {
|
|
25
|
+
// headless start check used by the swap-profile host during an upgrade
|
|
26
|
+
fs.writeSync(1, VERSION + "\\n");
|
|
27
|
+
process.exit(BEHAVIOR === "crash-on-start" ? 1 : 0);
|
|
28
|
+
}
|
|
29
|
+
if (args[0] === "greet") {
|
|
30
|
+
fs.writeSync(1, "Hello, " + (args[1] ?? "world") + "! (v" + VERSION + ")\\n");
|
|
31
|
+
process.exit(0);
|
|
32
|
+
}
|
|
33
|
+
if (args[0] === "--version") {
|
|
34
|
+
fs.writeSync(1, VERSION + "\\n");
|
|
35
|
+
process.exit(0);
|
|
36
|
+
}
|
|
37
|
+
if (args[0] === "confirm" && args[1] === "upgrade") {
|
|
38
|
+
confirmUpgrade(args[2]).catch((e) => { fs.writeSync(2, String(e) + "\\n"); process.exit(5); });
|
|
39
|
+
} else if (args[0] === "self" && args[1] === "upgrade") {
|
|
40
|
+
selfUpgrade().catch((e) => { fs.writeSync(2, String(e) + "\\n"); process.exit(5); });
|
|
41
|
+
} else {
|
|
42
|
+
switch (BEHAVIOR) {
|
|
43
|
+
case "crash-on-start": fs.writeSync(2, "swap-tool crashed\\n"); process.exit(1); break;
|
|
44
|
+
case "hang-on-quiesce": fs.writeSync(1, "swap-tool " + VERSION + "\\n"); setInterval(() => {}, 2147483647); break;
|
|
45
|
+
case "ok":
|
|
46
|
+
default: fs.writeSync(1, "swap-tool " + VERSION + "\\n"); process.exit(0); break;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async function selfUpgrade() {
|
|
51
|
+
if (!RELEASE_BASE) { fs.writeSync(2, "K_RELEASE_BASE not set\\n"); process.exit(2); }
|
|
52
|
+
if (!CORE_UPGRADER) { fs.writeSync(2, "K_CORE_UPGRADER not set (the example's @k-carrier/core wiring)\\n"); process.exit(2); }
|
|
53
|
+
const coreSrcUrl = new URL(".", CORE_UPGRADER).href;
|
|
54
|
+
const { createRunner } = await import(CORE_UPGRADER);
|
|
55
|
+
const { staticManifestSource } = await import(new URL("artifact/staticManifestSource.ts", coreSrcUrl).href);
|
|
56
|
+
const { atomicWriteFile } = await import(new URL("artifact/swap.ts", coreSrcUrl).href);
|
|
57
|
+
const { slotArtifactPath } = await import(new URL("txn/fileEffects.ts", coreSrcUrl).href);
|
|
58
|
+
|
|
59
|
+
// swap profile: the app IS the process. The probe verifies the
|
|
60
|
+
// experiment's bytes actually start (headless --probe run); a new
|
|
61
|
+
// version that fails to start makes the transaction roll back.
|
|
62
|
+
const host = {
|
|
63
|
+
async quiesce() {},
|
|
64
|
+
async stop() {},
|
|
65
|
+
async start() {},
|
|
66
|
+
async healthProbe() {
|
|
67
|
+
const experiment = path.join(STATE_DIR, "slots", "experiment", "artifact.bin");
|
|
68
|
+
if (fs.existsSync(experiment)) {
|
|
69
|
+
const r = spawnSync(process.execPath, [experiment, "--probe"], { encoding: "utf8", timeout: 5000 });
|
|
70
|
+
if (r.status !== 0) {
|
|
71
|
+
throw new Error("experiment artifact failed to start (status " + r.status + ")");
|
|
72
|
+
}
|
|
73
|
+
// The evidence belongs to the process that answered: the headless
|
|
74
|
+
// probe run, a fresh incarnation each time.
|
|
75
|
+
return { version: r.stdout.trim(), pid: r.pid, startId: r.pid + "-" + process.hrtime.bigint().toString(36) };
|
|
76
|
+
}
|
|
77
|
+
return { version: VERSION, pid: process.pid, startId };
|
|
78
|
+
},
|
|
79
|
+
async resume() {},
|
|
80
|
+
};
|
|
81
|
+
|
|
82
|
+
// K verifies integrity (sha256) but not authenticity: it does not check who
|
|
83
|
+
// produced these bytes (design-v1 §L0.5). For a real product that decision
|
|
84
|
+
// belongs to the adopter, in code like this one -- never to an option on the
|
|
85
|
+
// source and never to a field in the manifest, which is served by the very
|
|
86
|
+
// party such a check would exist to distrust.
|
|
87
|
+
const source = staticManifestSource({ baseUrl: RELEASE_BASE });
|
|
88
|
+
|
|
89
|
+
const upgrader = createRunner({
|
|
90
|
+
host,
|
|
91
|
+
source,
|
|
92
|
+
policy: process.env.K_POLICY ?? "auto",
|
|
93
|
+
notificationSink: async (ev) => {
|
|
94
|
+
const ver = ev.detail.version !== undefined ? " v" + ev.detail.version : "";
|
|
95
|
+
const reason = ev.detail.reason !== undefined ? ": " + ev.detail.reason : "";
|
|
96
|
+
fs.writeSync(2, "notify " + ev.kind + ver + reason + "\\n");
|
|
97
|
+
},
|
|
98
|
+
stateDir: STATE_DIR,
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
const outcome = await upgrader.upgrade();
|
|
102
|
+
await finish(outcome, upgrader, { slotArtifactPath, atomicWriteFile });
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Continue a policy=confirm flow AFTER the user approved the offered
|
|
106
|
+
* version: the consent WAS the gate; the version stays bound (the source
|
|
107
|
+
* must still serve it, or the continuation refuses). */
|
|
108
|
+
async function confirmUpgrade(version) {
|
|
109
|
+
if (!version) { fs.writeSync(2, "confirm upgrade <version>\\n"); process.exit(2); }
|
|
110
|
+
if (!RELEASE_BASE) { fs.writeSync(2, "K_RELEASE_BASE not set\\n"); process.exit(2); }
|
|
111
|
+
if (!CORE_UPGRADER) { fs.writeSync(2, "K_CORE_UPGRADER not set (the example's @k-carrier/core wiring)\\n"); process.exit(2); }
|
|
112
|
+
const coreSrcUrl = new URL(".", CORE_UPGRADER).href;
|
|
113
|
+
const { createRunner } = await import(CORE_UPGRADER);
|
|
114
|
+
const { staticManifestSource } = await import(new URL("artifact/staticManifestSource.ts", coreSrcUrl).href);
|
|
115
|
+
const { atomicWriteFile } = await import(new URL("artifact/swap.ts", coreSrcUrl).href);
|
|
116
|
+
const { slotArtifactPath } = await import(new URL("txn/fileEffects.ts", coreSrcUrl).href);
|
|
117
|
+
const host = {
|
|
118
|
+
async quiesce() {},
|
|
119
|
+
async stop() {},
|
|
120
|
+
async start() {},
|
|
121
|
+
async healthProbe() {
|
|
122
|
+
const experiment = path.join(STATE_DIR, "slots", "experiment", "artifact.bin");
|
|
123
|
+
if (fs.existsSync(experiment)) {
|
|
124
|
+
const r = spawnSync(process.execPath, [experiment, "--probe"], { encoding: "utf8", timeout: 5000 });
|
|
125
|
+
if (r.status !== 0) throw new Error("experiment artifact failed to start");
|
|
126
|
+
return { version: r.stdout.trim(), pid: r.pid, startId: r.pid + "-" + process.hrtime.bigint().toString(36) };
|
|
127
|
+
}
|
|
128
|
+
return { version: VERSION, pid: process.pid, startId };
|
|
129
|
+
},
|
|
130
|
+
async resume() {},
|
|
131
|
+
};
|
|
132
|
+
const upgrader = createRunner({
|
|
133
|
+
host,
|
|
134
|
+
source: staticManifestSource({ baseUrl: RELEASE_BASE }),
|
|
135
|
+
policy: "confirm",
|
|
136
|
+
notificationSink: async (ev) => {
|
|
137
|
+
const ver = ev.detail.version !== undefined ? " v" + ev.detail.version : "";
|
|
138
|
+
fs.writeSync(2, "notify " + ev.kind + ver + "\\n");
|
|
139
|
+
},
|
|
140
|
+
stateDir: STATE_DIR,
|
|
141
|
+
});
|
|
142
|
+
const outcome = await upgrader.upgradeTo(version, { consented: true });
|
|
143
|
+
await finish(outcome, upgrader, { slotArtifactPath, atomicWriteFile });
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async function finish(outcome, upgrader, helpers) {
|
|
147
|
+
if (outcome.result === "promoted") {
|
|
148
|
+
// install step: swap the promoted slot's bytes over ourselves
|
|
149
|
+
const promoted = fs.readFileSync(helpers.slotArtifactPath(STATE_DIR, "stable"));
|
|
150
|
+
await helpers.atomicWriteFile(process.argv[1], promoted);
|
|
151
|
+
const st = await upgrader.state();
|
|
152
|
+
fs.writeSync(1, "upgraded to " + st.stableVersion + "\\n");
|
|
153
|
+
process.exit(0);
|
|
154
|
+
}
|
|
155
|
+
if (outcome.result === "rolled-back") {
|
|
156
|
+
fs.writeSync(1, "rolled back: " + outcome.reason + "\\n");
|
|
157
|
+
process.exit(1);
|
|
158
|
+
}
|
|
159
|
+
if (outcome.result === "held") {
|
|
160
|
+
fs.writeSync(1, "held: " + outcome.reason + "\\n");
|
|
161
|
+
process.exit(0);
|
|
162
|
+
}
|
|
163
|
+
fs.writeSync(1, "up to date\\n");
|
|
164
|
+
process.exit(0);
|
|
165
|
+
}
|
|
166
|
+
`;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Fixture: pause the external runner AFTER stop completed, before start. */
|
|
2
|
+
import { promises as fs } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
5
|
+
import { createOptions } from "../../../examples/external-service/adapter.ts";
|
|
6
|
+
import { createRunner } from "../../../core/src/index.ts";
|
|
7
|
+
export default function create() {
|
|
8
|
+
const options = createOptions();
|
|
9
|
+
const stop = options.host.stop;
|
|
10
|
+
options.host.stop = async (slot) => {
|
|
11
|
+
await stop(slot);
|
|
12
|
+
const dir = path.dirname(options.stateDir);
|
|
13
|
+
if (slot === "stable" && await fs.stat(path.join(dir, "pause")).then(() => true, () => false)) {
|
|
14
|
+
await fs.writeFile(path.join(dir, "stopped"), String(process.pid));
|
|
15
|
+
for (;;) await sleep(100);
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
return createRunner(options);
|
|
19
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/** Internal host fixture for session ledger preservation and live probe checks.
|
|
2
|
+
* This deliberately uses the harness HostDriver contract; it is not a product
|
|
3
|
+
* integration example. See examples/external-service for application wiring.
|
|
4
|
+
*/
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
import { promises as fs } from "node:fs";
|
|
7
|
+
import * as path from "node:path";
|
|
8
|
+
import type { ProcessEvidence, Slot } from "../../../core/src/lifecycle/hostAdapter.ts";
|
|
9
|
+
import type { HostDriver, LedgerState } from "../fake-host/inproc.ts";
|
|
10
|
+
|
|
11
|
+
const SESSION_FILE = "session.bin";
|
|
12
|
+
|
|
13
|
+
export function createManagedHost(stateDir: string): HostDriver {
|
|
14
|
+
let counter = 0;
|
|
15
|
+
let checksum: Uint8Array = createHash("sha256").update("hosted-service-v1").digest();
|
|
16
|
+
let runningSlot: Slot | null = null;
|
|
17
|
+
let parked = false;
|
|
18
|
+
let incarnation = 0;
|
|
19
|
+
let currentStartId: string | null = null;
|
|
20
|
+
|
|
21
|
+
const sessionBytes = (): Uint8Array => {
|
|
22
|
+
const out = new Uint8Array(8 + 32);
|
|
23
|
+
new DataView(out.buffer).setBigUint64(0, BigInt(counter), false);
|
|
24
|
+
out.set(checksum, 8);
|
|
25
|
+
return out;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const writeSession = async (): Promise<void> => {
|
|
29
|
+
await fs.mkdir(stateDir, { recursive: true });
|
|
30
|
+
await fs.writeFile(path.join(stateDir, SESSION_FILE), sessionBytes());
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
return {
|
|
34
|
+
async quiesce() {
|
|
35
|
+
if (parked) return; // idempotent
|
|
36
|
+
if (!runningSlot) throw new Error("quiesce: no running slot");
|
|
37
|
+
parked = true;
|
|
38
|
+
await writeSession(); // durable park
|
|
39
|
+
},
|
|
40
|
+
async stop(slot: Slot) {
|
|
41
|
+
if (runningSlot !== slot) throw new Error(`stop: ${slot} is not the running slot`);
|
|
42
|
+
runningSlot = null;
|
|
43
|
+
},
|
|
44
|
+
async start(slot: Slot) {
|
|
45
|
+
if (runningSlot !== null) throw new Error(`start: ${runningSlot} already running`);
|
|
46
|
+
runningSlot = slot;
|
|
47
|
+
incarnation += 1;
|
|
48
|
+
currentStartId = `managed-inc:${incarnation}`;
|
|
49
|
+
},
|
|
50
|
+
async healthProbe(): Promise<ProcessEvidence> {
|
|
51
|
+
if (!runningSlot) throw new Error("probe: no running slot");
|
|
52
|
+
if (!currentStartId) throw new Error("probe: no startId");
|
|
53
|
+
return {
|
|
54
|
+
version: runningSlot === "stable" ? "1.0.0" : "2.0.0",
|
|
55
|
+
pid: process.pid,
|
|
56
|
+
startId: currentStartId,
|
|
57
|
+
};
|
|
58
|
+
},
|
|
59
|
+
async resume() {
|
|
60
|
+
if (!parked) throw new Error("resume: not quiesced");
|
|
61
|
+
parked = false;
|
|
62
|
+
// restore the session state from the parked file, byte-for-byte
|
|
63
|
+
const raw = new Uint8Array(await fs.readFile(path.join(stateDir, SESSION_FILE)));
|
|
64
|
+
counter = Number(new DataView(raw.buffer, raw.byteOffset, raw.byteLength).getBigUint64(0, false));
|
|
65
|
+
checksum = raw.slice(8);
|
|
66
|
+
},
|
|
67
|
+
get running(): Slot | null {
|
|
68
|
+
return runningSlot;
|
|
69
|
+
},
|
|
70
|
+
get parked(): boolean {
|
|
71
|
+
return parked;
|
|
72
|
+
},
|
|
73
|
+
get startId(): string | null {
|
|
74
|
+
return currentStartId;
|
|
75
|
+
},
|
|
76
|
+
async doWork(n: number) {
|
|
77
|
+
if (parked) throw new Error("sessions are parked (quiesced)");
|
|
78
|
+
if (!runningSlot) throw new Error("doWork: no running slot");
|
|
79
|
+
for (let i = 0; i < n; i++) {
|
|
80
|
+
counter += 1;
|
|
81
|
+
checksum = createHash("sha256").update(be64(counter)).update(checksum).digest();
|
|
82
|
+
}
|
|
83
|
+
await writeSession();
|
|
84
|
+
},
|
|
85
|
+
async ledger(): Promise<Uint8Array> {
|
|
86
|
+
return new Uint8Array(await fs.readFile(path.join(stateDir, SESSION_FILE)));
|
|
87
|
+
},
|
|
88
|
+
async ledgerState(): Promise<LedgerState> {
|
|
89
|
+
return { counter, checksum };
|
|
90
|
+
},
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function be64(n: number): Uint8Array {
|
|
95
|
+
const out = new Uint8Array(8);
|
|
96
|
+
new DataView(out.buffer).setBigUint64(0, BigInt(n), false);
|
|
97
|
+
return out;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export default createManagedHost;
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
/** Internal service fixture: an external driver owns upgrade/recovery.
|
|
2
|
+
* The resident role only serves probe/exit and never runs K.
|
|
3
|
+
*/
|
|
4
|
+
export const PLAIN_DAEMON_SOURCE = `#!/usr/bin/env node
|
|
5
|
+
// Internal service fixture built by ArtifactFactory.
|
|
6
|
+
"use strict";
|
|
7
|
+
const VERSION = "__K_VERSION__";
|
|
8
|
+
const BEHAVIOR = "__K_BEHAVIOR__";
|
|
9
|
+
const fs = require("node:fs");
|
|
10
|
+
const path = require("node:path");
|
|
11
|
+
const { spawn } = require("node:child_process");
|
|
12
|
+
const RELEASE_BASE = process.env.K_RELEASE_BASE;
|
|
13
|
+
const STATE_DIR = process.env.K_STATE_DIR ?? path.join(path.dirname(process.argv[1]), "state");
|
|
14
|
+
const CORE_UPGRADER = process.env.K_CORE_UPGRADER;
|
|
15
|
+
const args = process.argv.slice(2);
|
|
16
|
+
const startId = process.pid + "-" + process.hrtime.bigint().toString(36);
|
|
17
|
+
const INCARNATION_FILE = path.join(STATE_DIR, "incarnation.json");
|
|
18
|
+
|
|
19
|
+
function wsync(s, m) { fs.writeSync(s, m + "\\n"); }
|
|
20
|
+
function readIncarnation() {
|
|
21
|
+
try { return JSON.parse(fs.readFileSync(INCARNATION_FILE, "utf8")); } catch { return null; }
|
|
22
|
+
}
|
|
23
|
+
function writeIncarnation() {
|
|
24
|
+
fs.mkdirSync(STATE_DIR, { recursive: true });
|
|
25
|
+
fs.writeFileSync(INCARNATION_FILE, JSON.stringify({ version: VERSION, pid: process.pid, startId }));
|
|
26
|
+
}
|
|
27
|
+
function processAlive(pid) {
|
|
28
|
+
try { process.kill(pid, 0); return true; } catch { return false; }
|
|
29
|
+
}
|
|
30
|
+
function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); }
|
|
31
|
+
async function waitDead(pid, timeoutMs) {
|
|
32
|
+
const deadline = Date.now() + timeoutMs;
|
|
33
|
+
while (processAlive(pid)) {
|
|
34
|
+
if (Date.now() > deadline) throw new Error("pid " + pid + " still alive after " + timeoutMs + "ms");
|
|
35
|
+
await sleep(10);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
function readLine(child, prefix, timeoutMs) {
|
|
39
|
+
return new Promise((resolve, reject) => {
|
|
40
|
+
let buffer = "";
|
|
41
|
+
const timer = setTimeout(() => { cleanup(); reject(new Error("timed out waiting for \\"" + prefix + "\\" from service")); }, timeoutMs);
|
|
42
|
+
const onData = (chunk) => {
|
|
43
|
+
buffer += chunk.toString("utf8");
|
|
44
|
+
// Only COMPLETE lines (newline-terminated) are protocol messages; a
|
|
45
|
+
// partial chunk must never be mistaken for a full one (truncated JSON).
|
|
46
|
+
for (;;) {
|
|
47
|
+
const nl = buffer.indexOf("\\n");
|
|
48
|
+
if (nl === -1) break;
|
|
49
|
+
const line = buffer.slice(0, nl);
|
|
50
|
+
buffer = buffer.slice(nl + 1);
|
|
51
|
+
if (line.startsWith(prefix + " ")) {
|
|
52
|
+
cleanup();
|
|
53
|
+
resolve(line.slice(prefix.length + 1));
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
const cleanup = () => { clearTimeout(timer); child.stdout?.off("data", onData); };
|
|
59
|
+
child.stdout?.on("data", onData);
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
/** Wait for the ready line; null if the child died or never became ready. */
|
|
63
|
+
async function tryReady(child, timeoutMs) {
|
|
64
|
+
try { return JSON.parse(await readLine(child, "ready", timeoutMs)); }
|
|
65
|
+
catch { return null; }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
(async () => {
|
|
69
|
+
if (args[0] === "--version") {
|
|
70
|
+
wsync(1, VERSION);
|
|
71
|
+
process.exit(0);
|
|
72
|
+
}
|
|
73
|
+
if (args[0] === "self" && (args[1] === "upgrade" || args[1] === "recover")) {
|
|
74
|
+
try {
|
|
75
|
+
await selfUpgrade();
|
|
76
|
+
} catch (e) {
|
|
77
|
+
wsync(2, String(e));
|
|
78
|
+
process.exit(5);
|
|
79
|
+
}
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
// ---- SERVICE role ----
|
|
83
|
+
if (BEHAVIOR === "crash-on-start") { wsync(2, "service-daemon crashed"); process.exit(1); }
|
|
84
|
+
writeIncarnation();
|
|
85
|
+
wsync(1, "ready " + JSON.stringify({ version: VERSION, pid: process.pid, startId }));
|
|
86
|
+
process.stdin.on("data", (chunk) => {
|
|
87
|
+
for (const line of chunk.toString("utf8").split("\\n")) {
|
|
88
|
+
const cmd = line.trim();
|
|
89
|
+
if (cmd === "probe") {
|
|
90
|
+
wsync(1, "evidence " + JSON.stringify({ version: VERSION, pid: process.pid, startId }));
|
|
91
|
+
} else if (cmd === "exit") {
|
|
92
|
+
process.exit(0);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
});
|
|
96
|
+
setInterval(() => {}, 1 << 30);
|
|
97
|
+
})();
|
|
98
|
+
|
|
99
|
+
async function selfUpgrade() {
|
|
100
|
+
if (!RELEASE_BASE) { wsync(2, "K_RELEASE_BASE not set"); process.exit(2); }
|
|
101
|
+
if (!CORE_UPGRADER) { wsync(2, "K_CORE_UPGRADER not set"); process.exit(2); }
|
|
102
|
+
const coreSrcUrl = new URL(".", CORE_UPGRADER).href;
|
|
103
|
+
const { createRunner } = await import(CORE_UPGRADER);
|
|
104
|
+
const { staticManifestSource } = await import(new URL("artifact/staticManifestSource.ts", coreSrcUrl).href);
|
|
105
|
+
|
|
106
|
+
// The external driver starts and probes each replacement itself.
|
|
107
|
+
let requested = false;
|
|
108
|
+
let successor = null;
|
|
109
|
+
const host = {
|
|
110
|
+
async quiesce() {}, // service profile hosts no workloads
|
|
111
|
+
async stop() {
|
|
112
|
+
if (process.env.K_STUCK_DRIVER === "1") {
|
|
113
|
+
// Force a host-call timeout; a fresh external driver owns recovery.
|
|
114
|
+
await new Promise(() => {});
|
|
115
|
+
}
|
|
116
|
+
const inc = readIncarnation();
|
|
117
|
+
if (inc && inc.pid !== process.pid) {
|
|
118
|
+
try { process.kill(inc.pid, "SIGKILL"); } catch { /* already gone */ }
|
|
119
|
+
await waitDead(inc.pid, 5000);
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
async start(slot) {
|
|
123
|
+
requested = true;
|
|
124
|
+
const artifact = path.join(STATE_DIR, "slots", slot, "artifact.bin");
|
|
125
|
+
if (!fs.existsSync(artifact)) throw new Error("slot " + slot + " has no artifact");
|
|
126
|
+
const child = spawn(process.execPath, [artifact], {
|
|
127
|
+
env: { ...process.env, K_STATE_DIR: STATE_DIR },
|
|
128
|
+
stdio: ["pipe", "pipe", "ignore"],
|
|
129
|
+
detached: true,
|
|
130
|
+
});
|
|
131
|
+
child.unref(); // the service outlives the driver
|
|
132
|
+
const info = await tryReady(child, 5000);
|
|
133
|
+
successor = info === null ? null : { child, ...info };
|
|
134
|
+
},
|
|
135
|
+
async healthProbe() {
|
|
136
|
+
if (requested) {
|
|
137
|
+
// the successor must answer; silence or a dead child is a failure
|
|
138
|
+
if (successor === null || successor.child.exitCode !== null) {
|
|
139
|
+
throw new Error("no live successor to probe");
|
|
140
|
+
}
|
|
141
|
+
successor.child.stdin.write("probe\\n");
|
|
142
|
+
const line = await readLine(successor.child, "evidence", 5000);
|
|
143
|
+
return JSON.parse(line);
|
|
144
|
+
}
|
|
145
|
+
const inc = readIncarnation();
|
|
146
|
+
if (!inc) throw new Error("no registered service to probe");
|
|
147
|
+
return inc; // the app's own record of the incarnation being replaced
|
|
148
|
+
},
|
|
149
|
+
async resume() {},
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
const upgrader = createRunner({
|
|
153
|
+
host,
|
|
154
|
+
source: staticManifestSource({ baseUrl: RELEASE_BASE }),
|
|
155
|
+
policy: "auto",
|
|
156
|
+
notificationSink: async (ev) => {
|
|
157
|
+
const reason = ev.detail.reason !== undefined ? ": " + ev.detail.reason : "";
|
|
158
|
+
wsync(2, "notify " + ev.kind + reason);
|
|
159
|
+
},
|
|
160
|
+
stateDir: STATE_DIR,
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
if (args[1] === "recover") { await upgrader.recover(); process.exit(0); }
|
|
164
|
+
const outcome = await upgrader.upgrade();
|
|
165
|
+
if (outcome.result === "promoted") {
|
|
166
|
+
const st = await upgrader.state();
|
|
167
|
+
wsync(1, "upgraded to " + st.stableVersion);
|
|
168
|
+
process.exit(0);
|
|
169
|
+
}
|
|
170
|
+
if (outcome.result === "rolled-back") {
|
|
171
|
+
wsync(1, "rolled back: " + outcome.reason);
|
|
172
|
+
process.exit(1);
|
|
173
|
+
}
|
|
174
|
+
if (outcome.result === "held") {
|
|
175
|
+
wsync(1, "held: " + outcome.reason);
|
|
176
|
+
process.exit(0);
|
|
177
|
+
}
|
|
178
|
+
wsync(1, "up to date");
|
|
179
|
+
process.exit(0);
|
|
180
|
+
}
|
|
181
|
+
`;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/** Fault injection around real production host calls; used only by process tests. */
|
|
2
|
+
import { promises as fs } from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
5
|
+
import { createOptions } from "../../../examples/external-service/adapter.ts";
|
|
6
|
+
import { createRunner } from "../../../core/src/index.ts";
|
|
7
|
+
const hang = async (): Promise<never> => { for (;;) await sleep(100); };
|
|
8
|
+
export default function create() {
|
|
9
|
+
const options = createOptions();
|
|
10
|
+
const dir = path.dirname(options.stateDir);
|
|
11
|
+
const faultPath = path.join(dir, "fault");
|
|
12
|
+
const fault = () => fs.readFile(faultPath, "utf8").catch(() => "");
|
|
13
|
+
const stop = options.host.stop;
|
|
14
|
+
options.host.stop = async (slot) => {
|
|
15
|
+
if (slot === "stable" && await fault() === "external-controller") {
|
|
16
|
+
await fs.writeFile(path.join(dir, "needs-controller"), String(process.pid));
|
|
17
|
+
return hang();
|
|
18
|
+
}
|
|
19
|
+
await stop(slot);
|
|
20
|
+
const mode = await fault();
|
|
21
|
+
if (slot === "stable" && ["stop-crash", "stop-hang", "recovery-hang"].includes(mode)) {
|
|
22
|
+
await fs.writeFile(path.join(dir, "stopped"), String(process.pid));
|
|
23
|
+
if (mode === "stop-hang") return hang();
|
|
24
|
+
process.exit(77);
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
let attempting = false;
|
|
28
|
+
const resume = options.host.resume;
|
|
29
|
+
options.host.resume = async () => {
|
|
30
|
+
if (attempting && await fault() === "resume-once") {
|
|
31
|
+
await fs.unlink(faultPath);
|
|
32
|
+
await fs.writeFile(path.join(dir, "resuming"), String(process.pid));
|
|
33
|
+
return hang();
|
|
34
|
+
}
|
|
35
|
+
return resume();
|
|
36
|
+
};
|
|
37
|
+
const fence = options.host.fence!;
|
|
38
|
+
options.host.fence = async () => {
|
|
39
|
+
await fs.writeFile(path.join(dir, "fence-attempt"), String(process.pid));
|
|
40
|
+
if (await fault() === "recovery-hang" && await fs.stat(path.join(dir, "stopped")).then(() => true, () => false)) return hang();
|
|
41
|
+
return fence();
|
|
42
|
+
};
|
|
43
|
+
const fetch = options.source.fetchRelease;
|
|
44
|
+
options.source.fetchRelease = async (version) => {
|
|
45
|
+
attempting = true;
|
|
46
|
+
await fs.appendFile(path.join(dir, "fetches"), `${version}\n`);
|
|
47
|
+
return fetch(version);
|
|
48
|
+
};
|
|
49
|
+
const runner = createRunner(options);
|
|
50
|
+
const upgradeTo = runner.upgradeTo;
|
|
51
|
+
runner.upgradeTo = async (...args) => {
|
|
52
|
+
const outcome = await upgradeTo(...args);
|
|
53
|
+
if (await fault() === "report-crash") { await fs.unlink(faultPath); process.exit(78); }
|
|
54
|
+
return outcome;
|
|
55
|
+
};
|
|
56
|
+
return runner;
|
|
57
|
+
}
|
|
@@ -43,7 +43,9 @@ export function findPidsByMarkerToken(name: string, value: string): number[] {
|
|
|
43
43
|
}
|
|
44
44
|
return pids;
|
|
45
45
|
}
|
|
46
|
-
|
|
46
|
+
// `e` appends every process's environment; on a busy machine that exceeds
|
|
47
|
+
// execFileSync's 1 MiB default and fails with ENOBUFS. Same bound as Windows.
|
|
48
|
+
const out = execFileSync("ps", ["eaxo", "pid=,command="], { encoding: "utf8", maxBuffer: 32 * 1024 * 1024 });
|
|
47
49
|
const pids: number[] = [];
|
|
48
50
|
for (const line of out.split("\n")) {
|
|
49
51
|
if (!line.includes(token)) continue;
|
|
@@ -134,8 +134,8 @@ const SANDBOX_DIR_PREFIX = "k-harness-";
|
|
|
134
134
|
/**
|
|
135
135
|
* The marker a process must carry to be found by this sandbox's teardown.
|
|
136
136
|
*
|
|
137
|
-
* Teeth often derive a NESTED context (e.g. `<sandbox>/
|
|
138
|
-
* shape). Taking basename() of that nested dir yields "
|
|
137
|
+
* Teeth often derive a NESTED context (e.g. `<sandbox>/service` per host
|
|
138
|
+
* shape). Taking basename() of that nested dir yields "service", which no
|
|
139
139
|
* teardown scan will ever match -- so the scan returns zero, the tooth reports
|
|
140
140
|
* a clean teardown, and the leaked process is still running. The zero means
|
|
141
141
|
* "the query matched nothing", not "nothing leaked". (Found 08-05 by looking
|
|
@@ -145,14 +145,14 @@ registerTooth({
|
|
|
145
145
|
});
|
|
146
146
|
|
|
147
147
|
registerTooth({
|
|
148
|
-
id: "m3.stuck-driver-
|
|
148
|
+
id: "m3.stuck-driver-rollback-recovery",
|
|
149
149
|
profiles: ["service"],
|
|
150
150
|
layers: ["L0", "L1", "L2", "L3"],
|
|
151
151
|
kind: { kind: "invariant" },
|
|
152
152
|
mustRed: [
|
|
153
153
|
{
|
|
154
|
-
mutate: "the
|
|
155
|
-
caughtOnlyBy: "this", // only this tooth wedges the driver and demands
|
|
154
|
+
mutate: "the external driver skips recovery and leaves interrupted work unresolved",
|
|
155
|
+
caughtOnlyBy: "this", // only this tooth wedges the driver and demands external rollback recovery
|
|
156
156
|
},
|
|
157
157
|
],
|
|
158
158
|
run: checkM3StuckDriverEvidence,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Examples teeth — each demo is the credential for its profile's support
|
|
3
|
-
* claim (
|
|
3
|
+
* claim (harness/src/fixtures/README.md). Registration site only; check bodies live in
|
|
4
4
|
* harness/src/examples/checks.ts.
|
|
5
5
|
*/
|
|
6
6
|
import { registerTooth } from "./registry.ts";
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@botiverse/k-carrier",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"packageManager": "pnpm@11.18.0",
|
|
5
|
-
"description": "
|
|
5
|
+
"description": "Reliable application upgrades run by an independent installer: two-slot transactions with promote/rollback, a durable journal, live readback, and bounded crash recovery for self-distributed services.",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
8
8
|
"url": "https://github.com/botiverse/k-carrier"
|
|
@@ -37,10 +37,12 @@
|
|
|
37
37
|
"lint": "oxlint --deny-warnings core harness examples",
|
|
38
38
|
"ratchet": "bash scripts/ratchets.sh",
|
|
39
39
|
"sim": "node harness/src/cli.ts sim",
|
|
40
|
-
"check": "pnpm typecheck && pnpm lint && pnpm ratchet && pnpm test"
|
|
40
|
+
"check": "pnpm typecheck && pnpm lint && pnpm ratchet && pnpm test",
|
|
41
|
+
"test:runner": "node --test --test-timeout=60000 \"core/src/runner/*.test.ts\" \"core/src/launcher/*.test.ts\" \"core/src/protocol/*.test.ts\" \"core/src/lifecycle/commandHost.test.ts\""
|
|
41
42
|
},
|
|
42
43
|
"devDependencies": {
|
|
43
44
|
"@types/node": "^26.1.2",
|
|
45
|
+
"esbuild": "0.28.2",
|
|
44
46
|
"oxlint": "^1.0.0",
|
|
45
47
|
"oxlint-tsgolint": "^7.0.2001",
|
|
46
48
|
"typescript": "^5.6.0"
|