@parcha/pi-durable-disk 0.0.0-stage → 0.1.0-beta.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +329 -0
- package/LICENSE +202 -0
- package/NOTICE +2 -0
- package/README.md +695 -2
- package/bin/archil-scoped +165 -0
- package/dist/app.d.ts +40 -0
- package/dist/app.js +47 -0
- package/dist/claim.d.ts +222 -0
- package/dist/claim.js +722 -0
- package/dist/cli.d.ts +64 -0
- package/dist/cli.js +664 -0
- package/dist/commands.d.ts +67 -0
- package/dist/commands.js +299 -0
- package/dist/confine.d.ts +91 -0
- package/dist/confine.js +341 -0
- package/dist/env.d.ts +129 -0
- package/dist/env.js +608 -0
- package/dist/errors.d.ts +47 -0
- package/dist/errors.js +55 -0
- package/dist/fork.d.ts +41 -0
- package/dist/fork.js +181 -0
- package/dist/hosts/daytona-launch.d.ts +52 -0
- package/dist/hosts/daytona-launch.js +270 -0
- package/dist/hosts/daytona.d.ts +147 -0
- package/dist/hosts/daytona.js +368 -0
- package/dist/hosts/docker.d.ts +103 -0
- package/dist/hosts/docker.js +362 -0
- package/dist/hosts/local-host.d.ts +71 -0
- package/dist/hosts/local-host.js +356 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.js +17 -0
- package/dist/lease.d.ts +7 -0
- package/dist/lease.js +6 -0
- package/dist/local-run.d.ts +10 -0
- package/dist/local-run.js +63 -0
- package/dist/park.d.ts +92 -0
- package/dist/park.js +229 -0
- package/dist/run.d.ts +290 -0
- package/dist/run.js +785 -0
- package/dist/serve.d.ts +101 -0
- package/dist/serve.js +393 -0
- package/dist/status.d.ts +123 -0
- package/dist/status.js +264 -0
- package/dist/store.d.ts +103 -0
- package/dist/store.js +217 -0
- package/dist/supervise.d.ts +340 -0
- package/dist/supervise.js +528 -0
- package/dist/watchdog.d.ts +39 -0
- package/dist/watchdog.js +137 -0
- package/package.json +87 -4
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
// localHost: run an instance of a run on this machine. Default mode: a systemd transient service created
|
|
2
|
+
// through the system manager, running as an unprivileged user with KillMode=control-group, so stopping or restarting
|
|
3
|
+
// it kills every command the instance started, pi's detached ones included. Development mode ("child"): a detached
|
|
4
|
+
// child process, without that guarantee (commands of a crashed child can outlive it).
|
|
5
|
+
//
|
|
6
|
+
// Root: every archil verb needs root (client 0.8.42), so the instance runs them through `sudo -n`, as the claim does;
|
|
7
|
+
// the instance and the agent's tools run as `user`. The mount goes through `bin/archil-scoped`, which starts the FUSE
|
|
8
|
+
// daemon in its own scope (`<unit>-fuse-<n>.scope`), outside the instance's control group: a restart in place finds
|
|
9
|
+
// the claim still held. The mount token is reusable for a day, so no process of the run's user may read it:
|
|
10
|
+
// it reaches the instance on stdin from a root-only file (/run/pi-durable-disk/<unit>.mount-token) that systemd opens
|
|
11
|
+
// as root; the file is replaced by an empty one once the unit started (the instance's fd keeps the token, a restart in
|
|
12
|
+
// place reads nothing and reuses its live mount), the instance closes its stdin after reading, and stop removes it.
|
|
13
|
+
// The token is never in argv, the environment, a systemd credential directory or anything the run's user can open.
|
|
14
|
+
import { spawn } from "node:child_process";
|
|
15
|
+
import { readFileSync } from "node:fs";
|
|
16
|
+
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
17
|
+
import { hostname } from "node:os";
|
|
18
|
+
import { extname, join } from "node:path";
|
|
19
|
+
import { fileURLToPath } from "node:url";
|
|
20
|
+
import { ARCHIL_SCOPED, DEFAULT_MOUNT_ROOT, runPath } from "../claim.js";
|
|
21
|
+
import { EXIT_FENCED, EXIT_HELD, PdaError } from "../errors.js";
|
|
22
|
+
import { LOCAL_PARK_THRESHOLD_MS } from "../park.js";
|
|
23
|
+
export class LocalHostError extends PdaError {
|
|
24
|
+
constructor(code, message, options = {}) {
|
|
25
|
+
super(code, message, options);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
const SAFE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin";
|
|
29
|
+
const TOKEN_DIR = "/run/pi-durable-disk";
|
|
30
|
+
const SYSTEMCTL = "/usr/bin/systemctl";
|
|
31
|
+
const SYSTEMD_RUN = "/usr/bin/systemd-run";
|
|
32
|
+
const ARCHIL = "/usr/bin/archil";
|
|
33
|
+
const UNIT_SAFE = /^[A-Za-z0-9:_.-]{1,200}$/;
|
|
34
|
+
const POLITE_UNMOUNT_MS = 10_000;
|
|
35
|
+
const STAT = "/usr/bin/stat";
|
|
36
|
+
const FUSERMOUNT = "/usr/bin/fusermount";
|
|
37
|
+
const UMOUNT = "/usr/bin/umount";
|
|
38
|
+
/** How long a stat of the mountpoint may take before its daemon counts as alive but stuck (never removed). */
|
|
39
|
+
const STAT_TIMEOUT_MS = 3_000;
|
|
40
|
+
/** EX_DATAERR: a store behind its seal or a bad run.json (run.ts names it EXIT_DATAERR). */
|
|
41
|
+
const EXIT_DATAERR = 65;
|
|
42
|
+
/** EX_SOFTWARE: the store's head is unreadable (pi's schema moved); run.json says failed, STORE_HEAD_UNREADABLE. */
|
|
43
|
+
const EXIT_SOFTWARE = 70;
|
|
44
|
+
/** Exits a restart cannot fix: 65 data error, 70 store head unreadable, 75 fenced, 76 held. */
|
|
45
|
+
export const TERMINAL_EXITS = [EXIT_DATAERR, EXIT_SOFTWARE, EXIT_FENCED, EXIT_HELD];
|
|
46
|
+
// The package's own CLI, with the extension this module has: `.ts` from a checkout, `.js` once built.
|
|
47
|
+
const CLI = fileURLToPath(new URL(`../cli${extname(import.meta.url)}`, import.meta.url));
|
|
48
|
+
/** Minimal environment, a timeout that settles even if a grandchild keeps the pipes, and the input on stdin. */
|
|
49
|
+
export const runCommand = (argv, opts = {}) => new Promise((resolve, reject) => {
|
|
50
|
+
let stdout = "", stderr = "", settled = false;
|
|
51
|
+
const child = spawn(argv[0], argv.slice(1), { env: { PATH: SAFE_PATH, LANG: "C.UTF-8", HOME: process.env.HOME ?? "/" }, stdio: ["pipe", "pipe", "pipe"] });
|
|
52
|
+
child.stdout.setEncoding("utf8").on("data", (c) => (stdout += c));
|
|
53
|
+
child.stderr.setEncoding("utf8").on("data", (c) => (stderr += c));
|
|
54
|
+
const done = (r) => void (settled || ((settled = true), clearTimeout(timer), resolve(r)));
|
|
55
|
+
const timer = setTimeout(() => {
|
|
56
|
+
child.kill("SIGTERM");
|
|
57
|
+
child.stdout.destroy();
|
|
58
|
+
child.stderr.destroy();
|
|
59
|
+
done({ code: null, timedOut: true, stdout, stderr });
|
|
60
|
+
}, opts.timeoutMs ?? 30_000);
|
|
61
|
+
child.on("error", (err) => void (settled || ((settled = true), clearTimeout(timer), reject(err))));
|
|
62
|
+
child.on("close", (code) => done({ code, timedOut: false, stdout, stderr }));
|
|
63
|
+
child.stdin.on("error", () => { });
|
|
64
|
+
child.stdin.end(opts.input ?? "");
|
|
65
|
+
});
|
|
66
|
+
export function currentBootId() {
|
|
67
|
+
return readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim();
|
|
68
|
+
}
|
|
69
|
+
const firstLine = (r) => (r.stderr.trim().split("\n").at(-1) ?? "").slice(0, 300);
|
|
70
|
+
const okRan = (r) => r.code === 0 && !r.timedOut;
|
|
71
|
+
/** `systemctl show` output as a map. */
|
|
72
|
+
export function parseShow(text) {
|
|
73
|
+
const out = {};
|
|
74
|
+
for (const line of text.split("\n")) {
|
|
75
|
+
const i = line.indexOf("=");
|
|
76
|
+
if (i > 0)
|
|
77
|
+
out[line.slice(0, i)] = line.slice(i + 1);
|
|
78
|
+
}
|
|
79
|
+
return out;
|
|
80
|
+
}
|
|
81
|
+
/** A unit's state as a host status: loaded and active (or restarting) is running; unloaded is gone. */
|
|
82
|
+
export function unitStatus(show) {
|
|
83
|
+
if (show.LoadState === "not-found" || show.LoadState === undefined)
|
|
84
|
+
return "gone";
|
|
85
|
+
switch (show.ActiveState) {
|
|
86
|
+
case "active":
|
|
87
|
+
case "activating":
|
|
88
|
+
case "deactivating":
|
|
89
|
+
case "reloading":
|
|
90
|
+
case "refreshing":
|
|
91
|
+
return "running";
|
|
92
|
+
case "failed":
|
|
93
|
+
return "failed";
|
|
94
|
+
case "inactive":
|
|
95
|
+
return "stopped";
|
|
96
|
+
default:
|
|
97
|
+
return "unknown";
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
export function localHost(opts = {}) {
|
|
101
|
+
const mode = opts.mode ?? "systemd";
|
|
102
|
+
const mountRoot = opts.mountRoot ?? DEFAULT_MOUNT_ROOT;
|
|
103
|
+
const hostName = opts.hostName ?? hostname();
|
|
104
|
+
const bootId = currentBootId();
|
|
105
|
+
const exec = opts.exec ?? runCommand;
|
|
106
|
+
const isRoot = process.getuid?.() === 0;
|
|
107
|
+
const sudo = opts.sudo ?? (isRoot ? false : "/usr/bin/sudo");
|
|
108
|
+
const asRoot = (argv) => (sudo ? [sudo, "-n", ...argv] : argv);
|
|
109
|
+
const root = (argv, o) => exec(asRoot(argv), o);
|
|
110
|
+
if (isRoot && opts.user === undefined && mode === "systemd") {
|
|
111
|
+
throw new LocalHostError("INVALID_ARGUMENT", "running as root: name the unprivileged user the instance runs as (user)");
|
|
112
|
+
}
|
|
113
|
+
const user = String(opts.user ?? process.getuid?.() ?? "");
|
|
114
|
+
const group = String(opts.group ?? process.getgid?.() ?? "");
|
|
115
|
+
const command = opts.command ?? [process.execPath, CLI];
|
|
116
|
+
const archil = opts.archil ?? ARCHIL_SCOPED;
|
|
117
|
+
const stopTimeoutMs = opts.stopTimeoutMs ?? 30_000;
|
|
118
|
+
const prefix = opts.unitPrefix ?? "pda-";
|
|
119
|
+
const procMounts = opts.procMounts ?? "/proc/self/mounts";
|
|
120
|
+
const mine = (h) => h.driver === "local" && h.host === hostName;
|
|
121
|
+
const mountpointOf = (ref) => join(mountRoot, runPath(ref.id));
|
|
122
|
+
const mounted = async (mp) => (await readFile(procMounts, "utf8")).split("\n").some((l) => l.split(" ")[1] === mp);
|
|
123
|
+
const parkMs = opts.parkThresholdMs === undefined ? LOCAL_PARK_THRESHOLD_MS : opts.parkThresholdMs;
|
|
124
|
+
const drainMs = Math.max(0, stopTimeoutMs - Math.min(5_000, stopTimeoutMs / 2));
|
|
125
|
+
const runFlags = (ref) => [
|
|
126
|
+
"run", "--disk", ref.disk, "--region", ref.region, "--id", ref.id, "--mount-root", mountRoot, "--archil", archil,
|
|
127
|
+
...(parkMs ? ["--park-threshold", `${parkMs}ms`] : []),
|
|
128
|
+
"--drain-timeout", `${Math.floor(drainMs)}ms`,
|
|
129
|
+
...(opts.runArgs ?? []),
|
|
130
|
+
];
|
|
131
|
+
const holderEnv = (h) => JSON.stringify(h);
|
|
132
|
+
const tokenFile = (unit) => join(TOKEN_DIR, `${unit}.mount-token`);
|
|
133
|
+
async function writeTokenFile(path, token) {
|
|
134
|
+
if (!sudo) {
|
|
135
|
+
await mkdir(TOKEN_DIR, { recursive: true, mode: 0o700 });
|
|
136
|
+
await writeFile(path, token, { mode: 0o600 });
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
const r = await root(["/bin/sh", "-c", 'umask 077 && mkdir -p "$1" && cat > "$2"', "sh", TOKEN_DIR, path], { input: token });
|
|
140
|
+
if (!okRan(r))
|
|
141
|
+
throw new LocalHostError("START_FAILED", `cannot write the mount token file: ${firstLine(r)}`);
|
|
142
|
+
}
|
|
143
|
+
/** Replace the token file by an empty one: the started instance's open fd keeps the token, the path no longer has it. */
|
|
144
|
+
async function emptyTokenFile(path) {
|
|
145
|
+
const r = await root(["/bin/sh", "-c", 'umask 077 && : > "$1.tmp" && mv -f "$1.tmp" "$1"', "sh", path]);
|
|
146
|
+
if (!okRan(r))
|
|
147
|
+
await removeFile(path);
|
|
148
|
+
}
|
|
149
|
+
async function removeFile(path) {
|
|
150
|
+
if (!sudo)
|
|
151
|
+
return rm(path, { force: true });
|
|
152
|
+
await root(["/bin/rm", "-f", path]);
|
|
153
|
+
}
|
|
154
|
+
async function startUnit(ref, token) {
|
|
155
|
+
const unit = `${prefix}${ref.id}-${Date.now().toString(36)}${Math.floor(Math.random() * 1296).toString(36).padStart(2, "0")}`;
|
|
156
|
+
if (!UNIT_SAFE.test(unit))
|
|
157
|
+
throw new LocalHostError("INVALID_ARGUMENT", `unit name ${unit} is not safe`);
|
|
158
|
+
const mountpoint = mountpointOf(ref);
|
|
159
|
+
const handle = { driver: "local", mode: "systemd", host: hostName, bootId, unit, mountpoint };
|
|
160
|
+
const env = { ...opts.env, PATH: SAFE_PATH, PDA_HOLDER: holderEnv(handle) };
|
|
161
|
+
const path = tokenFile(unit);
|
|
162
|
+
await writeTokenFile(path, token);
|
|
163
|
+
let started = false;
|
|
164
|
+
try {
|
|
165
|
+
const argv = [
|
|
166
|
+
SYSTEMD_RUN,
|
|
167
|
+
`--unit=${unit}`,
|
|
168
|
+
`--description=pi-durable-disk run ${ref.id}`,
|
|
169
|
+
"--service-type=exec",
|
|
170
|
+
"--quiet",
|
|
171
|
+
"-p", "KillMode=control-group",
|
|
172
|
+
"-p", `TimeoutStopSec=${Math.max(1, Math.ceil(stopTimeoutMs / 1000))}s`,
|
|
173
|
+
...(opts.restart === false ? [] : ["-p", "Restart=on-failure", "-p", "RestartSec=1", "-p", `RestartPreventExitStatus=${TERMINAL_EXITS.join(" ")}`]),
|
|
174
|
+
"-p", `User=${user}`,
|
|
175
|
+
"-p", `Group=${group}`,
|
|
176
|
+
"-p", `StandardInput=file:${path}`,
|
|
177
|
+
...Object.entries(env).map(([k, v]) => `--setenv=${k}=${v}`),
|
|
178
|
+
"--",
|
|
179
|
+
...command,
|
|
180
|
+
...runFlags(ref),
|
|
181
|
+
"--token-stdin",
|
|
182
|
+
];
|
|
183
|
+
const r = await root(argv, { timeoutMs: 60_000 });
|
|
184
|
+
if (!okRan(r))
|
|
185
|
+
throw new LocalHostError("START_FAILED", `systemd-run ${unit}: ${r.timedOut ? "timed out" : `exit ${r.code}`}: ${firstLine(r)}`);
|
|
186
|
+
started = true;
|
|
187
|
+
return handle;
|
|
188
|
+
}
|
|
189
|
+
finally {
|
|
190
|
+
// Type=exec: systemd-run returns after the exec, so the instance's stdin is already open on the token. The path
|
|
191
|
+
// stays (empty) because a restart in place opens it again; stop removes it.
|
|
192
|
+
await (started ? emptyTokenFile(path) : removeFile(path)).catch(() => { });
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
async function startChild(ref, token) {
|
|
196
|
+
const tag = `${prefix}${ref.id}-${Date.now().toString(36)}`;
|
|
197
|
+
const mountpoint = mountpointOf(ref);
|
|
198
|
+
const base = { driver: "local", mode: "child", host: hostName, bootId, mountpoint, tag };
|
|
199
|
+
let out = "ignore";
|
|
200
|
+
if (opts.logDir) {
|
|
201
|
+
const { openSync } = await import("node:fs");
|
|
202
|
+
out = openSync(join(opts.logDir, `${tag}.log`), "a", 0o600);
|
|
203
|
+
}
|
|
204
|
+
const env = { ...opts.env, PATH: SAFE_PATH, LANG: "C.UTF-8", HOME: process.env.HOME ?? "/", PDA_HOLDER: holderEnv(base) };
|
|
205
|
+
const child = spawn(command[0], [...command.slice(1), ...runFlags(ref), "--token-stdin"], { detached: true, stdio: ["pipe", out, out], env });
|
|
206
|
+
await new Promise((resolve, reject) => {
|
|
207
|
+
child.once("spawn", resolve);
|
|
208
|
+
child.once("error", (err) => reject(new LocalHostError("START_FAILED", `cannot start ${command[0]}`, { cause: err })));
|
|
209
|
+
});
|
|
210
|
+
child.stdin.on("error", () => { });
|
|
211
|
+
child.stdin.end(`${token}\n`);
|
|
212
|
+
child.unref();
|
|
213
|
+
if (typeof out === "number")
|
|
214
|
+
(await import("node:fs")).closeSync(out);
|
|
215
|
+
const pid = child.pid;
|
|
216
|
+
return { ...base, pid, startTicks: procStartTicks(pid) ?? 0 };
|
|
217
|
+
}
|
|
218
|
+
/** The FUSE scopes this unit's mounts run in that are still active. */
|
|
219
|
+
async function fuseScopes(unit) {
|
|
220
|
+
const r = await exec([SYSTEMCTL, "list-units", "--all", "--plain", "--no-legend", "--type=scope", `${unit}-fuse-*`]);
|
|
221
|
+
return r.stdout
|
|
222
|
+
.split("\n")
|
|
223
|
+
.map((l) => l.trim().split(/\s+/))
|
|
224
|
+
.filter((f) => f.length >= 3 && f[2] === "active")
|
|
225
|
+
.map((f) => f[0]);
|
|
226
|
+
}
|
|
227
|
+
/** SIGKILL the unit's FUSE daemons (power-off semantics for its mount) and wait until their scopes are gone. */
|
|
228
|
+
async function killDaemons(unit) {
|
|
229
|
+
const alive = await fuseScopes(unit);
|
|
230
|
+
if (!alive.length)
|
|
231
|
+
return;
|
|
232
|
+
await root([SYSTEMCTL, "kill", "--signal=SIGKILL", ...alive]);
|
|
233
|
+
for (let i = 0; i < 50 && (await fuseScopes(unit)).length; i++)
|
|
234
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Whether the mount at `mp` is dead: its FUSE daemon is gone, so the kernel answers with ENOTCONN ("Transport endpoint
|
|
238
|
+
* is not connected"). The stat runs in a child with a timeout, so a live but stuck daemon never blocks this process; a
|
|
239
|
+
* stat that answers or does not finish is a live mount.
|
|
240
|
+
*/
|
|
241
|
+
async function deadMount(mp) {
|
|
242
|
+
const r = await exec([STAT, "-c", "%i", mp], { timeoutMs: STAT_TIMEOUT_MS });
|
|
243
|
+
return !r.timedOut && r.code !== 0 && /Transport endpoint is not connected/.test(r.stderr);
|
|
244
|
+
}
|
|
245
|
+
/** Remove a dead mount: `fusermount -u`, then a lazy `umount -l`. The mount table, not an exit code, decides. */
|
|
246
|
+
async function removeDeadMount(mp) {
|
|
247
|
+
await root([FUSERMOUNT, "-u", mp]);
|
|
248
|
+
if (await mounted(mp))
|
|
249
|
+
await root([UMOUNT, "-l", mp]);
|
|
250
|
+
if (await mounted(mp))
|
|
251
|
+
throw new LocalHostError("STOP_FAILED", `${mp} is still mounted after fusermount -u and umount -l`);
|
|
252
|
+
await root(["/bin/rmdir", mp]);
|
|
253
|
+
}
|
|
254
|
+
/**
|
|
255
|
+
* Clean the run's mount after its instance is gone. With the mount in the table and this unit's FUSE scope alive:
|
|
256
|
+
* `archil unmount` (flush, check the delegation in) in a child process with a short timeout. Then this unit's FUSE
|
|
257
|
+
* scope is SIGKILLed while it is still active, whatever the mount table says: a client whose filesystem FAILED (an
|
|
258
|
+
* auth failure at a token refresh) drops its mount but keeps its process and the mountpoint's control socket, and the
|
|
259
|
+
* next mount at the path is refused for as long as it lives. The scope must be gone afterwards. Then a mount still in
|
|
260
|
+
* the table whose daemon is gone (a power off, an OOM kill, the SIGKILL) is removed. A live mount without this unit's
|
|
261
|
+
* scope is a newer instance's and is never touched; nothing here stats the mount from this process.
|
|
262
|
+
*/
|
|
263
|
+
async function cleanMount(unit, mp) {
|
|
264
|
+
if ((await mounted(mp)) && (await fuseScopes(unit)).length)
|
|
265
|
+
await root([ARCHIL, "unmount", mp], { timeoutMs: POLITE_UNMOUNT_MS });
|
|
266
|
+
await killDaemons(unit);
|
|
267
|
+
const left = await fuseScopes(unit);
|
|
268
|
+
if (left.length)
|
|
269
|
+
throw new LocalHostError("STOP_FAILED", `${left.join(", ")} still active after SIGKILL`);
|
|
270
|
+
if ((await mounted(mp)) && (await deadMount(mp)))
|
|
271
|
+
await removeDeadMount(mp);
|
|
272
|
+
}
|
|
273
|
+
async function show(unit) {
|
|
274
|
+
const r = await exec([SYSTEMCTL, "show", `${unit}.service`, "--property=LoadState,ActiveState,SubState,Result,ExecMainStatus,MainPID"]);
|
|
275
|
+
return parseShow(r.stdout);
|
|
276
|
+
}
|
|
277
|
+
return {
|
|
278
|
+
hostName,
|
|
279
|
+
mountRoot,
|
|
280
|
+
start(ref, token) {
|
|
281
|
+
runPath(ref.id);
|
|
282
|
+
return mode === "systemd" ? startUnit(ref, token) : startChild(ref, token);
|
|
283
|
+
},
|
|
284
|
+
async status(h) {
|
|
285
|
+
if (!mine(h))
|
|
286
|
+
return "unknown";
|
|
287
|
+
if (h.bootId !== bootId)
|
|
288
|
+
return "gone";
|
|
289
|
+
if (h.mode === "child") {
|
|
290
|
+
const ticks = procStartTicks(Number(h.pid));
|
|
291
|
+
return ticks === null || ticks !== h.startTicks ? "stopped" : "running";
|
|
292
|
+
}
|
|
293
|
+
if (typeof h.unit !== "string" || !UNIT_SAFE.test(h.unit))
|
|
294
|
+
return "unknown";
|
|
295
|
+
return unitStatus(await show(h.unit));
|
|
296
|
+
},
|
|
297
|
+
async stop(h) {
|
|
298
|
+
if (!mine(h) || h.bootId !== bootId)
|
|
299
|
+
return;
|
|
300
|
+
if (h.mode === "child") {
|
|
301
|
+
const pid = Number(h.pid);
|
|
302
|
+
// A handle comes back from run.json with whatever was written there, and stop signals its pid as a group: only a
|
|
303
|
+
// real process, then. A pid of 1 or below names a group (`kill(-0)` is this supervisor's own, `kill(-1)` every
|
|
304
|
+
// process it may signal), and without start ticks nothing tells a reused pid from the instance.
|
|
305
|
+
if (!Number.isInteger(pid) || pid <= 1 || typeof h.startTicks !== "number")
|
|
306
|
+
return;
|
|
307
|
+
const alive = () => procStartTicks(pid) === h.startTicks;
|
|
308
|
+
if (!alive())
|
|
309
|
+
return;
|
|
310
|
+
try {
|
|
311
|
+
process.kill(-pid, "SIGTERM");
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
return;
|
|
315
|
+
}
|
|
316
|
+
for (let t = 0; t < stopTimeoutMs && alive(); t += 100)
|
|
317
|
+
await new Promise((r) => setTimeout(r, 100));
|
|
318
|
+
if (alive())
|
|
319
|
+
process.kill(-pid, "SIGKILL");
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
if (typeof h.unit !== "string" || !UNIT_SAFE.test(h.unit))
|
|
323
|
+
return;
|
|
324
|
+
const stopUnit = () => root([SYSTEMCTL, "stop", `${h.unit}.service`], { timeoutMs: stopTimeoutMs + 10_000 });
|
|
325
|
+
let r = await stopUnit();
|
|
326
|
+
if (!okRan(r) && unitStatus(await show(h.unit)) === "running") {
|
|
327
|
+
// A unit that outlives SIGKILL is wedged in its mount (a FUSE request its daemon never answers): kill the
|
|
328
|
+
// daemon so those requests fail and the processes can die, then stop once more.
|
|
329
|
+
await killDaemons(h.unit);
|
|
330
|
+
r = await stopUnit();
|
|
331
|
+
if (!okRan(r) && unitStatus(await show(h.unit)) === "running") {
|
|
332
|
+
throw new LocalHostError("STOP_FAILED", `systemctl stop ${h.unit}: ${r.timedOut ? "timed out" : firstLine(r)}`);
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
await cleanMount(h.unit, String(h.mountpoint));
|
|
336
|
+
await root([SYSTEMCTL, "reset-failed", `${h.unit}.service`]);
|
|
337
|
+
await removeFile(tokenFile(h.unit));
|
|
338
|
+
},
|
|
339
|
+
};
|
|
340
|
+
}
|
|
341
|
+
/** Field 22 of /proc/<pid>/stat (start time in clock ticks), so a reused pid is never mistaken for the instance. */
|
|
342
|
+
export function procStartTicks(pid) {
|
|
343
|
+
if (!Number.isInteger(pid) || pid <= 0)
|
|
344
|
+
return null;
|
|
345
|
+
let stat;
|
|
346
|
+
try {
|
|
347
|
+
stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
348
|
+
}
|
|
349
|
+
catch {
|
|
350
|
+
return null;
|
|
351
|
+
}
|
|
352
|
+
const rest = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
|
|
353
|
+
if (rest[0] === "Z" || rest[0] === "X")
|
|
354
|
+
return null;
|
|
355
|
+
return Number(rest[19]);
|
|
356
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export * from "./app.js";
|
|
2
|
+
export * from "./claim.js";
|
|
3
|
+
export { PathOutsideWorkError, type RefusalReason } from "./confine.js";
|
|
4
|
+
export * from "./env.js";
|
|
5
|
+
export * from "./errors.js";
|
|
6
|
+
export * from "./fork.js";
|
|
7
|
+
export * from "./hosts/daytona.js";
|
|
8
|
+
export * from "./hosts/docker.js";
|
|
9
|
+
export * from "./hosts/local-host.js";
|
|
10
|
+
export * from "./local-run.js";
|
|
11
|
+
export * from "./park.js";
|
|
12
|
+
export * from "./run.js";
|
|
13
|
+
export * from "./serve.js";
|
|
14
|
+
export * from "./status.js";
|
|
15
|
+
export * from "./store.js";
|
|
16
|
+
export * from "./supervise.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// Public API.
|
|
2
|
+
export * from "./app.js";
|
|
3
|
+
export * from "./claim.js";
|
|
4
|
+
export { PathOutsideWorkError } from "./confine.js";
|
|
5
|
+
export * from "./env.js";
|
|
6
|
+
export * from "./errors.js";
|
|
7
|
+
export * from "./fork.js";
|
|
8
|
+
export * from "./hosts/daytona.js";
|
|
9
|
+
export * from "./hosts/docker.js";
|
|
10
|
+
export * from "./hosts/local-host.js";
|
|
11
|
+
export * from "./local-run.js";
|
|
12
|
+
export * from "./park.js";
|
|
13
|
+
export * from "./run.js";
|
|
14
|
+
export * from "./serve.js";
|
|
15
|
+
export * from "./status.js";
|
|
16
|
+
export * from "./store.js";
|
|
17
|
+
export * from "./supervise.js";
|
package/dist/lease.d.ts
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { EXIT_DATAERR, EXIT_SOFTWARE, LeaseLapsedError, openRunLease, OWNER_LOCK, RunError, StoreBehindSealError, storeHead, STORE_FILE } from "./run.js";
|
|
2
|
+
export type { LeaseOptions, OpenRunLeaseOptions, OpenStep, ReleaseStep, RunClaim, RunErrorCode, RunLease, StoreHeadSource } from "./run.js";
|
|
3
|
+
export * from "./errors.js";
|
|
4
|
+
export { assertPragmas, Fence, FencedDatabase, openArchilStore, PROFILE_PRAGMAS, StoreBusyError, StoreFencedError, StorePragmaError } from "./store.js";
|
|
5
|
+
export type { ArchilStore, OpenOptions, PragmaMismatch, Profile } from "./store.js";
|
|
6
|
+
export type { AcquireOptions, ArchilHost, Claim, RunRef, Unmounted } from "./claim.js";
|
|
7
|
+
export type { ClaimDir, PersistRecord, RunHolder, RunRecord, RunStatus } from "./status.js";
|
package/dist/lease.js
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
// The narrow entry (`@parcha/pi-durable-disk/lease`) for a host that owns its own pi-durable Harness and store connections: the
|
|
2
|
+
// claim's lifecycle without the supervisor, the host drivers, the CLI or the app contract. Importing it must not pull those
|
|
3
|
+
// in (test/lease-entry.test.ts bundles it and checks); a name added here is part of that promise.
|
|
4
|
+
export { EXIT_DATAERR, EXIT_SOFTWARE, LeaseLapsedError, openRunLease, OWNER_LOCK, RunError, StoreBehindSealError, storeHead, STORE_FILE } from "./run.js";
|
|
5
|
+
export * from "./errors.js";
|
|
6
|
+
export { assertPragmas, Fence, FencedDatabase, openArchilStore, PROFILE_PRAGMAS, StoreBusyError, StoreFencedError, StorePragmaError } from "./store.js";
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { AcquireOptions, Claim } from "./claim.js";
|
|
2
|
+
import { type ClaimDir } from "./status.js";
|
|
3
|
+
export interface LocalRun {
|
|
4
|
+
/** Takes the claim: the directory, made when it is missing. */
|
|
5
|
+
readonly acquire: (options: AcquireOptions) => Promise<Claim>;
|
|
6
|
+
/** Holds the directory open as the run's root; a local directory is not an Archil mount, so its type is not checked. */
|
|
7
|
+
readonly claimDir: (root: string) => Promise<ClaimDir>;
|
|
8
|
+
}
|
|
9
|
+
/** `openDurableRun`'s `acquire` and `claimDir` for a run in `root`, a directory on this machine. */
|
|
10
|
+
export declare function localRun(root: string): LocalRun;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// A run on a plain local directory, for one machine: the claim is the directory itself. Nothing mounts and nothing is
|
|
2
|
+
// delegated, so nothing but this machine's owner lock stands between two processes and the run: it is for a laptop, a CI
|
|
3
|
+
// job or a test, never for a directory another machine can reach.
|
|
4
|
+
//
|
|
5
|
+
// const run = await openDurableRun(ref, { ...localRun("/var/runs/run-1"), harness: { models, registry } });
|
|
6
|
+
//
|
|
7
|
+
// No mount token is needed (`acquire` is replaced). The claim's barrier does not sync the files the agent wrote: the
|
|
8
|
+
// store's own commits are made durable by SQLite, and work/ is the operating system's to flush; a power loss can lose what
|
|
9
|
+
// it had not written yet. It refuses once the claim is fenced, as a mount's does.
|
|
10
|
+
//
|
|
11
|
+
// Linux only, for now: the run's root is held open through /proc/self/fd (openClaimDir), and the default environment
|
|
12
|
+
// (`archilEnv`) confines commands with `setpriv` (util-linux) and the file tools through /proc/self/fd. macOS has neither.
|
|
13
|
+
import { mkdir } from "node:fs/promises";
|
|
14
|
+
import { join, resolve } from "node:path";
|
|
15
|
+
import { FencedError } from "./errors.js";
|
|
16
|
+
import { openClaimDir } from "./status.js";
|
|
17
|
+
/** `openDurableRun`'s `acquire` and `claimDir` for a run in `root`, a directory on this machine. */
|
|
18
|
+
export function localRun(root) {
|
|
19
|
+
const dir = resolve(root);
|
|
20
|
+
return {
|
|
21
|
+
acquire: async ({ ref }) => {
|
|
22
|
+
await mkdir(dir, { recursive: true });
|
|
23
|
+
return new LocalClaim(ref, dir);
|
|
24
|
+
},
|
|
25
|
+
claimDir: (at) => openClaimDir(at, { fstype: null }),
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
class LocalClaim {
|
|
29
|
+
ref;
|
|
30
|
+
disk;
|
|
31
|
+
root;
|
|
32
|
+
work;
|
|
33
|
+
store;
|
|
34
|
+
/** `<root>/commands`: a local run has no unit cgroup, so the next open ends the commands a killed one left running. */
|
|
35
|
+
commands;
|
|
36
|
+
reused = false;
|
|
37
|
+
stray = null;
|
|
38
|
+
forced = false;
|
|
39
|
+
timings = { mountMs: 0, verifyMs: 0 };
|
|
40
|
+
#fence;
|
|
41
|
+
constructor(ref, root) {
|
|
42
|
+
this.ref = ref;
|
|
43
|
+
this.disk = ref.disk;
|
|
44
|
+
this.root = root;
|
|
45
|
+
this.work = join(root, "work");
|
|
46
|
+
this.store = join(root, "store");
|
|
47
|
+
this.commands = join(root, "commands");
|
|
48
|
+
}
|
|
49
|
+
get fenced() {
|
|
50
|
+
return this.#fence !== undefined;
|
|
51
|
+
}
|
|
52
|
+
markFenced(cause) {
|
|
53
|
+
this.#fence ??= { cause };
|
|
54
|
+
}
|
|
55
|
+
async barrier() {
|
|
56
|
+
if (this.#fence)
|
|
57
|
+
throw new FencedError(`${this.root} is fenced; nothing is retried on it`, { cause: this.#fence.cause });
|
|
58
|
+
return { ms: 0 };
|
|
59
|
+
}
|
|
60
|
+
async release() {
|
|
61
|
+
return { via: "none" };
|
|
62
|
+
}
|
|
63
|
+
}
|
package/dist/park.d.ts
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import type { Context, JsonValue } from "@earendil-works/chord";
|
|
2
|
+
import type { Harness, HarnessInspection } from "@earendil-works/pi-durable";
|
|
3
|
+
import type { DurableRun, RunLease } from "./run.js";
|
|
4
|
+
/**
|
|
5
|
+
* What parking and drain need of a run: its Harness, and the lease's status write and release. A `DurableRun` is one;
|
|
6
|
+
* a host that owns its Harness over `openRunLease` gets one from `leaseParkTarget`.
|
|
7
|
+
*/
|
|
8
|
+
export type ParkTarget = Pick<DurableRun, "harness" | "fenced" | "setStatus" | "release">;
|
|
9
|
+
/**
|
|
10
|
+
* The park target of a Harness the host opened over `lease`: the wake goes through `lease.setStatus`, and the release
|
|
11
|
+
* closes the Harness (running work is aborted with no outcome recorded) and then releases the lease. A rejecting close
|
|
12
|
+
* is a fence, as in `DurableRun.release`.
|
|
13
|
+
*/
|
|
14
|
+
export declare function leaseParkTarget(lease: RunLease, harness: Harness, context?: Context): ParkTarget;
|
|
15
|
+
/** localHost's threshold, far above twice its start (about 1.5 s); Cloudflare and Rivet park pi after 60 s too. */
|
|
16
|
+
export declare const LOCAL_PARK_THRESHOLD_MS = 60000;
|
|
17
|
+
export type BusyState = {
|
|
18
|
+
readonly kind: "busy";
|
|
19
|
+
} | {
|
|
20
|
+
readonly kind: "idle";
|
|
21
|
+
}
|
|
22
|
+
/** Every task that could run sleeps until at least `until` (epoch ms, pi's clock), further away than the threshold. */
|
|
23
|
+
| {
|
|
24
|
+
readonly kind: "waiting";
|
|
25
|
+
readonly until: number;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* The deadline of a task that is only sleeping: `until` in pi's retry phase (generation and compaction), `pollAt` in
|
|
29
|
+
* its deferred poll phase. This is pi-durable's internal checkpoint shape, the same in 1.0.4 and 1.1.0
|
|
30
|
+
* (harness/generation.js, compaction.js), read here and nowhere else. Any other shape is no deadline, which counts as
|
|
31
|
+
* busy, so a pi that changes the shape keeps instances up rather than parking them mid-work. pi#10325 asks for this as
|
|
32
|
+
* a contract.
|
|
33
|
+
*/
|
|
34
|
+
export declare function waitDeadline(checkpoint: JsonValue | undefined): number | undefined;
|
|
35
|
+
/**
|
|
36
|
+
* Whether the run's live work needs this instance. A task that runs or is ready to run does, unless it only sleeps
|
|
37
|
+
* until a deadline more than `thresholdMs` away. Waiting and completing tasks depend on other tasks, which decide; a
|
|
38
|
+
* blocked task (no registered definition can take it) never runs, so it never keeps the instance up. A closing
|
|
39
|
+
* Harness, a task state this code does not know, and unsettled submissions with no live task all count as busy.
|
|
40
|
+
*/
|
|
41
|
+
export declare function busyState(inspection: HarnessInspection, now: number, thresholdMs: number): BusyState;
|
|
42
|
+
/** The default wake: run.json `sleeping` with `wakeAt` (null: idle, started on demand), written durably. */
|
|
43
|
+
export declare function recordWake(run: Pick<ParkTarget, "setStatus">, at: number | null): Promise<void>;
|
|
44
|
+
export interface ParkOptions {
|
|
45
|
+
/** A wait no longer than this keeps the instance up: per driver, at least twice the time the driver takes to start one. */
|
|
46
|
+
readonly thresholdMs: number;
|
|
47
|
+
/** Park a run with no live work once it stayed so this long; default never (only a request can wake an idle run). */
|
|
48
|
+
readonly idleMs?: number;
|
|
49
|
+
/**
|
|
50
|
+
* Record the wake before the release; default `recordWake`. A rejection that is not a fence keeps the instance up
|
|
51
|
+
* and pi's own timer ends the wait. A failed run.json write is a fence, as every error writing the claim's own files
|
|
52
|
+
* is: the instance exits 75 and the supervisor resumes the run after the lease.
|
|
53
|
+
*/
|
|
54
|
+
readonly wake?: (run: ParkTarget, at: number | null) => Promise<void>;
|
|
55
|
+
/** Something outside pi needs the instance (an open request): it never parks while this returns true. */
|
|
56
|
+
readonly keepAwake?: () => boolean;
|
|
57
|
+
/** Stop admitting new work before the last check; returns the undo for a park that is called off. */
|
|
58
|
+
readonly quiesce?: () => () => void;
|
|
59
|
+
/** pi's clock (HarnessOptions.now); default Date.now. */
|
|
60
|
+
readonly now?: () => number;
|
|
61
|
+
readonly log?: (event: string, detail: Record<string, JsonValue>) => void;
|
|
62
|
+
}
|
|
63
|
+
export type ParkResult = {
|
|
64
|
+
readonly wakeAt: number | null;
|
|
65
|
+
readonly blocked: number;
|
|
66
|
+
};
|
|
67
|
+
export interface Parking {
|
|
68
|
+
/** Settles once the run parked and was released; stays pending while the instance stays up. */
|
|
69
|
+
readonly parked: Promise<ParkResult>;
|
|
70
|
+
/** Look again now (call it when `keepAwake` turns false). */
|
|
71
|
+
check(): void;
|
|
72
|
+
/** No new attempt; resolves with the park that was already under way, if any finished. */
|
|
73
|
+
stop(): Promise<ParkResult | undefined>;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Watch `run` and park it when its work allows. Checks after every commit (pi's listener must not call pi,
|
|
77
|
+
* so the check runs after it returns), on `check()`, and when an idle period ends. Parking stops admission, checks
|
|
78
|
+
* once more, records the wake, then releases: the wake is always written before the claim goes.
|
|
79
|
+
*/
|
|
80
|
+
export declare function watchParking(run: ParkTarget, options: ParkOptions): Parking;
|
|
81
|
+
export interface DrainOptions {
|
|
82
|
+
/** Epoch ms after which the drain gives up and the run is released with its work interrupted. */
|
|
83
|
+
readonly deadline: number;
|
|
84
|
+
readonly now?: () => number;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Wait until no task needs this instance, or until `deadline`, rechecking after every commit.
|
|
88
|
+
* Any sleeping task counts as waiting here, however short its wait: closing ends the sleep and the next open resumes it.
|
|
89
|
+
* Returns the last classification: `busy` means the deadline passed with work running, which the release interrupts
|
|
90
|
+
* and the next open resumes (safe tools rerun, unsafe ones report the interruption).
|
|
91
|
+
*/
|
|
92
|
+
export declare function drain(run: Pick<ParkTarget, "harness">, options: DrainOptions): Promise<BusyState>;
|