@parcha/pi-durable-disk 0.1.0-beta.13 → 0.1.0-beta.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +262 -0
- package/README.md +281 -47
- package/dist/app.d.ts +5 -1
- package/dist/claim.d.ts +51 -3
- package/dist/claim.js +157 -17
- package/dist/cli.d.ts +6 -0
- package/dist/cli.js +112 -47
- package/dist/commands.d.ts +4 -1
- package/dist/commands.js +48 -90
- package/dist/confine.d.ts +36 -7
- package/dist/confine.js +125 -18
- package/dist/env.d.ts +7 -22
- package/dist/env.js +25 -60
- package/dist/errors.d.ts +2 -1
- package/dist/fork.d.ts +23 -0
- package/dist/fork.js +103 -3
- package/dist/hosts/attempt-env.d.ts +3 -0
- package/dist/hosts/attempt-env.js +15 -0
- package/dist/hosts/daytona-launch.d.ts +3 -1
- package/dist/hosts/daytona-launch.js +35 -12
- package/dist/hosts/daytona.d.ts +2 -2
- package/dist/hosts/daytona.js +48 -5
- package/dist/hosts/docker.d.ts +2 -2
- package/dist/hosts/docker.js +22 -5
- package/dist/hosts/local-host.d.ts +9 -3
- package/dist/hosts/local-host.js +59 -22
- package/dist/index.d.ts +35 -16
- package/dist/index.js +27 -16
- package/dist/internals.d.ts +3 -0
- package/dist/internals.js +6 -0
- package/dist/lease.d.ts +4 -1
- package/dist/lease.js +2 -0
- package/dist/local-run.js +3 -2
- package/dist/place.d.ts +190 -0
- package/dist/place.js +526 -0
- package/dist/process-table.d.ts +38 -0
- package/dist/process-table.js +158 -0
- package/dist/run.d.ts +61 -10
- package/dist/run.js +69 -33
- package/dist/status.d.ts +19 -3
- package/dist/status.js +25 -13
- package/dist/store.d.ts +14 -2
- package/dist/store.js +16 -4
- package/dist/supervise.d.ts +43 -1
- package/dist/supervise.js +26 -1
- package/dist/watchdog.d.ts +3 -1
- package/dist/watchdog.js +43 -12
- package/dist/with-claim.d.ts +63 -0
- package/dist/with-claim.js +325 -0
- package/package.json +9 -3
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
// The process table a run reads to know its commands and its holder. On Linux it is /proc, read live. macOS has no /proc:
|
|
2
|
+
// there it is one ps(1) listing per look, ps for a process's environment, and sysctl(8) for the boot. A process's start
|
|
3
|
+
// tells it from a later process with the same pid: on Linux its start time in clock ticks after boot (field 22 of
|
|
4
|
+
// /proc/<pid>/stat), on macOS the second it started (ps's `lstart`, read in UTC). A start is only ever compared with
|
|
5
|
+
// another read on the same machine and boot. On macOS a pid freed and handed out again within the same second would pass
|
|
6
|
+
// for the old process; pids there are handed out in order up to 99999 before one is reused.
|
|
7
|
+
import { execFileSync } from "node:child_process";
|
|
8
|
+
import { readdirSync, readFileSync } from "node:fs";
|
|
9
|
+
import { join } from "node:path";
|
|
10
|
+
/** The table on Linux: `/proc` (or another directory laid out like it, for tests). */
|
|
11
|
+
export function procTable(proc = "/proc") {
|
|
12
|
+
const read = (...parts) => readFileSync(join(proc, ...parts), "utf8");
|
|
13
|
+
const snapshot = {
|
|
14
|
+
// `stat`'s fields after the last `)`, since the command name may hold spaces and parentheses.
|
|
15
|
+
info(pid) {
|
|
16
|
+
try {
|
|
17
|
+
const text = read(String(pid), "stat");
|
|
18
|
+
const fields = text.slice(text.lastIndexOf(")") + 2).split(" ");
|
|
19
|
+
if (fields[0] === "Z" || fields[0] === "X")
|
|
20
|
+
return undefined;
|
|
21
|
+
return { ppid: Number(fields[1]), pgrp: Number(fields[2]), start: Number(fields[19]) };
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
},
|
|
27
|
+
// From each of its threads' `children` list.
|
|
28
|
+
children(pid) {
|
|
29
|
+
let tasks;
|
|
30
|
+
try {
|
|
31
|
+
tasks = readdirSync(join(proc, String(pid), "task"));
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
return tasks.flatMap((tid) => {
|
|
37
|
+
try {
|
|
38
|
+
return read(String(pid), "task", tid, "children").split(" ").filter(Boolean).map(Number);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return [];
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
},
|
|
45
|
+
marked(pid, name, value) {
|
|
46
|
+
try {
|
|
47
|
+
return read(String(pid), "environ").split("\0").includes(`${name}=${value}`);
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
commandLine(pid) {
|
|
54
|
+
try {
|
|
55
|
+
return read(String(pid), "cmdline").split("\0").filter(Boolean).join(" ") || "?";
|
|
56
|
+
}
|
|
57
|
+
catch {
|
|
58
|
+
return "?";
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
return {
|
|
63
|
+
snapshot: () => snapshot,
|
|
64
|
+
bootId() {
|
|
65
|
+
try {
|
|
66
|
+
return read("sys/kernel/random/boot_id").trim() || undefined;
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
walkMs: 100,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
const PS = "/bin/ps";
|
|
76
|
+
const SYSCTL = "/usr/sbin/sysctl";
|
|
77
|
+
const MONTHS = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
|
78
|
+
/** In the C locale and UTC, so `lstart` reads the same whatever the user's settings. */
|
|
79
|
+
const runCommand = (file, args) => execFileSync(file, args, { encoding: "utf8", env: { LC_ALL: "C", TZ: "UTC" }, stdio: ["ignore", "pipe", "ignore"], timeout: 10_000, maxBuffer: 64 << 20 });
|
|
80
|
+
/**
|
|
81
|
+
* One line of `ps -A -o pid=,ppid=,pgid=,stat=,lstart=` in UTC: `pid ppid pgid stat Www Mmm dd hh:mm:ss yyyy`. Null
|
|
82
|
+
* for a zombie; throws on a line it does not read.
|
|
83
|
+
*/
|
|
84
|
+
function psLine(line) {
|
|
85
|
+
const f = line.trim().split(/\s+/);
|
|
86
|
+
const [pid, ppid, pgrp] = f.slice(0, 3).map(Number);
|
|
87
|
+
const month = MONTHS.indexOf(f[5] ?? "");
|
|
88
|
+
const [h, m, s] = (f[7] ?? "").split(":").map(Number);
|
|
89
|
+
const start = Date.UTC(Number(f[8]), month, Number(f[6]), h, m, s) / 1000;
|
|
90
|
+
if (f.length !== 9 || ![pid, ppid, pgrp, start].every(Number.isSafeInteger) || month < 0)
|
|
91
|
+
throw new Error(`ps printed a line this does not read: ${JSON.stringify(line)}`);
|
|
92
|
+
return f[3].startsWith("Z") ? null : [pid, { ppid: ppid, pgrp: pgrp, start }];
|
|
93
|
+
}
|
|
94
|
+
/** The table on macOS: `ps` and `sysctl`, through `run` (for tests). */
|
|
95
|
+
export function psTable(run = runCommand) {
|
|
96
|
+
let boot;
|
|
97
|
+
return {
|
|
98
|
+
snapshot() {
|
|
99
|
+
const rows = new Map();
|
|
100
|
+
for (const line of run(PS, ["-A", "-o", "pid=,ppid=,pgid=,stat=,lstart="]).split("\n")) {
|
|
101
|
+
if (line.trim() === "")
|
|
102
|
+
continue;
|
|
103
|
+
const row = psLine(line);
|
|
104
|
+
if (row !== null)
|
|
105
|
+
rows.set(...row);
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
info: (pid) => rows.get(pid),
|
|
109
|
+
children: (pid) => [...rows].filter(([, info]) => info.ppid === pid).map(([child]) => child),
|
|
110
|
+
// `-E` prints the environment the process was started with after its arguments, separated by spaces; a mark is
|
|
111
|
+
// a name and a UUID, which hold none.
|
|
112
|
+
marked(pid, name, value) {
|
|
113
|
+
try {
|
|
114
|
+
return run(PS, ["-ww", "-E", "-p", String(pid), "-o", "command="]).split(/\s+/).includes(`${name}=${value}`);
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
},
|
|
120
|
+
commandLine(pid) {
|
|
121
|
+
try {
|
|
122
|
+
return run(PS, ["-ww", "-p", String(pid), "-o", "command="]).trim() || "?";
|
|
123
|
+
}
|
|
124
|
+
catch {
|
|
125
|
+
return "?";
|
|
126
|
+
}
|
|
127
|
+
},
|
|
128
|
+
};
|
|
129
|
+
},
|
|
130
|
+
// Read once it can be: a boot's id does not change.
|
|
131
|
+
bootId() {
|
|
132
|
+
try {
|
|
133
|
+
boot ??= run(SYSCTL, ["-n", "kern.bootsessionuuid"]).trim() || undefined;
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
// not now; the next call asks again
|
|
137
|
+
}
|
|
138
|
+
return boot;
|
|
139
|
+
},
|
|
140
|
+
walkMs: 500,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
/** Test seam: a table that stands in for this machine's everywhere `processTable()` is read. */
|
|
144
|
+
export const processTableHooks = {};
|
|
145
|
+
let platformTable;
|
|
146
|
+
/** This machine's table: ps on macOS, /proc elsewhere. */
|
|
147
|
+
export function processTable() {
|
|
148
|
+
if (processTableHooks.table !== undefined)
|
|
149
|
+
return processTableHooks.table;
|
|
150
|
+
if (platformTable?.platform !== process.platform) {
|
|
151
|
+
platformTable = { platform: process.platform, table: process.platform === "darwin" ? psTable() : procTable() };
|
|
152
|
+
}
|
|
153
|
+
return platformTable.table;
|
|
154
|
+
}
|
|
155
|
+
/** This boot's id on this machine, null when it cannot be read. */
|
|
156
|
+
export function thisBoot() {
|
|
157
|
+
return processTable().bootId() ?? null;
|
|
158
|
+
}
|
package/dist/run.d.ts
CHANGED
|
@@ -57,7 +57,21 @@ export interface LeaseOptions {
|
|
|
57
57
|
/** How often the self-fence timer looks; default 1 s. */
|
|
58
58
|
readonly checkMs?: number;
|
|
59
59
|
}
|
|
60
|
-
export type OpenStep = "acquire" | "owner-lock" | "
|
|
60
|
+
export type OpenStep = "acquire" | "owner-lock" | "layout" | "commands" | "after-claim" | "seed" | "run.json" | "heartbeat" | "store" | "seal" | "harness" | "live" | "before-resume" | "resume";
|
|
61
|
+
/** Where a new run's seed writes: the claim's root, the store file the Harness will open, and the workspace. */
|
|
62
|
+
export interface RunSeedDir {
|
|
63
|
+
readonly root: string;
|
|
64
|
+
readonly storePath: string;
|
|
65
|
+
readonly workDir: string;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* What a new run starts with, written before anything can see the run: its store file (`storePath`, a SQLite database
|
|
69
|
+
* the Harness then opens) and files in its workspace. It runs inside the run's first claim, holding the owner lock, after
|
|
70
|
+
* `store/`, `work/` and `tmp/` exist and `afterClaim` has run, and before `run.json` publishes the run. It never runs on a
|
|
71
|
+
* later open. If it throws (or its process dies) the open fails and nothing is published, so the next open seeds again: it
|
|
72
|
+
* must overwrite what an earlier attempt wrote. A connection it opens on `storePath` must be closed before it returns.
|
|
73
|
+
*/
|
|
74
|
+
export type RunSeed = (dir: RunSeedDir) => void | Promise<void>;
|
|
61
75
|
export type ReleaseStep = "close" | "cleanup" | "barrier" | "seal" | "unlock" | "unmount";
|
|
62
76
|
export interface OpenRunLeaseOptions {
|
|
63
77
|
/**
|
|
@@ -104,12 +118,30 @@ export interface OpenRunLeaseOptions {
|
|
|
104
118
|
readonly claimDir?: (root: string) => Promise<ClaimDir>;
|
|
105
119
|
/** Replaceable for tests: the monotonic clock the self-fence measures with, in ms (default `performance.now`). */
|
|
106
120
|
readonly clock?: () => number;
|
|
121
|
+
/**
|
|
122
|
+
* A new run's starting state (`RunSeed`): run once, on the open that finds no `run.json`, before `run.json` is written
|
|
123
|
+
* and the store opened; what it wrote is made durable (the claim's barrier) before the run is published.
|
|
124
|
+
*/
|
|
125
|
+
readonly seed?: RunSeed;
|
|
107
126
|
}
|
|
108
127
|
export interface OpenDurableRunOptions<Tool extends ToolRegistration = ToolRegistration> extends Omit<OpenRunLeaseOptions, "cleanup"> {
|
|
109
128
|
/** Default `exclusive`. */
|
|
110
129
|
readonly profile?: Profile;
|
|
111
|
-
/**
|
|
112
|
-
|
|
130
|
+
/**
|
|
131
|
+
* pi's Harness options; the registry holds every extension before the Harness opens (recovered tasks need them).
|
|
132
|
+
* Required, here or from `afterClaim`.
|
|
133
|
+
*/
|
|
134
|
+
readonly harness?: Omit<HarnessOptions<Tool>, "env">;
|
|
135
|
+
/**
|
|
136
|
+
* Awaited once the claim is held (the run's directory mounted, its owner lock taken, `store/`, `work/` and `tmp/` made,
|
|
137
|
+
* what a killed incarnation's commands left running reaped) and before this incarnation's `run.json`, with the claim's
|
|
138
|
+
* paths: what it returns (`harness`, `beforeResume`, a new run's `seed`) goes into the open, over what these options
|
|
139
|
+
* give; the seed runs right after it, on a new run only. For code that
|
|
140
|
+
* must run on the live directory, as an app module's factory does under the CLI. It runs before the heartbeat, inside a
|
|
141
|
+
* supervisor's start grace. A rejection fails the open with `run.json` as the open found it (none on a new run), so a
|
|
142
|
+
* supervisor counts a failed start and backs off; the claim is released.
|
|
143
|
+
*/
|
|
144
|
+
readonly afterClaim?: (claim: Pick<Claim, "ref" | "root" | "work" | "store">) => Promise<AfterClaim<Tool>>;
|
|
113
145
|
/** The run's environment, built on the claim; default `archilEnv(claim)`. Its `cleanup` is the lease's. */
|
|
114
146
|
readonly env?: (claim: Claim) => RunEnv;
|
|
115
147
|
/** Wraps the raw database inside the store's fence (instrumentation, fault injection). */
|
|
@@ -122,6 +154,13 @@ export interface OpenDurableRunOptions<Tool extends ToolRegistration = ToolRegis
|
|
|
122
154
|
*/
|
|
123
155
|
readonly beforeResume?: (run: BeforeResume) => void | Promise<void>;
|
|
124
156
|
}
|
|
157
|
+
/** What `afterClaim` gives the open: the Harness options, the `beforeResume` hook and a new run's seed, when it holds them. */
|
|
158
|
+
export interface AfterClaim<Tool extends ToolRegistration = ToolRegistration> {
|
|
159
|
+
readonly harness?: Omit<HarnessOptions<Tool>, "env">;
|
|
160
|
+
readonly beforeResume?: (run: BeforeResume) => void | Promise<void>;
|
|
161
|
+
/** A new run's seed, over the options' `seed`; ignored on an open that finds `run.json` (a resume). */
|
|
162
|
+
readonly seed?: RunSeed;
|
|
163
|
+
}
|
|
125
164
|
/**
|
|
126
165
|
* What `beforeResume` sees: the open Harness, not yet resumed, the open's context, the incarnation it belongs to, and the
|
|
127
166
|
* store the Harness was opened over with its head (a recorder snapshots every document at `head` here).
|
|
@@ -130,7 +169,15 @@ export interface BeforeResume {
|
|
|
130
169
|
readonly harness: Harness;
|
|
131
170
|
readonly context: Context;
|
|
132
171
|
readonly generation: number;
|
|
172
|
+
/** This incarnation's `run.json`, which the open has already written (`running`, this holder, `detail` null). */
|
|
133
173
|
readonly record: RunRecord;
|
|
174
|
+
/**
|
|
175
|
+
* `run.json` as it was before this open wrote over it: how the run was left. After a park it is `sleeping` with its
|
|
176
|
+
* `wakeAt` and `detail` and the seal its release wrote; after a planned stop, `paused` or `done` with the `detail` it
|
|
177
|
+
* set; after a crash, `running` with the dead holder and no new seal. On a fork's first open it is the record `fork`
|
|
178
|
+
* wrote: `paused`, generation 0, `detail.forkedFrom`. Null only when there was no `run.json` (a new run's first open).
|
|
179
|
+
*/
|
|
180
|
+
readonly previous: RunRecord | null;
|
|
134
181
|
readonly claim: RunClaim;
|
|
135
182
|
/** The store the Harness was opened over, read-only: four reads on an object of their own, and no way to a write. */
|
|
136
183
|
readonly storage: StoreReads;
|
|
@@ -156,9 +203,10 @@ export interface RunClaim {
|
|
|
156
203
|
}
|
|
157
204
|
/**
|
|
158
205
|
* The claim's lifecycle for an app that owns its store and Harness (`openDurableRun` without the store and the Harness). Open
|
|
159
|
-
* order, all done by `openRunLease`: acquire and verify the claim, take the owner lock,
|
|
160
|
-
*
|
|
161
|
-
* a
|
|
206
|
+
* order, all done by `openRunLease`: acquire and verify the claim, take the owner lock, create `store/`, `work/` and
|
|
207
|
+
* `tmp/`, reap what a killed incarnation's commands left running, seed a new run (`seed`), write `run.json` (`running`, generation + 1, this
|
|
208
|
+
* holder, a heartbeat, the previous seal kept), start the heartbeat and the self-fence (a timer and a watchdog thread).
|
|
209
|
+
* The app then opens its store, calls `checkSeal` before
|
|
162
210
|
* anything commits, opens its Harness over `observe(storage)`, and calls `live()`. Once fenced, every member that
|
|
163
211
|
* writes rejects with the first fence; nothing is retried.
|
|
164
212
|
*/
|
|
@@ -171,6 +219,8 @@ export interface RunLease {
|
|
|
171
219
|
readonly record: RunRecord;
|
|
172
220
|
/** The seal the previous incarnation's clean release wrote; null on a first run and after a crash. */
|
|
173
221
|
readonly sealedSeq: number | null;
|
|
222
|
+
/** `run.json` as the open found it, before it wrote this incarnation's: null when there was none (a new run's first open). */
|
|
223
|
+
readonly previous: RunRecord | null;
|
|
174
224
|
readonly fenced: boolean;
|
|
175
225
|
/**
|
|
176
226
|
* The seal check on the app's store, before anything commits: the store's head (`storeHead`) below
|
|
@@ -272,15 +322,16 @@ export interface StoreHeadSource {
|
|
|
272
322
|
*/
|
|
273
323
|
export declare function storeHead(store: StoreHeadSource): Promise<number>;
|
|
274
324
|
/**
|
|
275
|
-
* Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock,
|
|
276
|
-
* `
|
|
325
|
+
* Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock, create
|
|
326
|
+
* `store/`, `work/` and `tmp/`, reap a killed incarnation's commands, seed a new run, write `run.json` and start the
|
|
327
|
+
* heartbeat and the self-fence. Errors are typed as for
|
|
277
328
|
* `openDurableRun`: HeldError (76), FencedError (75, also passed to `onFenced`), RunRecordError (65), RunError
|
|
278
329
|
* INVALID_ARGUMENT before any claim.
|
|
279
330
|
*/
|
|
280
331
|
export declare function openRunLease(ref: RunRef, options: OpenRunLeaseOptions): Promise<RunLease>;
|
|
281
332
|
/**
|
|
282
|
-
* Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock,
|
|
283
|
-
* self-fence
|
|
333
|
+
* Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock, layout, the commands reap, then
|
|
334
|
+
* `afterClaim`, a new run's seed, then `run.json`, heartbeat and self-fence), then the store, the seal check before anything commits, pi's Harness with the app's registry,
|
|
284
335
|
* the seal cleared in `run.json`, resume.
|
|
285
336
|
*
|
|
286
337
|
* Errors are typed: HeldError (76) when another client, another process on this host or another connection holds the
|
package/dist/run.js
CHANGED
|
@@ -278,6 +278,9 @@ class Lease {
|
|
|
278
278
|
get sealedSeq() {
|
|
279
279
|
return this.#previous?.sealedSeq ?? null;
|
|
280
280
|
}
|
|
281
|
+
get previous() {
|
|
282
|
+
return this.#previous ?? null;
|
|
283
|
+
}
|
|
281
284
|
get fenced() {
|
|
282
285
|
return this.#fence !== undefined;
|
|
283
286
|
}
|
|
@@ -308,12 +311,15 @@ class Lease {
|
|
|
308
311
|
return this.#fence?.done ?? Promise.resolve();
|
|
309
312
|
}
|
|
310
313
|
/**
|
|
311
|
-
* The order after the claim
|
|
312
|
-
*
|
|
313
|
-
*
|
|
314
|
-
*
|
|
314
|
+
* The order after the claim: the owner lock, the layout, the reap of a killed incarnation's commands, `afterClaim`
|
|
315
|
+
* (openDurableRun's), a new run's seed (afterClaim's over the options'), then `run.json` (`running`, generation + 1,
|
|
316
|
+
* this holder, a heartbeat) and the heartbeat, before
|
|
317
|
+
* the store and the Harness open. Nothing is published before that write: an open that fails earlier leaves `run.json`
|
|
318
|
+
* as it found it, so a supervisor counts a failed start and backs off; its start grace covers the steps before it.
|
|
319
|
+
* Once written, a supervisor never sees the previous incarnation's stale heartbeat on the live claim. That first write
|
|
320
|
+
* keeps the previous `sealedSeq`, which the seal check still needs; `live()` clears it once the store is open.
|
|
315
321
|
*/
|
|
316
|
-
async open(step, holder) {
|
|
322
|
+
async open(step, holder, afterClaim) {
|
|
317
323
|
const claim = this.rawClaim;
|
|
318
324
|
// SQLite resolves symbolic links in a file name, `/proc/self/fd/<n>` included, so `owner.lock` is the one file opened
|
|
319
325
|
// by path: right after the mount check, and the file it names must then be the one in the claim directory.
|
|
@@ -338,6 +344,25 @@ class Lease {
|
|
|
338
344
|
const previous = await this.#readPrevious();
|
|
339
345
|
this.#previous = previous;
|
|
340
346
|
this.#lastSeq = previous?.sealedSeq ?? 0;
|
|
347
|
+
await step("layout", () => this.#layout());
|
|
348
|
+
// A claim that records its commands (a local run's): end what a killed incarnation left running, before any code of
|
|
349
|
+
// this incarnation runs, so no command of it still writes in work/ once the run resumes. A process of it that outlives
|
|
350
|
+
// its SIGKILL refuses the open (HeldError, 76): the run never resumes beside it, and its record stays for the next open.
|
|
351
|
+
const commands = claim.commands;
|
|
352
|
+
if (commands !== undefined)
|
|
353
|
+
await step("commands", () => reapOrRefuse(commands));
|
|
354
|
+
const given = afterClaim ? await step("after-claim", () => afterClaim(claim)) : undefined;
|
|
355
|
+
// A new run (no run.json): its seed writes what it starts with, and only then is it published. afterClaim's seed
|
|
356
|
+
// goes over the options'.
|
|
357
|
+
const seed = given?.seed ?? this.#options.seed;
|
|
358
|
+
if (seed !== undefined && previous === undefined) {
|
|
359
|
+
await step("seed", async () => {
|
|
360
|
+
await this.#dir.assertMounted();
|
|
361
|
+
await seed({ root: claim.root, storePath: join(claim.store, STORE_FILE), workDir: claim.work });
|
|
362
|
+
await this.#dir.assertMounted();
|
|
363
|
+
await claim.barrier();
|
|
364
|
+
});
|
|
365
|
+
}
|
|
341
366
|
await step("run.json", async () => {
|
|
342
367
|
const now = new Date().toISOString();
|
|
343
368
|
this.#noteBeat(await this.#write(() => ({
|
|
@@ -363,19 +388,21 @@ class Lease {
|
|
|
363
388
|
ownCgroup: this.#options.ownCgroup === true,
|
|
364
389
|
});
|
|
365
390
|
});
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
}
|
|
391
|
+
}
|
|
392
|
+
/** `store/`, `work/` and `tmp/`, made through the held claim directory. */
|
|
393
|
+
async #layout() {
|
|
394
|
+
const claim = this.rawClaim;
|
|
395
|
+
for (const dir of [claim.store, claim.work, join(claim.root, "tmp")]) {
|
|
396
|
+
const under = relative(claim.root, dir);
|
|
397
|
+
if (under === "" || under.startsWith("..") || isAbsolute(under))
|
|
398
|
+
throw new RunError("INVALID_ARGUMENT", `${dir} is not under the claim root ${claim.root}`);
|
|
399
|
+
try {
|
|
400
|
+
await mkdir(join(this.#dir.at, under), { recursive: true });
|
|
377
401
|
}
|
|
378
|
-
|
|
402
|
+
catch (error) {
|
|
403
|
+
throw ownWriteFenced(dir, error);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
379
406
|
}
|
|
380
407
|
async checkSeal(store) {
|
|
381
408
|
const sealed = this.sealedSeq;
|
|
@@ -608,7 +635,7 @@ class Lease {
|
|
|
608
635
|
throw new RunError("RUN_RELEASED", `run ${this.ref.id} is released`);
|
|
609
636
|
}
|
|
610
637
|
}
|
|
611
|
-
async function openLease(ref, options, step) {
|
|
638
|
+
async function openLease(ref, options, step, afterClaim) {
|
|
612
639
|
const context = options.context ?? BACKGROUND_CONTEXT;
|
|
613
640
|
leaseOf(options.lease);
|
|
614
641
|
let holder;
|
|
@@ -630,7 +657,7 @@ async function openLease(ref, options, step) {
|
|
|
630
657
|
}));
|
|
631
658
|
const lease = new Lease(ref, claim, options, context);
|
|
632
659
|
try {
|
|
633
|
-
await lease.open(step, holder);
|
|
660
|
+
await lease.open(step, holder, afterClaim);
|
|
634
661
|
}
|
|
635
662
|
catch (error) {
|
|
636
663
|
await lease.abandon(error);
|
|
@@ -639,8 +666,9 @@ async function openLease(ref, options, step) {
|
|
|
639
666
|
return lease;
|
|
640
667
|
}
|
|
641
668
|
/**
|
|
642
|
-
* Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock,
|
|
643
|
-
* `
|
|
669
|
+
* Open the claim's lifecycle for run `ref` (see `RunLease`): acquire and verify the claim, take the owner lock, create
|
|
670
|
+
* `store/`, `work/` and `tmp/`, reap a killed incarnation's commands, seed a new run, write `run.json` and start the
|
|
671
|
+
* heartbeat and the self-fence. Errors are typed as for
|
|
644
672
|
* `openDurableRun`: HeldError (76), FencedError (75, also passed to `onFenced`), RunRecordError (65), RunError
|
|
645
673
|
* INVALID_ARGUMENT before any claim.
|
|
646
674
|
*/
|
|
@@ -694,6 +722,7 @@ class Run {
|
|
|
694
722
|
// Both refusals here are terminal: the run is marked `failed`, keeping its seal, so the supervisor stops on it.
|
|
695
723
|
await step("seal", () => lease.checkSeal(this.#store));
|
|
696
724
|
const storage = lease.observe(this.#store.storage);
|
|
725
|
+
// openDurableRun refused an open without Harness options (here or from afterClaim) before this runs.
|
|
697
726
|
this.#harness = await step("harness", () => Harness.open(storage, { ...this.#options.harness, env: this.env }, this.#context));
|
|
698
727
|
await step("live", () => lease.live());
|
|
699
728
|
const before = this.#options.beforeResume;
|
|
@@ -702,7 +731,7 @@ class Run {
|
|
|
702
731
|
// between the open and the hook, since the Harness has not resumed.
|
|
703
732
|
await step("before-resume", async () => {
|
|
704
733
|
const head = (await storeHead(this.#store));
|
|
705
|
-
await before({ harness: this.#harness, context: this.#context, generation: lease.generation, record: lease.record, claim: lease.claim, storage: storeReads(storage), head });
|
|
734
|
+
await before({ harness: this.#harness, context: this.#context, generation: lease.generation, record: lease.record, previous: lease.previous, claim: lease.claim, storage: storeReads(storage), head });
|
|
706
735
|
});
|
|
707
736
|
}
|
|
708
737
|
await step("resume", () => this.#harness.resume());
|
|
@@ -746,8 +775,8 @@ class Run {
|
|
|
746
775
|
}
|
|
747
776
|
}
|
|
748
777
|
/**
|
|
749
|
-
* Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock,
|
|
750
|
-
* self-fence
|
|
778
|
+
* Open run `ref` on this host: the lease (`openRunLease`: claim, owner lock, layout, the commands reap, then
|
|
779
|
+
* `afterClaim`, a new run's seed, then `run.json`, heartbeat and self-fence), then the store, the seal check before anything commits, pi's Harness with the app's registry,
|
|
751
780
|
* the seal cleared in `run.json`, resume.
|
|
752
781
|
*
|
|
753
782
|
* Errors are typed: HeldError (76) when another client, another process on this host or another connection holds the
|
|
@@ -755,25 +784,32 @@ class Run {
|
|
|
755
784
|
* RunError STORE_HEAD_UNREADABLE (70), each after marking the run `failed` and releasing the claim.
|
|
756
785
|
*/
|
|
757
786
|
export async function openDurableRun(ref, options) {
|
|
787
|
+
if (!options.harness && !options.afterClaim)
|
|
788
|
+
throw new RunError("INVALID_ARGUMENT", "openDurableRun needs pi's Harness options, from its options or from afterClaim");
|
|
758
789
|
const context = options.context ?? BACKGROUND_CONTEXT;
|
|
759
790
|
const step = stepper(options.onStep);
|
|
760
791
|
let env;
|
|
761
|
-
|
|
792
|
+
let opening = options;
|
|
793
|
+
const given = options.afterClaim;
|
|
794
|
+
// Inside the lease's open, before run.json: a factory that fails publishes nothing of this incarnation.
|
|
795
|
+
const afterClaim = given &&
|
|
796
|
+
(async ({ ref: claimRef, root, work, store }) => {
|
|
797
|
+
const got = await given({ ref: claimRef, root, work, store });
|
|
798
|
+
opening = { ...options, ...(got.harness ? { harness: got.harness } : {}), ...(got.beforeResume ? { beforeResume: got.beforeResume } : {}) };
|
|
799
|
+
if (!opening.harness)
|
|
800
|
+
throw new RunError("INVALID_ARGUMENT", "openDurableRun needs pi's Harness options: afterClaim returned none, and its options give none");
|
|
801
|
+
return got.seed ? { seed: got.seed } : {};
|
|
802
|
+
});
|
|
803
|
+
// The lease's cleanup is the environment's: abandoning the lease cleans both.
|
|
804
|
+
const lease = await openLease(ref, { ...options, context, cleanup: (c) => env?.cleanup(c) ?? Promise.resolve() }, step, afterClaim);
|
|
762
805
|
try {
|
|
763
|
-
// A claim that records its commands (a local run's): end what a killed incarnation left running, now that the owner
|
|
764
|
-
// lock is held and before the Harness opens, so no command of it still writes in work/ once the run resumes.
|
|
765
|
-
const commands = lease.rawClaim.commands;
|
|
766
|
-
// A process of a killed incarnation that outlives its SIGKILL refuses the open (HeldError, 76): the run never resumes
|
|
767
|
-
// beside it, and its record stays for the next open.
|
|
768
|
-
if (commands !== undefined)
|
|
769
|
-
await step("commands", () => reapOrRefuse(commands));
|
|
770
806
|
env = (options.env ?? ((c) => archilEnv(c)))(lease.rawClaim);
|
|
771
807
|
}
|
|
772
808
|
catch (error) {
|
|
773
809
|
await lease.abandon(error);
|
|
774
810
|
throw error;
|
|
775
811
|
}
|
|
776
|
-
const run = new Run(lease, env,
|
|
812
|
+
const run = new Run(lease, env, opening, context);
|
|
777
813
|
try {
|
|
778
814
|
await run.open(step);
|
|
779
815
|
}
|
package/dist/status.d.ts
CHANGED
|
@@ -19,7 +19,7 @@ export type RunStatus = (typeof RUN_STATUSES)[number];
|
|
|
19
19
|
export interface RunHolder {
|
|
20
20
|
readonly driver: string;
|
|
21
21
|
readonly host: string;
|
|
22
|
-
/** `/proc/sys/kernel/random/boot_id
|
|
22
|
+
/** `/proc/sys/kernel/random/boot_id` (`kern.bootsessionuuid` on macOS); null where it cannot be read. */
|
|
23
23
|
readonly bootId: string | null;
|
|
24
24
|
readonly pid: number;
|
|
25
25
|
/** When this incarnation claimed the run (ISO 8601). */
|
|
@@ -55,6 +55,14 @@ export declare class RunRecordError extends PdaError {
|
|
|
55
55
|
export declare function parseRunRecord(text: string): RunRecord;
|
|
56
56
|
/** The `run.json` under a run's root on a local mount (or in a claim directory), or undefined when the run has none yet. */
|
|
57
57
|
export declare function readRunRecord(root: string | ClaimDir): Promise<RunRecord | undefined>;
|
|
58
|
+
/**
|
|
59
|
+
* The status of a run on a local directory (`localRun`): `<root>/run.json`, where `root` is the directory `localRun(root)`
|
|
60
|
+
* takes (not its `work/`). Read as `readRunStatus` reads a disk's through the control API: the same strict parser and the
|
|
61
|
+
* same `RunRecord`. It only reads: no lock, no mount and no store are touched, so it is safe beside a live instance, whose
|
|
62
|
+
* next write replaces the file whole. A run with no `run.json` yet, or no directory, is `null`; a record that is malformed
|
|
63
|
+
* is `RunRecordError` (`RUN_JSON_INVALID`); any other read error is thrown.
|
|
64
|
+
*/
|
|
65
|
+
export declare function readLocalRunStatus(root: string): Promise<RunRecord | null>;
|
|
58
66
|
/**
|
|
59
67
|
* True while the record's last heartbeat is younger than `expiryMs` on the reader's clock. The supervisor allows for
|
|
60
68
|
* clock skew between hosts inside the expiry.
|
|
@@ -81,12 +89,14 @@ export declare const ARCHIL_FSTYPE = "fuse.archil";
|
|
|
81
89
|
* A run root held open as a directory for the life of the instance. `at` is `/proc/self/fd/<n>`, a path the kernel
|
|
82
90
|
* resolves through the open directory and never through the root's path (Node has no openat; the magic link stands in
|
|
83
91
|
* for it): a file made, written or renamed under `at` is in the directory opened at acquire, or the call fails (ENOTCONN
|
|
84
|
-
* once the FUSE daemon is gone). It never lands on the local disk under a mountpoint whose mount went away.
|
|
92
|
+
* once the FUSE daemon is gone). It never lands on the local disk under a mountpoint whose mount went away. macOS has no
|
|
93
|
+
* such link: there `at` is the root's path (`byPath`), and only the checks below stand between a write and a root that was
|
|
94
|
+
* replaced.
|
|
85
95
|
*/
|
|
86
96
|
export interface ClaimDir {
|
|
87
97
|
/** The root's path, as the claim names it. */
|
|
88
98
|
readonly root: string;
|
|
89
|
-
/** `/proc/self/fd/<n
|
|
99
|
+
/** `/proc/self/fd/<n>`, or the root's path `byPath`. Throws once closed: the number may then name another file. */
|
|
90
100
|
readonly at: string;
|
|
91
101
|
/**
|
|
92
102
|
* Resolves while the root's path still resolves to the directory held open (same device, same inode). A mount that was
|
|
@@ -107,6 +117,12 @@ export interface OpenClaimDirOptions {
|
|
|
107
117
|
readonly fstype?: string | null;
|
|
108
118
|
/** Default `/proc/self/mountinfo`. */
|
|
109
119
|
readonly mountinfo?: string;
|
|
120
|
+
/**
|
|
121
|
+
* `at` is the root's path, not `/proc/self/fd/<n>`: the default on macOS, which has none. A write is checked against the
|
|
122
|
+
* held directory first (`assertMounted`), but a root renamed and replaced between that check and the write gets the
|
|
123
|
+
* write. For a local directory only.
|
|
124
|
+
*/
|
|
125
|
+
readonly byPath?: boolean;
|
|
110
126
|
}
|
|
111
127
|
/** `major:minor` of a device number, as `/proc/self/mountinfo` writes it (glibc's encoding). */
|
|
112
128
|
export declare function deviceName(dev: bigint): string;
|
package/dist/status.js
CHANGED
|
@@ -2,11 +2,12 @@
|
|
|
2
2
|
// claim with write-to-temp, fsync, rename, and read by the supervisor (over the S3 API or a local mount). It is a hint
|
|
3
3
|
// for the supervisor, never the source of truth: the store is. Any error writing it is a fence. The instance writes it
|
|
4
4
|
// through its claim directory (`ClaimDir`), never by path, so a write cannot land under a mountpoint whose mount is gone.
|
|
5
|
-
import { constants
|
|
5
|
+
import { constants } from "node:fs";
|
|
6
6
|
import { open, readFile, rename, rm, stat } from "node:fs/promises";
|
|
7
7
|
import { hostname } from "node:os";
|
|
8
8
|
import { join } from "node:path";
|
|
9
9
|
import { FencedError, ownWriteFenced, PdaError } from "./errors.js";
|
|
10
|
+
import { thisBoot } from "./process-table.js";
|
|
10
11
|
export const RUN_JSON = "run.json";
|
|
11
12
|
/** The temporary file every write goes through; renamed over `run.json` once it is fsynced. */
|
|
12
13
|
export const RUN_JSON_TEMP = "run.json.tmp";
|
|
@@ -95,6 +96,16 @@ export async function readRunRecord(root) {
|
|
|
95
96
|
}
|
|
96
97
|
return parseRunRecord(text);
|
|
97
98
|
}
|
|
99
|
+
/**
|
|
100
|
+
* The status of a run on a local directory (`localRun`): `<root>/run.json`, where `root` is the directory `localRun(root)`
|
|
101
|
+
* takes (not its `work/`). Read as `readRunStatus` reads a disk's through the control API: the same strict parser and the
|
|
102
|
+
* same `RunRecord`. It only reads: no lock, no mount and no store are touched, so it is safe beside a live instance, whose
|
|
103
|
+
* next write replaces the file whole. A run with no `run.json` yet, or no directory, is `null`; a record that is malformed
|
|
104
|
+
* is `RunRecordError` (`RUN_JSON_INVALID`); any other read error is thrown.
|
|
105
|
+
*/
|
|
106
|
+
export async function readLocalRunStatus(root) {
|
|
107
|
+
return (await readRunRecord(root)) ?? null;
|
|
108
|
+
}
|
|
98
109
|
/**
|
|
99
110
|
* True while the record's last heartbeat is younger than `expiryMs` on the reader's clock. The supervisor allows for
|
|
100
111
|
* clock skew between hosts inside the expiry.
|
|
@@ -204,7 +215,8 @@ export async function openClaimDir(root, options = {}) {
|
|
|
204
215
|
await handle.close().catch(() => undefined);
|
|
205
216
|
throw error instanceof FencedError ? error : unmounted(`checking the claim root ${root} failed: ${error.message}`, error);
|
|
206
217
|
}
|
|
207
|
-
const
|
|
218
|
+
const byPath = options.byPath ?? process.platform === "darwin";
|
|
219
|
+
const at = byPath ? root : `/proc/self/fd/${handle.fd}`;
|
|
208
220
|
let closed = false;
|
|
209
221
|
const same = (path, now, claims) => {
|
|
210
222
|
if (now.dev === claims.dev && now.ino === claims.ino)
|
|
@@ -234,10 +246,18 @@ export async function openClaimDir(root, options = {}) {
|
|
|
234
246
|
async assertSame(name) {
|
|
235
247
|
if (closed)
|
|
236
248
|
throw unmounted(`the claim directory of ${root} is closed`);
|
|
237
|
-
|
|
249
|
+
if (byPath) {
|
|
250
|
+
// No second way to name the entry: the root must still be the directory held, and the entry on its device.
|
|
251
|
+
const [dir, entry] = await Promise.all([statOf(root), statOf(join(root, name))]);
|
|
252
|
+
same(root, dir, held);
|
|
253
|
+
if (entry.dev !== held.dev)
|
|
254
|
+
throw unmounted(`${name} in the claim directory is on device ${deviceName(entry.dev)}, not the claim's ${deviceName(held.dev)}`);
|
|
255
|
+
return;
|
|
256
|
+
}
|
|
257
|
+
const [named, inDir] = await Promise.all([statOf(join(root, name)), statOf(join(at, name))]);
|
|
238
258
|
if (inDir.dev !== held.dev)
|
|
239
259
|
throw unmounted(`${name} in the claim directory is on device ${deviceName(inDir.dev)}, not the claim's ${deviceName(held.dev)}`);
|
|
240
|
-
same(join(root, name),
|
|
260
|
+
same(join(root, name), named, inDir);
|
|
241
261
|
},
|
|
242
262
|
async close() {
|
|
243
263
|
if (closed)
|
|
@@ -247,18 +267,10 @@ export async function openClaimDir(root, options = {}) {
|
|
|
247
267
|
},
|
|
248
268
|
};
|
|
249
269
|
}
|
|
250
|
-
function bootId() {
|
|
251
|
-
try {
|
|
252
|
-
return readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim() || null;
|
|
253
|
-
}
|
|
254
|
-
catch {
|
|
255
|
-
return null;
|
|
256
|
-
}
|
|
257
|
-
}
|
|
258
270
|
/**
|
|
259
271
|
* This process as a holder: the host driver's `handle` (default driver `local`, this machine's host name and boot id)
|
|
260
272
|
* with this process's pid and `since`. A handle whose named fields have the wrong type is refused (RunRecordError).
|
|
261
273
|
*/
|
|
262
274
|
export function currentHolder(handle = {}, since = new Date()) {
|
|
263
|
-
return holderOf({ driver: "local", host: hostname(), bootId:
|
|
275
|
+
return holderOf({ driver: "local", host: hostname(), bootId: thisBoot(), ...handle, pid: process.pid, since: since.toISOString() });
|
|
264
276
|
}
|
package/dist/store.d.ts
CHANGED
|
@@ -39,8 +39,18 @@ export declare const PROFILE_PRAGMAS: {
|
|
|
39
39
|
readonly mmap_size: 0;
|
|
40
40
|
};
|
|
41
41
|
};
|
|
42
|
-
/**
|
|
43
|
-
|
|
42
|
+
/**
|
|
43
|
+
* On macOS, fsync does not flush the drive's cache, so a commit FULL syncs could still be lost to a power cut. These make
|
|
44
|
+
* SQLite sync with F_FULLFSYNC instead, at every commit and every checkpoint.
|
|
45
|
+
*/
|
|
46
|
+
export declare const FULLFSYNC_PRAGMAS: {
|
|
47
|
+
readonly fullfsync: 1;
|
|
48
|
+
readonly checkpoint_fullfsync: 1;
|
|
49
|
+
};
|
|
50
|
+
/** What `profile` must read back on `platform`: its own pragmas, and on macOS `FULLFSYNC_PRAGMAS` too. */
|
|
51
|
+
export declare function profilePragmas(profile: Profile, platform?: NodeJS.Platform): Readonly<Record<string, string | number>>;
|
|
52
|
+
/** Read every pragma of `profile` on `platform` (default this one) back and refuse (StorePragmaError) when any differs. */
|
|
53
|
+
export declare function assertPragmas(database: SqliteExecutor, profile: Profile, platform?: NodeJS.Platform): Promise<void>;
|
|
44
54
|
/**
|
|
45
55
|
* The fenced state of one handle. The first error whose result code is in the I/O, full or not-a-database class marks it
|
|
46
56
|
* fenced, runs `onFenced` once (before the error propagates), and from then on every operation fails with the same
|
|
@@ -84,6 +94,8 @@ export type OpenOptions = {
|
|
|
84
94
|
readonly onFenced?: (error: StoreFencedError) => void;
|
|
85
95
|
/** Wraps the raw database inside the fence, for instrumentation and fault injection. */
|
|
86
96
|
readonly decorate?: (database: SqliteDatabase) => SqliteDatabase;
|
|
97
|
+
/** The platform whose pragmas the store takes (`profilePragmas`); default this one. For tests. */
|
|
98
|
+
readonly platform?: NodeJS.Platform;
|
|
87
99
|
};
|
|
88
100
|
export type ArchilStore = {
|
|
89
101
|
readonly storage: SqliteStorage;
|