@parcha/pi-durable-disk 0.1.0-beta.12 → 0.1.0-beta.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +241 -0
- package/README.md +268 -28
- package/dist/app.d.ts +5 -0
- package/dist/app.js +3 -2
- package/dist/claim.d.ts +34 -2
- package/dist/claim.js +86 -5
- package/dist/cli.d.ts +6 -0
- package/dist/cli.js +98 -33
- package/dist/commands.d.ts +70 -0
- package/dist/commands.js +257 -0
- package/dist/confine.d.ts +47 -7
- package/dist/confine.js +148 -19
- package/dist/env.d.ts +17 -26
- package/dist/env.js +73 -68
- package/dist/errors.d.ts +2 -1
- package/dist/errors.js +1 -1
- package/dist/fork.d.ts +23 -0
- package/dist/fork.js +103 -3
- package/dist/hosts/daytona-launch.d.ts +3 -1
- package/dist/hosts/daytona-launch.js +30 -6
- package/dist/hosts/daytona.js +15 -3
- package/dist/hosts/local-host.d.ts +7 -1
- package/dist/hosts/local-host.js +30 -16
- package/dist/index.d.ts +31 -15
- package/dist/index.js +23 -15
- package/dist/internals.d.ts +3 -0
- package/dist/internals.js +6 -0
- package/dist/lease.d.ts +5 -2
- package/dist/lease.js +3 -1
- package/dist/local-run.d.ts +10 -0
- package/dist/local-run.js +64 -0
- package/dist/park.d.ts +4 -3
- package/dist/park.js +4 -3
- package/dist/process-table.d.ts +38 -0
- package/dist/process-table.js +158 -0
- package/dist/run.d.ts +52 -10
- package/dist/run.js +42 -7
- package/dist/status.d.ts +11 -3
- package/dist/status.js +15 -13
- package/dist/store.d.ts +14 -2
- package/dist/store.js +16 -4
- package/dist/supervise.d.ts +22 -3
- package/dist/supervise.js +40 -13
- package/dist/watchdog.d.ts +3 -1
- package/dist/watchdog.js +46 -13
- package/package.json +12 -7
package/dist/env.d.ts
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
1
|
import type { Context } from "@earendil-works/chord";
|
|
2
|
-
import type { Extension,
|
|
2
|
+
import type { Extension, ToolRegistration } from "@earendil-works/pi-durable";
|
|
3
3
|
import { ExecutionError } from "@earendil-works/pi-durable/env";
|
|
4
4
|
import { NodeExecutionEnv, type NodeWatchOptions } from "@earendil-works/pi-durable/env/node";
|
|
5
5
|
import { PdaError } from "./errors.js";
|
|
6
|
-
type ToolCall = Parameters<ToolHooks["afterTool"]>[0];
|
|
7
6
|
/** What the environment needs of a claim; the claim satisfies it structurally. */
|
|
8
7
|
export interface ArchilEnvClaim {
|
|
9
8
|
/** Absolute mount path of the run directory; the same on every host that claims the run. */
|
|
@@ -12,6 +11,11 @@ export interface ArchilEnvClaim {
|
|
|
12
11
|
readonly work: string;
|
|
13
12
|
/** The disk's id; part of the environment id when present. */
|
|
14
13
|
readonly disk?: string;
|
|
14
|
+
/**
|
|
15
|
+
* Where the commands this environment starts are recorded, so that the next open of the claim ends what a killed
|
|
16
|
+
* incarnation left running (`reapCommands`). Set by a claim that nothing else cleans up after: a local run's.
|
|
17
|
+
*/
|
|
18
|
+
readonly commands?: string;
|
|
15
19
|
}
|
|
16
20
|
export interface ArchilEnvOptions {
|
|
17
21
|
/** How `watch` runs. Polling is the default because a FUSE mount does not report changes made elsewhere. */
|
|
@@ -21,21 +25,23 @@ export interface ArchilEnvOptions {
|
|
|
21
25
|
/**
|
|
22
26
|
* Run every command with no_new_privs (`setpriv --no-new-privs --`): setuid programs and file capabilities stop
|
|
23
27
|
* granting privilege, so a command cannot `sudo` even when the run user has a sudo rule (the host needs one for the
|
|
24
|
-
* archil verbs). Default true
|
|
28
|
+
* archil verbs). Default true, but false on macOS, which has no no_new_privs: there a command keeps the user's rights,
|
|
29
|
+
* sudo included. Set false on a host that already confines commands, or where util-linux is absent.
|
|
25
30
|
*/
|
|
26
31
|
readonly noNewPrivs?: boolean;
|
|
27
32
|
/** The util-linux `setpriv` that `noNewPrivs` and `runAs` run; default `/usr/bin/setpriv`. */
|
|
28
33
|
readonly setprivPath?: string;
|
|
29
34
|
/**
|
|
30
35
|
* Confine every file operation to the workspace (`PathOutsideWorkError` otherwise): read, write, edit and every other
|
|
31
|
-
* path pi's file system takes. Default true.
|
|
32
|
-
*
|
|
36
|
+
* path pi's file system takes. Default true. On macOS, which has no `/proc/self/fd`, the confinement walks paths by
|
|
37
|
+
* name instead, and holds against an agent's paths and the links in place, not a command racing it (confine.ts). Set
|
|
38
|
+
* false only where the paths are confined some other way, or where `/proc/self/fd` is unavailable on Linux.
|
|
33
39
|
*/
|
|
34
40
|
readonly confineFiles?: boolean;
|
|
35
41
|
/**
|
|
36
42
|
* Directories file operations may read but never write, besides the workspace and the claim's `tmp/` (where a bash
|
|
37
|
-
* command's spilled output lives and which the bash tool names to the model
|
|
38
|
-
* material.
|
|
43
|
+
* command's spilled output lives and which the bash tool names to the model; a directory `createTempDir` returned is
|
|
44
|
+
* writable too, for the tool it was handed to). For an app's own skills or reference material.
|
|
39
45
|
*/
|
|
40
46
|
readonly readRoots?: readonly string[];
|
|
41
47
|
/**
|
|
@@ -72,7 +78,8 @@ export declare class EnvError extends PdaError {
|
|
|
72
78
|
* resumed run on another host has the same identity. Temporary files and directories (a bash command's spilled
|
|
73
79
|
* `pi-output-*.log` among them) live under `<root>/tmp/`, inside the claim and outside `work/`, so a path a result
|
|
74
80
|
* names still exists after a takeover on another host. Commands run under no_new_privs unless `noNewPrivs` is false, and
|
|
75
|
-
* every file operation is confined to the workspace (reads also to `tmp/` and `readRoots
|
|
81
|
+
* every file operation is confined to the workspace (reads also to `tmp/` and `readRoots`, writes also inside a directory
|
|
82
|
+
* `createTempDir` returned) unless `confineFiles` is false.
|
|
76
83
|
*/
|
|
77
84
|
export interface ArchilEnvFactory {
|
|
78
85
|
/** Rooted at the claim's `work/`; an agent `cwd` is resolved against it. */
|
|
@@ -93,7 +100,7 @@ export interface ArchilEnvFactory {
|
|
|
93
100
|
export declare function archilEnv(claim: ArchilEnvClaim, options?: ArchilEnvOptions): ArchilEnvFactory;
|
|
94
101
|
/** Built-in tools that only read. A tool in this set is declared `replay: "safe"`. */
|
|
95
102
|
export declare const READ_ONLY_TOOLS: ReadonlySet<string>;
|
|
96
|
-
/** Built-in tools that can change the workspace;
|
|
103
|
+
/** Built-in tools that can change the workspace; none of them is declared replay-safe. */
|
|
97
104
|
export declare const WRITING_TOOLS: ReadonlySet<string>;
|
|
98
105
|
/** The tool declared `replay: "safe"`: an interrupted call reruns on recovery instead of reporting `interrupted`. */
|
|
99
106
|
export declare function replaySafe<T extends ToolRegistration>(tool: T): T;
|
|
@@ -101,23 +108,7 @@ export declare function replaySafe<T extends ToolRegistration>(tool: T): T;
|
|
|
101
108
|
export declare function markReadOnlySafe<T extends ToolRegistration>(tools: readonly T[], readOnly?: ReadonlySet<string>): T[];
|
|
102
109
|
/**
|
|
103
110
|
* pi's `CodingTools` with `read` declared replay-safe. It keeps pi's extension name, so installing it replaces
|
|
104
|
-
* `CodingTools` in place. `write`, `edit` and `bash` stay as pi ships them (unsafe). pi 1.0
|
|
111
|
+
* `CodingTools` in place. `write`, `edit` and `bash` stay as pi ships them (unsafe). pi 1.1.0 ships no ls, find or
|
|
105
112
|
* grep; an app that adds them declares them safe with `markReadOnlySafe`.
|
|
106
113
|
*/
|
|
107
114
|
export declare const ArchilCodingTools: Extension;
|
|
108
|
-
export interface WorkspaceBarrierOptions {
|
|
109
|
-
/** Tools whose result follows a barrier; default `WRITING_TOOLS`. An app adds its own writing tools here. */
|
|
110
|
-
readonly tools?: Iterable<string>;
|
|
111
|
-
}
|
|
112
|
-
/**
|
|
113
|
-
* The hook point for the rule that the workspace changes a tool made are durable before its result is committed.
|
|
114
|
-
* Install this extension only when the result commit's own fsync does not already satisfy the rule; with it
|
|
115
|
-
* not installed nothing runs. `barrier` is the claim's (`claim.barrier()`), called after a writing tool ran and before
|
|
116
|
-
* pi appends its result entry, for the calls whose tool name is in `options.tools`.
|
|
117
|
-
*
|
|
118
|
-
* pi reports and ignores an `afterTool` hook that throws, so a failed barrier cannot hold the result back by throwing.
|
|
119
|
-
* It replaces the result with an error result carrying the diagnostic `workspace_not_durable`: a success never commits
|
|
120
|
-
* for a workspace that is not durable. The host decides separately whether a failed barrier means the claim is fenced.
|
|
121
|
-
*/
|
|
122
|
-
export declare function workspaceBarrier(barrier: (call: ToolCall, context: Context) => Promise<void>, options?: WorkspaceBarrierOptions): Extension;
|
|
123
|
-
export {};
|
package/dist/env.js
CHANGED
|
@@ -1,17 +1,18 @@
|
|
|
1
1
|
// The execution environment: pi's NodeExecutionEnv rooted at the run's work/ directory, with an environment id
|
|
2
2
|
// that names the run, commands that cannot gain privilege (no_new_privs), optionally commands run as another user
|
|
3
3
|
// (`runAs`, for an instance that runs as root in a container), file operations confined to work/ (confine.ts),
|
|
4
|
-
// read-only built-in tools declared replay-safe
|
|
5
|
-
//
|
|
4
|
+
// and read-only built-in tools declared replay-safe. Tools run on the host that holds the claim, against the claimed
|
|
5
|
+
// mount.
|
|
6
6
|
import { accessSync, constants, existsSync, statSync } from "node:fs";
|
|
7
|
-
import { access, chmod, lchown, lstat, mkdir, mkdtemp, realpath, stat } from "node:fs/promises";
|
|
8
|
-
import { delimiter, dirname, join, resolve } from "node:path";
|
|
7
|
+
import { access, chmod, lchown, lstat, mkdir, mkdtemp, open, realpath, stat } from "node:fs/promises";
|
|
8
|
+
import { basename, delimiter, dirname, join, resolve } from "node:path";
|
|
9
9
|
import { withAbortSignal } from "@earendil-works/chord/context";
|
|
10
|
-
import { defineExtension
|
|
10
|
+
import { defineExtension } from "@earendil-works/pi-durable";
|
|
11
11
|
import { err, ExecutionError, FileError, getOrThrow, ok } from "@earendil-works/pi-durable/env";
|
|
12
12
|
import { NodeExecutionEnv } from "@earendil-works/pi-durable/env/node";
|
|
13
13
|
import { CodingTools } from "@earendil-works/pi-durable/tools";
|
|
14
|
-
import {
|
|
14
|
+
import { COMMANDS_MARK, CommandRecorder } from "./commands.js";
|
|
15
|
+
import { canonicalizeSync, confineHooks, Confinement, fileError, inside, namesHeld, PathOutsideWorkError } from "./confine.js";
|
|
15
16
|
import { PdaError } from "./errors.js";
|
|
16
17
|
/**
|
|
17
18
|
* What `exec` returns once the factory's `cleanup()` ran: no command was started. Its `code` is pi's `spawn_error`
|
|
@@ -50,17 +51,22 @@ class ArchilExecutionEnv extends NodeExecutionEnv {
|
|
|
50
51
|
return err(new EnvClosedError());
|
|
51
52
|
this.#active++;
|
|
52
53
|
this.#scope.running.add(this);
|
|
54
|
+
this.#scope.recorder?.started();
|
|
53
55
|
try {
|
|
54
|
-
|
|
56
|
+
// A recorded environment marks its commands, so its recorder knows them from any other child of this process.
|
|
57
|
+
const recorder = this.#scope.recorder;
|
|
58
|
+
const run = recorder === undefined ? options : { ...options, env: { ...options?.env, [COMMANDS_MARK]: recorder.token } };
|
|
59
|
+
const confined = await this.#confine(command, run, context);
|
|
55
60
|
if (!confined.ok)
|
|
56
61
|
return confined;
|
|
57
62
|
// Again after the confinement lookups, which await; the signal then covers pi's own pre-spawn work: pi refuses
|
|
58
63
|
// before spawning once it is aborted, and kills a command that spawns after.
|
|
59
64
|
if (closing.signal.aborted)
|
|
60
65
|
return err(new EnvClosedError());
|
|
61
|
-
return await super.exec(confined.value,
|
|
66
|
+
return await super.exec(confined.value, run, withAbortSignal(closing.signal, context));
|
|
62
67
|
}
|
|
63
68
|
finally {
|
|
69
|
+
this.#scope.recorder?.finished();
|
|
64
70
|
if (--this.#active === 0)
|
|
65
71
|
this.#scope.running.delete(this);
|
|
66
72
|
}
|
|
@@ -124,7 +130,7 @@ class ArchilExecutionEnv extends NodeExecutionEnv {
|
|
|
124
130
|
return this.#entry(path, context, { access: spec.access, follow: true, ...(spec.parents ? { parents: true } : {}) }, async (pin) => {
|
|
125
131
|
const handle = await pin.openFile(spec.flags);
|
|
126
132
|
try {
|
|
127
|
-
return await run(handle);
|
|
133
|
+
return await run(handle, pin);
|
|
128
134
|
}
|
|
129
135
|
finally {
|
|
130
136
|
await handle.close().catch(() => undefined);
|
|
@@ -144,8 +150,10 @@ class ArchilExecutionEnv extends NodeExecutionEnv {
|
|
|
144
150
|
async openTextLineReader(path, context) {
|
|
145
151
|
if (this.#scope.confinement === undefined)
|
|
146
152
|
return super.openTextLineReader(path, context);
|
|
147
|
-
// pi opens the file again through the handle's own `/proc/self/fd` entry, which names that inode and no path
|
|
148
|
-
|
|
153
|
+
// pi opens the file again through the handle's own `/proc/self/fd` entry, which names that inode and no path; by path
|
|
154
|
+
// (macOS), through the path just checked.
|
|
155
|
+
const byPath = this.#scope.confinement.byPath;
|
|
156
|
+
return this.#file(path, context, { access: "read", flags: constants.O_RDONLY }, (handle, pin) => super.openTextLineReader(byPath ? pin.entry : `/proc/self/fd/${handle.fd}`, context));
|
|
149
157
|
}
|
|
150
158
|
async openBinaryReader(path, options, context) {
|
|
151
159
|
if (this.#scope.confinement === undefined)
|
|
@@ -259,7 +267,15 @@ class ArchilExecutionEnv extends NodeExecutionEnv {
|
|
|
259
267
|
async remove(path, options, context) {
|
|
260
268
|
if (this.#scope.confinement === undefined)
|
|
261
269
|
return super.remove(path, options, context);
|
|
262
|
-
|
|
270
|
+
let gone;
|
|
271
|
+
const removed = await this.#entry(path, context, { access: "write", follow: false, strict: true }, async (pin) => {
|
|
272
|
+
const result = pin.restore(await super.remove(pin.entry, options, context));
|
|
273
|
+
if (result.ok)
|
|
274
|
+
gone = pin.canonical;
|
|
275
|
+
return result;
|
|
276
|
+
});
|
|
277
|
+
if (gone !== undefined)
|
|
278
|
+
this.#scope.confinement.forgetWrites(gone); // a temp dir it was given, removed: its grant goes with it
|
|
263
279
|
// `force` ignores a missing target, a missing directory above it included.
|
|
264
280
|
return !removed.ok && removed.error.code === "not_found" && options?.force === true ? ok(undefined) : removed;
|
|
265
281
|
}
|
|
@@ -283,11 +299,30 @@ class ArchilExecutionEnv extends NodeExecutionEnv {
|
|
|
283
299
|
async createTempDir(prefix, context) {
|
|
284
300
|
if (context.abortSignal?.aborted)
|
|
285
301
|
return err(new FileError("aborted", "aborted"));
|
|
286
|
-
const { tempRoot } = this.#scope;
|
|
302
|
+
const { tempRoot, confinement, tempCanonical } = this.#scope;
|
|
287
303
|
try {
|
|
288
304
|
if ((await mkdir(tempRoot, { recursive: true, mode: 0o755 })) !== undefined)
|
|
289
305
|
await chmod(tempRoot, 0o755);
|
|
290
|
-
|
|
306
|
+
if (confinement === undefined || tempCanonical === undefined)
|
|
307
|
+
return ok(await mkdtemp(join(tempRoot, prefix ?? "tmp-")));
|
|
308
|
+
// The tool it is handed to may write in it (vision writes its source there); the rest of tmp/ stays read-only. It is
|
|
309
|
+
// made through a held handle of tmp/ that does not follow a link (a tmp/ a command replaced with one is refused here),
|
|
310
|
+
// and granted by name under the canonical tmp/, never resolved (a directory swapped for a link later grants nothing).
|
|
311
|
+
const held = await open(tempRoot, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
312
|
+
let name;
|
|
313
|
+
try {
|
|
314
|
+
// By path (macOS): made under tmp/'s path once that path is checked to still name the directory held.
|
|
315
|
+
if (confinement.byPath && !(await namesHeld(tempRoot, held)))
|
|
316
|
+
throw new PathOutsideWorkError(tempRoot, "changed", `${tempRoot}: changed while it was being resolved`);
|
|
317
|
+
name = basename(await mkdtemp(`${confinement.byPath ? tempRoot : `/proc/self/fd/${held.fd}`}/${prefix ?? "tmp-"}`));
|
|
318
|
+
}
|
|
319
|
+
finally {
|
|
320
|
+
await held.close();
|
|
321
|
+
}
|
|
322
|
+
const dir = join(tempRoot, name);
|
|
323
|
+
await confineHooks.afterTempDir?.(dir);
|
|
324
|
+
confinement.allowWrites(join(tempCanonical, name));
|
|
325
|
+
return ok(dir);
|
|
291
326
|
}
|
|
292
327
|
catch (error) {
|
|
293
328
|
return err(fileError(error, tempRoot));
|
|
@@ -450,17 +485,22 @@ function requireSetpriv(path) {
|
|
|
450
485
|
}
|
|
451
486
|
}
|
|
452
487
|
/**
|
|
453
|
-
* The roots, canonical once here: file operations may use the workspace, the claim's `tmp/` (read only
|
|
488
|
+
* The roots, canonical once here: file operations may use the workspace, the claim's `tmp/` (read only, but for the
|
|
489
|
+
* directories `createTempDir` hands out, which are writable) and `readRoots`.
|
|
454
490
|
* Taken at construction, before any agent runs, so a symlink an agent puts in place of `work` or `tmp` later cannot widen
|
|
455
491
|
* them: it resolves outside the roots and is refused.
|
|
456
492
|
*/
|
|
457
|
-
function requireConfinement(claim, readRoots) {
|
|
458
|
-
if (!existsSync("/proc/self/fd")) {
|
|
493
|
+
function requireConfinement(claim, readRoots, byPath) {
|
|
494
|
+
if (!byPath && !existsSync("/proc/self/fd")) {
|
|
459
495
|
throw new EnvError("PROC_UNAVAILABLE", "file operations are confined by walking paths through /proc/self/fd, which is not available here. " +
|
|
460
496
|
"Mount /proc, or pass confineFiles: false where the file paths are confined some other way.");
|
|
461
497
|
}
|
|
462
|
-
|
|
463
|
-
|
|
498
|
+
// Each root as the caller names it, where that differs from its canonical path (under /tmp or /var on macOS).
|
|
499
|
+
const names = new Map();
|
|
500
|
+
const named = (canonical, as) => (canonical === as ? canonical : (names.set(canonical, as), canonical));
|
|
501
|
+
const tmp = named(join(canonicalizeSync(claim.root), TEMP_DIR), join(claim.root, TEMP_DIR));
|
|
502
|
+
const work = named(canonicalizeSync(claim.work), claim.work);
|
|
503
|
+
return new Confinement(work, [tmp, ...readRoots.map((root) => named(canonicalizeSync(root), root))], { byPath, names });
|
|
464
504
|
}
|
|
465
505
|
export function archilEnv(claim, options = {}) {
|
|
466
506
|
const id = ["archil", claim.disk, claim.root].filter((part) => part !== undefined && part !== "").join(":");
|
|
@@ -469,9 +509,10 @@ export function archilEnv(claim, options = {}) {
|
|
|
469
509
|
if (runAs && ![runAs.uid, runAs.gid].every((n) => Number.isSafeInteger(n) && n >= 0)) {
|
|
470
510
|
throw new EnvError("INVALID_RUN_AS", `runAs needs a numeric uid and gid, got ${JSON.stringify(runAs)}`);
|
|
471
511
|
}
|
|
472
|
-
const
|
|
512
|
+
const darwin = process.platform === "darwin";
|
|
513
|
+
const noNewPrivs = options.noNewPrivs ?? !darwin;
|
|
473
514
|
const setpriv = noNewPrivs || runAs ? requireSetpriv(options.setprivPath ?? SETPRIV) : undefined;
|
|
474
|
-
const confinement = options.confineFiles === false ? undefined : requireConfinement(claim, options.readRoots ?? []);
|
|
515
|
+
const confinement = options.confineFiles === false ? undefined : requireConfinement(claim, options.readRoots ?? [], darwin);
|
|
475
516
|
const scope = {
|
|
476
517
|
id,
|
|
477
518
|
tempRoot: join(claim.root, TEMP_DIR),
|
|
@@ -483,6 +524,14 @@ export function archilEnv(claim, options = {}) {
|
|
|
483
524
|
running,
|
|
484
525
|
closing: new AbortController(),
|
|
485
526
|
confinement,
|
|
527
|
+
tempCanonical: confinement === undefined ? undefined : join(canonicalizeSync(claim.root), TEMP_DIR),
|
|
528
|
+
// A record that cannot be written (a full disk) leaves a command the next open could miss: said once on stderr, and
|
|
529
|
+
// the recorder keeps trying; the run goes on.
|
|
530
|
+
recorder: claim.commands === undefined
|
|
531
|
+
? undefined
|
|
532
|
+
: new CommandRecorder(claim.commands, {
|
|
533
|
+
onError: (error) => process.stderr.write(`pi-durable-disk: cannot record the run's commands in ${claim.commands}: ${String(error)}\n`),
|
|
534
|
+
}),
|
|
486
535
|
};
|
|
487
536
|
const workOwned = runAs ? ownedOnce(claim.work, runAs) : undefined;
|
|
488
537
|
const build = (target) => {
|
|
@@ -497,12 +546,13 @@ export function archilEnv(claim, options = {}) {
|
|
|
497
546
|
const cleanup = async (context) => {
|
|
498
547
|
scope.closing.abort();
|
|
499
548
|
await Promise.all([...running].map((env) => env.cleanup(context)));
|
|
549
|
+
scope.recorder?.close();
|
|
500
550
|
};
|
|
501
551
|
return Object.assign(build, { id, cleanup });
|
|
502
552
|
}
|
|
503
553
|
/** Built-in tools that only read. A tool in this set is declared `replay: "safe"`. */
|
|
504
554
|
export const READ_ONLY_TOOLS = new Set(["read", "ls", "find", "grep"]);
|
|
505
|
-
/** Built-in tools that can change the workspace;
|
|
555
|
+
/** Built-in tools that can change the workspace; none of them is declared replay-safe. */
|
|
506
556
|
export const WRITING_TOOLS = new Set(["write", "edit", "bash", "powershell"]);
|
|
507
557
|
/** The tool declared `replay: "safe"`: an interrupted call reruns on recovery instead of reporting `interrupted`. */
|
|
508
558
|
export function replaySafe(tool) {
|
|
@@ -514,55 +564,10 @@ export function markReadOnlySafe(tools, readOnly = READ_ONLY_TOOLS) {
|
|
|
514
564
|
}
|
|
515
565
|
/**
|
|
516
566
|
* pi's `CodingTools` with `read` declared replay-safe. It keeps pi's extension name, so installing it replaces
|
|
517
|
-
* `CodingTools` in place. `write`, `edit` and `bash` stay as pi ships them (unsafe). pi 1.0
|
|
567
|
+
* `CodingTools` in place. `write`, `edit` and `bash` stay as pi ships them (unsafe). pi 1.1.0 ships no ls, find or
|
|
518
568
|
* grep; an app that adds them declares them safe with `markReadOnlySafe`.
|
|
519
569
|
*/
|
|
520
570
|
export const ArchilCodingTools = defineExtension({
|
|
521
571
|
name: CodingTools.name,
|
|
522
572
|
tools: markReadOnlySafe(CodingTools.tools ?? []),
|
|
523
573
|
});
|
|
524
|
-
/**
|
|
525
|
-
* The hook point for the rule that the workspace changes a tool made are durable before its result is committed.
|
|
526
|
-
* Install this extension only when the result commit's own fsync does not already satisfy the rule; with it
|
|
527
|
-
* not installed nothing runs. `barrier` is the claim's (`claim.barrier()`), called after a writing tool ran and before
|
|
528
|
-
* pi appends its result entry, for the calls whose tool name is in `options.tools`.
|
|
529
|
-
*
|
|
530
|
-
* pi reports and ignores an `afterTool` hook that throws, so a failed barrier cannot hold the result back by throwing.
|
|
531
|
-
* It replaces the result with an error result carrying the diagnostic `workspace_not_durable`: a success never commits
|
|
532
|
-
* for a workspace that is not durable. The host decides separately whether a failed barrier means the claim is fenced.
|
|
533
|
-
*/
|
|
534
|
-
export function workspaceBarrier(barrier, options = {}) {
|
|
535
|
-
const writing = new Set(options.tools ?? WRITING_TOOLS);
|
|
536
|
-
return defineExtension({
|
|
537
|
-
name: "archil-workspace-barrier",
|
|
538
|
-
hooks: [
|
|
539
|
-
hook(ToolTask, {
|
|
540
|
-
afterTool: async (call, result, _api, context) => {
|
|
541
|
-
if (!writing.has(call.name))
|
|
542
|
-
return undefined;
|
|
543
|
-
try {
|
|
544
|
-
await barrier(call, context);
|
|
545
|
-
return undefined;
|
|
546
|
-
}
|
|
547
|
-
catch (error) {
|
|
548
|
-
// An aborted invocation must propagate: pi rethrows hook errors only while it is aborted.
|
|
549
|
-
if (context.abortSignal?.aborted)
|
|
550
|
-
throw error;
|
|
551
|
-
return notDurable(result, error);
|
|
552
|
-
}
|
|
553
|
-
},
|
|
554
|
-
}),
|
|
555
|
-
],
|
|
556
|
-
});
|
|
557
|
-
}
|
|
558
|
-
function notDurable(result, error) {
|
|
559
|
-
const reason = error instanceof Error ? error.message : String(error);
|
|
560
|
-
return {
|
|
561
|
-
...result,
|
|
562
|
-
isError: true,
|
|
563
|
-
diagnostics: [
|
|
564
|
-
...(result.diagnostics ?? []),
|
|
565
|
-
{ severity: "error", code: "workspace_not_durable", message: `The workspace could not be made durable: ${reason}` },
|
|
566
|
-
],
|
|
567
|
-
};
|
|
568
|
-
}
|
package/dist/errors.d.ts
CHANGED
|
@@ -20,7 +20,8 @@ export declare class FencedError extends PdaError {
|
|
|
20
20
|
code?: string;
|
|
21
21
|
});
|
|
22
22
|
}
|
|
23
|
-
|
|
23
|
+
/** `commands`: processes a killed incarnation's commands left that did not die when the next open killed them. */
|
|
24
|
+
export type Holder = "claim" | "owner-lock" | "store" | "commands";
|
|
24
25
|
/** Someone else holds the run (see `Holder`). The process exits 76. */
|
|
25
26
|
export declare class HeldError extends PdaError {
|
|
26
27
|
readonly holder: Holder;
|
package/dist/errors.js
CHANGED
|
@@ -25,7 +25,7 @@ export class FencedError extends PdaError {
|
|
|
25
25
|
super(options.code ?? "FENCED", message, { cause: options.cause, exitCode: EXIT_FENCED });
|
|
26
26
|
}
|
|
27
27
|
}
|
|
28
|
-
const HELD_CODES = { claim: "CLAIM_HELD", "owner-lock": "OWNER_LOCK_HELD", store: "STORE_BUSY" };
|
|
28
|
+
const HELD_CODES = { claim: "CLAIM_HELD", "owner-lock": "OWNER_LOCK_HELD", store: "STORE_BUSY", commands: "COMMANDS_ALIVE" };
|
|
29
29
|
/** Someone else holds the run (see `Holder`). The process exits 76. */
|
|
30
30
|
export class HeldError extends PdaError {
|
|
31
31
|
holder;
|
package/dist/fork.d.ts
CHANGED
|
@@ -39,3 +39,26 @@ export type ForkResult = {
|
|
|
39
39
|
* and mount of its own is removed either way.
|
|
40
40
|
*/
|
|
41
41
|
export declare function fork(ref: RunRef, newId: string, options: ForkOptions): Promise<ForkResult>;
|
|
42
|
+
export interface ForkLocalResult {
|
|
43
|
+
/** The new run's id, and the directory it lives in. */
|
|
44
|
+
readonly run: string;
|
|
45
|
+
readonly root: string;
|
|
46
|
+
readonly from: string;
|
|
47
|
+
readonly sealedSeq: number;
|
|
48
|
+
readonly sourceGeneration: number;
|
|
49
|
+
readonly files: number;
|
|
50
|
+
readonly bytes: number;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Fork a run on a local directory (`localRun`) into one new run directory per entry of `into`: the same copy as `fork`, for
|
|
54
|
+
* one machine and without a disk. `from` must hold a released, sealed run (run.json paused, sleeping, done or failed with a
|
|
55
|
+
* `sealedSeq`); anything else is `SOURCE_NOT_RELEASED`, and no target may exist (`TARGET_EXISTS`). The source is only read.
|
|
56
|
+
* Every copy is made beside its target under a temporary name and none is published until all are complete, so a failed
|
|
57
|
+
* batch leaves nothing behind and never takes back a run another process opened; each opens as
|
|
58
|
+
* generation 1 of a run of its own, refusing a copy that lost a committed sequence like any fork. Nothing guards a source
|
|
59
|
+
* that another process is opening at the same moment: a local directory has no mount to take it exclusively.
|
|
60
|
+
*/
|
|
61
|
+
export declare function forkLocal(from: string, into: readonly {
|
|
62
|
+
readonly id: string;
|
|
63
|
+
readonly root: string;
|
|
64
|
+
}[]): Promise<ForkLocalResult[]>;
|
package/dist/fork.js
CHANGED
|
@@ -3,8 +3,9 @@
|
|
|
3
3
|
// root of their own, never where an instance mounts. The source is only read (its claim probe aside, which every mount
|
|
4
4
|
// writes). The new run's run.json starts at generation 0 and carries the source's seal: its first open is generation 1
|
|
5
5
|
// and refuses a copy that lost a committed sequence (STORE_BEHIND_SEAL). Supervisor side: it needs the API key.
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { randomUUID } from "node:crypto";
|
|
7
|
+
import { chmod, copyFile, lchown, lstat, mkdir, readdir, readlink, realpath, rename, rm, rmdir, symlink, utimes } from "node:fs/promises";
|
|
8
|
+
import { basename, dirname, join, resolve, sep } from "node:path";
|
|
8
9
|
import { acquire, createRunDir, findDelegations, mintMountToken, removeMountToken, runPath } from "./claim.js";
|
|
9
10
|
import { HeldError, PdaError } from "./errors.js";
|
|
10
11
|
import { persistRecord, readRunRecord, RUN_JSON, RUN_JSON_TEMP } from "./status.js";
|
|
@@ -16,7 +17,7 @@ const RELEASED = ["paused", "sleeping", "done", "failed"];
|
|
|
16
17
|
* Not copied: the run's own lifecycle files (the fork writes its own run.json), the supervisor's start mark (copied, it
|
|
17
18
|
* would hold the new run in a start grace for a start that never happened), and the scratch directory.
|
|
18
19
|
*/
|
|
19
|
-
const SKIP = new Set([RUN_JSON, RUN_JSON_TEMP, OWNER_LOCK, `${OWNER_LOCK}-journal`, ".claim", START_MARK, "tmp"]);
|
|
20
|
+
const SKIP = new Set([RUN_JSON, RUN_JSON_TEMP, OWNER_LOCK, `${OWNER_LOCK}-journal`, ".claim", START_MARK, "tmp", "commands"]);
|
|
20
21
|
export class ForkError extends PdaError {
|
|
21
22
|
constructor(code, message, options = {}) {
|
|
22
23
|
super(code, message, options);
|
|
@@ -179,3 +180,102 @@ export async function fork(ref, newId, options) {
|
|
|
179
180
|
await rmdir(base).catch(() => { });
|
|
180
181
|
}
|
|
181
182
|
}
|
|
183
|
+
/**
|
|
184
|
+
* Fork a run on a local directory (`localRun`) into one new run directory per entry of `into`: the same copy as `fork`, for
|
|
185
|
+
* one machine and without a disk. `from` must hold a released, sealed run (run.json paused, sleeping, done or failed with a
|
|
186
|
+
* `sealedSeq`); anything else is `SOURCE_NOT_RELEASED`, and no target may exist (`TARGET_EXISTS`). The source is only read.
|
|
187
|
+
* Every copy is made beside its target under a temporary name and none is published until all are complete, so a failed
|
|
188
|
+
* batch leaves nothing behind and never takes back a run another process opened; each opens as
|
|
189
|
+
* generation 1 of a run of its own, refusing a copy that lost a committed sequence like any fork. Nothing guards a source
|
|
190
|
+
* that another process is opening at the same moment: a local directory has no mount to take it exclusively.
|
|
191
|
+
*/
|
|
192
|
+
export async function forkLocal(from, into) {
|
|
193
|
+
for (const t of into)
|
|
194
|
+
runPath(t.id); // a run id the disk would refuse is refused here too
|
|
195
|
+
const source = await realpath(resolve(from)).catch(() => resolve(from));
|
|
196
|
+
const targets = await Promise.all(into.map(async (t) => ({ id: t.id, root: await resolvedPath(t.root) })));
|
|
197
|
+
if (targets.length === 0)
|
|
198
|
+
throw new ForkError("INVALID_ARGUMENT", "forkLocal needs at least one target");
|
|
199
|
+
// Two targets the same, or one beneath another (`/runs/a` and `/runs/a/b`): staging the inner one would make the outer one's directory.
|
|
200
|
+
if (targets.some((t, i) => targets.some((u, j) => i !== j && (t.root === u.root || t.root.startsWith(`${u.root}${sep}`))))) {
|
|
201
|
+
throw new ForkError("INVALID_ARGUMENT", "two targets are the same directory, or one is inside another");
|
|
202
|
+
}
|
|
203
|
+
// A target inside the source (by a symlink too) would be copied into itself and would write into a source that is only read.
|
|
204
|
+
if (targets.some((t) => t.root === source || t.root.startsWith(`${source}${sep}`)))
|
|
205
|
+
throw new ForkError("INVALID_ARGUMENT", "a fork needs a directory of its own, outside the source");
|
|
206
|
+
const record = await readRunRecord(source).catch((cause) => {
|
|
207
|
+
throw new ForkError("SOURCE_NOT_RELEASED", `${source} holds no readable run.json`, { cause });
|
|
208
|
+
});
|
|
209
|
+
if (!record || !RELEASED.includes(record.status) || record.sealedSeq === null) {
|
|
210
|
+
throw new ForkError("SOURCE_NOT_RELEASED", `the run in ${source} is not released and sealed (${record ? `${record.status}, sealedSeq ${record.sealedSeq}` : "no run.json"})`);
|
|
211
|
+
}
|
|
212
|
+
if (targets.some((t) => t.id === record.run))
|
|
213
|
+
throw new ForkError("INVALID_ARGUMENT", "a fork needs a new run id");
|
|
214
|
+
for (const t of targets) {
|
|
215
|
+
if (await lstat(t.root).then(() => true, () => false))
|
|
216
|
+
throw new ForkError("TARGET_EXISTS", `${t.root} already exists`);
|
|
217
|
+
}
|
|
218
|
+
const asRoot = process.getuid?.() === 0;
|
|
219
|
+
const st = await lstat(source);
|
|
220
|
+
// Every copy is made beside its target under a temporary name and none is published until all are complete: a failed
|
|
221
|
+
// batch removes only temporary directories, never a run another process could already have opened.
|
|
222
|
+
const staged = [];
|
|
223
|
+
const published = [];
|
|
224
|
+
try {
|
|
225
|
+
for (const t of targets) {
|
|
226
|
+
const temp = `${t.root}.fork-${randomUUID()}`; // unique per call: two forks of one target never share a temporary directory
|
|
227
|
+
const entry = { ...t, temp, files: 0, bytes: 0 };
|
|
228
|
+
await mkdir(dirname(t.root), { recursive: true });
|
|
229
|
+
await mkdir(temp);
|
|
230
|
+
staged.push(entry); // tracked for cleanup only once this call made it
|
|
231
|
+
const count = { files: 0, bytes: 0 };
|
|
232
|
+
await copyTree(source, temp, true, asRoot, count);
|
|
233
|
+
Object.assign(entry, count);
|
|
234
|
+
const forked = {
|
|
235
|
+
run: t.id,
|
|
236
|
+
status: "paused",
|
|
237
|
+
generation: 0,
|
|
238
|
+
sealedSeq: record.sealedSeq,
|
|
239
|
+
wakeAt: null,
|
|
240
|
+
holder: null,
|
|
241
|
+
heartbeatAt: null,
|
|
242
|
+
updatedAt: new Date().toISOString(),
|
|
243
|
+
detail: { forkedFrom: { run: record.run, generation: record.generation, sealedSeq: record.sealedSeq } },
|
|
244
|
+
};
|
|
245
|
+
await persistRecord(temp, `${JSON.stringify(forked, null, 2)}\n`);
|
|
246
|
+
// The run's directory is the source's: its owner and mode, so whoever owned the source can open the fork.
|
|
247
|
+
if (asRoot)
|
|
248
|
+
await lchown(temp, st.uid, st.gid);
|
|
249
|
+
await chmod(temp, st.mode & 0o7777);
|
|
250
|
+
}
|
|
251
|
+
for (const entry of staged) {
|
|
252
|
+
await rename(entry.temp, entry.root);
|
|
253
|
+
published.push(entry.root);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
catch (error) {
|
|
257
|
+
for (const entry of staged)
|
|
258
|
+
await rm(entry.temp, { recursive: true, force: true }).catch(() => undefined);
|
|
259
|
+
// Only a copy no one has opened is taken back: an owner lock means a process already runs it.
|
|
260
|
+
for (const root of published)
|
|
261
|
+
if (!(await lstat(join(root, OWNER_LOCK)).then(() => true, () => false)))
|
|
262
|
+
await rm(root, { recursive: true, force: true }).catch(() => undefined);
|
|
263
|
+
throw error instanceof ForkError ? error : new ForkError("COPY_FAILED", `forking ${source} failed: ${error.message}`, { cause: error });
|
|
264
|
+
}
|
|
265
|
+
return staged.map((t) => ({ run: t.id, root: t.root, from: record.run, sealedSeq: record.sealedSeq, sourceGeneration: record.generation, files: t.files, bytes: t.bytes }));
|
|
266
|
+
}
|
|
267
|
+
/** `path` made absolute with every symlink of its existing part resolved, so two spellings of one place compare equal. */
|
|
268
|
+
async function resolvedPath(path) {
|
|
269
|
+
const absolute = resolve(path);
|
|
270
|
+
const missing = [];
|
|
271
|
+
for (let at = absolute;; at = dirname(at)) {
|
|
272
|
+
try {
|
|
273
|
+
return join(await realpath(at), ...missing.reverse());
|
|
274
|
+
}
|
|
275
|
+
catch {
|
|
276
|
+
if (dirname(at) === at)
|
|
277
|
+
return absolute;
|
|
278
|
+
missing.push(basename(at));
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
@@ -42,7 +42,9 @@ export type LaunchStatus = {
|
|
|
42
42
|
instanceAlive: boolean;
|
|
43
43
|
};
|
|
44
44
|
export declare function readState(dir: string, name: string): LaunchState | null;
|
|
45
|
-
/** running while either process lives; otherwise stopped after a clean exit,
|
|
45
|
+
/** running while either process lives and the launcher has not recorded its exit; otherwise stopped after a clean exit,
|
|
46
|
+
* failed after any other end. Every "exited" the launcher saves is final and comes after its instance ended, so once it is
|
|
47
|
+
* saved (and no instance lives) the record is the answer, even while the launcher's own process is still exiting. */
|
|
46
48
|
export declare function launchStatus(dir: string, name: string): LaunchStatus;
|
|
47
49
|
/** The loop `serve` runs; resolves with the final state. */
|
|
48
50
|
export declare function serve(dir: string, name: string): Promise<LaunchState>;
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// incarnation gets the token; a restart in place reads an empty stdin and reuses its live mount, as under localHost.
|
|
14
14
|
// No cgroup per incarnation: commands a crashed instance detached can outlive it until the box is deleted.
|
|
15
15
|
import { spawn } from "node:child_process";
|
|
16
|
-
import { closeSync, existsSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
16
|
+
import { closeSync, existsSync, openSync, readFileSync, realpathSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
17
17
|
import { join, resolve } from "node:path";
|
|
18
18
|
import { fileURLToPath } from "node:url";
|
|
19
19
|
import { procStartTicks, TERMINAL_EXITS } from "./local-host.js";
|
|
@@ -40,15 +40,20 @@ function writeState(path, state) {
|
|
|
40
40
|
writeFileSync(`${path}.tmp`, `${JSON.stringify(state)}\n`, { mode: 0o600 });
|
|
41
41
|
renameSync(`${path}.tmp`, path);
|
|
42
42
|
}
|
|
43
|
-
|
|
44
|
-
|
|
43
|
+
// Only a real process with its start ticks: `stop` signals these pids, the instance's as a group, and a pid of 1 or below
|
|
44
|
+
// names a group (`kill(-1)` is every process this user may signal). The state file holds whatever was written there.
|
|
45
|
+
const alive = (pid, ticks) => Number.isInteger(pid) && pid > 1 && typeof ticks === "number" && procStartTicks(pid) === ticks;
|
|
46
|
+
/** running while either process lives and the launcher has not recorded its exit; otherwise stopped after a clean exit,
|
|
47
|
+
* failed after any other end. Every "exited" the launcher saves is final and comes after its instance ended, so once it is
|
|
48
|
+
* saved (and no instance lives) the record is the answer, even while the launcher's own process is still exiting. */
|
|
45
49
|
export function launchStatus(dir, name) {
|
|
46
50
|
const state = readState(dir, name);
|
|
47
51
|
if (!state)
|
|
48
52
|
return { status: "failed", state: null, launcherAlive: false, instanceAlive: false };
|
|
49
53
|
const launcherAlive = alive(state.launcher, state.launcherTicks);
|
|
50
54
|
const instanceAlive = alive(state.instance, state.instanceTicks);
|
|
51
|
-
|
|
55
|
+
const ended = state.phase === "exited" && !instanceAlive;
|
|
56
|
+
if (!ended && (launcherAlive || instanceAlive))
|
|
52
57
|
return { status: "running", state, launcherAlive, instanceAlive };
|
|
53
58
|
const clean = state.phase === "exited" && state.exit === 0;
|
|
54
59
|
return { status: clean ? "stopped" : "failed", state, launcherAlive, instanceAlive };
|
|
@@ -214,8 +219,12 @@ export async function stop(dir, name, timeoutMs) {
|
|
|
214
219
|
}
|
|
215
220
|
};
|
|
216
221
|
let s = launchStatus(dir, name);
|
|
217
|
-
if (s.status !== "running")
|
|
222
|
+
if (s.status !== "running") {
|
|
223
|
+
// The launcher recorded its exit and is ending on its own; a launcher process that outstays its record is still sent away.
|
|
224
|
+
if (s.launcherAlive)
|
|
225
|
+
kill(s.state.launcher, s.state.launcherTicks, "SIGTERM");
|
|
218
226
|
return s;
|
|
227
|
+
}
|
|
219
228
|
if (s.launcherAlive)
|
|
220
229
|
kill(s.state.launcher, s.state.launcherTicks, "SIGTERM");
|
|
221
230
|
else
|
|
@@ -260,7 +269,22 @@ async function main(argv) {
|
|
|
260
269
|
return 2;
|
|
261
270
|
}
|
|
262
271
|
}
|
|
263
|
-
|
|
272
|
+
/**
|
|
273
|
+
* Whether Node was started with this module. LAUNCH_SCRIPT comes from import.meta.url, which is the real path; argv[1] is
|
|
274
|
+
* the path as given, which may go through a link (a symlinked package directory, pnpm's linked node_modules).
|
|
275
|
+
*/
|
|
276
|
+
function invokedAsMain() {
|
|
277
|
+
const arg = process.argv[1];
|
|
278
|
+
if (!arg)
|
|
279
|
+
return false;
|
|
280
|
+
try {
|
|
281
|
+
return realpathSync(arg) === LAUNCH_SCRIPT;
|
|
282
|
+
}
|
|
283
|
+
catch {
|
|
284
|
+
return resolve(arg) === LAUNCH_SCRIPT;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
if (invokedAsMain()) {
|
|
264
288
|
main(process.argv.slice(2)).then((code) => process.exit(code), (err) => {
|
|
265
289
|
process.stderr.write(`${err.message}\n`);
|
|
266
290
|
process.exit(1);
|