@edgehero/pi-dispatch 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +160 -0
- package/deploy/com.pi-dispatch.worker.plist +66 -0
- package/deploy/nssm-install.cmd +59 -0
- package/deploy/receiver.service +36 -0
- package/deploy/worker-env-wrapper.cmd +50 -0
- package/deploy/worker-env-wrapper.sh +63 -0
- package/deploy/worker.service +55 -0
- package/package.json +83 -0
- package/src/azure-auth.mjs +61 -0
- package/src/azure-host.mjs +236 -0
- package/src/azure-identity.mjs +63 -0
- package/src/azure-prompt.mjs +118 -0
- package/src/branch.mjs +80 -0
- package/src/budget.mjs +179 -0
- package/src/cli.mjs +208 -0
- package/src/config.mjs +329 -0
- package/src/connection.mjs +40 -0
- package/src/cron.mjs +94 -0
- package/src/docker-run.mjs +119 -0
- package/src/doctor.mjs +1127 -0
- package/src/env-allowlist.mjs +198 -0
- package/src/env-file.mjs +153 -0
- package/src/exit-code.mjs +32 -0
- package/src/flow-gate.mjs +82 -0
- package/src/forgejo-auth.mjs +77 -0
- package/src/forgejo-host.mjs +172 -0
- package/src/forgejo-identity.mjs +74 -0
- package/src/forgejo-prompt.mjs +123 -0
- package/src/forges.mjs +148 -0
- package/src/get-token.mjs +226 -0
- package/src/git-dirty.mjs +16 -0
- package/src/github-app-setup.mjs +517 -0
- package/src/github-host.mjs +159 -0
- package/src/github-prompt.mjs +286 -0
- package/src/gitlab-auth.mjs +72 -0
- package/src/gitlab-host.mjs +200 -0
- package/src/gitlab-identity.mjs +61 -0
- package/src/gitlab-prompt.mjs +123 -0
- package/src/identity.mjs +57 -0
- package/src/image-preflight.mjs +180 -0
- package/src/import-pi.mjs +451 -0
- package/src/index.mjs +177 -0
- package/src/init.mjs +77 -0
- package/src/job-id.mjs +100 -0
- package/src/materialize.mjs +138 -0
- package/src/outbox.mjs +179 -0
- package/src/packages.mjs +188 -0
- package/src/pause-windows.mjs +218 -0
- package/src/prepare-github.mjs +260 -0
- package/src/prepare-local.mjs +76 -0
- package/src/prepare.mjs +199 -0
- package/src/pricing.mjs +168 -0
- package/src/processor.mjs +360 -0
- package/src/queue.mjs +152 -0
- package/src/run-container.mjs +133 -0
- package/src/run-history.mjs +534 -0
- package/src/runtime-settings.mjs +188 -0
- package/src/sandbox-cli.mjs +156 -0
- package/src/sandbox-store.mjs +269 -0
- package/src/sandbox.mjs +171 -0
- package/src/scheduler-stall-guard.mjs +67 -0
- package/src/schedules.mjs +62 -0
- package/src/service.mjs +677 -0
- package/src/session-key.mjs +108 -0
- package/src/session-store.mjs +249 -0
- package/src/start.mjs +502 -0
- package/src/subscriptions.mjs +208 -0
- package/src/triggers.mjs +491 -0
- package/src/up.mjs +315 -0
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import * as nodeFs from "node:fs";
|
|
2
|
+
import { dirname } from "node:path";
|
|
3
|
+
import { defaultSettingsFile } from "./config.mjs";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Runtime-settings overlay: the shared, durable truth between the admin extension and the worker
|
|
7
|
+
* (INT-CONFIG-OVERLAY-CONTRACT, DES-RUNTIME-SETTINGS-FILE-OVERLAY). A flat `settings.json` with ten
|
|
8
|
+
* optional keys -- `model`, `provider` (non-empty strings), `maxTurns`, `dailyCap`, `weeklyCap`,
|
|
9
|
+
* `monthlyCap`, `maxTokens`, `dailyTokenCap` (int >= 1), `concurrency` (int 1-10), `softHoldPct`
|
|
10
|
+
* (int 1-99) -- read by the worker at each job start and written atomically by the admin extension
|
|
11
|
+
* (tmp + rename). `maxTokens`/`dailyTokenCap` are the optional token controls (issue #25).
|
|
12
|
+
*
|
|
13
|
+
* `readOverlay` NEVER throws: a bad settings file returns a discriminated `{ invalid }` rather than an
|
|
14
|
+
* exception, so the processor RETURNS a policy refusal (`settings-overlay-invalid`) instead of letting
|
|
15
|
+
* its catch classify the file as retryable infra and pay to retry a job that can never succeed
|
|
16
|
+
* (CONST-RETRY-INFRA-ONLY). A present-but-unreadable file fails closed -- it never degrades to an empty
|
|
17
|
+
* overlay that would silently restore env's higher `dailyCap` (DES-RUNTIME-SETTINGS-FILE-OVERLAY).
|
|
18
|
+
*
|
|
19
|
+
* Precedence resolved here is overlay > env > default only; `config` already encodes env > default. The
|
|
20
|
+
* `job.data` precedence layer belongs to the processor, not here.
|
|
21
|
+
*
|
|
22
|
+
* Invalid reasons name the offending KEY and its constraint, never the value: settings values may echo
|
|
23
|
+
* operator input, so only key names are safe to surface in logs and refusals (no-pii-in-logs).
|
|
24
|
+
*
|
|
25
|
+
* Custom: overlay validated inline per config.mjs precedent; zod not in deps
|
|
26
|
+
*/
|
|
27
|
+
|
|
28
|
+
export const KNOWN_KEYS = ["model", "provider", "maxTurns", "dailyCap", "weeklyCap", "monthlyCap", "maxTokens", "dailyTokenCap", "concurrency", "softHoldPct"];
|
|
29
|
+
|
|
30
|
+
function isNonEmptyString(value) {
|
|
31
|
+
return typeof value === "string" && value.trim() !== "";
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function isIntAtLeast(value, min) {
|
|
35
|
+
return Number.isInteger(value) && value >= min;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function isIntInRange(value, min, max) {
|
|
39
|
+
return Number.isInteger(value) && value >= min && value <= max;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* The absolute path of the settings overlay. Mirrors config.mjs: `PI_SETTINGS_FILE` wins, an unset or
|
|
44
|
+
* empty value falls back to the shared default so the admin extension and the worker resolve the same
|
|
45
|
+
* path without either coupling to the other.
|
|
46
|
+
*/
|
|
47
|
+
export function settingsFilePath(env = process.env) {
|
|
48
|
+
return env.PI_SETTINGS_FILE || defaultSettingsFile();
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Validate a parsed overlay against the key contract, returning the sanitized overlay (known keys only)
|
|
53
|
+
* or `{ invalid }`. Shared by `readOverlay` (post-parse) and `writeOverlay` (pre-write) so a single
|
|
54
|
+
* definition governs both directions. Unknown keys are dropped and logged, leaving the overlay valid;
|
|
55
|
+
* any invalid known key fails the WHOLE object.
|
|
56
|
+
*/
|
|
57
|
+
function validateOverlay(candidate, log) {
|
|
58
|
+
if (candidate === null || typeof candidate !== "object" || Array.isArray(candidate)) {
|
|
59
|
+
return { invalid: "root must be a JSON object" };
|
|
60
|
+
}
|
|
61
|
+
const overlay = {};
|
|
62
|
+
for (const key of Object.keys(candidate)) {
|
|
63
|
+
const value = candidate[key];
|
|
64
|
+
switch (key) {
|
|
65
|
+
case "model":
|
|
66
|
+
case "provider":
|
|
67
|
+
if (!isNonEmptyString(value)) return { invalid: `${key} must be a non-empty string` };
|
|
68
|
+
overlay[key] = value;
|
|
69
|
+
break;
|
|
70
|
+
case "maxTurns":
|
|
71
|
+
case "dailyCap":
|
|
72
|
+
case "weeklyCap":
|
|
73
|
+
case "monthlyCap":
|
|
74
|
+
case "maxTokens":
|
|
75
|
+
case "dailyTokenCap":
|
|
76
|
+
// Overlay caps are positive ints. A window/cap is DISABLED by absence, not by a 0 -- `unset weeklyCap`
|
|
77
|
+
// drops the key so it falls through to env, which unset means the window is off (config.mjs). The
|
|
78
|
+
// token knobs (maxTokens, dailyTokenCap) follow the same "absent = disabled" shape (issue #25).
|
|
79
|
+
if (!isIntAtLeast(value, 1)) return { invalid: `${key} must be an integer >= 1` };
|
|
80
|
+
overlay[key] = value;
|
|
81
|
+
break;
|
|
82
|
+
case "concurrency":
|
|
83
|
+
// Range-checked here, not via config's positiveInt: that helper has no upper bound and parses
|
|
84
|
+
// env strings, so it cannot enforce the 1-10 ceiling this key carries over a JSON number.
|
|
85
|
+
if (!isIntInRange(value, 1, 10)) return { invalid: "concurrency must be an integer 1-10" };
|
|
86
|
+
overlay[key] = value;
|
|
87
|
+
break;
|
|
88
|
+
case "softHoldPct":
|
|
89
|
+
// A percentage of each active cap; 100 would equal the hard wall (no band) and 0 has no meaning,
|
|
90
|
+
// so the enforced band is 1-99. Absence disables the soft-hold entirely.
|
|
91
|
+
if (!isIntInRange(value, 1, 99)) return { invalid: "softHoldPct must be an integer 1-99" };
|
|
92
|
+
overlay[key] = value;
|
|
93
|
+
break;
|
|
94
|
+
default:
|
|
95
|
+
log("settings_overlay_unknown_key", { key });
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return { overlay };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Read the overlay at `path`. Returns `{ overlay }` (known keys only) on success, or `{ invalid }` with
|
|
103
|
+
* a key-only reason on failure. NEVER throws.
|
|
104
|
+
*
|
|
105
|
+
* A missing file (ENOENT) is the normal empty overlay `{ overlay: {} }`. Any OTHER read error
|
|
106
|
+
* (EACCES, EISDIR, ...) is `{ invalid }`: a present-but-unreadable file must fail closed, not silently
|
|
107
|
+
* become an empty overlay that restores env's higher `dailyCap`. Unparseable JSON, a non-object root,
|
|
108
|
+
* or an invalid known key is `{ invalid }`.
|
|
109
|
+
*/
|
|
110
|
+
export function readOverlay(path, { fs = nodeFs, log = () => {} } = {}) {
|
|
111
|
+
let text;
|
|
112
|
+
try {
|
|
113
|
+
text = fs.readFileSync(path, "utf8");
|
|
114
|
+
} catch (err) {
|
|
115
|
+
if (err?.code === "ENOENT") return { overlay: {} }; // missing file is a normal empty overlay
|
|
116
|
+
return { invalid: `settings file unreadable (${err?.code ?? "read-error"})` };
|
|
117
|
+
}
|
|
118
|
+
let parsed;
|
|
119
|
+
try {
|
|
120
|
+
parsed = JSON.parse(text);
|
|
121
|
+
} catch {
|
|
122
|
+
return { invalid: "settings file is not valid JSON" };
|
|
123
|
+
}
|
|
124
|
+
return validateOverlay(parsed, log);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Resolve the ten effective settings from `config` and a validated `overlay`: overlay value where the
|
|
129
|
+
* overlay sets it, else the config value. Precedence is overlay > env > default only -- `config`
|
|
130
|
+
* already carries env > default. `weeklyCap`/`monthlyCap`/`maxTokens`/`dailyTokenCap`/`softHoldPct` may
|
|
131
|
+
* resolve to `null` (their config value when unset), meaning that window / cap / band is disabled.
|
|
132
|
+
* `job.data` is NOT merged here; that layer is the processor's.
|
|
133
|
+
*/
|
|
134
|
+
export function effectiveSettings(config, overlay) {
|
|
135
|
+
const o = overlay ?? {};
|
|
136
|
+
return {
|
|
137
|
+
provider: o.provider ?? config.provider,
|
|
138
|
+
model: o.model ?? config.model,
|
|
139
|
+
maxTurns: o.maxTurns ?? config.maxTurns,
|
|
140
|
+
dailyCap: o.dailyCap ?? config.dailyCap,
|
|
141
|
+
weeklyCap: o.weeklyCap ?? config.weeklyCap,
|
|
142
|
+
monthlyCap: o.monthlyCap ?? config.monthlyCap,
|
|
143
|
+
maxTokens: o.maxTokens ?? config.maxTokens,
|
|
144
|
+
dailyTokenCap: o.dailyTokenCap ?? config.dailyTokenCap,
|
|
145
|
+
concurrency: o.concurrency ?? config.concurrency,
|
|
146
|
+
softHoldPct: o.softHoldPct ?? config.softHoldPct,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/**
|
|
151
|
+
* Write `candidate` to `path` atomically. Validates with the same rules as `readOverlay` (an invalid
|
|
152
|
+
* candidate returns `{ invalid }` and touches no file; an empty `{}` candidate is valid), serialises
|
|
153
|
+
* the validated overlay with 2-space indent and a trailing newline, writes a same-directory
|
|
154
|
+
* `<path>.tmp`, then renames it over `path`. NEVER throws.
|
|
155
|
+
*
|
|
156
|
+
* The rename is retried once on EPERM to ride out a Windows AV/indexer briefly locking the destination;
|
|
157
|
+
* a second failure surfaces as `{ invalid }`, never a throw. Returns `{ ok: true }` on success.
|
|
158
|
+
*/
|
|
159
|
+
export function writeOverlay(path, candidate, { fs = nodeFs, log = () => {} } = {}) {
|
|
160
|
+
const result = validateOverlay(candidate, log);
|
|
161
|
+
if (result.invalid) return { invalid: result.invalid };
|
|
162
|
+
|
|
163
|
+
try {
|
|
164
|
+
fs.mkdirSync(dirname(path), { recursive: true });
|
|
165
|
+
} catch (err) {
|
|
166
|
+
return { invalid: `settings dir unwritable (${err?.code ?? "mkdir-error"})` };
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
const tmp = `${path}.tmp`;
|
|
170
|
+
const data = `${JSON.stringify(result.overlay, null, 2)}\n`;
|
|
171
|
+
try {
|
|
172
|
+
fs.writeFileSync(tmp, data);
|
|
173
|
+
} catch (err) {
|
|
174
|
+
return { invalid: `settings write failed (${err?.code ?? "write-error"})` };
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
try {
|
|
178
|
+
fs.renameSync(tmp, path);
|
|
179
|
+
} catch (err) {
|
|
180
|
+
if (err?.code !== "EPERM") return { invalid: `settings rename failed (${err?.code ?? "rename-error"})` };
|
|
181
|
+
try {
|
|
182
|
+
fs.renameSync(tmp, path); // single retry: Windows AV/indexer lock is transient
|
|
183
|
+
} catch (retryErr) {
|
|
184
|
+
return { invalid: `settings rename failed (${retryErr?.code ?? "rename-error"})` };
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return { ok: true };
|
|
188
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
import { loadConfig } from "./config.mjs";
|
|
3
|
+
import { sanitizeJobId } from "./run-history.mjs";
|
|
4
|
+
import { buildSandboxRunArgs, launchSandbox, listRunningSandboxes, parsePublish, resolveSandbox, sandboxContainerName } from "./sandbox.mjs";
|
|
5
|
+
import { listSandboxes, pinSandbox } from "./sandbox-store.mjs";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `pi-dispatch sandbox` -- re-open a finished run's sandbox as an interactive shell
|
|
9
|
+
* (REQ-RESURRECTABLE-SANDBOX).
|
|
10
|
+
*
|
|
11
|
+
* A command module beside doctor.mjs and import-pi.mjs, with the same posture: the whole I/O surface is
|
|
12
|
+
* injected so the decision paths are testable without docker, a terminal, or a disk, and every refusal
|
|
13
|
+
* names what to do about it rather than only what went wrong.
|
|
14
|
+
*
|
|
15
|
+
* The container this launches is NOT a job container. It carries the same isolation flags and the same
|
|
16
|
+
* mounts, and no credentials at all -- see sandbox.mjs, which owns that shape.
|
|
17
|
+
*/
|
|
18
|
+
export async function runSandbox(argv = [], { env = process.env, deps = {} } = {}) {
|
|
19
|
+
const {
|
|
20
|
+
out = (s) => process.stdout.write(s),
|
|
21
|
+
err = (s) => process.stderr.write(s),
|
|
22
|
+
isTty = Boolean(process.stdin.isTTY && process.stdout.isTTY),
|
|
23
|
+
running = listRunningSandboxes,
|
|
24
|
+
launch = launchSandbox,
|
|
25
|
+
now = () => Date.now(),
|
|
26
|
+
} = deps;
|
|
27
|
+
|
|
28
|
+
let values;
|
|
29
|
+
let positionals;
|
|
30
|
+
try {
|
|
31
|
+
({ values, positionals } = parseArgs({
|
|
32
|
+
args: argv,
|
|
33
|
+
allowPositionals: true,
|
|
34
|
+
options: {
|
|
35
|
+
list: { type: "boolean", default: false },
|
|
36
|
+
publish: { type: "string", multiple: true }, // repeatable; always bound to 127.0.0.1
|
|
37
|
+
pin: { type: "boolean", default: false },
|
|
38
|
+
},
|
|
39
|
+
}));
|
|
40
|
+
} catch (error) {
|
|
41
|
+
return fail(err, error.message);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
const config = loadConfig(env);
|
|
45
|
+
// A docker that cannot be reached costs a column, never the command: `listRunningSandboxes` throws so
|
|
46
|
+
// the REAPER can tell "none" from "could not ask", and these callers only draw a marker. Asked only
|
|
47
|
+
// where it is used, and never before the arguments are known good -- a typo should not shell out.
|
|
48
|
+
const liveSandboxes = async () => new Set(await running().catch(() => []));
|
|
49
|
+
|
|
50
|
+
if (values.list) {
|
|
51
|
+
return renderList({ config, live: await liveSandboxes(), out, now });
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const jobId = positionals[0];
|
|
55
|
+
if (!jobId) return fail(err, "a job id is required — `pi-dispatch sandbox --list` shows what is still re-openable");
|
|
56
|
+
|
|
57
|
+
// Refused BEFORE anything else that could half-succeed. `-t` against a pipe fails inside docker with
|
|
58
|
+
// "the input device is not a TTY", which names neither the cause nor the fix; a sandbox is an operator
|
|
59
|
+
// session by definition, so the absence of an operator is a refusal rather than a fallback.
|
|
60
|
+
if (!isTty) {
|
|
61
|
+
return fail(err, "`pi-dispatch sandbox` needs a terminal — it opens an interactive shell, so it cannot run from a pipe, a script without a TTY, or CI");
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// `listRunningSandboxes` yields ids, already sanitized, so this compares like with like.
|
|
65
|
+
if ((await liveSandboxes()).has(sanitizeJobId(jobId))) {
|
|
66
|
+
return fail(err, `a sandbox for ${jobId} is already running — attach to it with \`docker attach ${sandboxContainerName(jobId)}\`, or exit it first`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
let publish;
|
|
70
|
+
try {
|
|
71
|
+
publish = parsePublish(values.publish ?? []);
|
|
72
|
+
} catch (error) {
|
|
73
|
+
return fail(err, error.message);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const resolved = resolveSandbox({
|
|
77
|
+
jobId,
|
|
78
|
+
sandboxDir: config.sandboxDir,
|
|
79
|
+
retentionHours: config.sandboxRetentionHours,
|
|
80
|
+
publish,
|
|
81
|
+
});
|
|
82
|
+
if (resolved.refused) return fail(err, resolved.message);
|
|
83
|
+
|
|
84
|
+
// Pin BEFORE the shell, not after: the operator asked to keep this one, and a session that ends in a
|
|
85
|
+
// crashed terminal or a closed laptop lid must not be the reason the pin never landed.
|
|
86
|
+
if (values.pin) {
|
|
87
|
+
const pinned = pinSandbox({ sandboxDir: config.sandboxDir, jobId, pinDays: config.sandboxPinDays, now });
|
|
88
|
+
if (pinned.pinned) out(`pinned ${jobId} until ${pinned.keepUntil} (${config.sandboxPinDays}d)\n`);
|
|
89
|
+
else err(`warning: could not pin ${jobId}: ${pinned.reason}\n`);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
const args = buildSandboxRunArgs({
|
|
93
|
+
image: resolved.manifest.image,
|
|
94
|
+
name: resolved.name,
|
|
95
|
+
workspace: resolved.manifest.workspace,
|
|
96
|
+
jobDir: resolved.manifest.dir,
|
|
97
|
+
publish,
|
|
98
|
+
term: env.TERM,
|
|
99
|
+
idleSeconds: config.sandboxIdleMinutes * 60,
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
out(`opening ${resolved.name} — image ${resolved.manifest.image}, workspace ${resolved.manifest.workspace}\n`);
|
|
103
|
+
out("no credentials are set in this container. exit the shell to dispose of it.\n");
|
|
104
|
+
if (publish.length > 0) out(`published: ${publish.filter((f) => f !== "-p").join(", ")}\n`);
|
|
105
|
+
|
|
106
|
+
const { code, error } = await launch({ args });
|
|
107
|
+
if (error) return fail(err, `could not start docker: ${error.message}`);
|
|
108
|
+
return code ?? 0;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* What is still re-openable, newest first, plus what is running right now.
|
|
113
|
+
*
|
|
114
|
+
* The running column is the honest answer to `TMOUT`'s one gap: an idle timeout does not tick while a
|
|
115
|
+
* foreground command runs, so a sandbox left serving an app stays up. Making it findable is the least
|
|
116
|
+
* this can do about that.
|
|
117
|
+
*/
|
|
118
|
+
function renderList({ config, live, out, now }) {
|
|
119
|
+
if (config.sandboxRetentionHours === 0) {
|
|
120
|
+
out("workspace retention is off (PI_SANDBOX_RETENTION_HOURS=0) — finished runs are deleted as before\n");
|
|
121
|
+
}
|
|
122
|
+
const rows = listSandboxes({ sandboxDir: config.sandboxDir });
|
|
123
|
+
if (rows.length === 0) {
|
|
124
|
+
out("no retained workspaces\n");
|
|
125
|
+
return 0;
|
|
126
|
+
}
|
|
127
|
+
const width = Math.max(...rows.map((r) => String(r.jobId ?? "").length), 5);
|
|
128
|
+
for (const row of rows) {
|
|
129
|
+
const id = String(row.jobId ?? "?").padEnd(width);
|
|
130
|
+
const kind = String(row.kind ?? "?").padEnd(8);
|
|
131
|
+
const state = live.has(sanitizeJobId(row.jobId)) ? "RUNNING" : remaining(row, config.sandboxRetentionHours, now());
|
|
132
|
+
out(`${id} ${kind} ${state}\n`);
|
|
133
|
+
}
|
|
134
|
+
return 0;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/** How long this one has left, from the manifest's own timestamps -- never from mtime, which a live sandbox moves. */
|
|
138
|
+
function remaining(row, retentionHours, at) {
|
|
139
|
+
const keepUntil = Date.parse(row.keepUntil ?? "");
|
|
140
|
+
if (Number.isFinite(keepUntil)) return `pinned, ${humanise(keepUntil - at)} left`;
|
|
141
|
+
const createdAt = Date.parse(row.createdAt ?? "");
|
|
142
|
+
if (!Number.isFinite(createdAt)) return "expired";
|
|
143
|
+
return `${humanise(createdAt + retentionHours * 3600000 - at)} left`;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function humanise(ms) {
|
|
147
|
+
if (ms <= 0) return "0h";
|
|
148
|
+
const hours = Math.round(ms / 3600000);
|
|
149
|
+
if (hours < 48) return `${Math.max(1, hours)}h`;
|
|
150
|
+
return `${Math.round(hours / 24)}d`;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function fail(err, message) {
|
|
154
|
+
err(`error: ${message}\n`);
|
|
155
|
+
return 1;
|
|
156
|
+
}
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import { lstatSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { isAbsolute, join, relative } from "node:path";
|
|
3
|
+
import { sanitizeJobId } from "./run-history.mjs";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* sandbox-store.mjs -- the host side of a resurrectable sandbox (REQ-RESURRECTABLE-SANDBOX,
|
|
7
|
+
* INT-SANDBOX-CONTRACT).
|
|
8
|
+
*
|
|
9
|
+
* A job container is still single-use and still `--rm`s. What survives it, for a bounded window, is the
|
|
10
|
+
* per-job DIRECTORY: `cleanup` renames it here instead of deleting it, and `pi-dispatch sandbox` later
|
|
11
|
+
* mounts it into a fresh container. Nothing about the job path changes -- with the window at 0 this
|
|
12
|
+
* module is never reached and `cleanup` is byte-for-byte the `rm -rf` it always was.
|
|
13
|
+
*
|
|
14
|
+
* A SIBLING of makeLogReaper and of session-store's reapSessions rather than a widening of either: that
|
|
15
|
+
* one's `.log`/`.json` filter and logsDir scope are a documented contract, and these directories have a
|
|
16
|
+
* different retention policy and a different PII class again. Same never-throws shape, and three
|
|
17
|
+
* DELIBERATE divergences from makeLogReaper, each of which would be a silent bug if copied from it:
|
|
18
|
+
*
|
|
19
|
+
* - `lstatSync`, never `statSync`. The retained tree is agent-written; a symlink planted in it resolves
|
|
20
|
+
* on the HOST when the reaper stats it. session-store.mjs:192-201 records this lesson and
|
|
21
|
+
* makeLogReaper is the habit that predates it.
|
|
22
|
+
* - Age comes from the manifest's `createdAt`, never from mtime. makeLogReaper calls mtime "the
|
|
23
|
+
* authority" and is right about an append-once log file. Here an operator working inside a resurrected
|
|
24
|
+
* sandbox writes into the directory, so mtime would keep moving and the window would never close --
|
|
25
|
+
* for exactly the directories most likely to be large.
|
|
26
|
+
* - A directory whose sandbox container is RUNNING is skipped. The sweep runs at worker boot, an
|
|
27
|
+
* operator's shell can outlive a worker restart by design (the container is named outside the
|
|
28
|
+
* `pi-job-` reaper's filter), and deleting a live bind mount underneath it is a confusing failure
|
|
29
|
+
* with a boring cause.
|
|
30
|
+
*
|
|
31
|
+
* NEVER THROWS, on any path. Retention is a convenience layered onto a job that has already finished and
|
|
32
|
+
* already been paid for; a disk fault here must degrade to "not resurrectable" and never to a failed run.
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
/** The manifest filename inside a retained directory. */
|
|
36
|
+
export const SANDBOX_MANIFEST = "manifest.json";
|
|
37
|
+
|
|
38
|
+
const HOUR_MS = 3600000;
|
|
39
|
+
const DAY_MS = 86400000;
|
|
40
|
+
|
|
41
|
+
const defaultFs = { lstatSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync };
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Retain one finished job's directory, or delete it.
|
|
45
|
+
*
|
|
46
|
+
* Called by `makeCleanup` in place of the bare `rm -rf`. Returns the written manifest, or `null` when
|
|
47
|
+
* nothing was retained -- and on `null` the caller has nothing left to do, because every failure path
|
|
48
|
+
* here removes `jobDir` itself. Retention must never leave debris behind.
|
|
49
|
+
*
|
|
50
|
+
* `prepared.sandbox` is `{ jobId, kind, image }`, stamped by `makePrepareWorkspace`. Absent (a bare
|
|
51
|
+
* construction, a test, an unwired dispatcher) means no retention, which keeps such a caller on exactly
|
|
52
|
+
* the pre-feature path.
|
|
53
|
+
*/
|
|
54
|
+
export function retainJobDir(prepared, { sandboxDir, fs = defaultFs, log = () => {}, now = () => Date.now() } = {}) {
|
|
55
|
+
const jobDir = prepared?.jobDir;
|
|
56
|
+
const meta = prepared?.sandbox;
|
|
57
|
+
if (!jobDir) return null;
|
|
58
|
+
if (!sandboxDir || !meta?.jobId) {
|
|
59
|
+
discard(jobDir, fs);
|
|
60
|
+
return null;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const dest = join(sandboxDir, sanitizeJobId(meta.jobId));
|
|
64
|
+
try {
|
|
65
|
+
// FIRST, and load-bearing rather than hygiene. The per-job transcript copy is the most PII-bearing
|
|
66
|
+
// artifact this system holds -- tool output, file contents, the agent's own reasoning -- and it
|
|
67
|
+
// belongs to PI_SESSIONS_DIR's own TTL (INT-SESSION-STORE-CONTRACT). Carrying it into a directory
|
|
68
|
+
// with a different, operator-extendable lifetime would silently extend that TTL, which is not a
|
|
69
|
+
// weakening of the session policy so much as an end-run around it.
|
|
70
|
+
fs.rmSync(join(jobDir, "session"), { recursive: true, force: true });
|
|
71
|
+
|
|
72
|
+
fs.mkdirSync(sandboxDir, { recursive: true, mode: 0o700 });
|
|
73
|
+
// A BullMQ retry reuses the job id, so the previous attempt may already be sitting at `dest`. Last
|
|
74
|
+
// attempt wins: it is the one whose workspace matches the run the operator just watched.
|
|
75
|
+
fs.rmSync(dest, { recursive: true, force: true });
|
|
76
|
+
fs.renameSync(jobDir, dest);
|
|
77
|
+
|
|
78
|
+
const manifest = {
|
|
79
|
+
jobId: meta.jobId,
|
|
80
|
+
kind: meta.kind ?? null,
|
|
81
|
+
image: meta.image ?? null,
|
|
82
|
+
workspace: rebaseWorkspace(prepared.workspace, jobDir, dest),
|
|
83
|
+
createdAt: new Date(now()).toISOString(),
|
|
84
|
+
keepUntil: null,
|
|
85
|
+
};
|
|
86
|
+
fs.writeFileSync(join(dest, SANDBOX_MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`, { mode: 0o600 });
|
|
87
|
+
log("sandbox_retained", { jobId: meta.jobId, kind: manifest.kind });
|
|
88
|
+
return manifest;
|
|
89
|
+
} catch (err) {
|
|
90
|
+
// Fall back to the behaviour retention replaced. Both paths are attempted because the rename may
|
|
91
|
+
// have already moved the tree, in which case `jobDir` no longer exists and `dest` is the debris.
|
|
92
|
+
log("sandbox_retain_failed", { jobId: meta.jobId, reason: err?.message });
|
|
93
|
+
discard(jobDir, fs);
|
|
94
|
+
discard(dest, fs);
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Where the sandbox's `/workspace` lives once the directory has moved.
|
|
101
|
+
*
|
|
102
|
+
* A forge job's workspace is a subdirectory of jobDir (prepare-github.mjs), so it travels with the rename
|
|
103
|
+
* and its recorded path must be rebased. A local job's workspace IS the operator's own folder, outside
|
|
104
|
+
* jobDir entirely, and must be recorded verbatim -- it was never ours to move.
|
|
105
|
+
*
|
|
106
|
+
* Decided by path containment rather than by `kind`, so a preparer that changes where it puts a clone
|
|
107
|
+
* cannot silently record a path that does not exist.
|
|
108
|
+
*/
|
|
109
|
+
function rebaseWorkspace(workspace, jobDir, dest) {
|
|
110
|
+
if (!workspace) return null;
|
|
111
|
+
const rel = relative(jobDir, workspace);
|
|
112
|
+
if (rel === "" || rel.startsWith("..") || isAbsolute(rel)) return workspace;
|
|
113
|
+
return join(dest, rel);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Best-effort removal. Swallows everything: this is already the failure path. */
|
|
117
|
+
function discard(dir, fs) {
|
|
118
|
+
try {
|
|
119
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
120
|
+
} catch {
|
|
121
|
+
// nothing left to try
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** One retained run by (raw) job id, or null when absent, unreadable or not JSON. */
|
|
126
|
+
export function readManifest({ sandboxDir, jobId, fs = defaultFs }) {
|
|
127
|
+
if (!sandboxDir || jobId === undefined || jobId === null) return null;
|
|
128
|
+
const dir = join(sandboxDir, sanitizeJobId(jobId));
|
|
129
|
+
try {
|
|
130
|
+
const manifest = JSON.parse(fs.readFileSync(join(dir, SANDBOX_MANIFEST), "utf8"));
|
|
131
|
+
return { ...manifest, dir };
|
|
132
|
+
} catch {
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Every retained run, newest first. A filename-keyed scan of one directory, exactly like
|
|
139
|
+
* makeFindPreviousRun's -- no index, no database, no new query surface (DES-RUN-HISTORY-FLAT-FILES-NO-DB).
|
|
140
|
+
* An entry with no readable manifest is skipped rather than surfaced: it cannot be resurrected, and the
|
|
141
|
+
* reaper removes it on the next sweep.
|
|
142
|
+
*/
|
|
143
|
+
export function listSandboxes({ sandboxDir, fs = defaultFs }) {
|
|
144
|
+
if (!sandboxDir) return [];
|
|
145
|
+
let names;
|
|
146
|
+
try {
|
|
147
|
+
names = fs.readdirSync(sandboxDir);
|
|
148
|
+
} catch {
|
|
149
|
+
return [];
|
|
150
|
+
}
|
|
151
|
+
const out = [];
|
|
152
|
+
for (const name of names) {
|
|
153
|
+
const dir = join(sandboxDir, name);
|
|
154
|
+
try {
|
|
155
|
+
const manifest = JSON.parse(fs.readFileSync(join(dir, SANDBOX_MANIFEST), "utf8"));
|
|
156
|
+
out.push({ ...manifest, dir });
|
|
157
|
+
} catch {
|
|
158
|
+
// unreadable or not a retained directory
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return out.sort((a, b) => String(b.createdAt ?? "").localeCompare(String(a.createdAt ?? "")));
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Extend one run's retention to `now + pinDays`, and say so on disk.
|
|
166
|
+
*
|
|
167
|
+
* Bounded on purpose: `keepUntil` is a timestamp, never a boolean. "Keep this one" that means "forever"
|
|
168
|
+
* is how a directory holding a full repository clone per run becomes unbounded, and the acceptance this
|
|
169
|
+
* feature was written against says retention stays swept.
|
|
170
|
+
*/
|
|
171
|
+
export function pinSandbox({ sandboxDir, jobId, pinDays, fs = defaultFs, now = () => Date.now() }) {
|
|
172
|
+
const manifest = readManifest({ sandboxDir, jobId, fs });
|
|
173
|
+
if (!manifest) return { pinned: false, reason: "absent" };
|
|
174
|
+
const keepUntil = new Date(now() + pinDays * DAY_MS).toISOString();
|
|
175
|
+
const { dir, ...body } = manifest;
|
|
176
|
+
try {
|
|
177
|
+
fs.writeFileSync(join(dir, SANDBOX_MANIFEST), `${JSON.stringify({ ...body, keepUntil }, null, 2)}\n`, { mode: 0o600 });
|
|
178
|
+
return { pinned: true, keepUntil };
|
|
179
|
+
} catch (err) {
|
|
180
|
+
return { pinned: false, reason: err?.message ?? "write-failed" };
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* The boot sweep. Fault isolation is the contract, mirroring makeLogReaper and makeReaper: `reapSandboxes`
|
|
186
|
+
* NEVER throws under any input, and one bad entry cannot abort the rest of the sweep.
|
|
187
|
+
*
|
|
188
|
+
* There is NO keep-forever sentinel here, unlike PI_LOG_RETENTION_DAYS and PI_SESSIONS_TTL_DAYS.
|
|
189
|
+
* `retentionHours === 0` is the feature being OFF, and it needs no special case: the cutoff becomes `now`,
|
|
190
|
+
* so every unpinned directory is already expired and gets swept. Turning retention off therefore also
|
|
191
|
+
* cleans up what an earlier setting retained, while an explicit `--pin` still runs to its own deadline --
|
|
192
|
+
* an operator's deliberate act outliving a config change is the behaviour worth having.
|
|
193
|
+
*
|
|
194
|
+
* `listRunning` yields the JOB IDS of live sandboxes -- ids, not container names, so this module needs to
|
|
195
|
+
* know nothing about how a container is named and the two files stay acyclic. It defaults to none, so an
|
|
196
|
+
* unwired reaper still sweeps; start.mjs injects the docker-backed one.
|
|
197
|
+
*/
|
|
198
|
+
export function makeSandboxReaper({
|
|
199
|
+
sandboxDir,
|
|
200
|
+
retentionHours,
|
|
201
|
+
fs = defaultFs,
|
|
202
|
+
log = () => {},
|
|
203
|
+
now = () => Date.now(),
|
|
204
|
+
listRunning = async () => [],
|
|
205
|
+
}) {
|
|
206
|
+
return async function reapSandboxes() {
|
|
207
|
+
if (!sandboxDir) return;
|
|
208
|
+
let running = new Set();
|
|
209
|
+
try {
|
|
210
|
+
running = new Set(await listRunning());
|
|
211
|
+
} catch (err) {
|
|
212
|
+
// Could not ask docker. Sweeping blind risks pulling a mount out from under a live shell, so
|
|
213
|
+
// skip this sweep entirely: a directory kept one boot too long is the cheaper mistake.
|
|
214
|
+
log("sandbox_reaper_skipped", { reason: err?.message ?? "running-lookup-failed" });
|
|
215
|
+
return;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
let names;
|
|
219
|
+
try {
|
|
220
|
+
names = fs.readdirSync(sandboxDir);
|
|
221
|
+
} catch (err) {
|
|
222
|
+
log("sandbox_reaper_skipped", { reason: err?.message });
|
|
223
|
+
return;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
const at = now();
|
|
227
|
+
const cutoff = at - retentionHours * HOUR_MS;
|
|
228
|
+
for (const name of names) {
|
|
229
|
+
const dir = join(sandboxDir, name);
|
|
230
|
+
try {
|
|
231
|
+
// lstat: a symlink here resolves on the host, and this tree is agent-written.
|
|
232
|
+
if (!fs.lstatSync(dir).isDirectory()) {
|
|
233
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
234
|
+
log("reaped_sandbox", { entry: name, reason: "not-a-directory" });
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
if (running.has(name)) continue; // an operator is inside it
|
|
238
|
+
const verdict = expiry(dir, fs, at, cutoff);
|
|
239
|
+
if (!verdict.expired) continue;
|
|
240
|
+
fs.rmSync(dir, { recursive: true, force: true });
|
|
241
|
+
log("reaped_sandbox", { entry: name, reason: verdict.reason });
|
|
242
|
+
} catch (err) {
|
|
243
|
+
log("sandbox_reaper_skipped", { entry: name, reason: err?.message });
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Whether one retained directory is past its window.
|
|
251
|
+
*
|
|
252
|
+
* An unreadable, absent or unparseable manifest is EXPIRED, not skipped: without it the directory names no
|
|
253
|
+
* image, no workspace and no job, so nothing can resurrect it and keeping it is just disk. A pin wins over
|
|
254
|
+
* the base window while it lasts, and an unparseable `keepUntil` is treated as no pin rather than as
|
|
255
|
+
* forever -- the direction that stays bounded.
|
|
256
|
+
*/
|
|
257
|
+
function expiry(dir, fs, at, cutoff) {
|
|
258
|
+
let manifest;
|
|
259
|
+
try {
|
|
260
|
+
manifest = JSON.parse(fs.readFileSync(join(dir, SANDBOX_MANIFEST), "utf8"));
|
|
261
|
+
} catch {
|
|
262
|
+
return { expired: true, reason: "no-manifest" };
|
|
263
|
+
}
|
|
264
|
+
const keepUntil = Date.parse(manifest?.keepUntil ?? "");
|
|
265
|
+
if (Number.isFinite(keepUntil)) return keepUntil <= at ? { expired: true, reason: "pin-expired" } : { expired: false };
|
|
266
|
+
const createdAt = Date.parse(manifest?.createdAt ?? "");
|
|
267
|
+
if (!Number.isFinite(createdAt)) return { expired: true, reason: "no-created-at" };
|
|
268
|
+
return createdAt < cutoff ? { expired: true, reason: "window" } : { expired: false };
|
|
269
|
+
}
|