@zenera/sandbox 0.0.0-stage → 1.1.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +60 -2
- package/dist/contract.d.ts +230 -0
- package/dist/contract.js +27 -0
- package/dist/fs.d.ts +46 -0
- package/dist/fs.js +39 -0
- package/dist/image.d.ts +62 -0
- package/dist/image.js +210 -0
- package/dist/in-memory-fs.d.ts +25 -0
- package/dist/in-memory-fs.js +165 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +12 -0
- package/dist/limits.d.ts +19 -0
- package/dist/limits.js +33 -0
- package/dist/mounts.d.ts +84 -0
- package/dist/mounts.js +63 -0
- package/dist/podman.d.ts +80 -0
- package/dist/podman.js +575 -0
- package/dist/process.d.ts +29 -0
- package/dist/process.js +122 -0
- package/dist/testing.d.ts +25 -0
- package/dist/testing.js +118 -0
- package/package.json +58 -5
package/dist/podman.d.ts
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
import { type FsDriver } from './fs.ts';
|
|
2
|
+
import { type FilesView, type Mount } from './mounts.ts';
|
|
3
|
+
import { type Runner } from './process.ts';
|
|
4
|
+
import { type BuildSpec, type PrepareOptions, type Sandbox, type SandboxFactory, type ExecOptions, type ExecResult, type JobOutput, type JobStopped, type SandboxOptions, type SandboxSpec } from './contract.ts';
|
|
5
|
+
/** What only a podman sandbox needs: which binary to drive, and how to spawn it. */
|
|
6
|
+
export interface PodmanHost {
|
|
7
|
+
/** the container engine binary; `podman` when unset */
|
|
8
|
+
engine?: string;
|
|
9
|
+
/** how processes are spawned — the seam tests replace */
|
|
10
|
+
exec?: Runner;
|
|
11
|
+
}
|
|
12
|
+
interface Resolved extends Required<Omit<SandboxSpec, 'user' | 'build'>> {
|
|
13
|
+
user?: string;
|
|
14
|
+
/** the Dockerfile `image` is built from, when it is */
|
|
15
|
+
build?: BuildSpec;
|
|
16
|
+
root: string;
|
|
17
|
+
key: string;
|
|
18
|
+
readOnly: boolean;
|
|
19
|
+
mounts: readonly Mount[];
|
|
20
|
+
execEnv: Record<string, string>;
|
|
21
|
+
execPaths: Record<string, readonly string[]>;
|
|
22
|
+
explicitCpus?: boolean;
|
|
23
|
+
explicitMemory?: boolean;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* The container, started when it is first needed and not before. A project
|
|
27
|
+
* whose agents never shell out never pays for an image pull, and a session
|
|
28
|
+
* that only ever asks a question never leaves anything behind.
|
|
29
|
+
*/
|
|
30
|
+
export declare class PodmanSandbox implements Sandbox {
|
|
31
|
+
#private;
|
|
32
|
+
readonly name: string;
|
|
33
|
+
/** the container binds the host directories, so the disk is where the files are */
|
|
34
|
+
readonly fs: FsDriver;
|
|
35
|
+
readonly spec: Resolved;
|
|
36
|
+
readonly engine: string;
|
|
37
|
+
constructor(opts: SandboxOptions, host?: PodmanHost);
|
|
38
|
+
/** Whether anything was actually started — `close` is free when not. */
|
|
39
|
+
get started(): boolean;
|
|
40
|
+
/** The host paths, as they are: the container binds exactly these. */
|
|
41
|
+
files(opts: FilesView): FilesView;
|
|
42
|
+
/** Pulls the image, or builds it from the Dockerfile, unless the engine already has it. */
|
|
43
|
+
prepare(opts?: PrepareOptions): Promise<void>;
|
|
44
|
+
/**
|
|
45
|
+
* Idempotent, and single-flight: several tool calls in one batch all await
|
|
46
|
+
* the same start rather than racing to create the same container.
|
|
47
|
+
*/
|
|
48
|
+
start(): Promise<void>;
|
|
49
|
+
/**
|
|
50
|
+
* Runs one command and waits for it. The script goes down stdin, so no part
|
|
51
|
+
* of it is ever parsed by a shell on this side of the container.
|
|
52
|
+
*/
|
|
53
|
+
exec(command: string, opts?: ExecOptions): Promise<ExecResult>;
|
|
54
|
+
/**
|
|
55
|
+
* Starts a command and returns. The wrapper writes the log and the exit
|
|
56
|
+
* code to files the container owns, which is what makes the job readable
|
|
57
|
+
* after the `exec` that started it has long returned.
|
|
58
|
+
*/
|
|
59
|
+
startJob(command: string, opts?: ExecOptions): Promise<string>;
|
|
60
|
+
/** A window of a job's log, plus whether it is still going. */
|
|
61
|
+
readJob(id: string, from?: number): Promise<JobOutput>;
|
|
62
|
+
/** Signals a job's whole process group, so its children go too. */
|
|
63
|
+
stopJob(id: string): Promise<JobStopped>;
|
|
64
|
+
/**
|
|
65
|
+
* Stops a persisted container and removes an ephemeral one. Stopping
|
|
66
|
+
* rather than removing is what makes `persist` mean anything: a stopped
|
|
67
|
+
* container costs disk, a running one costs the machine's whole budget.
|
|
68
|
+
*/
|
|
69
|
+
close(): Promise<void>;
|
|
70
|
+
/**
|
|
71
|
+
* A path inside the container, contained to the working directory. Purely
|
|
72
|
+
* lexical, and correctly so: this is the container's path space, not this
|
|
73
|
+
* machine's, and there is nothing here to resolve a symlink against.
|
|
74
|
+
*/
|
|
75
|
+
inside(path?: string): string;
|
|
76
|
+
}
|
|
77
|
+
/** Podman on this machine: what `zen run` uses, and the engine's default. */
|
|
78
|
+
export declare const podmanSandbox: SandboxFactory;
|
|
79
|
+
export {};
|
|
80
|
+
//# sourceMappingURL=podman.d.ts.map
|
package/dist/podman.js
ADDED
|
@@ -0,0 +1,575 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { posix } from 'node:path';
|
|
3
|
+
import { ensurePodmanImage, localImageTag } from "./image.js";
|
|
4
|
+
import { localFs } from "./fs.js";
|
|
5
|
+
import { sandboxLimits } from "./limits.js";
|
|
6
|
+
import { WORKSPACE_MOUNT } from "./mounts.js";
|
|
7
|
+
import { MAX_TIMEOUT_MS, resilient, runProcess } from "./process.js";
|
|
8
|
+
import { DEFAULT_SANDBOX_IMAGE, SandboxError, } from "./contract.js";
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
// The podman sandbox
|
|
11
|
+
//
|
|
12
|
+
// `Sandbox` on this machine: one podman container per spec, driven by spawning the
|
|
13
|
+
// engine binary. Everything here is about that engine — its argv, its exit
|
|
14
|
+
// codes, its lock table, the ssh hop into its virtual machine — and none of it
|
|
15
|
+
// is something another backend would share.
|
|
16
|
+
// ---------------------------------------------------------------------------
|
|
17
|
+
/** Where background jobs keep their logs. Inside the container, not the host. */
|
|
18
|
+
const JOBS = '/tmp/zenera-jobs';
|
|
19
|
+
/** ...and a job's log is read in windows of this many lines. */
|
|
20
|
+
const MAX_LOG_LINES = 400;
|
|
21
|
+
const DEFAULT_TIMEOUT_MS = 120_000;
|
|
22
|
+
const DEFAULT_PIDS = 1024;
|
|
23
|
+
/** How long a container gets to stop itself before it is killed. */
|
|
24
|
+
const STOP_GRACE_S = 2;
|
|
25
|
+
/** SIGKILL, which is how the kernel's out-of-memory killer arrives. */
|
|
26
|
+
const KILLED = 137;
|
|
27
|
+
// Strict mode's filesystem limits; its cores and memory are in `limits.ts`.
|
|
28
|
+
const STRICT_TMPFS = 'rw,noexec,nosuid,nodev';
|
|
29
|
+
const STRICT_TMP_SIZE = '64m';
|
|
30
|
+
/**
|
|
31
|
+
* Asked of the container after a kill. Builtins only: a slim image may have no
|
|
32
|
+
* `awk`, and strict hardening leaves nowhere writable to install one. `oom_kill`
|
|
33
|
+
* counts kills in this container's cgroup and no other, which is what separates
|
|
34
|
+
* the two cases. Unreadable values print empty, and the caller reads that as
|
|
35
|
+
* "could not tell".
|
|
36
|
+
*/
|
|
37
|
+
const CGROUP = [
|
|
38
|
+
`peak=; kills=`,
|
|
39
|
+
`read peak < /sys/fs/cgroup/memory.peak 2>/dev/null`,
|
|
40
|
+
`while read -r k v; do [ "$k" = oom_kill ] && kills=$v; done < /sys/fs/cgroup/memory.events 2>/dev/null`,
|
|
41
|
+
`echo "peak=$peak kills=$kills"`,
|
|
42
|
+
].join('\n');
|
|
43
|
+
/**
|
|
44
|
+
* The container, started when it is first needed and not before. A project
|
|
45
|
+
* whose agents never shell out never pays for an image pull, and a session
|
|
46
|
+
* that only ever asks a question never leaves anything behind.
|
|
47
|
+
*/
|
|
48
|
+
export class PodmanSandbox {
|
|
49
|
+
name;
|
|
50
|
+
/** the container binds the host directories, so the disk is where the files are */
|
|
51
|
+
fs = localFs;
|
|
52
|
+
spec;
|
|
53
|
+
engine;
|
|
54
|
+
#run;
|
|
55
|
+
#spawn;
|
|
56
|
+
#jobs = new Map();
|
|
57
|
+
#ready;
|
|
58
|
+
#prepared;
|
|
59
|
+
#created = false;
|
|
60
|
+
/** kills counted in this container's cgroup as of the last probe */
|
|
61
|
+
#kills;
|
|
62
|
+
constructor(opts, host = {}) {
|
|
63
|
+
this.spec = resolveSpec(opts);
|
|
64
|
+
this.engine = host.engine ?? 'podman';
|
|
65
|
+
this.#spawn = host.exec ?? runProcess;
|
|
66
|
+
this.#run = resilient(this.#spawn);
|
|
67
|
+
this.name = containerName(this.spec);
|
|
68
|
+
}
|
|
69
|
+
/** Whether anything was actually started — `close` is free when not. */
|
|
70
|
+
get started() {
|
|
71
|
+
return this.#created;
|
|
72
|
+
}
|
|
73
|
+
/** The host paths, as they are: the container binds exactly these. */
|
|
74
|
+
files(opts) {
|
|
75
|
+
return { ...opts, fs: this.fs };
|
|
76
|
+
}
|
|
77
|
+
/** Pulls the image, or builds it from the Dockerfile, unless the engine already has it. */
|
|
78
|
+
prepare(opts = {}) {
|
|
79
|
+
if (this.#prepared && !opts.force) {
|
|
80
|
+
return this.#prepared;
|
|
81
|
+
}
|
|
82
|
+
this.#prepared = ensurePodmanImage({ image: this.spec.image, build: this.spec.build }, { engine: this.engine, exec: this.#spawn }, opts).catch((err) => {
|
|
83
|
+
this.#prepared = undefined;
|
|
84
|
+
throw err;
|
|
85
|
+
});
|
|
86
|
+
return this.#prepared;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Idempotent, and single-flight: several tool calls in one batch all await
|
|
90
|
+
* the same start rather than racing to create the same container.
|
|
91
|
+
*/
|
|
92
|
+
start() {
|
|
93
|
+
this.#ready ??= this.#bring().catch((err) => {
|
|
94
|
+
// A failed start must not be remembered as a start, or every later
|
|
95
|
+
// call in the session inherits a failure it could have retried.
|
|
96
|
+
this.#ready = undefined;
|
|
97
|
+
throw err;
|
|
98
|
+
});
|
|
99
|
+
return this.#ready;
|
|
100
|
+
}
|
|
101
|
+
async #bring() {
|
|
102
|
+
const status = await this.#podman([
|
|
103
|
+
'container',
|
|
104
|
+
'inspect',
|
|
105
|
+
this.name,
|
|
106
|
+
'--format',
|
|
107
|
+
'{{.State.Status}}',
|
|
108
|
+
]);
|
|
109
|
+
if (status.code === 0) {
|
|
110
|
+
const state = status.stdout.trim();
|
|
111
|
+
if (state !== 'running') {
|
|
112
|
+
await this.#must(['start', this.name], `could not start container ${this.name}`);
|
|
113
|
+
}
|
|
114
|
+
this.#created = true;
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
// `run` pulls a registry image by itself; a built one has to exist first.
|
|
118
|
+
if (this.spec.build) {
|
|
119
|
+
await this.prepare();
|
|
120
|
+
}
|
|
121
|
+
const made = await this.#podman(this.#createArgs());
|
|
122
|
+
if (made.code !== 0) {
|
|
123
|
+
throw new SandboxError(`could not create container ${this.name}: ${message(made)}`, LOCKS.test(made.stderr) ? locksHint(this.engine) : undefined);
|
|
124
|
+
}
|
|
125
|
+
this.#created = true;
|
|
126
|
+
}
|
|
127
|
+
#createArgs() {
|
|
128
|
+
const s = this.spec;
|
|
129
|
+
const args = ['run', '--detach', '--name', this.name];
|
|
130
|
+
args.push('--label', 'zenera=1', '--label', `zenera.key=${s.key}`);
|
|
131
|
+
args.push('--volume', mount({ host: s.root, at: s.workdir, readOnly: s.readOnly }));
|
|
132
|
+
for (const m of s.mounts) {
|
|
133
|
+
args.push('--volume', mount(m));
|
|
134
|
+
}
|
|
135
|
+
args.push('--workdir', s.workdir);
|
|
136
|
+
args.push('--network', s.network);
|
|
137
|
+
if (s.cpus > 0) {
|
|
138
|
+
args.push('--cpus', String(s.cpus));
|
|
139
|
+
}
|
|
140
|
+
if (s.memory > 0) {
|
|
141
|
+
args.push('--memory', `${s.memory}m`);
|
|
142
|
+
}
|
|
143
|
+
args.push('--pids-limit', String(DEFAULT_PIDS));
|
|
144
|
+
// `no-new-privileges` costs nothing here and closes setuid escalation.
|
|
145
|
+
// Under `standard` the capabilities are deliberately *not* dropped
|
|
146
|
+
// wholesale: installing a package is the ordinary use of this tool, and
|
|
147
|
+
// `--cap-drop=ALL` breaks every package manager there is. The container
|
|
148
|
+
// is the boundary. `strict` makes the opposite trade, below.
|
|
149
|
+
args.push('--security-opt', 'no-new-privileges');
|
|
150
|
+
// Background jobs orphan processes; without an init they accumulate as
|
|
151
|
+
// zombies until the pids limit stops the sandbox dead.
|
|
152
|
+
args.push('--init');
|
|
153
|
+
args.push('--restart', 'no');
|
|
154
|
+
if (s.hardening === 'strict') {
|
|
155
|
+
args.push('--cap-drop', 'ALL');
|
|
156
|
+
// The image's own files become reference material: a command can
|
|
157
|
+
// read a binary and run it, and can write nowhere but the mounts
|
|
158
|
+
// and the tmpfs below.
|
|
159
|
+
args.push('--read-only');
|
|
160
|
+
// `noexec` is safe for the job launcher because a job script is
|
|
161
|
+
// *interpreted* (`/bin/sh <file>`), never executed as a program.
|
|
162
|
+
// `/run` is left to podman's own read-only tmpfs: `--init` mounts
|
|
163
|
+
// catatonit under it and has to be able to exec it.
|
|
164
|
+
args.push('--tmpfs', `/tmp:${STRICT_TMPFS},size=${STRICT_TMP_SIZE}`);
|
|
165
|
+
if (!s.user) {
|
|
166
|
+
// Rootless podman maps container uid 0 to the host account, so
|
|
167
|
+
// the default container root is already unprivileged on the
|
|
168
|
+
// host — but it is uid 0 *inside*, where the setuid binaries
|
|
169
|
+
// are. `keep-id` runs the process as the host's own uid, which
|
|
170
|
+
// is both non-zero and the owner of every bind mount, so the
|
|
171
|
+
// workspace stays writable without a chown.
|
|
172
|
+
args.push('--userns', 'keep-id');
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
if (s.user) {
|
|
176
|
+
args.push('--user', s.user);
|
|
177
|
+
}
|
|
178
|
+
for (const [k, v] of Object.entries(s.env)) {
|
|
179
|
+
args.push('--env', `${k}=${v}`);
|
|
180
|
+
}
|
|
181
|
+
// No `=`, and so no value: podman reads each one from its own
|
|
182
|
+
// environment. A key passed this way is not in the argv of a process
|
|
183
|
+
// any user on the machine can list.
|
|
184
|
+
for (const name of s.secrets) {
|
|
185
|
+
if (process.env[name]) {
|
|
186
|
+
args.push('--env', name);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
args.push(s.image, 'sleep', 'infinity');
|
|
190
|
+
return args;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* `podman exec` for a command the agent asked for, with the run's own
|
|
194
|
+
* environment. The script still arrives on stdin; when a path has to be
|
|
195
|
+
* extended, the shell that reads it is started by a fixed `-c` line that
|
|
196
|
+
* exports the variable and `exec`s it — built from host-chosen names and
|
|
197
|
+
* paths only, so no text of the agent's is ever part of a parse here.
|
|
198
|
+
*/
|
|
199
|
+
#exec(options = []) {
|
|
200
|
+
const env = Object.entries(this.spec.execEnv).flatMap(([k, v]) => ['--env', `${k}=${v}`]);
|
|
201
|
+
const paths = Object.entries(this.spec.execPaths).filter(([, dirs]) => dirs.length);
|
|
202
|
+
const shell = paths.length
|
|
203
|
+
? [
|
|
204
|
+
'/bin/sh',
|
|
205
|
+
'-c',
|
|
206
|
+
[
|
|
207
|
+
...paths.map(([name, dirs]) => `export ${name}=${quote(dirs.join(':'))}"\${${name}:+:\$${name}}"`),
|
|
208
|
+
'exec /bin/sh -s',
|
|
209
|
+
].join('; '),
|
|
210
|
+
]
|
|
211
|
+
: ['/bin/sh', '-s'];
|
|
212
|
+
return ['exec', '--interactive', ...env, ...options, this.name, ...shell];
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Runs one command and waits for it. The script goes down stdin, so no part
|
|
216
|
+
* of it is ever parsed by a shell on this side of the container.
|
|
217
|
+
*/
|
|
218
|
+
async exec(command, opts = {}) {
|
|
219
|
+
await this.start();
|
|
220
|
+
const cwd = this.inside(opts.cwd);
|
|
221
|
+
const timeoutMs = this.#timeout(opts.timeout);
|
|
222
|
+
const startedAt = Date.now();
|
|
223
|
+
const once = () => this.#run(this.engine, this.#exec(['--workdir', cwd]), {
|
|
224
|
+
input: command,
|
|
225
|
+
timeoutMs,
|
|
226
|
+
signal: opts.signal,
|
|
227
|
+
});
|
|
228
|
+
let res = await once();
|
|
229
|
+
if (gone(res)) {
|
|
230
|
+
// podman refused before running anything, so the retry cannot run the command twice.
|
|
231
|
+
await this.#revive(res);
|
|
232
|
+
res = await once();
|
|
233
|
+
}
|
|
234
|
+
return {
|
|
235
|
+
exit_code: res.code,
|
|
236
|
+
stdout: res.stdout,
|
|
237
|
+
stderr: res.stderr,
|
|
238
|
+
truncated: res.truncated || undefined,
|
|
239
|
+
timed_out: res.timedOut || undefined,
|
|
240
|
+
duration_ms: Date.now() - startedAt,
|
|
241
|
+
out_of_memory: await this.#oom(res.code, res.timedOut, opts.signal),
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
/**
|
|
245
|
+
* Starts a command and returns. The wrapper writes the log and the exit
|
|
246
|
+
* code to files the container owns, which is what makes the job readable
|
|
247
|
+
* after the `exec` that started it has long returned.
|
|
248
|
+
*/
|
|
249
|
+
async startJob(command, opts = {}) {
|
|
250
|
+
await this.start();
|
|
251
|
+
const cwd = this.inside(opts.cwd);
|
|
252
|
+
const id = jobId();
|
|
253
|
+
// The command reaches the container as a *file*, written by `cat` from
|
|
254
|
+
// stdin. Interpolating it into the launcher script would put user text
|
|
255
|
+
// back into a shell parse, which is the one thing this design avoids.
|
|
256
|
+
const body = await this.#run(this.engine, [
|
|
257
|
+
'exec',
|
|
258
|
+
'--interactive',
|
|
259
|
+
this.name,
|
|
260
|
+
'/bin/sh',
|
|
261
|
+
'-c',
|
|
262
|
+
`mkdir -p ${JOBS} && cat > ${JOBS}/${id}.sh`,
|
|
263
|
+
], { input: command, timeoutMs: 30_000 });
|
|
264
|
+
if (body.code !== 0) {
|
|
265
|
+
throw new SandboxError(`could not write the job: ${message(body)}`);
|
|
266
|
+
}
|
|
267
|
+
// One logical line, because a newline would end the group command and
|
|
268
|
+
// leave the redirection and the `&` on a statement of their own — the
|
|
269
|
+
// job would then run in the foreground of this `exec`, with its output
|
|
270
|
+
// going to the caller instead of the log.
|
|
271
|
+
//
|
|
272
|
+
// `setsid` puts the job in a session of its own so that stopping it can
|
|
273
|
+
// signal a process group rather than one process; a shell without job
|
|
274
|
+
// control leaves background jobs in its own group, and signalling that
|
|
275
|
+
// would take down the `exec` shell with them. It is not required: where
|
|
276
|
+
// it is missing the job still runs, and stopping falls back to the
|
|
277
|
+
// single pid. The leader writes its own pid, since with `setsid` the
|
|
278
|
+
// launching shell's `$!` is no longer the process to signal.
|
|
279
|
+
const inner = `echo $$ > ${JOBS}/${id}.pid; ` +
|
|
280
|
+
`/bin/sh ${JOBS}/${id}.sh; echo $? > ${JOBS}/${id}.exit`;
|
|
281
|
+
const launch = [
|
|
282
|
+
`cd ${quote(cwd)}`,
|
|
283
|
+
`S=; command -v setsid > /dev/null 2>&1 && S=setsid`,
|
|
284
|
+
`$S /bin/sh -c ${quote(inner)} > ${JOBS}/${id}.log 2>&1 < /dev/null &`,
|
|
285
|
+
// The pid file is written by the child, so the launcher waits for
|
|
286
|
+
// it: a read or a stop arriving immediately after must not find
|
|
287
|
+
// the job missing.
|
|
288
|
+
`i=0; while [ ! -s ${JOBS}/${id}.pid ] && [ $i -lt 50 ]; do i=$((i+1)); sleep 0.1; done`,
|
|
289
|
+
].join('\n');
|
|
290
|
+
const started = await this.#run(this.engine, this.#exec(), {
|
|
291
|
+
input: launch,
|
|
292
|
+
timeoutMs: 30_000,
|
|
293
|
+
});
|
|
294
|
+
if (started.code !== 0) {
|
|
295
|
+
throw new SandboxError(`could not start the job: ${message(started)}`);
|
|
296
|
+
}
|
|
297
|
+
const job = { id, command, cwd, startedAt: Date.now() };
|
|
298
|
+
this.#jobs.set(id, job);
|
|
299
|
+
return id;
|
|
300
|
+
}
|
|
301
|
+
/** A window of a job's log, plus whether it is still going. */
|
|
302
|
+
async readJob(id, from = 1) {
|
|
303
|
+
const job = this.#job(id);
|
|
304
|
+
await this.start();
|
|
305
|
+
const first = Math.max(1, Math.trunc(from));
|
|
306
|
+
const script = [
|
|
307
|
+
`cd ${JOBS} || exit 0`,
|
|
308
|
+
`printf 'lines=%s\\n' "$(wc -l < ${id}.log 2>/dev/null || echo 0)"`,
|
|
309
|
+
`if [ -f ${id}.exit ]; then printf 'exit=%s\\n' "$(cat ${id}.exit)"; fi`,
|
|
310
|
+
`printf -- '---\\n'`,
|
|
311
|
+
`tail -n +${first} ${id}.log 2>/dev/null | head -n ${MAX_LOG_LINES}`,
|
|
312
|
+
].join('\n');
|
|
313
|
+
const res = await this.#run(this.engine, ['exec', '--interactive', this.name, '/bin/sh', '-s'], { input: script, timeoutMs: 30_000 });
|
|
314
|
+
const [head, ...rest] = res.stdout.split('\n---\n');
|
|
315
|
+
const meta = Object.fromEntries(head
|
|
316
|
+
.split('\n')
|
|
317
|
+
.filter(Boolean)
|
|
318
|
+
.map((l) => l.split('=', 2)));
|
|
319
|
+
const output = rest.join('\n---\n');
|
|
320
|
+
const lines = Number(meta.lines ?? 0);
|
|
321
|
+
const shown = output ? output.replace(/\n$/, '').split('\n').length : 0;
|
|
322
|
+
const exit = meta.exit === undefined ? undefined : Number(meta.exit);
|
|
323
|
+
// Kept on the job: a second read would find the kill counter unchanged
|
|
324
|
+
// and report this container's own overrun as the machine's doing.
|
|
325
|
+
if (exit === KILLED && !job.oom) {
|
|
326
|
+
job.oom = await this.#oom(exit, false);
|
|
327
|
+
}
|
|
328
|
+
return {
|
|
329
|
+
job_id: id,
|
|
330
|
+
command: job.command,
|
|
331
|
+
running: meta.exit === undefined,
|
|
332
|
+
exit_code: exit,
|
|
333
|
+
duration_ms: Date.now() - job.startedAt,
|
|
334
|
+
start_line: first,
|
|
335
|
+
end_line: first + shown - 1,
|
|
336
|
+
lines,
|
|
337
|
+
truncated: first + shown - 1 < lines || undefined,
|
|
338
|
+
output,
|
|
339
|
+
out_of_memory: job.oom,
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
/** Signals a job's whole process group, so its children go too. */
|
|
343
|
+
async stopJob(id) {
|
|
344
|
+
this.#job(id);
|
|
345
|
+
await this.start();
|
|
346
|
+
const script = [
|
|
347
|
+
`pid=$(cat ${JOBS}/${id}.pid 2>/dev/null)`,
|
|
348
|
+
`[ -n "$pid" ] || { echo "already gone"; exit 0; }`,
|
|
349
|
+
`kill -TERM -"$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true`,
|
|
350
|
+
`sleep 1`,
|
|
351
|
+
`kill -KILL -"$pid" 2>/dev/null || true`,
|
|
352
|
+
`echo stopped`,
|
|
353
|
+
].join('\n');
|
|
354
|
+
const res = await this.#run(this.engine, ['exec', '--interactive', this.name, '/bin/sh', '-s'], { input: script, timeoutMs: 30_000 });
|
|
355
|
+
this.#jobs.delete(id);
|
|
356
|
+
return { job_id: id, stopped: res.code === 0 };
|
|
357
|
+
}
|
|
358
|
+
#job(id) {
|
|
359
|
+
const job = this.#jobs.get(id);
|
|
360
|
+
if (!job) {
|
|
361
|
+
throw new SandboxError(`no such job: ${id}`, 'ids come from run_command_background');
|
|
362
|
+
}
|
|
363
|
+
return job;
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* Stops a persisted container and removes an ephemeral one. Stopping
|
|
367
|
+
* rather than removing is what makes `persist` mean anything: a stopped
|
|
368
|
+
* container costs disk, a running one costs the machine's whole budget.
|
|
369
|
+
*/
|
|
370
|
+
async close() {
|
|
371
|
+
if (!this.#created) {
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
this.#created = false;
|
|
375
|
+
this.#ready = undefined;
|
|
376
|
+
await this.#podman(this.spec.persist
|
|
377
|
+
? ['stop', '--time', String(STOP_GRACE_S), this.name]
|
|
378
|
+
: ['rm', '--force', '--volumes', this.name]).catch(() => undefined);
|
|
379
|
+
}
|
|
380
|
+
/**
|
|
381
|
+
* A path inside the container, contained to the working directory. Purely
|
|
382
|
+
* lexical, and correctly so: this is the container's path space, not this
|
|
383
|
+
* machine's, and there is nothing here to resolve a symlink against.
|
|
384
|
+
*/
|
|
385
|
+
inside(path) {
|
|
386
|
+
const base = this.spec.workdir;
|
|
387
|
+
if (!path) {
|
|
388
|
+
return base;
|
|
389
|
+
}
|
|
390
|
+
if (path.includes('\0')) {
|
|
391
|
+
throw new SandboxError('path contains a null byte');
|
|
392
|
+
}
|
|
393
|
+
const at = posix.resolve(base, path);
|
|
394
|
+
if (at !== base && !at.startsWith(`${base}/`)) {
|
|
395
|
+
throw new SandboxError(`outside ${base}: ${path}`, `paths are relative to ${base}`);
|
|
396
|
+
}
|
|
397
|
+
return at;
|
|
398
|
+
}
|
|
399
|
+
#timeout(seconds) {
|
|
400
|
+
const wanted = seconds && seconds > 0 ? seconds * 1000 : this.spec.timeout * 1000;
|
|
401
|
+
return Math.min(wanted || DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS);
|
|
402
|
+
}
|
|
403
|
+
/**
|
|
404
|
+
* Turns a bare 137 into a reason, by asking the container which cgroup did
|
|
405
|
+
* the killing. Only a kill counted against *this* container is the caller's
|
|
406
|
+
* to fix; the rest is the machine being busy, and telling someone to shrink
|
|
407
|
+
* a command that already fit would send them off after the wrong thing.
|
|
408
|
+
*/
|
|
409
|
+
async #oom(code, timedOut, signal) {
|
|
410
|
+
// A timeout or a cancel kills podman on this side, not the container,
|
|
411
|
+
// and an uncapped container has no limit it could have exceeded.
|
|
412
|
+
if (code !== KILLED || timedOut || signal?.aborted || this.spec.memory <= 0) {
|
|
413
|
+
return undefined;
|
|
414
|
+
}
|
|
415
|
+
const res = await this.#run(this.engine, ['exec', '--interactive', this.name, '/bin/sh', '-s'], { input: CGROUP, timeoutMs: 15_000 }).catch(() => undefined);
|
|
416
|
+
const out = res?.stdout ?? '';
|
|
417
|
+
const peak = Number(/peak=(\d+)/.exec(out)?.[1]);
|
|
418
|
+
const kills = Number(/kills=(\d+)/.exec(out)?.[1]);
|
|
419
|
+
const counted = Number.isFinite(kills);
|
|
420
|
+
// Nothing readable is read as our own doing: that costs one wasted
|
|
421
|
+
// retry, where the other guess costs an agent retrying forever a
|
|
422
|
+
// command that was never going to fit.
|
|
423
|
+
const own = !counted || kills > (this.#kills ?? 0);
|
|
424
|
+
if (counted) {
|
|
425
|
+
this.#kills = kills;
|
|
426
|
+
}
|
|
427
|
+
const limit = this.spec.memory;
|
|
428
|
+
return {
|
|
429
|
+
limit_mib: limit,
|
|
430
|
+
peak_mib: Number.isFinite(peak) ? Math.round(peak / (1024 * 1024)) : undefined,
|
|
431
|
+
retryable: !own,
|
|
432
|
+
hint: own
|
|
433
|
+
? `this container is limited to ${limit} MiB and the command wanted more. ` +
|
|
434
|
+
`Running it again unchanged will end the same way: give the sandbox more ` +
|
|
435
|
+
`memory in the project configuration, or do the work in smaller pieces.`
|
|
436
|
+
: `the machine ran out of memory while this container was inside its ` +
|
|
437
|
+
`${limit} MiB limit, so it was killed for something another container did. ` +
|
|
438
|
+
`The command itself does not need to change, and may work once the machine ` +
|
|
439
|
+
`is less busy.`,
|
|
440
|
+
};
|
|
441
|
+
}
|
|
442
|
+
#podman(args) {
|
|
443
|
+
return this.#run(this.engine, args, { timeoutMs: 120_000 });
|
|
444
|
+
}
|
|
445
|
+
/** The container stopped or vanished under us (machine slept, restarted, or someone removed it). */
|
|
446
|
+
async #revive(res) {
|
|
447
|
+
this.#ready = undefined;
|
|
448
|
+
try {
|
|
449
|
+
await this.start();
|
|
450
|
+
}
|
|
451
|
+
catch (err) {
|
|
452
|
+
throw new SandboxError(`the sandbox container ${this.name} stopped (${message(res)}) and could not be restarted: ` +
|
|
453
|
+
(err instanceof Error ? err.message : String(err)), `the ${this.engine} machine may have stopped or restarted — check \`${this.engine} machine list\`, ` +
|
|
454
|
+
`start it with \`${this.engine} machine start\`, then retry`);
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
async #must(args, what) {
|
|
458
|
+
const res = await this.#podman(args);
|
|
459
|
+
if (res.code !== 0) {
|
|
460
|
+
throw new SandboxError(`${what}: ${message(res)}`);
|
|
461
|
+
}
|
|
462
|
+
return res;
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
function mount(m) {
|
|
466
|
+
return `${m.host}:${m.at}${m.readOnly ? ':ro' : ''}`;
|
|
467
|
+
}
|
|
468
|
+
function message(res) {
|
|
469
|
+
return (res.stderr.trim() || res.stdout.trim() || `exit ${res.code}`).split('\n')[0];
|
|
470
|
+
}
|
|
471
|
+
/** podman's words for "every lock is taken": `allocating lock … exceeded num_locks (2048)`. */
|
|
472
|
+
const LOCKS = /exceeded num_locks|allocating lock/i;
|
|
473
|
+
/**
|
|
474
|
+
* A stopped container holds a lock as surely as a running one, and a
|
|
475
|
+
* persisted sandbox is stopped, never removed — so they pile up one per
|
|
476
|
+
* session until the engine can create nothing at all, for anyone.
|
|
477
|
+
*/
|
|
478
|
+
function locksHint(engine) {
|
|
479
|
+
return (`the ${engine} engine has no locks left: every container it holds, stopped ones ` +
|
|
480
|
+
`included, takes one. Remove the stopped sandbox containers with ` +
|
|
481
|
+
`\`${engine} container prune --force --filter label=zenera=1\`, then retry`);
|
|
482
|
+
}
|
|
483
|
+
/** podman's own exit code for "could not exec at all", with the two reasons that mean the container is not there to exec in. */
|
|
484
|
+
function gone(res) {
|
|
485
|
+
return (res.code === 125 &&
|
|
486
|
+
/can only create exec sessions on running containers|container state improper|no such container/i.test(res.stderr));
|
|
487
|
+
}
|
|
488
|
+
/** Only ever applied to paths this module derived, never to a model's text. */
|
|
489
|
+
function quote(s) {
|
|
490
|
+
return `'${s.replaceAll("'", `'\\''`)}'`;
|
|
491
|
+
}
|
|
492
|
+
function jobId() {
|
|
493
|
+
return `job-${createHash('sha256').update(`${Date.now()}${Math.random()}`).digest('hex').slice(0, 6)}`;
|
|
494
|
+
}
|
|
495
|
+
function resolveSpec(opts) {
|
|
496
|
+
const workdir = opts.workdir ?? WORKSPACE_MOUNT;
|
|
497
|
+
if (!posix.isAbsolute(workdir)) {
|
|
498
|
+
throw new SandboxError(`sandbox workdir must be absolute: ${workdir}`);
|
|
499
|
+
}
|
|
500
|
+
const limits = sandboxLimits(opts);
|
|
501
|
+
const { hardening } = limits;
|
|
502
|
+
const strict = hardening === 'strict';
|
|
503
|
+
for (const name of [...Object.keys(opts.execEnv ?? {}), ...Object.keys(opts.execPaths ?? {})]) {
|
|
504
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)) {
|
|
505
|
+
throw new SandboxError(`not a variable name: ${JSON.stringify(name)}`);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
return {
|
|
509
|
+
root: opts.root,
|
|
510
|
+
key: opts.key,
|
|
511
|
+
image: opts.build ? localImageTag(opts.build) : (opts.image ?? DEFAULT_SANDBOX_IMAGE),
|
|
512
|
+
build: opts.build,
|
|
513
|
+
cpus: limits.cpus,
|
|
514
|
+
memory: limits.memory,
|
|
515
|
+
explicitCpus: opts.cpus !== undefined,
|
|
516
|
+
explicitMemory: opts.memory !== undefined,
|
|
517
|
+
network: opts.network ?? (strict ? 'none' : 'bridge'),
|
|
518
|
+
workdir,
|
|
519
|
+
timeout: opts.timeout ?? DEFAULT_TIMEOUT_MS / 1000,
|
|
520
|
+
hardening,
|
|
521
|
+
user: opts.user,
|
|
522
|
+
persist: opts.persist ?? false,
|
|
523
|
+
env: opts.env ?? {},
|
|
524
|
+
secrets: opts.secrets ?? [],
|
|
525
|
+
readOnly: opts.readOnly ?? false,
|
|
526
|
+
mounts: opts.mounts ?? [],
|
|
527
|
+
execEnv: opts.execEnv ?? {},
|
|
528
|
+
execPaths: opts.execPaths ?? {},
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
/**
|
|
532
|
+
* The container's name is a function of its configuration. Change the image
|
|
533
|
+
* and a new container appears rather than an old one quietly persisting with
|
|
534
|
+
* the wrong rootfs — which is the failure mode `persist` would otherwise have.
|
|
535
|
+
*
|
|
536
|
+
* Secrets contribute their names and not their values: what a container is
|
|
537
|
+
* depends on which credentials reach it, not on what they happen to say today,
|
|
538
|
+
* and hashing the values would throw away a persisted container's filesystem
|
|
539
|
+
* every time a key was rotated.
|
|
540
|
+
*/
|
|
541
|
+
function containerName(spec) {
|
|
542
|
+
// Unconfigured cpus and memory hash 0 under standard and strict's own
|
|
543
|
+
// defaults under strict, so introducing defaults did not rename every
|
|
544
|
+
// existing container.
|
|
545
|
+
const strict = spec.hardening === 'strict' ? sandboxLimits({ hardening: 'strict' }) : undefined;
|
|
546
|
+
const hashCpus = spec.explicitCpus ? spec.cpus : (strict?.cpus ?? 0);
|
|
547
|
+
const hashMemory = spec.explicitMemory ? spec.memory : (strict?.memory ?? 0);
|
|
548
|
+
const digest = createHash('sha256')
|
|
549
|
+
.update(JSON.stringify([
|
|
550
|
+
spec.root,
|
|
551
|
+
spec.image,
|
|
552
|
+
hashCpus,
|
|
553
|
+
hashMemory,
|
|
554
|
+
spec.network,
|
|
555
|
+
spec.workdir,
|
|
556
|
+
spec.user ?? null,
|
|
557
|
+
spec.readOnly,
|
|
558
|
+
spec.mounts,
|
|
559
|
+
Object.entries(spec.env).sort(),
|
|
560
|
+
[...spec.secrets].sort(),
|
|
561
|
+
// Appended only when it is set, so adding the profile did not
|
|
562
|
+
// rename every container that existed before it.
|
|
563
|
+
...(spec.hardening === 'strict' ? ['strict'] : []),
|
|
564
|
+
]))
|
|
565
|
+
.digest('hex')
|
|
566
|
+
.slice(0, 10);
|
|
567
|
+
const key = spec.key
|
|
568
|
+
.toLowerCase()
|
|
569
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
570
|
+
.replace(/^-|-$/g, '');
|
|
571
|
+
return `zn-${key || 'session'}-${digest}`;
|
|
572
|
+
}
|
|
573
|
+
/** Podman on this machine: what `zen run` uses, and the engine's default. */
|
|
574
|
+
export const podmanSandbox = (opts) => new PodmanSandbox(opts);
|
|
575
|
+
//# sourceMappingURL=podman.js.map
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
/** Nothing runs longer than this, whatever it asked for. */
|
|
2
|
+
export declare const MAX_TIMEOUT_MS = 3600000;
|
|
3
|
+
export interface ProcOptions {
|
|
4
|
+
/** written to stdin, which is then closed */
|
|
5
|
+
input?: string;
|
|
6
|
+
timeoutMs?: number;
|
|
7
|
+
signal?: AbortSignal;
|
|
8
|
+
/** per stream */
|
|
9
|
+
maxBytes?: number;
|
|
10
|
+
}
|
|
11
|
+
export interface ProcResult {
|
|
12
|
+
code: number;
|
|
13
|
+
stdout: string;
|
|
14
|
+
stderr: string;
|
|
15
|
+
truncated: boolean;
|
|
16
|
+
timedOut: boolean;
|
|
17
|
+
}
|
|
18
|
+
export type Runner = (bin: string, args: readonly string[], opts?: ProcOptions) => Promise<ProcResult>;
|
|
19
|
+
export declare const runProcess: Runner;
|
|
20
|
+
/**
|
|
21
|
+
* `run`, retried while the engine could not be reached at all.
|
|
22
|
+
*
|
|
23
|
+
* Empty stdout is the guard that keeps this from retrying real work: a command
|
|
24
|
+
* that produced output ran, whatever its stderr says afterwards. The wait is
|
|
25
|
+
* jittered because the burst being backed off from is N clients arriving
|
|
26
|
+
* together, and a fixed delay would only reassemble it.
|
|
27
|
+
*/
|
|
28
|
+
export declare function resilient(run: Runner, attempts?: number): Runner;
|
|
29
|
+
//# sourceMappingURL=process.d.ts.map
|