@justjoshn/pi-pstack-v2 0.1.7 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -7
- package/docs/guide/01-setup.md +3 -3
- package/docs/guide/02-poteto-mode.md +1 -1
- package/docs/guide/07-overnight.md +3 -3
- package/docs/pi-mechanisms.md +20 -38
- package/package.json +5 -3
- package/skills/benchmark-checklist/SKILL.md +1 -1
- package/skills/control-cli/SKILL.md +1 -1
- package/skills/make-bot-ui/SKILL.md +2 -2
- package/skills/poteto-mode/SKILL.md +2 -2
- package/skills/poteto-mode/playbooks/autopilot-full.md +2 -2
- package/skills/poteto-mode/playbooks/babysit.md +1 -1
- package/skills/poteto-mode/playbooks/shipping.md +2 -2
- package/skills/statechart/SKILL.md +1 -1
- package/skills/statechart/references/render.md +1 -1
- package/src/gondolin/bash.ts +51 -0
- package/src/gondolin/fs-operations.ts +56 -0
- package/src/gondolin/grep.ts +96 -0
- package/src/gondolin/index.ts +51 -0
- package/src/gondolin/paths.ts +34 -0
- package/src/gondolin/search.ts +67 -0
- package/src/gondolin/tools.ts +40 -0
- package/src/gondolin/vm.ts +62 -0
- package/src/index.ts +31 -130
- package/src/jobs.ts +58 -315
- package/src/credentials.ts +0 -171
- package/src/git.ts +0 -94
- package/src/paths.ts +0 -16
- package/src/requests.ts +0 -182
- package/src/sandbox.ts +0 -284
- package/src/sweep.ts +0 -187
- package/src/transcript.ts +0 -79
package/src/jobs.ts
CHANGED
|
@@ -1,40 +1,17 @@
|
|
|
1
1
|
import { type SpawnSyncReturns, spawnSync } from 'node:child_process';
|
|
2
2
|
import { createHash, randomBytes } from 'node:crypto';
|
|
3
|
-
import { closeSync,
|
|
4
|
-
import { createRequire } from 'node:module';
|
|
3
|
+
import { closeSync, existsSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, statSync, writeFileSync } from 'node:fs';
|
|
5
4
|
import { basename, dirname, join } from 'node:path';
|
|
6
|
-
import { fileURLToPath } from 'node:url';
|
|
7
5
|
|
|
8
|
-
import { jobAgentDir } from './credentials.ts';
|
|
9
6
|
import { hostEnv } from './host-env.ts';
|
|
10
7
|
import { passthroughObject } from './object-schema.ts';
|
|
11
|
-
import { JOB_DIR_ENV, SANDBOX_ENV, type SandboxPolicy, TOP_DIR_ENV } from './sandbox.ts';
|
|
12
|
-
import { type Inventory, inventory, inventoryFile } from './sweep.ts';
|
|
13
|
-
|
|
14
|
-
const sweepSource = fileURLToPath(new URL('./sweep.ts', import.meta.url));
|
|
15
|
-
|
|
16
|
-
export type JobKind = 'pi' | 'command';
|
|
17
8
|
|
|
18
9
|
export interface JobSpec {
|
|
19
10
|
id: string;
|
|
20
11
|
name: string;
|
|
21
|
-
kind: JobKind;
|
|
22
12
|
cwd: string;
|
|
23
|
-
/**
|
|
24
|
-
argv: string[];
|
|
25
|
-
/** The command before sandboxing, and the policy it asked for. A broker launches from these after clamping the policy. */
|
|
13
|
+
/** The argv tmux runs, such as `bash -c <command>`. */
|
|
26
14
|
command: string[];
|
|
27
|
-
policy: SandboxPolicy;
|
|
28
|
-
/**
|
|
29
|
-
* Launched by the root Pi on a sandboxed worker's behalf, because macOS cannot nest Seatbelt profiles. The worker cannot
|
|
30
|
-
* reach the root's tmux server, so it reads the job's state from files only.
|
|
31
|
-
*/
|
|
32
|
-
brokered: boolean;
|
|
33
|
-
/**
|
|
34
|
-
* Where the job's output, exit code, and launch markers live. It is the job directory itself, except for a brokered job,
|
|
35
|
-
* whose results the root writes under the top-level job, where no worker can plant a link for the root to write through.
|
|
36
|
-
*/
|
|
37
|
-
out: string;
|
|
38
15
|
/** Session id of the Pi process that owns the job and receives its wake message. */
|
|
39
16
|
owner: string;
|
|
40
17
|
/** Message delivered to the owner when the job exits. Null means the owner is not woken. */
|
|
@@ -43,17 +20,12 @@ export interface JobSpec {
|
|
|
43
20
|
repeat: boolean;
|
|
44
21
|
/** Id of the first job in a repeat series. Stopping any job in the series stops the series. */
|
|
45
22
|
series: string;
|
|
46
|
-
/** Path of the tmux server socket the job runs under.
|
|
23
|
+
/** Path of the tmux server socket the job runs under. */
|
|
47
24
|
socket: string;
|
|
48
|
-
/** Protected paths and checkout roots the wrapper sweeps after the job, for code that appeared while it ran. */
|
|
49
|
-
sweep: { roots: string[]; protect: string[] };
|
|
50
|
-
/** The $TMPDIR of the worker that started this job, shared so paths it hands the job exist. Unset for a top-level job. */
|
|
51
|
-
tmp?: string;
|
|
52
|
-
createWorktree?: { branch: string; base?: string };
|
|
53
25
|
createdAt: string;
|
|
54
26
|
}
|
|
55
27
|
|
|
56
|
-
/** A job.json is read as written: older and newer versions of it
|
|
28
|
+
/** A job.json is read as written: older and newer versions of it differ in which keys they hold. */
|
|
57
29
|
const JobSpecSchema = passthroughObject<JobSpec>();
|
|
58
30
|
|
|
59
31
|
export type JobStatus = { state: 'running' } | { state: 'exited'; code: number } | { state: 'lost' };
|
|
@@ -65,7 +37,6 @@ export interface Job {
|
|
|
65
37
|
}
|
|
66
38
|
|
|
67
39
|
export const STOPPED_EXIT_CODE = 143;
|
|
68
|
-
const LAUNCH_FAILED_EXIT_CODE = 127;
|
|
69
40
|
const HASH_LENGTH = 16;
|
|
70
41
|
const JOB_ID_SUFFIX_BYTES = 3;
|
|
71
42
|
const STOP_GRACE_MS = 6000;
|
|
@@ -74,31 +45,13 @@ const STOP_POLL_MS = 50;
|
|
|
74
45
|
const TAIL_WINDOW_KIBIBYTES = 256;
|
|
75
46
|
const BYTES_PER_KIBIBYTE = 1024;
|
|
76
47
|
const TAIL_WINDOW_BYTES = TAIL_WINDOW_KIBIBYTES * BYTES_PER_KIBIBYTE;
|
|
77
|
-
const UTF8_CONTINUATION_MASK = 0xc0;
|
|
78
|
-
const UTF8_CONTINUATION_TAG = 0x80;
|
|
79
|
-
const UTF8_LEAD_4 = 0xf0;
|
|
80
|
-
const UTF8_LEAD_3 = 0xe0;
|
|
81
|
-
const UTF8_LEAD_2 = 0xc0;
|
|
82
|
-
const FOUR_BYTES = 4;
|
|
83
|
-
const THREE_BYTES = 3;
|
|
84
|
-
const TWO_BYTES = 2;
|
|
85
|
-
// A UTF-8 lead byte at or above each threshold starts a sequence of that many bytes.
|
|
86
|
-
const UTF8_LEAD_WIDTHS: readonly (readonly [number, number])[] = [
|
|
87
|
-
[UTF8_LEAD_4, FOUR_BYTES],
|
|
88
|
-
[UTF8_LEAD_3, THREE_BYTES],
|
|
89
|
-
[UTF8_LEAD_2, TWO_BYTES],
|
|
90
|
-
];
|
|
91
48
|
|
|
92
49
|
const shellQuote = (word: string): string => `'${word.replaceAll("'", "'\\''")}'`;
|
|
93
50
|
const hash = (text: string): string => createHash('sha256').update(text).digest('hex').slice(0, HASH_LENGTH);
|
|
94
51
|
|
|
95
52
|
export const jobDir = (root: string, id: string): string => join(root, id);
|
|
96
|
-
export const outputPath = (
|
|
97
|
-
export const exitPath = (
|
|
98
|
-
/** Where the root writes a brokered job's results: under the top-level job, which every descendant can read and none can write. */
|
|
99
|
-
export const resultsDir = (top: string, dir: string): string => join(top, 'brokered', hash(dir));
|
|
100
|
-
/** The tmux session a brokered job runs as on the root's server, named from its directory so a worker cannot pick another job's name. */
|
|
101
|
-
export const brokeredSession = (dir: string): string => `brokered-${hash(dir)}`;
|
|
53
|
+
export const outputPath = (dir: string): string => join(dir, 'output.log');
|
|
54
|
+
export const exitPath = (dir: string): string => join(dir, 'exit');
|
|
102
55
|
|
|
103
56
|
export function newJobId(name: string): string {
|
|
104
57
|
const slug = name
|
|
@@ -109,10 +62,7 @@ export function newJobId(name: string): string {
|
|
|
109
62
|
return `${slug || 'job'}-${randomBytes(JOB_ID_SUFFIX_BYTES).toString('hex')}`;
|
|
110
63
|
}
|
|
111
64
|
|
|
112
|
-
/**
|
|
113
|
-
* One tmux server per owning session, started with that session's environment so jobs inherit PATH and credentials.
|
|
114
|
-
* Its socket sits where sandboxed jobs cannot connect, since the server runs whatever it is sent outside the sandbox.
|
|
115
|
-
*/
|
|
65
|
+
/** One tmux server per owning session, started with that session's environment so jobs inherit PATH and credentials. */
|
|
116
66
|
export function tmuxSocket(owner: string): string {
|
|
117
67
|
const uid = process.getuid?.() ?? 0;
|
|
118
68
|
const dir = `/tmp/pstack-v2-${uid}`;
|
|
@@ -140,131 +90,27 @@ function tmux(socket: string, args: string[]): void {
|
|
|
140
90
|
if (result.status !== 0) throw new Error(`Command failed: tmux -S ${basename(socket)} ${args.join(' ')}`);
|
|
141
91
|
}
|
|
142
92
|
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
/** Package caches in a job's temp directory, deleted when it exits. One job's installs can fill hundreds of megabytes. */
|
|
150
|
-
const CACHES: Record<string, string> = {
|
|
151
|
-
XDG_CACHE_HOME: 'cache',
|
|
152
|
-
npm_config_cache: 'npm',
|
|
153
|
-
BUN_INSTALL_CACHE_DIR: 'bun',
|
|
154
|
-
PIP_CACHE_DIR: 'pip',
|
|
155
|
-
UV_CACHE_DIR: 'uv',
|
|
156
|
-
GOCACHE: 'go-build',
|
|
157
|
-
GOMODCACHE: 'go-mod',
|
|
158
|
-
YARN_CACHE_FOLDER: 'yarn',
|
|
159
|
-
DENO_DIR: 'deno',
|
|
160
|
-
GRADLE_USER_HOME: 'gradle',
|
|
161
|
-
npm_config_store_dir: 'pnpm',
|
|
162
|
-
};
|
|
163
|
-
|
|
164
|
-
// zsh writes here-documents under TMPPREFIX, which defaults to the shared /tmp.
|
|
165
|
-
const tempEnv = (tmp: string): Record<string, string> => ({ TMPDIR: tmp, TMP: tmp, TEMP: tmp, MAC_CHROMIUM_TMPDIR: tmp, TMPPREFIX: join(tmp, 'zsh') });
|
|
166
|
-
|
|
167
|
-
/** macOS mktemp prefers the per-user temp directory to $TMPDIR, and a job may not write there, so jobs find this one first on PATH. */
|
|
168
|
-
const MKTEMP = `#!/bin/sh
|
|
169
|
-
flags=
|
|
170
|
-
prefix=
|
|
171
|
-
while getopts :dqut:p: opt; do
|
|
172
|
-
case $opt in
|
|
173
|
-
d|q|u) flags="$flags -$opt" ;;
|
|
174
|
-
t) prefix=$OPTARG ;;
|
|
175
|
-
*) exec /usr/bin/mktemp "$@" ;;
|
|
176
|
-
esac
|
|
177
|
-
done
|
|
178
|
-
shift $((OPTIND - 1))
|
|
179
|
-
[ $# -gt 0 ] && [ -z "$prefix" ] && exec /usr/bin/mktemp $flags "$@"
|
|
180
|
-
exec /usr/bin/mktemp $flags "\${TMPDIR%/}/\${prefix:-tmp}.XXXXXXXXXX"
|
|
181
|
-
`;
|
|
182
|
-
|
|
183
|
-
/**
|
|
184
|
-
* A private temp directory and caches for the job. Shared temp directories and user caches hold code that unsandboxed
|
|
185
|
-
* programs later load, such as Pi's compiled-extension cache, so a job may write none of them.
|
|
186
|
-
*/
|
|
187
|
-
export const privateEnv = (tmp: string): Record<string, string> => ({
|
|
188
|
-
...tempEnv(tmp),
|
|
189
|
-
...Object.fromEntries(Object.entries(CACHES).map(([name, dir]) => [name, join(tmp, dir)])),
|
|
190
|
-
});
|
|
191
|
-
|
|
192
|
-
/**
|
|
193
|
-
* Firefox and pnpm run their own sandbox or store-locking steps, which fail inside the job's sandbox. Firefox aborts when it
|
|
194
|
-
* cannot sandbox its child processes, and pnpm cannot lock the shared store it switches engines from.
|
|
195
|
-
*/
|
|
196
|
-
const SANDBOX_COMPAT: Record<string, string> = {
|
|
197
|
-
...Object.fromEntries(['CONTENT', 'GMP', 'RDD', 'SOCKET_PROCESS', 'GPU', 'UTILITY'].map((name) => [`MOZ_DISABLE_${name}_SANDBOX`, '1'])),
|
|
198
|
-
npm_config_manage_package_manager_versions: 'false',
|
|
199
|
-
};
|
|
200
|
-
|
|
201
|
-
interface Run {
|
|
202
|
-
kind: JobKind;
|
|
203
|
-
cwd: string;
|
|
204
|
-
argv: string[];
|
|
205
|
-
inventory: Inventory;
|
|
206
|
-
socket: string;
|
|
207
|
-
session: string;
|
|
208
|
-
/** The job's own directory, exported to it as its job directory. */
|
|
209
|
-
dir: string;
|
|
210
|
-
/** Where output and the exit code go. */
|
|
211
|
-
out: string;
|
|
212
|
-
top: string;
|
|
213
|
-
tmp: string | undefined;
|
|
214
|
-
}
|
|
215
|
-
|
|
216
|
-
const requireFromHere = createRequire(import.meta.url);
|
|
217
|
-
|
|
218
|
-
/** The copied sweep script imports zod, which resolves from a node_modules beside it, so one is linked there. */
|
|
219
|
-
function linkZod(out: string): void {
|
|
220
|
-
const modules = join(out, 'node_modules');
|
|
221
|
-
mkdirSync(modules, { recursive: true });
|
|
222
|
-
symlinkSync(dirname(requireFromHere.resolve('zod/package.json')), join(modules, 'zod'), 'dir');
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
/** The assignments that put a job in its own Pi agent directory, sandbox markers, and private temp directories. */
|
|
226
|
-
function jobEnv(job: Run): string {
|
|
227
|
-
// Pi's own variable points every job at its top-level job's agent directory, so no job's Pi writes your real credentials.
|
|
228
|
-
return Object.entries({
|
|
229
|
-
[SANDBOX_ENV]: '1',
|
|
230
|
-
[JOB_DIR_ENV]: job.dir,
|
|
231
|
-
[TOP_DIR_ENV]: job.top,
|
|
232
|
-
PI_CODING_AGENT_DIR: jobAgentDir(job.top),
|
|
233
|
-
...SANDBOX_COMPAT,
|
|
234
|
-
...privateEnv(join(job.dir, 'tmp')),
|
|
235
|
-
...(job.tmp ? tempEnv(job.tmp) : {}),
|
|
236
|
-
})
|
|
237
|
-
.map(([name, value]) => `${name}=${shellQuote(value)}`)
|
|
238
|
-
.join(' ');
|
|
239
|
-
}
|
|
240
|
-
|
|
241
|
-
/** The shell script that runs the job in its own process group, mirrors its output, and cleans up on every exit path. */
|
|
242
|
-
function wrapperScript(job: Run, body: string, sweepCopy: string): string {
|
|
243
|
-
const out = shellQuote(outputPath(job.out));
|
|
244
|
-
const exit = shellQuote(exitPath(job.out));
|
|
245
|
-
const caches = Object.values(CACHES)
|
|
246
|
-
.map((dir) => shellQuote(join(job.dir, 'tmp', dir)))
|
|
247
|
-
.join(' ');
|
|
93
|
+
/** The shell script that runs the job in its own process group, mirrors its output, and records its exit on every exit path. */
|
|
94
|
+
function wrapperScript(dir: string, command: string[]): string {
|
|
95
|
+
const out = shellQuote(outputPath(dir));
|
|
96
|
+
const exit = shellQuote(exitPath(dir));
|
|
248
97
|
return [
|
|
249
98
|
'set -m',
|
|
250
|
-
`echo $$ > ${shellQuote(join(
|
|
99
|
+
`echo $$ > ${shellQuote(join(dir, 'wrapper.pid'))}`,
|
|
251
100
|
'finished=0',
|
|
252
101
|
'finish() {',
|
|
253
102
|
' [ "$finished" = 1 ] && return',
|
|
254
103
|
' finished=1',
|
|
255
104
|
' kill -KILL -- -$pid 2>/dev/null; wait $pid 2>/dev/null',
|
|
256
105
|
' kill $mirror 2>/dev/null',
|
|
257
|
-
// Off the terminal: once tmux is gone, Node reopening the terminal to report an error blocks forever and holds a pty.
|
|
258
|
-
` ${shellQuote(process.execPath)} ${shellQuote(sweepCopy)} ${shellQuote(inventoryFile(job.out))} < /dev/null >> ${out} 2>&1`,
|
|
259
|
-
// Go leaves its module cache read-only. find never follows a link the job planted, so only the job's own directories change.
|
|
260
|
-
` find ${caches} -type d ! -perm -u+w -exec chmod u+w {} + 2>/dev/null; rm -rf ${caches}`,
|
|
261
106
|
` echo "$1" > ${exit}.tmp && mv ${exit}.tmp ${exit}`,
|
|
262
107
|
'}',
|
|
263
108
|
`trap 'kill -TERM -- -$pid 2>/dev/null; for i in $(seq 30); do kill -0 -- -$pid 2>/dev/null || break; sleep 0.1; done; finish ${STOPPED_EXIT_CODE}; exit ${STOPPED_EXIT_CODE}' HUP TERM INT`,
|
|
264
|
-
|
|
109
|
+
// Output goes straight to its file, not through a pipe, so a child the job leaves running cannot hold the job open.
|
|
110
|
+
`${command.map(shellQuote).join(' ')} < /dev/null > ${out} 2>&1 &`,
|
|
265
111
|
'pid=$!',
|
|
266
112
|
// The session's pane shows the output as it is written, so attaching to it is a live view of the job.
|
|
267
|
-
`tail -n +1 -f ${
|
|
113
|
+
`tail -n +1 -f ${out} 2>/dev/null &`,
|
|
268
114
|
'mirror=$!',
|
|
269
115
|
'wait $pid',
|
|
270
116
|
'finish "$?"',
|
|
@@ -272,91 +118,29 @@ function wrapperScript(job: Run, body: string, sweepCopy: string): string {
|
|
|
272
118
|
}
|
|
273
119
|
|
|
274
120
|
/**
|
|
275
|
-
*
|
|
276
|
-
*
|
|
277
|
-
*
|
|
278
|
-
*
|
|
279
|
-
* gets KILL after a short grace period.
|
|
121
|
+
* Creates the job directory, writes its spec and wrapper, and starts it in a detached tmux session. The job runs in its
|
|
122
|
+
* own process group while the wrapper waits on it, so when `tmux_jobs stop` hangs the session up, the trap fires at once.
|
|
123
|
+
* Bash would defer a trap until a foreground command finished. On every exit path the whole group is killed, and a job
|
|
124
|
+
* that ignores TERM gets KILL after a short grace period.
|
|
280
125
|
*/
|
|
281
|
-
function
|
|
282
|
-
const env = jobEnv(job);
|
|
283
|
-
// The job cannot write its results directory, so it cannot replace what it finds there on PATH.
|
|
284
|
-
const bin = join(job.out, 'bin');
|
|
285
|
-
if (process.platform === 'darwin') {
|
|
286
|
-
mkdirSync(bin, { recursive: true });
|
|
287
|
-
writeFileSync(join(bin, 'mktemp'), MKTEMP, { mode: 0o755 });
|
|
288
|
-
}
|
|
289
|
-
const path = process.platform === 'darwin' ? `PATH=${shellQuote(bin)}:"$PATH" ` : '';
|
|
290
|
-
const command = `${path}${env} ${job.argv.map(shellQuote).join(' ')} < /dev/null`;
|
|
291
|
-
const out = shellQuote(outputPath(job.out));
|
|
292
|
-
// Output goes straight to its file, not through a pipe, so a child the job leaves running cannot hold the job open.
|
|
293
|
-
const body = job.kind === 'pi' ? `${command} > ${out} 2> ${shellQuote(join(job.out, 'stderr.log'))}` : `${command} > ${out} 2>&1`;
|
|
294
|
-
// Node refuses to strip types from a file under node_modules, where an npm install keeps this one. The copy sits in the
|
|
295
|
-
// job's results directory, which the job cannot write.
|
|
296
|
-
const sweepCopy = join(job.out, 'sweep.ts');
|
|
297
|
-
const script = wrapperScript(job, body, sweepCopy);
|
|
298
|
-
// The script lives beside the job's results, where only the process that launched it can write, and runs from there,
|
|
299
|
-
// since a long list of protected paths would not fit on a tmux command line.
|
|
300
|
-
const file = join(job.out, 'wrapper.sh');
|
|
301
|
-
writeFileSync(inventoryFile(job.out), JSON.stringify(job.inventory));
|
|
302
|
-
copyFileSync(sweepSource, sweepCopy);
|
|
303
|
-
linkZod(job.out);
|
|
304
|
-
writeFileSync(file, `${script}\n`);
|
|
305
|
-
tmux(job.socket, ['new-session', '-d', '-s', job.session, '-c', job.cwd, 'bash', file]);
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
const takeInventory = (spec: JobSpec, out: string): Inventory => inventory(spec.sweep.roots, spec.sweep.protect, join(out, 'quarantine'), outputPath(out));
|
|
309
|
-
|
|
310
|
-
const inheritedTop = (dir: string): string => (hostEnv()[SANDBOX_ENV] === '1' ? (hostEnv()[TOP_DIR_ENV] ?? dir) : dir);
|
|
311
|
-
|
|
312
|
-
/** Creates the job directory, writes its spec, and starts it in a detached tmux session. */
|
|
313
|
-
export function launch(root: string, spec: JobSpec, files: Record<string, string> = {}): Job {
|
|
126
|
+
export function launch(root: string, spec: JobSpec): Job {
|
|
314
127
|
const dir = jobDir(root, spec.id);
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
export function request(root: string, spec: JobSpec, files: Record<string, string> = {}): Job {
|
|
323
|
-
const dir = jobDir(root, spec.id);
|
|
324
|
-
const top = hostEnv()[TOP_DIR_ENV];
|
|
325
|
-
if (!top) throw new Error('A sandboxed worker needs its top-level job directory to request a job.');
|
|
326
|
-
const full = { ...spec, argv: [], socket: '', brokered: true, out: resultsDir(top, dir) };
|
|
327
|
-
write(dir, full, files);
|
|
328
|
-
return { spec: full, dir, status: { state: 'running' } };
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
/** Records a brokered job's result in its root-owned results directory, for a launch the root refused or could not start. */
|
|
332
|
-
export function settleBrokered(out: string, code: number, message: string): void {
|
|
333
|
-
mkdirSync(out, { recursive: true });
|
|
334
|
-
writeFileSync(outputPath(out), `${message}\n`);
|
|
335
|
-
writeFileSync(join(out, 'launched'), '');
|
|
336
|
-
writeFileSync(exitPath(out), `${code}\n`);
|
|
337
|
-
}
|
|
338
|
-
|
|
339
|
-
/** Launches a brokered request on the root's tmux server. The caller has already sandboxed argv under a policy it clamped. */
|
|
340
|
-
export function launchBrokered(top: string, dir: string, spec: JobSpec, sweepWhat: JobSpec['sweep'], argv: string[], socket: string): boolean {
|
|
341
|
-
const out = resultsDir(top, dir);
|
|
342
|
-
mkdirSync(out, { recursive: true });
|
|
343
|
-
try {
|
|
344
|
-
run({ kind: spec.kind, cwd: spec.cwd, argv, inventory: takeInventory({ ...spec, sweep: sweepWhat }, out), socket, session: brokeredSession(dir), dir, out, top, tmp: spec.tmp });
|
|
345
|
-
writeFileSync(join(out, 'launched'), '');
|
|
346
|
-
return true;
|
|
347
|
-
} catch (error) {
|
|
348
|
-
settleBrokered(out, LAUNCH_FAILED_EXIT_CODE, `pstack could not launch this job: ${error instanceof Error ? error.message : String(error)}`);
|
|
349
|
-
return false;
|
|
350
|
-
}
|
|
128
|
+
mkdirSync(dir, { recursive: true });
|
|
129
|
+
writeFileSync(join(dir, 'job.json'), `${JSON.stringify(spec, null, 2)}\n`);
|
|
130
|
+
// The script runs from a file, since a long command would not fit on a tmux command line.
|
|
131
|
+
const file = join(dir, 'wrapper.sh');
|
|
132
|
+
writeFileSync(file, `${wrapperScript(dir, spec.command)}\n`);
|
|
133
|
+
tmux(spec.socket, ['new-session', '-d', '-s', spec.id, '-c', spec.cwd, 'bash', file]);
|
|
134
|
+
return { spec, dir, status: { state: 'running' } };
|
|
351
135
|
}
|
|
352
136
|
|
|
353
137
|
export function status(dir: string, spec: JobSpec): JobStatus {
|
|
354
|
-
const exit = exitPath(
|
|
138
|
+
const exit = exitPath(dir);
|
|
355
139
|
const exited = (): JobStatus | undefined => (existsSync(exit) ? { state: 'exited', code: Number(readFileSync(exit, 'utf8').trim()) } : undefined);
|
|
356
140
|
const before = exited();
|
|
357
141
|
if (before) return before;
|
|
358
|
-
|
|
359
|
-
if (tmuxOk(spec.socket, ['has-session', '-t', `=${spec.id}`])) return { state: 'running' };
|
|
142
|
+
// A job.json an older version wrote may name no socket, and no tmux server can still run it.
|
|
143
|
+
if (spec.socket && tmuxOk(spec.socket, ['has-session', '-t', `=${spec.id}`])) return { state: 'running' };
|
|
360
144
|
// The job may have written its exit code and closed its session between the two checks.
|
|
361
145
|
return exited() ?? { state: 'lost' };
|
|
362
146
|
}
|
|
@@ -384,37 +168,30 @@ export function list(root: string, owner: string): Job[] {
|
|
|
384
168
|
.toSorted((a, b) => a.spec.createdAt.localeCompare(b.spec.createdAt));
|
|
385
169
|
}
|
|
386
170
|
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
171
|
+
const wrapperAlive = (pid: number): boolean => {
|
|
172
|
+
try {
|
|
173
|
+
process.kill(pid, 0);
|
|
174
|
+
return pid > 0;
|
|
175
|
+
} catch {
|
|
176
|
+
return false;
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
/** Ends a job. Hanging up its tmux session runs the wrapper's cleanup, which ends every process in the job's group. */
|
|
391
181
|
export async function stop(job: Job): Promise<void> {
|
|
392
182
|
// Claimed first, since the session is gone before the wrapper records the exit, and the watcher would report the job lost.
|
|
393
183
|
claimWake(job);
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
try {
|
|
406
|
-
process.kill(wrapper, 0);
|
|
407
|
-
return wrapper > 0;
|
|
408
|
-
} catch {
|
|
409
|
-
return false;
|
|
410
|
-
}
|
|
411
|
-
};
|
|
412
|
-
// The wrapper keeps cleaning up after its session is gone, so waiting ends once it records the exit or is itself gone.
|
|
413
|
-
while (!existsSync(exit) && Date.now() < deadline && alive()) await new Promise((done) => setTimeout(done, STOP_POLL_MS));
|
|
414
|
-
if (!existsSync(exit)) {
|
|
415
|
-
writeFileSync(`${exit}.tmp`, `${STOPPED_EXIT_CODE}\n`);
|
|
416
|
-
renameSync(`${exit}.tmp`, exit);
|
|
417
|
-
}
|
|
184
|
+
// A session that is already gone makes this fail, which is expected.
|
|
185
|
+
tmuxOk(job.spec.socket, ['kill-session', '-t', `=${job.spec.id}`]);
|
|
186
|
+
// The wrapper records the exit once it has ended the job. Only a wrapper that is already gone, and so can never record
|
|
187
|
+
// it, leaves that to stop.
|
|
188
|
+
const exit = exitPath(job.dir);
|
|
189
|
+
const deadline = Date.now() + STOP_GRACE_MS;
|
|
190
|
+
const wrapper = Number(readEnd(join(job.dir, 'wrapper.pid'), PID_FILE_BYTES).trim());
|
|
191
|
+
while (!existsSync(exit) && Date.now() < deadline && wrapperAlive(wrapper)) await new Promise((done) => setTimeout(done, STOP_POLL_MS));
|
|
192
|
+
if (!existsSync(exit)) {
|
|
193
|
+
writeFileSync(`${exit}.tmp`, `${STOPPED_EXIT_CODE}\n`);
|
|
194
|
+
renameSync(`${exit}.tmp`, exit);
|
|
418
195
|
}
|
|
419
196
|
}
|
|
420
197
|
|
|
@@ -428,53 +205,19 @@ export function claimWake(job: Job): boolean {
|
|
|
428
205
|
}
|
|
429
206
|
}
|
|
430
207
|
|
|
431
|
-
|
|
208
|
+
/** The end of a file, read without loading the whole of it, since a job's output can be very large. */
|
|
209
|
+
export function readEnd(path: string, bytes: number): string {
|
|
210
|
+
if (!existsSync(path)) return '';
|
|
211
|
+
const size = statSync(path).size;
|
|
212
|
+
const length = Math.min(size, bytes);
|
|
432
213
|
const buffer = Buffer.alloc(length);
|
|
433
214
|
const fd = openSync(path, 'r');
|
|
434
215
|
try {
|
|
435
|
-
readSync(fd, buffer, 0, length,
|
|
216
|
+
readSync(fd, buffer, 0, length, size - length);
|
|
436
217
|
} finally {
|
|
437
218
|
closeSync(fd);
|
|
438
219
|
}
|
|
439
|
-
return buffer;
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
/** The end of a file, read without loading the whole of it, since a job's output can be very large. */
|
|
443
|
-
export function readEnd(path: string, bytes: number): string {
|
|
444
|
-
if (!existsSync(path)) return '';
|
|
445
|
-
const size = statSync(path).size;
|
|
446
|
-
const length = Math.min(size, bytes);
|
|
447
|
-
return readRange(path, size - length, length).toString('utf8');
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
const isContinuation = (byte: number | undefined): boolean => byte !== undefined && (byte & UTF8_CONTINUATION_MASK) === UTF8_CONTINUATION_TAG;
|
|
451
|
-
|
|
452
|
-
/**
|
|
453
|
-
* The start and the end of a file with a note of how many bytes lie between, so a verdict on its first line survives a
|
|
454
|
-
* long output. Both cuts fall on character boundaries.
|
|
455
|
-
*/
|
|
456
|
-
export function excerpt(path: string, head: number, tailBytes: number): string {
|
|
457
|
-
if (!existsSync(path)) return '';
|
|
458
|
-
const size = statSync(path).size;
|
|
459
|
-
if (size <= head + tailBytes) return readEnd(path, size);
|
|
460
|
-
return clipRanges(readRange(path, 0, head), readRange(path, size - tailBytes, tailBytes), size, path);
|
|
461
|
-
}
|
|
462
|
-
|
|
463
|
-
export function clip(text: string, head: number, tailBytes: number, source: string): string {
|
|
464
|
-
const buffer = Buffer.from(text);
|
|
465
|
-
if (buffer.length <= head + tailBytes) return text;
|
|
466
|
-
return clipRanges(buffer.subarray(0, head), buffer.subarray(buffer.length - tailBytes), buffer.length, source);
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
function clipRanges(first: Buffer, last: Buffer, size: number, path: string): string {
|
|
470
|
-
let lead = first.length - 1;
|
|
471
|
-
while (lead > 0 && isContinuation(first[lead])) lead--;
|
|
472
|
-
const byte = first[lead] ?? 0;
|
|
473
|
-
const end = lead + (UTF8_LEAD_WIDTHS.find(([threshold]) => byte >= threshold)?.[1] ?? 1) > first.length ? lead : first.length;
|
|
474
|
-
let from = 0;
|
|
475
|
-
while (isContinuation(last[from])) from++;
|
|
476
|
-
const omitted = size - end - (last.length - from);
|
|
477
|
-
return `${first.subarray(0, end).toString('utf8')}\n[... ${omitted} bytes omitted, full output at ${path} ...]\n${last.subarray(from).toString('utf8')}`;
|
|
220
|
+
return buffer.toString('utf8');
|
|
478
221
|
}
|
|
479
222
|
|
|
480
223
|
export function tail(path: string, lines: number): string {
|
package/src/credentials.ts
DELETED
|
@@ -1,171 +0,0 @@
|
|
|
1
|
-
import { copyFileSync, existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, realpathSync, renameSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
|
2
|
-
import { isAbsolute, join, resolve } from 'node:path';
|
|
3
|
-
|
|
4
|
-
import { z } from 'zod';
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Jobs never write your real `auth.json`. Pi runs a credential whose key starts with `!` as a shell command, so a job able
|
|
8
|
-
* to write that file could plant a command for your next unsandboxed Pi to run. Instead each top-level job, and every job
|
|
9
|
-
* it starts, uses its own Pi agent directory through Pi's `PI_CODING_AGENT_DIR`. It links to your real configuration and
|
|
10
|
-
* holds a copy of your credentials without commands. Only refreshed OAuth tokens move between the copy and the real file.
|
|
11
|
-
*/
|
|
12
|
-
|
|
13
|
-
type Credentials = Record<string, Record<string, unknown>>;
|
|
14
|
-
/** Any JSON object is accepted, as Pi's own reader does. Entries are checked where they are used. */
|
|
15
|
-
const CredentialsSchema = z.record(z.string(), z.any());
|
|
16
|
-
|
|
17
|
-
export const jobAgentDir = (top: string): string => join(top, 'agent');
|
|
18
|
-
|
|
19
|
-
const PRIVATE = new Set(['auth.json', 'auth.json.lock', 'models-store.json', 'models-store.json.lock', 'pstack-v2', 'sessions', 'settings.json', 'settings.json.lock']);
|
|
20
|
-
|
|
21
|
-
/** A settings path as it resolves from your real agent directory: local ones made absolute, sources and names kept. */
|
|
22
|
-
function anchored(realAgentDir: string, entry: string): string {
|
|
23
|
-
const [, prefix = '', path = ''] = /^([!+-]?)(.*)$/s.exec(entry) ?? [];
|
|
24
|
-
if (/^[a-z][a-z0-9+.-]*:/i.test(path) || isAbsolute(path) || path.startsWith('~')) return entry;
|
|
25
|
-
return `${prefix}${resolve(realAgentDir, path)}`;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
/** Anchors one packages entry, a path string or an object with a source path, to the real agent directory. */
|
|
29
|
-
function anchoredPackage(realAgentDir: string, entry: unknown): unknown {
|
|
30
|
-
if (typeof entry === 'string') return anchored(realAgentDir, entry);
|
|
31
|
-
if (entry && typeof entry === 'object' && typeof (entry as { source?: unknown }).source === 'string') {
|
|
32
|
-
return { ...entry, source: anchored(realAgentDir, (entry as { source: string }).source) };
|
|
33
|
-
}
|
|
34
|
-
return entry;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
function anchoredSettings(realAgentDir: string, settings: Record<string, unknown>): Record<string, unknown> {
|
|
38
|
-
const listed = Object.fromEntries(
|
|
39
|
-
['extensions', 'skills', 'prompts', 'themes'].flatMap((key) => {
|
|
40
|
-
const list = settings[key];
|
|
41
|
-
return Array.isArray(list) ? [[key, list.map((entry) => (typeof entry === 'string' ? anchored(realAgentDir, entry) : entry))]] : [];
|
|
42
|
-
}),
|
|
43
|
-
);
|
|
44
|
-
const copy = { ...settings, ...listed };
|
|
45
|
-
if (Array.isArray(copy['packages'])) {
|
|
46
|
-
return {
|
|
47
|
-
...copy,
|
|
48
|
-
packages: copy['packages'].map((entry: unknown) => anchoredPackage(realAgentDir, entry)),
|
|
49
|
-
};
|
|
50
|
-
}
|
|
51
|
-
return copy;
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function parse(text: string | undefined): Credentials | undefined {
|
|
55
|
-
if (text === undefined) return undefined;
|
|
56
|
-
try {
|
|
57
|
-
const parsed = CredentialsSchema.safeParse(JSON.parse(text.replace(/^/, '')));
|
|
58
|
-
return parsed.success ? parsed.data : undefined;
|
|
59
|
-
} catch {
|
|
60
|
-
return undefined;
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
const readCredentials = (dir: string): Credentials | undefined => {
|
|
65
|
-
const path = join(dir, 'auth.json');
|
|
66
|
-
return existsSync(path) ? parse(readFileSync(path, 'utf8')) : undefined;
|
|
67
|
-
};
|
|
68
|
-
|
|
69
|
-
const isCommand = (entry: Record<string, unknown> | undefined): boolean => entry?.['type'] === 'api_key' && typeof entry['key'] === 'string' && entry['key'].startsWith('!');
|
|
70
|
-
|
|
71
|
-
/** Providers whose credential in the real auth.json is a command, which the root resolves before handing it to a job. */
|
|
72
|
-
export const commandProviders = (realAgentDir: string): string[] =>
|
|
73
|
-
Object.entries(readCredentials(realAgentDir) ?? {})
|
|
74
|
-
.filter(([, entry]) => isCommand(entry))
|
|
75
|
-
.map(([provider]) => provider);
|
|
76
|
-
|
|
77
|
-
/**
|
|
78
|
-
* Creates the job's agent directory: links to each real entry Pi reads, and a credentials copy in which each command is
|
|
79
|
-
* replaced by its resolved key when given, and dropped otherwise.
|
|
80
|
-
*/
|
|
81
|
-
export function makeJobAgentDir(realAgentDir: string, dir: string, resolved: Record<string, string> = {}): void {
|
|
82
|
-
mkdirSync(dir, { recursive: true });
|
|
83
|
-
for (const name of readdirSync(realAgentDir)) {
|
|
84
|
-
const link = join(dir, name);
|
|
85
|
-
if (!PRIVATE.has(name) && !existsSync(link) && !isLink(link)) symlinkSync(join(realAgentDir, name), link);
|
|
86
|
-
}
|
|
87
|
-
const catalog = join(realAgentDir, 'models-store.json');
|
|
88
|
-
if (existsSync(catalog)) copyFileSync(catalog, join(dir, 'models-store.json'));
|
|
89
|
-
const settings = join(realAgentDir, 'settings.json');
|
|
90
|
-
const parsed = existsSync(settings) ? parse(readFileSync(settings, 'utf8')) : undefined;
|
|
91
|
-
if (parsed) writeFileSync(join(dir, 'settings.json'), `${JSON.stringify(anchoredSettings(realAgentDir, parsed), null, 2)}\n`);
|
|
92
|
-
const copy: Credentials = Object.fromEntries(
|
|
93
|
-
Object.entries(readCredentials(realAgentDir) ?? {}).flatMap(([provider, entry]) => {
|
|
94
|
-
if (!isCommand(entry)) return [[provider, entry]];
|
|
95
|
-
return resolved[provider] !== undefined ? [[provider, { ...entry, key: resolved[provider] }]] : [];
|
|
96
|
-
}),
|
|
97
|
-
);
|
|
98
|
-
writeFileSync(join(dir, 'auth.json'), `${JSON.stringify(copy, null, 2)}\n`, { mode: 0o600 });
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
function isLink(path: string): boolean {
|
|
102
|
-
try {
|
|
103
|
-
return lstatSync(path).isSymbolicLink();
|
|
104
|
-
} catch {
|
|
105
|
-
return false;
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
// Pi guards auth.json with a lock directory beside the file, resolved through links as proper-lockfile does, so a symlinked
|
|
110
|
-
// auth.json shares one lock. Both files are locked before either is written, and a busy lock means waiting for the next
|
|
111
|
-
// sync rather than writing over a refresh in progress.
|
|
112
|
-
const resolved = (file: string): string => (existsSync(file) ? realpathSync(file) : file);
|
|
113
|
-
|
|
114
|
-
export function withLocks(files: string[], fn: () => void): boolean {
|
|
115
|
-
let held: readonly string[] = [];
|
|
116
|
-
try {
|
|
117
|
-
for (const file of files) {
|
|
118
|
-
const lock = `${resolved(file)}.lock`;
|
|
119
|
-
mkdirSync(lock);
|
|
120
|
-
held = [...held, lock];
|
|
121
|
-
}
|
|
122
|
-
fn();
|
|
123
|
-
return true;
|
|
124
|
-
} catch {
|
|
125
|
-
return false;
|
|
126
|
-
} finally {
|
|
127
|
-
// A job that shares a lock directory may have removed it already, which force tolerates.
|
|
128
|
-
for (const lock of held) rmSync(lock, { recursive: true, force: true });
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
// Pi rejects the whole file when an OAuth entry lacks string tokens or a finite expiry.
|
|
133
|
-
const validOauth = (entry: Record<string, unknown> | undefined): entry is Record<string, unknown> =>
|
|
134
|
-
entry?.['type'] === 'oauth' && typeof entry['access'] === 'string' && entry['access'] !== '' && typeof entry['refresh'] === 'string' && entry['refresh'] !== '' && typeof entry['expires'] === 'number' && Number.isFinite(entry['expires']);
|
|
135
|
-
|
|
136
|
-
const expiry = (entry: Record<string, unknown> | undefined): number => (validOauth(entry) ? (entry['expires'] as number) : -1);
|
|
137
|
-
|
|
138
|
-
/**
|
|
139
|
-
* Carries refreshed OAuth tokens both ways between the real auth.json and a job's copy, the later expiry winning. Only OAuth
|
|
140
|
-
* entries move, so nothing a job writes as a key or command ever reaches the real file. Returns false when a lock was busy.
|
|
141
|
-
*/
|
|
142
|
-
export function syncTokens(realAgentDir: string, jobDir: string): boolean {
|
|
143
|
-
return withLocks([join(realAgentDir, 'auth.json'), join(jobDir, 'auth.json')], () => {
|
|
144
|
-
const real = readCredentials(realAgentDir);
|
|
145
|
-
const copy = readCredentials(jobDir);
|
|
146
|
-
if (!real || !copy) return;
|
|
147
|
-
const providers = [...new Set([...Object.keys(real), ...Object.keys(copy)])];
|
|
148
|
-
const realUpdates = providers.flatMap((provider) => {
|
|
149
|
-
const [mine, theirs] = [real[provider], copy[provider]];
|
|
150
|
-
// A provider you logged out of, or switched to a key, is never brought back from a job's copy.
|
|
151
|
-
return expiry(theirs) > expiry(mine) && mine?.['type'] === 'oauth' ? [[provider, theirs as Record<string, unknown>] as const] : [];
|
|
152
|
-
});
|
|
153
|
-
const copyUpdates = providers.flatMap((provider) => {
|
|
154
|
-
const [mine, theirs] = [real[provider], copy[provider]];
|
|
155
|
-
const toReal = expiry(theirs) > expiry(mine) && mine?.['type'] === 'oauth';
|
|
156
|
-
return !toReal && expiry(mine) > expiry(theirs) && (theirs === undefined || theirs['type'] === 'oauth') ? [[provider, mine as Record<string, unknown>] as const] : [];
|
|
157
|
-
});
|
|
158
|
-
const realChanged = realUpdates.length > 0;
|
|
159
|
-
const copyChanged = copyUpdates.length > 0;
|
|
160
|
-
const newReal = { ...real, ...Object.fromEntries(realUpdates) };
|
|
161
|
-
const newCopy = { ...copy, ...Object.fromEntries(copyUpdates) };
|
|
162
|
-
if (realChanged) {
|
|
163
|
-
// Renamed into place so Pi never reads a half-written file. The job's copy is written in place instead, since the
|
|
164
|
-
// job may have that file bound to its own view, and a new file would leave it holding the old one.
|
|
165
|
-
const path = resolved(join(realAgentDir, 'auth.json'));
|
|
166
|
-
writeFileSync(`${path}.pstack-tmp`, `${JSON.stringify(newReal, null, 2)}\n`, { mode: 0o600 });
|
|
167
|
-
renameSync(`${path}.pstack-tmp`, path);
|
|
168
|
-
}
|
|
169
|
-
if (copyChanged) writeFileSync(join(jobDir, 'auth.json'), `${JSON.stringify(newCopy, null, 2)}\n`, { mode: 0o600 });
|
|
170
|
-
});
|
|
171
|
-
}
|