@join-pantheon/cli 0.31.1 → 0.31.2-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/worker.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { spawn as procSpawn } from "node:child_process";
2
2
  import { execFileSync } from "node:child_process";
3
- import { chmodSync, mkdirSync, writeFileSync, existsSync, readdirSync, readFileSync, realpathSync, rmSync } from "node:fs";
3
+ import { chmodSync, lstatSync, mkdirSync, writeFileSync, existsSync, readdirSync, readFileSync, readlinkSync, realpathSync, rmSync, statSync } from "node:fs";
4
4
  import { homedir } from "node:os";
5
5
  import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
6
6
  import { detectClaudeFeatures, describeFeatures, ensureRunTmp, NO_CLAUDE_FEATURES, runTmpDir } from "./claude-confine.js";
@@ -8,19 +8,22 @@ import { snapshotHome, restoreHome, describeChange, trustWrite, askedHere } from
8
8
  import { PantheonClient } from "./client.js";
9
9
  import { createHash, randomBytes } from "node:crypto";
10
10
  import { newKeypair, signDelegate } from "./identity.js";
11
- import { loadPins, pinScope, savePins, teamPinId } from "./pins.js";
11
+ import { loadPins, pinScope, savePins, teamPinId, writeWhole } from "./pins.js";
12
12
  import { answeredIn, markAnswered, noteNonce } from "./nonces.js";
13
- import { brokenStateFiles, brokenStateFix } from "./file-lock.js";
13
+ import { brokenStateFiles, brokenStateFix, pidAlive, readStateFile, withFileLock } from "./file-lock.js";
14
+ import { mkdirInside, writeInside } from "./safe-write.js";
15
+ import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
14
16
  import { trusted } from "./verify-act.js";
15
- import { headlessSpawn, materializeSpawnFiles, chooseAgent, describeChoice, unattendedRefusal, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
17
+ import { headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
18
+ import { activeOutages, detectOutage, earliest, whenAgain } from "./agent-outage.js";
16
19
  import { agentSpawn, CmdLineUnsafe } from "./which.js";
17
20
  import { VERSION } from "./version.js";
18
- import { applyAnswer, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, } from "./worker-policy.js";
21
+ import { applyAnswer, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, } from "./worker-policy.js";
19
22
  import { agentHookStates, repairHook } from "./notify.js";
20
23
  import { fence, assess } from "./untrusted.js";
21
- import { allowed, recordRun } from "./spend.js";
24
+ import { allowed, closeStartedRun, finishRun, recordRun, startRun } from "./spend.js";
22
25
  import { handledIn, loadContinued, markContinued, unmarkContinued } from "./continued.js";
23
- import { loadProfile, reservedDecisionsNote, saveProfile, wakeMode } from "./profile.js";
26
+ import { loadProfile, profileForTeam, reservedDecisionsNote, saveProfile, wakeMode } from "./profile.js";
24
27
  import { clip } from "./brief.js";
25
28
  import { composeWorker } from "./notice.js";
26
29
  import { diagnoseRun, gitProbe, workNeedsCommands } from "./run-failure.js";
@@ -57,7 +60,7 @@ function configuredSshCommand(cwd) {
57
60
  return hit;
58
61
  let set = false;
59
62
  try {
60
- set = execFileSync("git", ["-C", cwd, "config", "--get", "core.sshCommand"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 10_000 }).trim().length > 0;
63
+ set = execFileSync("git", safeGitArgs(cwd, ["-C", cwd, "config", "--get", "core.sshCommand"]), { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 10_000 }).trim().length > 0;
61
64
  }
62
65
  catch { }
63
66
  sshCommandSeen.set(cwd, set);
@@ -74,7 +77,7 @@ export function agentBaseEnv(base, proxy) {
74
77
  const GIT_TIMEOUT_MS = 60_000;
75
78
  export const WORKTREE_ADD_TIMEOUT_MS = 15 * 60_000;
76
79
  export const FETCH_TIMEOUT_MS = 90_000;
77
- const git = (cwd, ...args) => execFileSync("git", args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: gitEnv(process.env, proxyEnvFor(), cwd), timeout: GIT_TIMEOUT_MS, killSignal: "SIGKILL" }).trim();
80
+ const git = (cwd, ...args) => execFileSync("git", safeGitArgs(cwd, args), { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: gitCallEnv(args, gitEnv(process.env, proxyEnvFor(), cwd)), timeout: GIT_TIMEOUT_MS, killSignal: "SIGKILL" }).trim();
78
81
  const gitVersions = new Map();
79
82
  export function gitVersion() {
80
83
  const key = process.env.PATH ?? "";
@@ -97,7 +100,7 @@ export function fetchOrigin(project, timeoutMs = FETCH_TIMEOUT_MS) {
97
100
  return new Promise((res) => {
98
101
  let child;
99
102
  try {
100
- child = procSpawn("git", ["fetch", "--quiet", "origin"], { cwd: project, env: gitEnv(process.env, proxyEnvFor(), project), stdio: ["ignore", "ignore", "pipe"] });
103
+ child = procSpawn("git", safeGitArgs(project, ["fetch", "--quiet", "origin"]), { cwd: project, env: gitEnv(process.env, proxyEnvFor(), project), stdio: ["ignore", "ignore", "pipe"] });
101
104
  }
102
105
  catch (e) {
103
106
  res({ ok: false, why: e.message });
@@ -113,7 +116,7 @@ export function fetchOrigin(project, timeoutMs = FETCH_TIMEOUT_MS) {
113
116
  }
114
117
  function fetchOriginSync(project) {
115
118
  try {
116
- execFileSync("git", ["fetch", "--quiet", "origin"], { cwd: project, stdio: ["ignore", "ignore", "pipe"], env: gitEnv(process.env, proxyEnvFor(), project), timeout: FETCH_TIMEOUT_MS, killSignal: "SIGKILL" });
119
+ execFileSync("git", safeGitArgs(project, ["fetch", "--quiet", "origin"]), { cwd: project, stdio: ["ignore", "ignore", "pipe"], env: gitEnv(process.env, proxyEnvFor(), project), timeout: FETCH_TIMEOUT_MS, killSignal: "SIGKILL" });
117
120
  return { ok: true };
118
121
  }
119
122
  catch (e) {
@@ -133,7 +136,157 @@ const isRepo = (cwd) => {
133
136
  return false;
134
137
  }
135
138
  };
139
+ export const worktreeLabel = (wt) => `${basename(dirname(wt))}/${basename(wt)}`;
140
+ const LINK_SCAN_BUDGET = 1_000_000;
141
+ const GIT_SCAN_MAXBUFFER = 1 << 28;
142
+ const gitScan = (cwd, args) => execFileSync("git", safeGitArgs(cwd, args), { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: gitCallEnv(args, gitEnv(process.env, proxyEnvFor(), cwd)), timeout: GIT_TIMEOUT_MS, killSignal: "SIGKILL", maxBuffer: GIT_SCAN_MAXBUFFER });
143
+ const ENV_DIRS = new Set([".venv", "venv", "env", ".tox", "__pypackages__", ".direnv"]);
144
+ const AGENT_FILE = /^(CLAUDE.*\.md|AGENTS?.*\.md|GEMINI\.md|CONVENTIONS\.md|\.mcp\.json|\.cursorrules|\.windsurfrules|\.clinerules)$/i;
145
+ const AGENT_DIRS = new Set([".claude", ".cursor", ".github", ".kiro", ".codex", ".vscode", ".agents", ".gemini", ".junie", ".amazonq", ".continue", ".clinerules", ".windsurf"]);
146
+ function inEnvDir(rel) {
147
+ const parts = rel.split("/");
148
+ const base = parts[parts.length - 1];
149
+ if (AGENT_FILE.test(base) || parts.some((x) => AGENT_DIRS.has(x)))
150
+ return false;
151
+ const dirs = parts.slice(0, -1);
152
+ if (dirs.some((x) => ENV_DIRS.has(x)))
153
+ return true;
154
+ const nm = dirs.indexOf("node_modules");
155
+ return nm >= 0 && dirs.slice(nm + 1).includes(".bin");
156
+ }
157
+ function executableFile(abs) {
158
+ try {
159
+ const st = statSync(abs);
160
+ return st.isFile() && (st.mode & 0o111) !== 0;
161
+ }
162
+ catch {
163
+ return false;
164
+ }
165
+ }
166
+ export function outsideLinks(worktree, budgetMax = LINK_SCAN_BUDGET) {
167
+ const root = (() => { try {
168
+ return realpathSync.native(worktree);
169
+ }
170
+ catch {
171
+ return null;
172
+ } })();
173
+ if (!root)
174
+ return { links: [], unreadable: null };
175
+ const inside = (p) => p === root || p.startsWith(root + sep);
176
+ const candidates = new Map();
177
+ let repo = false;
178
+ try {
179
+ repo = git(worktree, "rev-parse", "--is-inside-work-tree") === "true";
180
+ }
181
+ catch {
182
+ repo = false;
183
+ }
184
+ if (repo) {
185
+ let tracked;
186
+ try {
187
+ tracked = gitScan(worktree, ["ls-files", "-s", "-z"]);
188
+ }
189
+ catch (e) {
190
+ return { links: [], unreadable: `the repo's tracked files could not be listed (${e.message.split("\n")[0]})` };
191
+ }
192
+ for (const e of tracked.split("\0")) {
193
+ const m = /^120000 ([0-9a-f]+) \d+\t([\s\S]+)$/.exec(e);
194
+ if (m)
195
+ candidates.set(m[2], m[1]);
196
+ }
197
+ }
198
+ let budget = budgetMax;
199
+ const walk = (rel) => {
200
+ let ents;
201
+ try {
202
+ ents = readdirSync(join(root, rel), { withFileTypes: true });
203
+ }
204
+ catch {
205
+ return null;
206
+ }
207
+ for (const d of ents) {
208
+ if (--budget < 0)
209
+ return `the checkout has more files than can be checked for links (over ${budgetMax})`;
210
+ if (d.name === ".git")
211
+ continue;
212
+ const r = rel ? `${rel}/${d.name}` : d.name;
213
+ if (d.isSymbolicLink()) {
214
+ if (!candidates.has(r))
215
+ candidates.set(r, null);
216
+ }
217
+ else if (d.isDirectory()) {
218
+ const why = walk(r);
219
+ if (why)
220
+ return why;
221
+ }
222
+ }
223
+ return null;
224
+ };
225
+ const why = walk("");
226
+ if (why)
227
+ return { links: [], unreadable: why };
228
+ const out = [];
229
+ for (const [rel, sha] of candidates) {
230
+ const p = join(root, rel);
231
+ let target = null;
232
+ try {
233
+ target = realpathSync.native(p);
234
+ }
235
+ catch {
236
+ try {
237
+ const dir = realpathSync.native(dirname(p));
238
+ target = resolve(dir, readlinkSync(p));
239
+ }
240
+ catch {
241
+ if (sha) {
242
+ try {
243
+ target = resolve(dirname(p), gitScan(worktree, ["cat-file", "blob", sha]).replace(/\n$/, ""));
244
+ }
245
+ catch { }
246
+ }
247
+ }
248
+ }
249
+ if (target === null || inside(target))
250
+ continue;
251
+ if (inEnvDir(rel) && executableFile(target))
252
+ continue;
253
+ out.push(rel);
254
+ }
255
+ return { links: out.sort(), unreadable: null };
256
+ }
257
+ export function outsideLinksDiagnosis(member, worktree, scan) {
258
+ const names = scan.links.slice(0, 8).join(", ") + (scan.links.length > 8 ? `, and ${scan.links.length - 8} more` : "");
259
+ const where = worktreeLabel(worktree);
260
+ const says = scan.unreadable
261
+ ? `the checkout for this could not be fully checked for links leading out of it (${scan.unreadable}) — a run could read a file outside it as the repo's own, so none was started`
262
+ : `the checkout holds links that lead outside it (${names}) — a run would read whatever they point at as the repo's own, and could send it on, so none was started`;
263
+ return {
264
+ cause: "outside_links",
265
+ says,
266
+ fix: scan.unreadable
267
+ ? `Look at this machine's worktree ${where}: a very large or unreadable checkout cannot be vouched for. Take this request in a live session, or clear it out (pantheon stops at that worktree; its branch and commits are kept in the repo).`
268
+ : `On this machine, delete these links from the worktree ${where} — not from the repo's branch, where they are not: ${scan.links.map((l) => `${where}/${l}`).join(", ")}. Or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it.`,
269
+ peer: {
270
+ summary: `${member}'s worker did not start a run: its checkout for this could not be vouched for`,
271
+ body: `${member}'s machine could not confirm its checkout for this holds no link leading outside it, so its worker did not start an unattended run: a run would read whatever such a link points at. It is still owed and waits for ${member}. Nothing was run.`,
272
+ },
273
+ };
274
+ }
275
+ export function tamperedDiagnosis(member, e) {
276
+ return {
277
+ cause: "worktree_tampered",
278
+ says: `the worktree's .git was changed after Pantheon made it (${e.why}) — git would obey that gitdir's config outside any sandbox, so git is not run in it`,
279
+ fix: /hooks folder/.test(e.why)
280
+ ? `Look at the repo's own git hooks folder (.git/hooks in your checkout) and remove anything you did not put there; the next request checks out a fresh worktree.`
281
+ : `Look at what is in that worktree, then remove its folder and run git worktree prune in the repo; the next request checks out a fresh one.`,
282
+ peer: {
283
+ summary: `${member}'s worker stopped: its checkout for this was changed underneath it`,
284
+ body: `The checkout ${member}'s worker uses for this was changed in a way it does not run git in, so it stopped. It is still owed and waits for ${member}.`,
285
+ },
286
+ };
287
+ }
136
288
  const CHECKOUT_MARK = "pantheon-checkout-complete";
289
+ function writeMark(mark) { writeInside(dirname(mark), mark, `${new Date().toISOString()}\n`, 0o644); }
137
290
  function checkoutMark(worktree) {
138
291
  try {
139
292
  return join(git(worktree, "rev-parse", "--absolute-git-dir"), CHECKOUT_MARK);
@@ -142,9 +295,23 @@ function checkoutMark(worktree) {
142
295
  return null;
143
296
  }
144
297
  }
298
+ const isLinkAt = (p) => { try {
299
+ return lstatSync(p).isSymbolicLink();
300
+ }
301
+ catch {
302
+ return false;
303
+ } };
145
304
  export function settleWorktree(project, worktree, log) {
146
- if (!existsSync(join(worktree, ".git")))
305
+ if (!existsSync(join(worktree, ".git")) && !isLinkAt(join(worktree, ".git")))
147
306
  return false;
307
+ const adopt = adoptWorktree(project, worktree);
308
+ if (adopt?.kind === "tampered")
309
+ throw new WorktreeTampered(worktree, adopt.why);
310
+ if (adopt?.kind === "stale") {
311
+ log(` ! the worktree ${worktree} no longer points at this repo (${adopt.why}) — removing it and checking out again`);
312
+ removeWorktree(project, worktree);
313
+ return false;
314
+ }
148
315
  const mark = checkoutMark(worktree);
149
316
  if (mark && existsSync(mark))
150
317
  return true;
@@ -159,7 +326,7 @@ export function settleWorktree(project, worktree, log) {
159
326
  }
160
327
  if (whole && mark) {
161
328
  try {
162
- writeFileSync(mark, `${new Date().toISOString()}\n`);
329
+ writeMark(mark);
163
330
  }
164
331
  catch { }
165
332
  return true;
@@ -242,11 +409,14 @@ function planWorktree(project, scratch, goal, log, branch, base, fetch) {
242
409
  export function worktreeAddArgs(add, os = process.platform) {
243
410
  return os === "win32" ? ["-c", "core.longpaths=true", ...add] : add;
244
411
  }
245
- function finishWorktree(r, log) {
412
+ function finishWorktree(project, r, log) {
413
+ const adopt = adoptWorktree(project, r.worktree);
414
+ if (adopt)
415
+ throw new WorktreeTampered(r.worktree, adopt.why);
246
416
  const mark = checkoutMark(r.worktree);
247
417
  if (mark)
248
- writeFileSync(mark, `${new Date().toISOString()}\n`);
249
- excludeRunFiles(r.worktree);
418
+ writeMark(mark);
419
+ excludeRunFiles(r.worktree, log);
250
420
  log(` worktree ${r.worktree} on ${r.branch} (from ${r.baseRef} ${r.baseSha.slice(0, 7)})`);
251
421
  return r;
252
422
  }
@@ -259,13 +429,13 @@ export function prepareWorktree(project, scratch, goal, log, branch = `pantheon/
259
429
  if ("done" in p)
260
430
  return p.done;
261
431
  try {
262
- execFileSync("git", p.add, { cwd: project, stdio: ["ignore", "ignore", "pipe"], env: gitEnv(process.env, proxyEnvFor(), project), timeout: opts.addTimeoutMs ?? WORKTREE_ADD_TIMEOUT_MS, killSignal: "SIGKILL" });
432
+ execFileSync("git", safeGitArgs(project, p.add), { cwd: project, stdio: ["ignore", "ignore", "pipe"], env: gitCallEnv(p.add, gitEnv(process.env, proxyEnvFor(), project)), timeout: opts.addTimeoutMs ?? WORKTREE_ADD_TIMEOUT_MS, killSignal: "SIGKILL" });
263
433
  }
264
434
  catch (e) {
265
435
  const err = e;
266
436
  throw addFailed(project, p.result.worktree, err.killed || err.signal ? `stopped after ${Math.round((opts.addTimeoutMs ?? WORKTREE_ADD_TIMEOUT_MS) / 1000)}s` : (String(err.stderr ?? "").trim().split("\n")[0] || ((err.message ?? "failed").split("\n")[0] ?? "failed")));
267
437
  }
268
- return finishWorktree(p.result, log);
438
+ return finishWorktree(project, p.result, log);
269
439
  }
270
440
  export async function prepareWorktreeAsync(project, scratch, goal, log, branch = `pantheon/${goal}`, base = null, opts = {}) {
271
441
  const inRepo = isRepo(project) && safeSegment(goal) !== null;
@@ -280,7 +450,7 @@ export async function prepareWorktreeAsync(project, scratch, goal, log, branch =
280
450
  const why = await new Promise((res) => {
281
451
  let child;
282
452
  try {
283
- child = procSpawn("git", p.add, { cwd: project, env: gitEnv(process.env, proxyEnvFor(), project), stdio: ["ignore", "ignore", "pipe"] });
453
+ child = procSpawn("git", safeGitArgs(project, p.add), { cwd: project, env: gitCallEnv(p.add, gitEnv(process.env, proxyEnvFor(), project)), stdio: ["ignore", "ignore", "pipe"] });
284
454
  }
285
455
  catch (e) {
286
456
  res(e.message);
@@ -295,7 +465,7 @@ export async function prepareWorktreeAsync(project, scratch, goal, log, branch =
295
465
  });
296
466
  if (why !== null)
297
467
  throw addFailed(project, p.result.worktree, why);
298
- return finishWorktree(p.result, log);
468
+ return finishWorktree(project, p.result, log);
299
469
  }
300
470
  export function branchName(space, goalId, title) {
301
471
  const slug = (title ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40).replace(/-+$/, "");
@@ -327,17 +497,47 @@ YOUR BRANCH STARTS AT ${baseSha.slice(0, 12)} (${baseRef}, ${fresh}).
327
497
  const RUN_DROPPINGS = [".pantheon-task.md", ".pantheon-run.log", ".pantheon-mcp.json", ".pantheon/*", "!.pantheon/commands", "!.pantheon/personas/",
328
498
  `*${AGENT_EDIT_SUFFIX}`, `*${AGENT_EDIT_SUFFIX}-*`];
329
499
  const RUN_DROPPINGS_OLD = [".pantheon/"];
330
- export function excludeRunFiles(worktree) {
500
+ const plainFile = (p) => { try {
501
+ const st = lstatSync(p);
502
+ return st.isFile() && st.nlink === 1;
503
+ }
504
+ catch {
505
+ return false;
506
+ } };
507
+ const excludeSaid = new Set();
508
+ export function excludeRunFiles(worktree, log) {
331
509
  try {
332
510
  const common = git(worktree, "rev-parse", "--git-common-dir");
333
- const file = resolve(worktree, common, "info", "exclude");
334
- const had = existsSync(file) ? readFileSync(file, "utf8") : "";
511
+ const root = resolve(worktree, common);
512
+ const info = join(root, "info");
513
+ const file = join(info, "exclude");
514
+ const infoOk = (() => { try {
515
+ const st = lstatSync(info);
516
+ return st.isDirectory();
517
+ }
518
+ catch {
519
+ return !existsSync(info);
520
+ } })();
521
+ const fileOk = (() => { try {
522
+ lstatSync(file);
523
+ return plainFile(file);
524
+ }
525
+ catch {
526
+ return true;
527
+ } })();
528
+ if (!infoOk || !fileOk) {
529
+ if (log && !excludeSaid.has(file)) {
530
+ excludeSaid.add(file);
531
+ log(` exclude ${file} is not a plain file in a plain folder (a link, or a hard link) — left as it is; the run's own files may show as untracked in this repo`);
532
+ }
533
+ return;
534
+ }
535
+ const had = plainFile(file) ? readFileSync(file, "utf8") : "";
335
536
  const cur = had.split("\n").filter((l) => !RUN_DROPPINGS_OLD.includes(l)).join("\n");
336
537
  const missing = RUN_DROPPINGS.filter((l) => !cur.split("\n").includes(l));
337
538
  if (!missing.length && cur === had)
338
539
  return;
339
- mkdirSync(resolve(worktree, common, "info"), { recursive: true });
340
- writeFileSync(file, missing.length ? `${cur.replace(/\n?$/, "\n")}${missing.join("\n")}\n` : cur);
540
+ writeInside(root, file, missing.length ? `${cur.replace(/\n?$/, "\n")}${missing.join("\n")}\n` : cur, 0o644);
341
541
  }
342
542
  catch { }
343
543
  }
@@ -384,10 +584,11 @@ export function workspaceNote(route, attached, sha, outbox) {
384
584
  return lines.join("\n");
385
585
  }
386
586
  export function gitOut(cwd, args, timeoutMs = GIT_TIMEOUT_MS) {
587
+ const argv = safeGitArgs(cwd, args);
387
588
  return new Promise((res) => {
388
589
  let child;
389
590
  try {
390
- child = procSpawn("git", args, { cwd, env: gitEnv(process.env, proxyEnvFor(), cwd), stdio: ["ignore", "pipe", "ignore"] });
591
+ child = procSpawn("git", argv, { cwd, env: gitCallEnv(args, gitEnv(process.env, proxyEnvFor(), cwd)), stdio: ["ignore", "pipe", "ignore"] });
391
592
  }
392
593
  catch {
393
594
  res(null);
@@ -413,6 +614,7 @@ export async function onRemote(cwd, branch, commit) {
413
614
  return (await gitOut(cwd, ["merge-base", "--is-ancestor", commit, tip])) !== null;
414
615
  }
415
616
  export const MASS_DELETE_SHARE = 0.5;
617
+ const ONCE_GIVE_BACK_MS = 120_000;
416
618
  export const asksForMassDelete = (text) => /\b(?:delete|remove|wipe|drop|rm)\b[^\n]{0,40}\b(?:all|every|entire|whole|most)\b/i.test(text);
417
619
  const GIT_HOOK_NAMES = [
418
620
  "applypatch-msg", "pre-applypatch", "post-applypatch", "prepare-commit-msg", "commit-msg", "post-commit",
@@ -466,10 +668,10 @@ exit 0
466
668
  `;
467
669
  export function massDeleteGuardEnv(runHome, worktree, allow, env = process.env) {
468
670
  const dir = join(runHome, "git-hooks");
469
- mkdirSync(dir, { recursive: true, mode: 0o700 });
671
+ mkdirInside(runHome, dir);
470
672
  for (const [name, body] of [["pre-commit", PRE_COMMIT], ["pre-push", PRE_PUSH], ...GIT_HOOK_NAMES.map((n) => [n, passThrough(n)])]) {
471
673
  const f = join(dir, name);
472
- writeFileSync(f, body);
674
+ writeInside(runHome, f, body, 0o755);
473
675
  try {
474
676
  chmodSync(f, 0o755);
475
677
  }
@@ -477,9 +679,13 @@ export function massDeleteGuardEnv(runHome, worktree, allow, env = process.env)
477
679
  }
478
680
  let orig = "";
479
681
  try {
480
- orig = resolve(worktree, git(worktree, "rev-parse", "--git-path", "hooks"));
682
+ const out = execFileSync("git", repoHooksArgs(worktree), { cwd: worktree, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: gitCallEnv(["rev-parse"], gitEnv(process.env, proxyEnvFor(), worktree)), timeout: GIT_TIMEOUT_MS }).trim();
683
+ orig = resolve(worktree, out);
684
+ }
685
+ catch (e) {
686
+ if (e instanceof WorktreeTampered)
687
+ throw e;
481
688
  }
482
- catch { }
483
689
  const n = Number.parseInt(env.GIT_CONFIG_COUNT ?? "0", 10) || 0;
484
690
  return {
485
691
  GIT_CONFIG_COUNT: String(n + 1),
@@ -491,12 +697,14 @@ export function massDeleteGuardEnv(runHome, worktree, allow, env = process.env)
491
697
  }
492
698
  export function deletedSinceBase(worktree, baseSha) {
493
699
  try {
494
- const total = git(worktree, "ls-tree", "-r", "--name-only", baseSha).split("\n").filter(Boolean).length;
495
- const gone = git(worktree, "diff", "--name-only", "--diff-filter=D", baseSha).split("\n").filter(Boolean).length;
496
- return { gone, total };
700
+ const total = gitScan(worktree, ["ls-tree", "-r", "--name-only", baseSha]).split("\n").filter(Boolean).length;
701
+ const gone = gitScan(worktree, ["diff", "--name-only", "--diff-filter=D", baseSha]).split("\n").filter(Boolean).length;
702
+ return { gone, total, unreadable: null };
497
703
  }
498
- catch {
499
- return null;
704
+ catch (e) {
705
+ if (e instanceof WorktreeTampered)
706
+ throw e;
707
+ return { gone: 0, total: 0, unreadable: e.message.split("\n")[0] || "git failed" };
500
708
  }
501
709
  }
502
710
  export const printable = (s) => typeof s === "string" && /^[A-Za-z0-9_][A-Za-z0-9_:.\-]{0,119}$/.test(s) ? s : "(an id this worker will not print)";
@@ -597,6 +805,70 @@ Act, reply if a reply is owed, then finish.`;
597
805
  export const TASK_POINTER = "Your task for this run is in the file .pantheon-task.md in the current directory. Read that file first and follow it: it is the whole brief.";
598
806
  let fleetLock = null;
599
807
  export const RUN_TIMEOUT_MS = 20 * 60_000;
808
+ export const RUN_SLOT = "run-slot.json";
809
+ export const RUN_LEASE_RENEW_MS = 60_000;
810
+ const RUN_LEASE_FRESH_MS = 10 * 60_000;
811
+ function readRunLease(f) {
812
+ let text;
813
+ try {
814
+ text = readStateFile(f);
815
+ }
816
+ catch {
817
+ rmSync(f, { recursive: true, force: true });
818
+ return null;
819
+ }
820
+ if (text === null)
821
+ return null;
822
+ try {
823
+ const v = JSON.parse(text);
824
+ if (v && typeof v === "object")
825
+ return v;
826
+ }
827
+ catch { }
828
+ rmSync(f, { recursive: true, force: true });
829
+ return null;
830
+ }
831
+ export function takeRunLease(dir, space, now = Date.now()) {
832
+ const f = join(dir, RUN_SLOT);
833
+ return withFileLock(f, () => {
834
+ const cur = readRunLease(f);
835
+ if (cur && Number.isInteger(cur.pid) && cur.pid !== process.pid && typeof cur.at === "number" && Number.isFinite(cur.at)
836
+ && Math.abs(now - cur.at) <= RUN_LEASE_FRESH_MS && pidAlive(cur.pid)) {
837
+ return { held: typeof cur.space === "string" ? cur.space : "another team" };
838
+ }
839
+ const token = randomBytes(12).toString("hex");
840
+ writeWhole(f, JSON.stringify({ pid: process.pid, token, space, at: now }) + "\n");
841
+ return { token };
842
+ });
843
+ }
844
+ export function renewRunLease(dir, token, now = Date.now()) {
845
+ const f = join(dir, RUN_SLOT);
846
+ try {
847
+ withFileLock(f, () => {
848
+ const cur = readRunLease(f);
849
+ if (cur?.token === token)
850
+ writeWhole(f, JSON.stringify({ ...cur, at: now }) + "\n");
851
+ });
852
+ }
853
+ catch { }
854
+ }
855
+ export function releaseRunLease(dir, token) {
856
+ const f = join(dir, RUN_SLOT);
857
+ try {
858
+ withFileLock(f, () => { if (readRunLease(f)?.token === token)
859
+ rmSync(f, { force: true }); });
860
+ }
861
+ catch { }
862
+ }
863
+ export function repairRunLease(dir) {
864
+ const f = join(dir, RUN_SLOT);
865
+ if (!existsSync(f))
866
+ return;
867
+ try {
868
+ withFileLock(f, () => { readRunLease(f); });
869
+ }
870
+ catch { }
871
+ }
600
872
  export function killRunTree(pid, os = process.platform) {
601
873
  if (!Number.isInteger(pid) || pid <= 1)
602
874
  return;
@@ -782,6 +1054,10 @@ export class Worker {
782
1054
  }
783
1055
  parkedNotHere = new Set();
784
1056
  parkedNoSandbox = new Map();
1057
+ outages = new Map();
1058
+ outageHold = new Map();
1059
+ outageHeldUntil = new Map();
1060
+ outageRetries = new Map();
785
1061
  parkedNoSandboxConts = new Map();
786
1062
  ranNoSandboxCheckedAt = 0;
787
1063
  noSandboxDiagnosis(choice) {
@@ -809,14 +1085,16 @@ export class Worker {
809
1085
  if (this.opts.agentCmd)
810
1086
  return null;
811
1087
  const ok = this.unsandboxedOk();
812
- const key = `${agent ?? "auto"}|${ok}`;
1088
+ const key = `${agent ?? "auto"}|${ok}|${Object.entries(activeOutages(this.outages)).map(([k, v]) => `${k}@${v.until}`).join(",")}`;
813
1089
  const hit = this.sandboxHold.get(key);
814
1090
  if (hit && Date.now() - hit.at < 60_000)
815
1091
  return hit.d;
816
1092
  let d = null;
817
1093
  try {
818
- const c = (this.opts.choose ?? chooseAgent)(agent, { prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: ok });
819
- if (NOT_FOR_UNATTENDED_RUNS.has(c.backend.id))
1094
+ const c = (this.opts.choose ?? chooseAgent)(agent, { prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: ok, outages: activeOutages(this.outages) });
1095
+ if (c.outage)
1096
+ d = this.outageDiagnosis(c.backend.label, c.outage, []);
1097
+ else if (NOT_FOR_UNATTENDED_RUNS.has(c.backend.id))
820
1098
  d = this.noUnattendedAgentDiagnosis(unattendedRefusal(c, { unsandboxedOk: ok }) ?? c.backend.label);
821
1099
  else if (!SANDBOXED_FOR_UNATTENDED.has(c.backend.id) && !ok)
822
1100
  d = this.noSandboxDiagnosis(c);
@@ -842,6 +1120,45 @@ export class Worker {
842
1120
  return false;
843
1121
  }
844
1122
  }
1123
+ outageDiagnosis(label, out, others, meanwhile) {
1124
+ const at = whenAgain(out.until);
1125
+ const rest = meanwhile ? `; the agent standing in for it cannot take this one (${meanwhile.replace(/\.$/, "")})`
1126
+ : others.length ? `; ${others.join("; ")}` : "; no other agent is installed for runs while you are away";
1127
+ return {
1128
+ cause: "agent_unavailable",
1129
+ says: `${label} is unavailable here until about ${at} — ${out.why}${rest}.`,
1130
+ fix: `It runs by itself after ${at}. Sooner: sign in to another agent (cursor-agent, codex or copilot) — and for an agent without a sandbox, pantheon policy grant unsandboxed-runs — or answer it in a live session.`,
1131
+ peer: {
1132
+ summary: `${this.cfg.member}'s agent is unavailable until about ${at} — this waits`,
1133
+ body: `${this.cfg.member}'s worker could not start a run: ${label} ${out.why}, and no other agent on that machine could take it. It is still owed; the worker tries again after ${at}.`,
1134
+ },
1135
+ };
1136
+ }
1137
+ outagesPassed(c) {
1138
+ const now = activeOutages(this.outages);
1139
+ return c.skipped.flatMap((x) => { const out = now[x.backend.id]; return out ? [{ label: x.backend.label, out }] : []; });
1140
+ }
1141
+ fallbackSaid = new Set();
1142
+ sayFallbackUnsandboxed(c) {
1143
+ if (SANDBOXED_FOR_UNATTENDED.has(c.backend.id) || !this.opts.notify)
1144
+ return;
1145
+ const passed = this.outagesPassed(c).filter((p) => p.label === "Claude Code");
1146
+ const first = passed[0];
1147
+ if (!first)
1148
+ return;
1149
+ const key = `${c.backend.id}|${first.out.until}`;
1150
+ if (this.fallbackSaid.has(key))
1151
+ return;
1152
+ this.fallbackSaid.add(key);
1153
+ try {
1154
+ this.opts.notify({
1155
+ kind: "worker", who: "Your worker", heading: `${c.backend.label} is answering for Claude Code`, context: this.cfg.space,
1156
+ body: `Claude Code is unavailable until about ${whenAgain(first.out.until)} — ${first.out.why}. Until then ${c.backend.label} takes teammates' requests on this machine, without Claude Code's sandbox, as your settings allow. To have them wait instead: pantheon policy revoke unsandboxed-runs`,
1157
+ ttl: 0, sound: false,
1158
+ });
1159
+ }
1160
+ catch { }
1161
+ }
845
1162
  claudeSandboxableNow() {
846
1163
  if (this.opts.agentCmd)
847
1164
  return false;
@@ -983,6 +1300,12 @@ export class Worker {
983
1300
  }
984
1301
  if (!ok && now - h.at < every)
985
1302
  continue;
1303
+ const ou = this.outageHeldUntil.get(id);
1304
+ if (ou !== undefined) {
1305
+ if (ou > now)
1306
+ continue;
1307
+ this.outageHeldUntil.delete(id);
1308
+ }
986
1309
  if (this.heldForSandbox(h.agent)) {
987
1310
  h.at = now;
988
1311
  continue;
@@ -1208,13 +1531,18 @@ export class Worker {
1208
1531
  const seen = noteNonce(this.pinTeam, act.from, sig.nonce, act.id, this.opts.home, sig.s);
1209
1532
  if (this.opts.home !== undefined) {
1210
1533
  const also = noteNonce(this.pinTeam, act.from, sig.nonce, act.id, undefined, sig.s);
1534
+ const forged = [seen, also].some((x) => x.replayOf === act.id && !x.unvouched);
1535
+ const unvouched = !forged && [seen, also].some((x) => x.replayOf === act.id && x.unvouched);
1211
1536
  seen.replayOf ??= also.replayOf;
1212
1537
  seen.answered ||= also.answered;
1538
+ seen.unvouched = unvouched || undefined;
1213
1539
  }
1540
+ if (seen.replayOf === act.id && seen.answered && seen.unvouched)
1541
+ return { why: "this machine already answered that request", trust: "answered", from: act.from };
1214
1542
  if (seen.replayOf)
1215
1543
  return { why: `a copy of ${seen.replayOf}, which this machine already had — the same signed request under a new id`, trust: "replayed", from: act.from };
1216
1544
  if (seen.answered)
1217
- return { why: "this machine already answered that signed request", trust: "replayed", from: act.from };
1545
+ return { why: "this machine already answered that signed request", trust: "answered", from: act.from };
1218
1546
  if (sig.role === "run" && sig.cert && seen.firstSeen > sig.cert.expires + 6 * 3600_000) {
1219
1547
  return { why: "a run's request that first reached this machine hours after that run ended", trust: "stale", from: act.from };
1220
1548
  }
@@ -1262,8 +1590,11 @@ export class Worker {
1262
1590
  const pol = loadPolicy(this.opts.home);
1263
1591
  if (isApprovedWaker(pol, [this.cfg.spaceId, this.cfg.space], from))
1264
1592
  return;
1593
+ const before = { ...(pol.wakeOnce ?? {}) };
1265
1594
  if (takeWakeOnce(pol, [this.cfg.spaceId, this.cfg.space], from)) {
1266
1595
  savePolicy(pol, this.opts.home);
1596
+ const keys = Object.keys(before).filter((k) => (pol.wakeOnce?.[k] ?? 0) < before[k]);
1597
+ this.spentOnce = { ...this.spentOnce, wake: keys, file: this.policyBytes() };
1267
1598
  this.opts.log(` wake ${from} may start this one run (you said "just this once")`);
1268
1599
  }
1269
1600
  }
@@ -1378,6 +1709,11 @@ export class Worker {
1378
1709
  }
1379
1710
  openRunHome(cfg = this.cfg, features = []) {
1380
1711
  const dir = this.runHomePath();
1712
+ try {
1713
+ if (!lstatSync(dir).isDirectory())
1714
+ rmSync(dir, { recursive: true, force: true });
1715
+ }
1716
+ catch { }
1381
1717
  mkdirSync(dir, { recursive: true, mode: 0o700 });
1382
1718
  const file = join(dir, "config.json");
1383
1719
  let written = cfg;
@@ -1396,11 +1732,7 @@ export class Worker {
1396
1732
  this.opts.log(` sign this relay cannot check a run's own key yet, so woken runs sign as ${cfg.member} (upgrade the relay to 0.31)`);
1397
1733
  }
1398
1734
  written = { ...written, pinTeam: this.pinTeam };
1399
- writeFileSync(file, JSON.stringify(written, null, 2) + "\n", { mode: 0o600 });
1400
- try {
1401
- chmodSync(file, 0o600);
1402
- }
1403
- catch { }
1735
+ writeInside(dir, file, JSON.stringify(written, null, 2) + "\n", 0o600);
1404
1736
  try {
1405
1737
  const p = loadPins(this.opts.home);
1406
1738
  savePins(p, dir);
@@ -1408,6 +1740,14 @@ export class Worker {
1408
1740
  catch { }
1409
1741
  return dir;
1410
1742
  }
1743
+ cleanupStep(what, step) {
1744
+ try {
1745
+ step();
1746
+ }
1747
+ catch (e) {
1748
+ this.opts.log(`! ${what} failed: ${e.message.split("\n")[0]}`);
1749
+ }
1750
+ }
1411
1751
  sealRunHome(dir) {
1412
1752
  try {
1413
1753
  for (const a of answeredIn(dir))
@@ -1421,7 +1761,7 @@ export class Worker {
1421
1761
  catch { }
1422
1762
  const { token: _t, privateKey: _k, pendingAuth: _p, delegate: _d, ...stub } = this.cfg;
1423
1763
  try {
1424
- writeFileSync(join(dir, "config.json"), JSON.stringify({ ...stub, sealed: "the token and key are written here only while a woken run holds them" }, null, 2) + "\n", { mode: 0o600 });
1764
+ writeInside(dir, join(dir, "config.json"), JSON.stringify({ ...stub, sealed: "the token and key are written here only while a woken run holds them" }, null, 2) + "\n", 0o600);
1425
1765
  }
1426
1766
  catch {
1427
1767
  try {
@@ -1477,10 +1817,18 @@ export class Worker {
1477
1817
  });
1478
1818
  await client.connect();
1479
1819
  this.client = client;
1820
+ try {
1821
+ await this.serve(client);
1822
+ }
1823
+ finally {
1824
+ client.close();
1825
+ }
1826
+ }
1827
+ async serve(client) {
1480
1828
  const agentLabel = this.opts.agentCmd
1481
1829
  ? this.opts.agentCmd
1482
1830
  : (() => { try {
1483
- return describeChoice((this.opts.choose ?? chooseAgent)(this.opts.agent, { prefer: loadProfile(this.cfg).agent.primary_surface }));
1831
+ return describeChoice((this.opts.choose ?? chooseAgent)(this.opts.agent, { prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: this.unsandboxedOk() }));
1484
1832
  }
1485
1833
  catch {
1486
1834
  return "auto";
@@ -1512,6 +1860,7 @@ export class Worker {
1512
1860
  for (const w of p.warnings)
1513
1861
  this.opts.log(` ! ${p.name}: ${w}`);
1514
1862
  }
1863
+ this.recoverInterruptedRun();
1515
1864
  if (this.opts.continuations !== false)
1516
1865
  await this.restoreHeldContinuations(client);
1517
1866
  this.writeState("watching", "waiting for work");
@@ -1566,8 +1915,11 @@ export class Worker {
1566
1915
  this.sayKeyChanges(client);
1567
1916
  const aged = client
1568
1917
  .myWork()
1569
- .filter((o) => o.kind !== "HUMAN" && !o.claimed_by)
1918
+ .filter((o) => o.kind !== "HUMAN" && o.kind !== "DELIVER" && !o.claimed_by)
1570
1919
  .filter((o) => Date.now() - o.created_at >= grace)
1920
+ .filter((o) => { const t = this.outageHold.get(o.id); if (t === undefined)
1921
+ return true; if (t > Date.now())
1922
+ return false; this.outageHold.delete(o.id); return true; })
1571
1923
  .filter((o) => {
1572
1924
  if (this.failuresOf(o.id) < 2)
1573
1925
  return true;
@@ -1719,111 +2071,251 @@ export class Worker {
1719
2071
  maxRunsPerHour: this.opts.maxRunsPerHour,
1720
2072
  maxRunsPerDay: this.opts.maxRunsPerDay,
1721
2073
  });
1722
- const gate = allowed({ perHour: policy.maxRunsPerHour, perDay: policy.maxRunsPerDay }, this.opts.home);
1723
- if (!gate.ok) {
1724
- if (!this.skipLogged.has(`cap:${candidate?.id ?? continuation.id}`)) {
1725
- this.skipLogged.add(`cap:${candidate?.id ?? continuation.id}`);
1726
- this.opts.log(`! not spawning: ${gate.why}. Raise it with --max-runs / --max-runs-day${persona ? ` (persona "${persona.name}" may cap it lower)` : ""}, or leave it.`);
1727
- }
1728
- this.writeState("capped", gate.why);
2074
+ const slot = this.takeSlot();
2075
+ if ("held" in slot) {
2076
+ this.writeState("watching", `waiting: ${slot.held}`);
1729
2077
  if (continuation)
1730
- this.requeueContinuation(continuation, 60_000);
2078
+ this.requeueContinuation(continuation, 5_000);
1731
2079
  continue;
1732
2080
  }
1733
- if (continuation) {
2081
+ try {
2082
+ const gate = allowed({ perHour: policy.maxRunsPerHour, perDay: policy.maxRunsPerDay }, this.opts.home);
2083
+ if (!gate.ok) {
2084
+ if (!this.skipLogged.has(`cap:${candidate?.id ?? continuation.id}`)) {
2085
+ this.skipLogged.add(`cap:${candidate?.id ?? continuation.id}`);
2086
+ this.opts.log(gate.broken ? `! not spawning: ${gate.why}`
2087
+ : `! not spawning: ${gate.why}. Raise it with --max-runs / --max-runs-day${persona ? ` (persona "${persona.name}" may cap it lower)` : ""}, or leave it.`);
2088
+ }
2089
+ this.writeState("capped", gate.why);
2090
+ if (continuation)
2091
+ this.requeueContinuation(continuation, 60_000);
2092
+ continue;
2093
+ }
2094
+ if (this.stopped)
2095
+ break;
2096
+ if (continuation) {
2097
+ try {
2098
+ await client.claim(`cont:${continuation.id}`);
2099
+ }
2100
+ catch (e) {
2101
+ const msg = e.message;
2102
+ if (/^DEFER/.test(msg)) {
2103
+ this.requeueContinuation(continuation, 10_000);
2104
+ }
2105
+ else {
2106
+ this.markDone(continuation.id);
2107
+ if (!this.skipLogged.has(`cont:${continuation.id}`)) {
2108
+ this.skipLogged.add(`cont:${continuation.id}`);
2109
+ this.opts.log(`- not continuing from ${continuation.id}: ${msg.split("\n")[0]}`);
2110
+ }
2111
+ }
2112
+ continue;
2113
+ }
2114
+ if (this.stopped) {
2115
+ this.opts.log(`- not continuing from ${continuation.id}: the worker is stopping`);
2116
+ break;
2117
+ }
2118
+ }
2119
+ this.running = true;
2120
+ this.writeState("running", candidate ? `${candidate.kind} ${candidate.id}` : `CONTINUE ${continuation.id}`);
2121
+ let job;
1734
2122
  try {
1735
- await client.claim(`cont:${continuation.id}`);
2123
+ job = candidate
2124
+ ? await this.handle(client, candidate, persona, policy, candidateAct)
2125
+ : await this.handleContinuation(client, continuation, persona, policy);
1736
2126
  }
1737
2127
  catch (e) {
1738
- const msg = e.message;
1739
- if (/^DEFER/.test(msg)) {
1740
- this.requeueContinuation(continuation, 10_000);
2128
+ const msg = e?.message?.split("\n")[0] ?? String(e);
2129
+ const ob = candidate ?? {
2130
+ id: `cont:${continuation.id}`, kind: "CONTINUE", owner: this.cfg.member, ref: continuation.id,
2131
+ goal: continuation.goal, created_at: continuation.ts, body: `${continuation.act} from ${continuation.from}`,
2132
+ };
2133
+ if (e instanceof WorktreeTampered) {
2134
+ this.opts.log(`! ${ob.id}: ${msg}`);
2135
+ job = { obligation: ob, branch: "", worktree: e.worktree, ok: false, refused: true, detail: msg, diagnosis: tamperedDiagnosis(this.cfg.member, e) };
1741
2136
  }
1742
2137
  else {
1743
- this.markDone(continuation.id);
1744
- if (!this.skipLogged.has(`cont:${continuation.id}`)) {
1745
- this.skipLogged.add(`cont:${continuation.id}`);
1746
- this.opts.log(`- not continuing from ${continuation.id}: ${msg.split("\n")[0]}`);
1747
- }
2138
+ this.opts.log(`! ${ob.id}: the run failed with an error — ${msg}`);
2139
+ job = { obligation: ob, branch: "", worktree: "", ok: false, detail: `the run failed with an error: ${msg}` };
2140
+ }
2141
+ }
2142
+ this.runProject = null;
2143
+ this.done.push(job);
2144
+ this.client?.reportWorkerState("idle");
2145
+ if (candidate && job.ok)
2146
+ this.markDone(`handled:${candidate.ref}`);
2147
+ const ran = candidate ? candidateAct : continuation;
2148
+ if (job.ok && ran?.sig) {
2149
+ try {
2150
+ markAnswered(this.pinTeam, ran.from, ran.sig.nonce, ran.id, this.opts.home, ran.sig.s);
2151
+ if (this.opts.home !== undefined)
2152
+ markAnswered(this.pinTeam, ran.from, ran.sig.nonce, ran.id, undefined, ran.sig.s);
2153
+ }
2154
+ catch { }
2155
+ }
2156
+ {
2157
+ const soon = earliest(activeOutages(this.outages));
2158
+ if (soon && job.refused && job.forOutage && job.diagnosis && !job.waitUntil) {
2159
+ const label = [...this.outages.entries()].find(([, v]) => v === soon)?.[0];
2160
+ Object.assign(job, { waitUntil: soon.until, diagnosis: this.outageDiagnosis(label ? agentById(label)?.label ?? label : "The agent", soon, [], job.diagnosis.says) });
2161
+ }
2162
+ }
2163
+ if (job.agentOut) {
2164
+ if (continuation) {
2165
+ this.contDone.delete(continuation.id);
2166
+ for (const k of this.contKeys())
2167
+ unmarkContinued(k, continuation.id, this.opts.home);
2168
+ this.requeueContinuation(continuation, 0);
1748
2169
  }
1749
2170
  continue;
1750
2171
  }
1751
- }
1752
- this.running = true;
1753
- fleetLock = this;
1754
- this.writeState("running", candidate ? `${candidate.kind} ${candidate.id}` : `CONTINUE ${continuation.id}`);
1755
- const job = candidate
1756
- ? await this.handle(client, candidate, persona, policy, candidateAct)
1757
- : await this.handleContinuation(client, continuation, persona, policy);
1758
- this.runProject = null;
1759
- this.done.push(job);
1760
- this.client?.reportWorkerState("idle");
1761
- if (candidate && job.ok)
1762
- this.markDone(`handled:${candidate.ref}`);
1763
- const ran = candidate ? candidateAct : continuation;
1764
- if (job.ok && ran?.sig) {
1765
- try {
1766
- markAnswered(this.pinTeam, ran.from, ran.sig.nonce, ran.id, this.opts.home, ran.sig.s);
1767
- if (this.opts.home !== undefined)
1768
- markAnswered(this.pinTeam, ran.from, ran.sig.nonce, ran.id, undefined, ran.sig.s);
2172
+ if (job.refused && job.diagnosis?.cause === "agent_unavailable" && job.waitUntil) {
2173
+ if (candidate)
2174
+ this.outageHold.set(candidate.id, job.waitUntil);
2175
+ if (this.outageHold.size > 500)
2176
+ this.outageHold.delete(this.outageHold.keys().next().value);
2177
+ if (continuation) {
2178
+ this.holdContinuation(continuation, policy.agent);
2179
+ this.outageHeldUntil.set(`cont:${continuation.id}`, job.waitUntil);
2180
+ }
2181
+ if (!this.skipLogged.has(`out-told:${job.obligation.id}`)) {
2182
+ this.skipLogged.add(`out-told:${job.obligation.id}`);
2183
+ await this.reportRefused(client, job.obligation, job.diagnosis, !!candidate);
2184
+ }
1769
2185
  }
1770
- catch { }
1771
- }
1772
- if (job.refused && job.diagnosis) {
1773
- if (candidate) {
1774
- this.failedRuns.set(candidate.id, 2);
1775
- this.skipLogged.add(`parked:${candidate.id}`);
1776
- if (job.diagnosis.cause === "repo_not_here")
1777
- this.parkedNotHere.add(candidate.id);
2186
+ else if (job.refused && job.diagnosis) {
2187
+ if (candidate) {
2188
+ this.failedRuns.set(candidate.id, 2);
2189
+ this.skipLogged.add(`parked:${candidate.id}`);
2190
+ if (job.diagnosis.cause === "repo_not_here")
2191
+ this.parkedNotHere.add(candidate.id);
2192
+ }
2193
+ if (!candidate && continuation && this.parkedNoSandbox.has(job.obligation.id))
2194
+ this.holdContinuation(continuation, policy.agent);
2195
+ await this.reportRefused(client, job.obligation, job.diagnosis, !!candidate);
2196
+ if (job.diagnosis.cause === "outside_links" || job.diagnosis.cause === "worktree_tampered")
2197
+ await this.reportStuck(client, job.obligation, job.diagnosis, 0);
1778
2198
  }
1779
- if (!candidate && continuation && this.parkedNoSandbox.has(job.obligation.id))
1780
- this.holdContinuation(continuation, policy.agent);
1781
- await this.reportRefused(client, job.obligation, job.diagnosis, !!candidate);
1782
- }
1783
- else if (candidate && !job.ok) {
1784
- const attempts = this.failuresOf(candidate.id) + 1;
1785
- const failures = job.diagnosis?.cause === "no_commands" || job.diagnosis?.cause === "no_sandbox_commands" ? 2 : attempts;
1786
- if (job.diagnosis?.cause === "no_sandbox_commands")
1787
- this.markDone(`parked-ns:${candidate.id}`);
1788
- if (this.contDone.has(`unparked-ns:${candidate.id}`))
1789
- this.markDone(`parked2-ns:${candidate.id}`);
1790
- this.failedRuns.set(candidate.id, failures);
1791
- this.markDone(failures >= 2 ? `parked:${candidate.id}` : `failed1:${candidate.id}`);
1792
- if (this.contDone.has(`unparked:${candidate.id}`))
1793
- this.markDone(`parked2:${candidate.id}`);
1794
- else if (job.diagnosis?.cause === "no_commands")
1795
- this.markDone(`parked-nc:${candidate.id}`);
1796
- if (this.failedRuns.size > 200) {
1797
- this.failedRuns.delete(this.failedRuns.keys().next().value);
2199
+ else if (candidate && !job.ok) {
2200
+ const attempts = this.failuresOf(candidate.id) + 1;
2201
+ const failures = job.diagnosis?.cause === "no_commands" || job.diagnosis?.cause === "no_sandbox_commands" ? 2 : attempts;
2202
+ if (job.diagnosis?.cause === "no_sandbox_commands")
2203
+ this.markDone(`parked-ns:${candidate.id}`);
2204
+ if (this.contDone.has(`unparked-ns:${candidate.id}`))
2205
+ this.markDone(`parked2-ns:${candidate.id}`);
2206
+ this.failedRuns.set(candidate.id, failures);
2207
+ this.markDone(failures >= 2 ? `parked:${candidate.id}` : `failed1:${candidate.id}`);
2208
+ if (this.contDone.has(`unparked:${candidate.id}`))
2209
+ this.markDone(`parked2:${candidate.id}`);
2210
+ else if (job.diagnosis?.cause === "no_commands")
2211
+ this.markDone(`parked-nc:${candidate.id}`);
2212
+ if (this.failedRuns.size > 200) {
2213
+ this.failedRuns.delete(this.failedRuns.keys().next().value);
2214
+ }
2215
+ if (failures >= 2 && job.diagnosis)
2216
+ await this.reportStuck(client, candidate, job.diagnosis, attempts);
1798
2217
  }
1799
- if (failures >= 2 && job.diagnosis)
1800
- await this.reportStuck(client, candidate, job.diagnosis, attempts);
1801
2218
  }
1802
- this.running = false;
1803
- if (fleetLock === this)
1804
- fleetLock = null;
2219
+ finally {
2220
+ this.running = false;
2221
+ this.endAttempt();
2222
+ this.giveSlot(slot.token);
2223
+ }
1805
2224
  this.writeState("watching", "waiting for work");
1806
2225
  if (this.opts.once)
1807
2226
  break;
1808
2227
  }
1809
- client.close();
1810
2228
  }
1811
2229
  stop() {
1812
2230
  this.stopped = true;
1813
2231
  for (const pid of this.runPids)
1814
2232
  killRunTree(pid);
1815
2233
  }
2234
+ takeSlot() {
2235
+ if (fleetLock && fleetLock !== this)
2236
+ return { held: `a run for "${fleetLock.cfg.space}" is in progress` };
2237
+ let lease;
2238
+ try {
2239
+ lease = takeRunLease(this.stateDir(), this.cfg.space);
2240
+ }
2241
+ catch (e) {
2242
+ const why = `${RUN_SLOT} cannot be read and written (${e.message.split("\n")[0]}) — repair or delete it`;
2243
+ if (!this.skipLogged.has(`slot:${why}`)) {
2244
+ this.skipLogged.add(`slot:${why}`);
2245
+ this.opts.log(`! holding unattended runs: ${why}`);
2246
+ }
2247
+ return { held: why };
2248
+ }
2249
+ if ("held" in lease)
2250
+ return { held: `a run for "${lease.held}" is in progress in another pantheon worker on this machine` };
2251
+ fleetLock = this;
2252
+ const dir = this.stateDir(), token = lease.token;
2253
+ this.leaseTimer = setInterval(() => renewRunLease(dir, token), this.opts.runLeaseRenewMs ?? RUN_LEASE_RENEW_MS);
2254
+ this.leaseTimer.unref?.();
2255
+ return lease;
2256
+ }
2257
+ leaseTimer = null;
2258
+ giveSlot(token) {
2259
+ if (this.leaseTimer) {
2260
+ clearInterval(this.leaseTimer);
2261
+ this.leaseTimer = null;
2262
+ }
2263
+ releaseRunLease(this.stateDir(), token);
2264
+ if (fleetLock === this)
2265
+ fleetLock = null;
2266
+ }
1816
2267
  runPids = new Set();
1817
- trackRun(pid) {
2268
+ trackRun(pid, attempt) {
1818
2269
  this.runPids.add(pid);
1819
2270
  liveRuns.add(pid);
1820
2271
  installRunReaper();
1821
- writeRunRecord(this.stateDir(), { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now() });
2272
+ this.attemptPid = pid;
2273
+ writeRunRecord(this.stateDir(), { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, obligation: attempt.obligation, ...(attempt.spend ? { spend: attempt.spend } : {}) });
1822
2274
  }
1823
2275
  untrackRun(pid) {
1824
2276
  this.runPids.delete(pid);
1825
2277
  liveRuns.delete(pid);
1826
- clearRunRecord(this.stateDir(), pid);
2278
+ }
2279
+ attemptPid = null;
2280
+ endAttempt() {
2281
+ if (this.attemptPid === null)
2282
+ return;
2283
+ clearRunRecord(this.stateDir(), this.attemptPid);
2284
+ this.attemptPid = null;
2285
+ }
2286
+ recoverInterruptedRun() {
2287
+ const dir = this.stateDir(), f = join(dir, RUN_RECORD);
2288
+ let r;
2289
+ try {
2290
+ r = JSON.parse(readFileSync(f, "utf8"));
2291
+ }
2292
+ catch {
2293
+ return;
2294
+ }
2295
+ if (!r || r.space !== this.cfg.space || r.member !== this.cfg.member || typeof r.obligation !== "string")
2296
+ return;
2297
+ if (!Number.isInteger(r.worker) || r.worker === process.pid || pidAlive(r.worker))
2298
+ return;
2299
+ if (Number.isInteger(r.pid) && typeof r.at === "number" && Date.now() - r.at <= RUN_TIMEOUT_MS + 5 * 60_000 && recordedRunAlive(r.pid))
2300
+ killRunTree(r.pid);
2301
+ try {
2302
+ rmSync(f, { force: true });
2303
+ }
2304
+ catch { }
2305
+ if (typeof r.spend === "string") {
2306
+ try {
2307
+ closeStartedRun(r.spend, "the worker went before the run ended", { at: r.at, goal: null, obligation: r.obligation, ok: false, triggeredBy: "unknown", summary: "the worker went before the run ended", space: this.spaceKey(), member: r.member }, this.opts.home);
2308
+ }
2309
+ catch { }
2310
+ }
2311
+ if (r.obligation.startsWith("cont:")) {
2312
+ this.opts.log(`! the last run (${r.obligation}) ended with its worker — counted, not run again`);
2313
+ return;
2314
+ }
2315
+ const failures = this.failuresOf(r.obligation) + 1;
2316
+ this.failedRuns.set(r.obligation, failures);
2317
+ this.markDone(failures >= 2 ? `parked:${r.obligation}` : `failed1:${r.obligation}`);
2318
+ this.opts.log(`! the last run for ${r.obligation} ended with its worker (stopped, restarted or killed mid-run) — counted as a failed attempt${failures >= 2 ? "; parked: it is not run again" : ""}`);
1827
2319
  }
1828
2320
  stateDir() {
1829
2321
  return this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
@@ -1841,7 +2333,10 @@ export class Worker {
1841
2333
  }
1842
2334
  catch { }
1843
2335
  }
1844
- brokenStateFiles() { return brokenStateFiles(this.stateDir()); }
2336
+ brokenStateFiles() {
2337
+ repairRunLease(this.stateDir());
2338
+ return brokenStateFiles(this.stateDir());
2339
+ }
1845
2340
  hooksInstalledHere() {
1846
2341
  try {
1847
2342
  return agentHookStates(this.opts.project).some((s) => s.installed);
@@ -1877,13 +2372,45 @@ export class Worker {
1877
2372
  const needs = workNeedsCommands("CONTINUE", trigger, act.act);
1878
2373
  return this.runJob(client, o, persona, policy, (branch, canPush) => buildContinuationBrief(act, trigger, branch, persona, canPush), false, needs, trigger, act);
1879
2374
  }
2375
+ spentOnce = {};
2376
+ policyBytes() { try {
2377
+ return readFileSync(policyPath(this.opts.home), "utf8");
2378
+ }
2379
+ catch {
2380
+ return null;
2381
+ } }
2382
+ restoreOnces() {
2383
+ const spent = this.spentOnce;
2384
+ this.spentOnce = {};
2385
+ if (!spent.cmd && !spent.wake?.length)
2386
+ return;
2387
+ try {
2388
+ if (this.policyBytes() !== spent.file) {
2389
+ this.opts.log(` grants not giving back the "just this once": your grants changed during the run`);
2390
+ return;
2391
+ }
2392
+ const p = loadPolicy(this.opts.home);
2393
+ if (spent.cmd && !p.grants.run_commands)
2394
+ p.grants.run_commands = spent.cmd;
2395
+ for (const k of spent.wake ?? [])
2396
+ (p.wakeOnce ??= {})[k] = Math.min(99, (p.wakeOnce[k] ?? 0) + 1);
2397
+ savePolicy(p, this.opts.home);
2398
+ this.opts.log(` grants the "just this once" spent on that run is given back for the next agent`);
2399
+ }
2400
+ catch (e) {
2401
+ this.opts.log(`! could not give back a "just this once" grant: ${e.message.split("\n")[0]}`);
2402
+ }
2403
+ }
1880
2404
  commandGrant(persona) {
1881
2405
  if (persona?.allowBash === false)
1882
2406
  return false;
1883
2407
  const policy = loadPolicy(this.opts.home);
2408
+ const was = policy.grants.run_commands;
1884
2409
  if (!useGrant(policy, "run_commands"))
1885
2410
  return false;
1886
2411
  savePolicy(policy, this.opts.home);
2412
+ if (was?.mode === "once")
2413
+ this.spentOnce = { ...this.spentOnce, cmd: was, file: this.policyBytes() };
1887
2414
  this.opts.log(` bash on for this run — your human granted run_commands`);
1888
2415
  return true;
1889
2416
  }
@@ -1928,7 +2455,7 @@ export class Worker {
1928
2455
  let choice;
1929
2456
  if (!this.opts.agentCmd) {
1930
2457
  try {
1931
- choice = (this.opts.choose ?? chooseAgent)(policy.agent ?? this.opts.agent, { prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: this.unsandboxedOk() });
2458
+ choice = (this.opts.choose ?? chooseAgent)(policy.agent ?? this.opts.agent, { prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: this.unsandboxedOk(), outages: activeOutages(this.outages) });
1932
2459
  }
1933
2460
  catch (e) {
1934
2461
  const detail = `could not plan the run: ${e.message}`;
@@ -1941,6 +2468,14 @@ export class Worker {
1941
2468
  this.opts.log(` agent ${said}`);
1942
2469
  this.lastAgentSaid = said;
1943
2470
  }
2471
+ if (choice.outage) {
2472
+ const diagnosis = this.outageDiagnosis(choice.backend.label, choice.outage, choice.skipped.filter((x) => x.backend.id !== choice.backend.id).map((x) => `${x.backend.label}: ${x.why}`));
2473
+ if (!this.skipLogged.has(`out:${o.id}:${choice.outage.until}`)) {
2474
+ this.skipLogged.add(`out:${o.id}:${choice.outage.until}`);
2475
+ this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${diagnosis.says} — it waits (not counted as a run)`);
2476
+ }
2477
+ return { obligation: o, branch, worktree, ok: false, refused: true, detail: diagnosis.says, diagnosis, waitUntil: choice.outage.until };
2478
+ }
1944
2479
  const refused = unattendedRefusal(choice, { unsandboxedOk: this.unsandboxedOk() });
1945
2480
  if (refused) {
1946
2481
  const noSandbox = !SANDBOXED_FOR_UNATTENDED.has(choice.backend.id) && !NOT_FOR_UNATTENDED_RUNS.has(choice.backend.id);
@@ -1957,7 +2492,7 @@ export class Worker {
1957
2492
  this.skipLogged.add(`refused:${o.id}`);
1958
2493
  this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${refused} — it waits for a live session (not counted as a run)`);
1959
2494
  }
1960
- return { obligation: o, branch, worktree, ok: false, refused: true, detail: refused, diagnosis };
2495
+ return { obligation: o, branch, worktree, ok: false, refused: true, detail: refused, diagnosis, ...(this.outagesPassed(choice).length ? { forOutage: true } : {}) };
1961
2496
  }
1962
2497
  }
1963
2498
  if (this.opts.allowBash && !policy.allowBash && persona) {
@@ -1989,13 +2524,14 @@ export class Worker {
1989
2524
  this.skipLogged.add(`refused:${o.id}`);
1990
2525
  this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${says}`);
1991
2526
  }
1992
- return { obligation: o, branch, worktree, ok: false, refused: true, detail: says, diagnosis };
2527
+ return { obligation: o, branch, worktree, ok: false, refused: true, detail: says, diagnosis, ...(choice && this.outagesPassed(choice).length ? { forOutage: true } : {}) };
1993
2528
  }
1994
2529
  const byCli = !!this.opts.agentCmd && !!this.opts.agentCmdFromCli;
1995
2530
  const noCommandsHere = features?.unknown ? "unknown"
1996
2531
  : features && ((choice?.backend.id === "claude-code" && !this.opts.agentCmd) || byCli) && !claudeSandboxable(features) && !this.unsandboxedOk() ? "unsandboxed"
1997
2532
  : null;
1998
2533
  const wantedBash = policy.allowBash || (persona?.allowBash !== false && !!loadPolicy(this.opts.home).grants.run_commands);
2534
+ this.spentOnce = {};
1999
2535
  let allowBash = noCommandsHere ? false : policy.allowBash || this.commandGrant(persona);
2000
2536
  const bashDroppedForSandbox = noCommandsHere === "unsandboxed" && wantedBash;
2001
2537
  let baseRef = "", baseSha = "", tipSha = "";
@@ -2004,11 +2540,21 @@ export class Worker {
2004
2540
  ({ branch, worktree, baseRef, baseSha, tipSha, fetched } = await prepareWorktreeAsync(project, scratch, goal, this.opts.log, branch, this.repoBase(place.fp)));
2005
2541
  }
2006
2542
  catch (e) {
2543
+ if (e instanceof WorktreeTampered)
2544
+ throw e;
2007
2545
  const detail = `could not prepare a worktree: ${e.message}`;
2008
2546
  this.opts.log(`! ${detail}`);
2009
2547
  return { obligation: o, branch, worktree, ok: false, detail };
2010
2548
  }
2549
+ const scan = outsideLinks(worktree);
2550
+ if (scan.links.length || scan.unreadable) {
2551
+ const diagnosis = outsideLinksDiagnosis(this.cfg.member, worktree, scan);
2552
+ this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${diagnosis.says}`);
2553
+ return { obligation: o, branch, worktree, ok: false, refused: true, detail: diagnosis.says, diagnosis };
2554
+ }
2011
2555
  await this.spendWakeOnce(client, o);
2556
+ if (choice)
2557
+ this.sayFallbackUnsandboxed(choice);
2012
2558
  if (noCommandsHere === "unknown" && wantedBash) {
2013
2559
  this.opts.log(` bash off for this run — \`${binary} --help\` did not answer, so its sandbox and deny rules cannot be confirmed`);
2014
2560
  }
@@ -2040,12 +2586,12 @@ export class Worker {
2040
2586
  }
2041
2587
  const multi = this.workspaceList().length > 1 || place.kind === "folder" || route.notHere.length > 0;
2042
2588
  const whereNote = multi ? workspaceNote(route, attached, tipSha || baseSha, place.kind === "folder" ? worktree : null) : "";
2043
- const reserved = reservedDecisionsNote(loadProfile(this.cfg));
2589
+ const reserved = reservedDecisionsNote(profileForTeam(this.cfg));
2044
2590
  const brief = makeBrief(branch, canPush) + whereNote + baseNote(baseRef, baseSha, fetched) +
2045
2591
  (allowBash ? (agentId === "claude-code" ? commandNote(worktree, tipSha || baseSha, (m) => this.opts.log(m)) : "") : holdRepoConfig ? heldNote() : "") +
2046
2592
  (reserved ? `\n\n${reserved}` : "");
2047
2593
  const startedAt = Date.now();
2048
- writeFileSync(join(worktree, ".pantheon-task.md"), brief);
2594
+ writeInside(worktree, join(worktree, ".pantheon-task.md"), brief, 0o644);
2049
2595
  let command;
2050
2596
  let args;
2051
2597
  let env = agentBaseEnv(process.env, proxyEnvFor());
@@ -2087,14 +2633,14 @@ export class Worker {
2087
2633
  backend, binary, prompt: brief, worktree, cfg: runCfg, branch, allowBash, runHome,
2088
2634
  ...(attached.length ? { readOnlyDirs: attached.map((a) => a.path) } : {}),
2089
2635
  ...(baseSha ? { base: baseSha } : {}), ...(tipSha ? { commandsRev: tipSha } : {}), goal,
2090
- ...(features ? { claude: { features, pantheonHome: this.stateDir(), home: personHome, userSettings: join(personHome, ".claude", "settings.json") } } : {}),
2636
+ ...(features ? { claude: { features, pantheonHome: this.stateDir(), home: personHome, userSettings: join(personHome, ".claude", "settings.json"), ...(() => { const c = adoptedCommon(worktree); return c ? { gitCommon: c } : {}; })() } } : {}),
2091
2637
  });
2092
2638
  for (const n of plan.notes ?? [])
2093
2639
  if (!this.notesSaid.has(n)) {
2094
2640
  this.notesSaid.add(n);
2095
2641
  this.opts.log(` confine ${n}`);
2096
2642
  }
2097
- materializeSpawnFiles(worktree, plan);
2643
+ materializeSpawnFiles(worktree, plan, runHome);
2098
2644
  for (const f of plan.writeFiles ?? []) {
2099
2645
  const rel = isAbsolute(f.path) ? relative(worktree, f.path) : f.path;
2100
2646
  if (!rel.startsWith("..") && !isAbsolute(rel))
@@ -2118,10 +2664,10 @@ export class Worker {
2118
2664
  }
2119
2665
  }
2120
2666
  catch (e) {
2121
- if (fenced)
2667
+ this.cleanupStep("putting the repo's agent config back", () => { if (fenced)
2122
2668
  for (const l of fenced.restore().lines)
2123
- this.opts.log(` config ${l}`);
2124
- this.sealRunHome(runHome);
2669
+ this.opts.log(` config ${l}`); });
2670
+ this.cleanupStep("sealing the run home", () => this.sealRunHome(runHome));
2125
2671
  const detail = `could not plan the run: ${e.message}`;
2126
2672
  this.opts.log(`! ${detail}`);
2127
2673
  recordRun({ at: Date.now(), goal: o.goal, obligation: o.id, ok: false, triggeredBy: (await this.actOf(client, o.ref))?.from ?? "unknown", summary: clip(o.body), space: this.spaceKey(), member: this.cfg.member }, this.opts.home);
@@ -2130,137 +2676,190 @@ export class Worker {
2130
2676
  let detail;
2131
2677
  let transcript = "";
2132
2678
  const homeBefore = snapshotHome(this.stateDir());
2679
+ let spendId = null;
2133
2680
  try {
2134
- detail = await new Promise((res) => {
2135
- let child;
2136
- try {
2137
- child = procSpawn(sp.command, sp.args, { cwd: worktree, stdio: ["ignore", "pipe", "pipe"], env, ...sp.options, detached: process.platform !== "win32" });
2138
- }
2139
- catch (e) {
2140
- res(`could not start "${command}": ${e.message}`);
2141
- return;
2142
- }
2143
- const pid = child.pid;
2144
- if (pid)
2145
- this.trackRun(pid);
2146
- let head = "", tail = "";
2147
- const take = (b) => {
2148
- const s = b.toString();
2149
- if (head.length < 16_000)
2150
- head += s.slice(0, 16_000 - head.length);
2151
- tail = (tail + s).slice(-16_000);
2152
- };
2153
- child.stdout.on("data", take);
2154
- child.stderr.on("data", take);
2155
- let timedOut = false;
2156
- let dead = false;
2157
- const markDead = () => { if (dead)
2158
- return; dead = true; if (pid)
2159
- this.untrackRun(pid); };
2160
- let settled = false;
2161
- let exitGrace = null;
2162
- let exitCode = null;
2163
- const limit = this.opts.runTimeoutMs ?? RUN_TIMEOUT_MS;
2164
- const finish = (code, signal) => {
2165
- if (settled)
2681
+ spendId = startRun({ at: Date.now(), goal: o.goal, obligation: o.id, ok: false, triggeredBy: client.recentFrom(o.ref) ?? "unknown", summary: clip(o.body), space: this.spaceKey(), member: this.cfg.member }, this.opts.home);
2682
+ }
2683
+ catch (e) {
2684
+ this.opts.log(`! could not record the run as started: ${e.message.split("\n")[0]}`);
2685
+ }
2686
+ let recorded = false;
2687
+ try {
2688
+ try {
2689
+ detail = await new Promise((res) => {
2690
+ let child;
2691
+ try {
2692
+ child = procSpawn(sp.command, sp.args, { cwd: worktree, stdio: ["ignore", "pipe", "pipe"], env, ...sp.options, detached: process.platform !== "win32" });
2693
+ }
2694
+ catch (e) {
2695
+ res(`could not start "${command}": ${e.message}`);
2166
2696
  return;
2167
- settled = true;
2168
- clearTimeout(kill);
2169
- if (exitGrace)
2170
- clearTimeout(exitGrace);
2171
- markDead();
2172
- transcript = `${head}\n${tail}`;
2173
- res(`exit ${code ?? signal}${timedOut ? ` (stopped after ${Math.round(limit / 1000)}s)` : ""}\n${tail.slice(-4000)}`);
2174
- };
2175
- const kill = setTimeout(() => { if (dead)
2176
- return; timedOut = true; if (pid)
2177
- killRunTree(pid); }, limit);
2178
- child.on("exit", (code, signal) => {
2179
- clearTimeout(kill);
2180
- if (pid && !dead && killTreeOnExit())
2181
- killRunTree(pid);
2182
- markDead();
2183
- const c = child;
2184
- exitCode = { code, signal };
2185
- exitGrace = setTimeout(() => { try {
2186
- c.stdout.destroy();
2187
- c.stderr.destroy();
2188
2697
  }
2189
- catch { } finish(code, signal); }, this.opts.exitGraceMs ?? EXIT_GRACE_MS);
2698
+ const pid = child.pid;
2699
+ if (pid)
2700
+ this.trackRun(pid, { obligation: o.id, spend: spendId });
2701
+ let head = "", tail = "";
2702
+ const take = (b) => {
2703
+ const s = b.toString();
2704
+ if (head.length < 16_000)
2705
+ head += s.slice(0, 16_000 - head.length);
2706
+ tail = (tail + s).slice(-16_000);
2707
+ };
2708
+ child.stdout.on("data", take);
2709
+ child.stderr.on("data", take);
2710
+ let timedOut = false;
2711
+ let dead = false;
2712
+ const markDead = () => { if (dead)
2713
+ return; dead = true; if (pid)
2714
+ this.untrackRun(pid); };
2715
+ let settled = false;
2716
+ let exitGrace = null;
2717
+ let exitCode = null;
2718
+ const limit = this.opts.runTimeoutMs ?? RUN_TIMEOUT_MS;
2719
+ const finish = (code, signal) => {
2720
+ if (settled)
2721
+ return;
2722
+ settled = true;
2723
+ clearTimeout(kill);
2724
+ if (exitGrace)
2725
+ clearTimeout(exitGrace);
2726
+ markDead();
2727
+ transcript = `${head}\n${tail}`;
2728
+ res(`exit ${code ?? signal}${timedOut ? ` (stopped after ${Math.round(limit / 1000)}s)` : ""}\n${tail.slice(-4000)}`);
2729
+ };
2730
+ const kill = setTimeout(() => { if (dead)
2731
+ return; timedOut = true; if (pid)
2732
+ killRunTree(pid); }, limit);
2733
+ child.on("exit", (code, signal) => {
2734
+ clearTimeout(kill);
2735
+ if (pid && !dead && killTreeOnExit())
2736
+ killRunTree(pid);
2737
+ markDead();
2738
+ const c = child;
2739
+ exitCode = { code, signal };
2740
+ exitGrace = setTimeout(() => { try {
2741
+ c.stdout.destroy();
2742
+ c.stderr.destroy();
2743
+ }
2744
+ catch { } finish(code, signal); }, this.opts.exitGraceMs ?? EXIT_GRACE_MS);
2745
+ });
2746
+ child.on("close", (code, signal) => finish(exitCode?.code ?? code, exitCode?.signal ?? signal));
2747
+ child.on("error", (e) => { if (settled)
2748
+ return; settled = true; clearTimeout(kill); if (exitGrace)
2749
+ clearTimeout(exitGrace); markDead(); res(`could not start "${command}": ${e.message}`); });
2190
2750
  });
2191
- child.on("close", (code, signal) => finish(exitCode?.code ?? code, exitCode?.signal ?? signal));
2192
- child.on("error", (e) => { if (settled)
2193
- return; settled = true; clearTimeout(kill); if (exitGrace)
2194
- clearTimeout(exitGrace); markDead(); res(`could not start "${command}": ${e.message}`); });
2195
- });
2196
- }
2197
- finally {
2198
- if (fenced) {
2199
- const r = fenced.restore();
2200
- for (const l of r.lines)
2201
- this.opts.log(` config ${l}`);
2202
2751
  }
2203
- this.sealRunHome(runHome);
2204
- this.restoreHomeAfterRun(homeBefore, o);
2205
- }
2206
- const runLog = join(worktree, ".pantheon-run.log");
2207
- let extra = "";
2208
- let massDelete = false;
2209
- if (allowBash && baseSha) {
2210
- const d = deletedSinceBase(worktree, baseSha);
2211
- if (d && d.total >= 4 && d.gone > d.total * MASS_DELETE_SHARE) {
2212
- massDelete = true;
2213
- extra = `\n! this branch deletes ${d.gone} of the ${d.total} files tracked at its base — do not merge it without looking`;
2214
- this.opts.log(`! ${o.id}: the branch ${branch} deletes ${d.gone} of the ${d.total} files tracked at ${baseSha.slice(0, 7)} — do not merge it without looking`);
2215
- }
2216
- }
2217
- writeFileSync(runLog, `obligation ${o.id}\nbranch ${branch}\n\n${detail}\n${extra}`);
2218
- const exited = /^exit 0/.test(detail);
2219
- const sent = client.mineSince(startedAt, o.goal);
2220
- const repliedWith = sent.find((a) => ["DELIVERED", "ANSWER", "ESCALATE", "ACCEPT", "OBJECT", "PROPOSE", "CONTRACT", "CLOSE", "WITHDRAW"].includes(a.act) || (a.act === "NOTE" && !!a.ref));
2221
- const replied = requireReply ? client.repliedTo(o) || !!repliedWith : true;
2222
- if (requireReply && replied && !client.repliedTo(o) && repliedWith) {
2223
- this.opts.log(` reply ${repliedWith.act} ${repliedWith.id} answers ${o.id}, though the relay still lists it as owed${repliedWith.ref ? "" : " (the act named no ref)"}`);
2224
- }
2225
- const ok = exited && replied;
2226
- await this.checkDeliveries(client, sent, { branch, worktree, canPush: canPush && !massDelete, goal: o.goal ?? null });
2227
- this.runProject = null;
2228
- if (place.kind === "folder") {
2229
- let drafts = [];
2230
- try {
2231
- drafts = readdirSync(worktree).filter((f) => !f.startsWith(".") && f !== "AGENTS.md" && f !== "CLAUDE.md");
2752
+ finally {
2753
+ this.cleanupStep("putting the repo's agent config back", () => { if (fenced)
2754
+ for (const l of fenced.restore().lines)
2755
+ this.opts.log(` config ${l}`); });
2756
+ this.cleanupStep("sealing the run home", () => this.sealRunHome(runHome));
2757
+ this.cleanupStep("checking Pantheon's own files", () => this.restoreHomeAfterRun(homeBefore, o));
2758
+ }
2759
+ const runLog = join(worktree, ".pantheon-run.log");
2760
+ let extra = "";
2761
+ let massDelete = false;
2762
+ if (allowBash && baseSha) {
2763
+ const d = deletedSinceBase(worktree, baseSha);
2764
+ if (d.unreadable) {
2765
+ massDelete = true;
2766
+ extra = `\n! could not check how many files this branch deletes (${d.unreadable}) — it is not pushed from here; look before merging`;
2767
+ this.opts.log(`! ${o.id}: could not check the branch ${branch} for a mass deletion (${d.unreadable}) — not pushing it`);
2768
+ }
2769
+ else if (d.total >= 4 && d.gone > d.total * MASS_DELETE_SHARE) {
2770
+ massDelete = true;
2771
+ extra = `\n! this branch deletes ${d.gone} of the ${d.total} files tracked at its base — do not merge it without looking`;
2772
+ this.opts.log(`! ${o.id}: the branch ${branch} deletes ${d.gone} of the ${d.total} files tracked at ${baseSha.slice(0, 7)} — do not merge it without looking`);
2773
+ }
2232
2774
  }
2233
- catch { }
2234
- if (drafts.length) {
2235
- this.opts.log(` drafts ${drafts.length} file${drafts.length === 1 ? "" : "s"} for ${place.name} in ${worktree}: ${drafts.slice(0, 5).join(", ")}${drafts.length > 5 ? ", …" : ""}`);
2236
- if (this.opts.notify) {
2237
- try {
2238
- this.opts.notify({ kind: "worker", who: "Your worker", heading: `Drafts for ${place.name}`, context: this.cfg.space, body: `A request from ${(await this.actOf(client, o.ref))?.from ?? "a teammate"} left ${drafts.length} draft file${drafts.length === 1 ? "" : "s"} for you to review in ${worktree}. Nothing in ${place.path} was changed.`, ttl: 0, sound: false });
2775
+ this.writeRunLog(worktree, runLog, `obligation ${o.id}\nbranch ${branch}\n\n${detail}\n${extra}`);
2776
+ const exited = /^exit 0/.test(detail);
2777
+ const sent = client.mineSince(startedAt, o.goal);
2778
+ const repliedWith = sent.find((a) => ["DELIVERED", "ANSWER", "ESCALATE", "ACCEPT", "OBJECT", "PROPOSE", "CONTRACT", "CLOSE", "WITHDRAW"].includes(a.act) || (a.act === "NOTE" && !!a.ref));
2779
+ const replied = requireReply ? client.repliedTo(o) || !!repliedWith : true;
2780
+ if (requireReply && replied && !client.repliedTo(o) && repliedWith) {
2781
+ this.opts.log(` reply ${repliedWith.act} ${repliedWith.id} answers ${o.id}, though the relay still lists it as owed${repliedWith.ref ? "" : " (the act named no ref)"}`);
2782
+ }
2783
+ const ok = exited && replied;
2784
+ await this.checkDeliveries(client, sent, { branch, worktree, canPush: canPush && !massDelete, goal: o.goal ?? null });
2785
+ this.runProject = null;
2786
+ if (place.kind === "folder") {
2787
+ let drafts = [];
2788
+ try {
2789
+ drafts = readdirSync(worktree).filter((f) => !f.startsWith(".") && f !== "AGENTS.md" && f !== "CLAUDE.md");
2790
+ }
2791
+ catch { }
2792
+ if (drafts.length) {
2793
+ this.opts.log(` drafts ${drafts.length} file${drafts.length === 1 ? "" : "s"} for ${place.name} in ${worktree}: ${drafts.slice(0, 5).join(", ")}${drafts.length > 5 ? ", …" : ""}`);
2794
+ if (this.opts.notify) {
2795
+ try {
2796
+ this.opts.notify({ kind: "worker", who: "Your worker", heading: `Drafts for ${place.name}`, context: this.cfg.space, body: `A request from ${(await this.actOf(client, o.ref))?.from ?? "a teammate"} left ${drafts.length} draft file${drafts.length === 1 ? "" : "s"} for you to review in ${worktree}. Nothing in ${place.path} was changed.`, ttl: 0, sound: false });
2797
+ }
2798
+ catch { }
2239
2799
  }
2240
- catch { }
2241
2800
  }
2242
2801
  }
2802
+ const triggeredBy = (await this.actOf(client, o.ref))?.from ?? "unknown";
2803
+ const summary = clip(o.body);
2804
+ const end = { at: Date.now(), goal: o.goal, obligation: o.id, ok, triggeredBy, summary, space: this.spaceKey(), member: this.cfg.member };
2805
+ if (spendId)
2806
+ finishRun(spendId, end, this.opts.home);
2807
+ else
2808
+ recordRun(end, this.opts.home);
2809
+ recorded = true;
2810
+ const runKeys = client.features.includes("delegate");
2811
+ const sentAnywhere = client.mineSince(startedAt).some((a) => !runKeys || !a.sig || a.sig.role === "run");
2812
+ const outage = !ok && !exited && !sentAnywhere && choice && !/\(stopped after \d+s\)/.test(detail.split("\n")[0] ?? "") ? detectOutage(choice.backend.id, detail) : null;
2813
+ if (outage) {
2814
+ this.outages.set(choice.backend.id, outage);
2815
+ const refusedLog = join(worktree, `.pantheon-run.${choice.backend.id}-refused.log`);
2816
+ this.writeRunLog(worktree, refusedLog, `obligation ${o.id}\nbranch ${branch}\nagent ${choice.backend.label}\n\n${detail}\n`);
2817
+ if (Date.now() - startedAt < ONCE_GIVE_BACK_MS)
2818
+ this.restoreOnces();
2819
+ }
2820
+ if (outage && (this.outageRetries.get(o.id) ?? 0) < 3) {
2821
+ this.outageRetries.set(o.id, (this.outageRetries.get(o.id) ?? 0) + 1);
2822
+ if (this.outageRetries.size > 200)
2823
+ this.outageRetries.delete(this.outageRetries.keys().next().value);
2824
+ this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over until about ${whenAgain(outage.until)}; the request goes to the next agent here (transcript: ${join(worktree, `.pantheon-run.${choice.backend.id}-refused.log`)})`);
2825
+ return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
2826
+ }
2827
+ const quiet = ok && !requireReply && sent.length === 0;
2828
+ const diagnosis = ok ? undefined : outage ? this.outageDiagnosis(choice.backend.label, outage, [], "it was handed to another agent three times already") : diagnoseRun({ exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands, ...(bashDroppedForSandbox ? { commandsNeedSandbox: byCli ? "agent-cmd" : "claude" } : {}), ...(agentId ? { agent: agentId } : {}), ...(place.kind === "folder" ? { folder: true } : {}),
2829
+ ...(agentId === "claude-code" && allowBash ? { allowedRules: bashRules } : {}), ...(baseRef ? { baseRef } : {}) }, this.opts.probe ?? gitProbe);
2830
+ if (this.opts.notify && !quiet && (!ok || loadProfile(this.cfg).notifications?.worker_done)) {
2831
+ try {
2832
+ const reply = sent[sent.length - 1] ?? null;
2833
+ this.opts.notify(composeWorker({ ok, triggeredBy, obligationBody: o.body, goal: o.goal ? client.goals.get(o.goal) : undefined, reply, obligationRef: o.ref, ...(diagnosis ? { diagnosis } : {}) }, (() => { const tools = installedTools(); return { tools, preferred: preferredTool(loadProfile(this.cfg), tools, this.opts.home) }; })()));
2834
+ }
2835
+ catch { }
2836
+ }
2837
+ this.opts.log(quiet ? `< read ${o.id} on ${branch} — the thread was already settled, nothing sent`
2838
+ : ok ? `< handled ${o.id} on ${branch}`
2839
+ : `! ${o.id} stopped [${diagnosis.cause}]: ${diagnosis.says} (transcript: ${runLog})`);
2840
+ return { obligation: o, branch, worktree, ok, detail, ...(diagnosis ? { diagnosis } : {}) };
2243
2841
  }
2244
- const triggeredBy = (await this.actOf(client, o.ref))?.from ?? "unknown";
2245
- const summary = clip(o.body);
2246
- recordRun({
2247
- at: Date.now(), goal: o.goal, obligation: o.id, ok,
2248
- triggeredBy, summary, space: this.spaceKey(), member: this.cfg.member,
2249
- }, this.opts.home);
2250
- const quiet = ok && !requireReply && sent.length === 0;
2251
- const diagnosis = ok ? undefined : diagnoseRun({ exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands, ...(bashDroppedForSandbox ? { commandsNeedSandbox: byCli ? "agent-cmd" : "claude" } : {}), ...(agentId ? { agent: agentId } : {}), ...(place.kind === "folder" ? { folder: true } : {}),
2252
- ...(agentId === "claude-code" && allowBash ? { allowedRules: bashRules } : {}), ...(baseRef ? { baseRef } : {}) }, this.opts.probe ?? gitProbe);
2253
- if (this.opts.notify && !quiet && (!ok || loadProfile(this.cfg).notifications?.worker_done)) {
2254
- try {
2255
- const reply = sent[sent.length - 1] ?? null;
2256
- this.opts.notify(composeWorker({ ok, triggeredBy, obligationBody: o.body, goal: o.goal ? client.goals.get(o.goal) : undefined, reply, obligationRef: o.ref, ...(diagnosis ? { diagnosis } : {}) }, (() => { const tools = installedTools(); return { tools, preferred: preferredTool(loadProfile(this.cfg), tools, this.opts.home) }; })()));
2842
+ catch (e) {
2843
+ if (!recorded) {
2844
+ try {
2845
+ const end = { at: Date.now(), goal: o.goal, obligation: o.id, ok: false, triggeredBy: client.recentFrom(o.ref) ?? "unknown", summary: clip(o.body), space: this.spaceKey(), member: this.cfg.member };
2846
+ if (spendId)
2847
+ finishRun(spendId, end, this.opts.home);
2848
+ else
2849
+ recordRun(end, this.opts.home);
2850
+ }
2851
+ catch { }
2257
2852
  }
2258
- catch { }
2853
+ throw e;
2854
+ }
2855
+ }
2856
+ writeRunLog(worktree, f, text) {
2857
+ try {
2858
+ writeInside(worktree, f, text, 0o644);
2859
+ }
2860
+ catch (e) {
2861
+ this.opts.log(`! could not write the run log ${f}: ${e.message.split("\n")[0]}`);
2259
2862
  }
2260
- this.opts.log(quiet ? `< read ${o.id} on ${branch} — the thread was already settled, nothing sent`
2261
- : ok ? `< handled ${o.id} on ${branch}`
2262
- : `! ${o.id} stopped [${diagnosis.cause}]: ${diagnosis.says} (transcript: ${runLog})`);
2263
- return { obligation: o, branch, worktree, ok, detail, ...(diagnosis ? { diagnosis } : {}) };
2264
2863
  }
2265
2864
  lastFeaturesSaid = "";
2266
2865
  notesSaid = new Set();
@@ -2312,7 +2911,7 @@ export class Worker {
2312
2911
  const r = await gitOut(run.worktree, ["push", "--quiet", "origin", `refs/heads/${br}:refs/heads/${br}`], FETCH_TIMEOUT_MS);
2313
2912
  pushed = r !== null && await onRemote(run.worktree, br, commit);
2314
2913
  if (pushed)
2315
- this.opts.log(`* pushed ${br} for the run: its DELIVERED ${d.id} named ${commit.slice(0, 7)}, which origin did not have`);
2914
+ this.opts.log(`* pushed ${br} for the run: its DELIVERED ${d.id} named ${commit.slice(0, 7)}, which origin did not have — repo hooks not run on Pantheon's push${lfsInUse(run.worktree) ? "; LFS objects uploaded" : ""}`);
2316
2915
  }
2317
2916
  if (pushed)
2318
2917
  continue;
@@ -2375,7 +2974,8 @@ export class Worker {
2375
2974
  ].join("\n"),
2376
2975
  });
2377
2976
  markContinued(space, key, this.opts.home);
2378
- this.opts.log(`* told ${peer ?? "the teammate"} that ${o.id} waits for a live session: ${d.cause === "repo_not_here" ? d.says : "no agent here can take unattended work"}`);
2977
+ this.opts.log(d.cause === "agent_unavailable" ? `* told ${peer ?? "the teammate"} that ${o.id} waits: ${d.says}`
2978
+ : `* told ${peer ?? "the teammate"} that ${o.id} waits for a live session: ${d.cause === "repo_not_here" ? d.says : "no agent here can take unattended work"}`);
2379
2979
  }
2380
2980
  catch (e) {
2381
2981
  this.opts.log(`! could not tell the teammate that ${o.id} is waiting: ${e.message}`);
@@ -2390,7 +2990,8 @@ export class Worker {
2390
2990
  const asking = !!d.capability && mayAsk(policy, d.capability);
2391
2991
  const what = clip(o.body, 120);
2392
2992
  const lines = [
2393
- `${attempts === 1 ? "A run" : `${attempts === 2 ? "Two" : attempts} runs`} on this machine could not finish ${o.kind === "CONTINUE" ? "a follow-up" : `the ${o.kind}`} it owes: ${what}`,
2993
+ attempts === 0 ? `No run was started on this machine for ${o.kind === "CONTINUE" ? "a follow-up" : `the ${o.kind}`} it owes: ${what}`
2994
+ : `${attempts === 1 ? "A run" : `${attempts === 2 ? "Two" : attempts} runs`} on this machine could not finish ${o.kind === "CONTINUE" ? "a follow-up" : `the ${o.kind}`} it owes: ${what}`,
2394
2995
  ``,
2395
2996
  `WHY IT STOPPED. ${d.says}`,
2396
2997
  `WHAT UNBLOCKS IT. ${d.fix}`,
@@ -2542,39 +3143,48 @@ export class Worker {
2542
3143
  const lastAttempt = policy.attempts.self_upgrade ?? null;
2543
3144
  if (!needsSelfUpgrade(VERSION, relayPkg, lastAttempt))
2544
3145
  return;
2545
- if (useGrant(policy, "self_upgrade")) {
2546
- policy.attempts.self_upgrade = Date.now();
2547
- savePolicy(policy, home);
2548
- this.writeState("upgrading", `pantheon ${VERSION} -> ${relayPkg} (relay is ahead)`);
2549
- this.opts.log(`* self-upgrade: relay runs ${relayPkg}, this machine runs ${VERSION} — upgrading (granted by this machine's human)`);
2550
- let result;
3146
+ if (policy.grants.self_upgrade) {
3147
+ const slot = this.takeSlot();
3148
+ if ("held" in slot)
3149
+ return;
2551
3150
  try {
2552
- result = this.opts.selfUpgrade?.run ? this.opts.selfUpgrade.run() : await this.runRealUpgrade(relayPkg ?? undefined);
2553
- }
2554
- catch (e) {
2555
- result = { ok: false, alreadyCurrent: false, from: VERSION, to: relayPkg ?? "?" };
2556
- this.opts.log(`! self-upgrade failed: ${e.message}`);
2557
- }
2558
- if (result.ok && result.to === result.from) {
2559
- delete policy.attempts.self_upgrade;
3151
+ useGrant(policy, "self_upgrade");
3152
+ policy.attempts.self_upgrade = Date.now();
2560
3153
  savePolicy(policy, home);
2561
- this.opts.log(`* self-upgrade: nothing newer than ${result.to} to install yet (the relay runs ${relayPkg}); will check again`);
3154
+ this.writeState("upgrading", `pantheon ${VERSION} -> ${relayPkg} (relay is ahead)`);
3155
+ this.opts.log(`* self-upgrade: relay runs ${relayPkg}, this machine runs ${VERSION} — upgrading (granted by this machine's human)`);
3156
+ let result;
3157
+ try {
3158
+ result = this.opts.selfUpgrade?.run ? this.opts.selfUpgrade.run() : await this.runRealUpgrade(relayPkg ?? undefined);
3159
+ }
3160
+ catch (e) {
3161
+ result = { ok: false, alreadyCurrent: false, from: VERSION, to: relayPkg ?? "?" };
3162
+ this.opts.log(`! self-upgrade failed: ${e.message}`);
3163
+ }
3164
+ if (result.ok && result.to === result.from) {
3165
+ delete policy.attempts.self_upgrade;
3166
+ savePolicy(policy, home);
3167
+ this.opts.log(`* self-upgrade: nothing newer than ${result.to} to install yet (the relay runs ${relayPkg}); will check again`);
3168
+ return;
3169
+ }
3170
+ if (result.ok && !result.alreadyCurrent) {
3171
+ await client.send({
3172
+ act: "NOTE", goal: null, after_seq: Math.max(client.readSeq, client.seq),
3173
+ summary: `self-upgraded pantheon ${result.from} -> ${result.to}; worker restarting on the new build`,
3174
+ body: `This machine's worker upgraded pantheon ${result.from} -> ${result.to} (fast-forward pull + rebuild, granted by its human) and is restarting on the new build. Agent windows on this machine opened on 0.24 or later switch to the new version by themselves; a window opened on an older pantheon needs one reconnect (/mcp → pantheon → Reconnect, or reopen it).`,
3175
+ }).catch((e) => this.opts.log(`! could not report the upgrade on the ledger: ${e.message}`));
3176
+ await this.bringMachineAlong();
3177
+ this.opts.log(`* self-upgrade done: ${result.from} -> ${result.to}; restarting worker to load it`);
3178
+ (this.opts.selfUpgrade?.restart ?? (() => process.exit(0)))();
3179
+ }
3180
+ else if (!result.ok) {
3181
+ this.opts.log("! self-upgrade did not complete — will not retry for an hour; run `pantheon upgrade` by hand to see why");
3182
+ }
2562
3183
  return;
2563
3184
  }
2564
- if (result.ok && !result.alreadyCurrent) {
2565
- await client.send({
2566
- act: "NOTE", goal: null, after_seq: Math.max(client.readSeq, client.seq),
2567
- summary: `self-upgraded pantheon ${result.from} -> ${result.to}; worker restarting on the new build`,
2568
- body: `This machine's worker upgraded pantheon ${result.from} -> ${result.to} (fast-forward pull + rebuild, granted by its human) and is restarting on the new build. Agent windows on this machine opened on 0.24 or later switch to the new version by themselves; a window opened on an older pantheon needs one reconnect (/mcp → pantheon → Reconnect, or reopen it).`,
2569
- }).catch((e) => this.opts.log(`! could not report the upgrade on the ledger: ${e.message}`));
2570
- await this.bringMachineAlong();
2571
- this.opts.log(`* self-upgrade done: ${result.from} -> ${result.to}; restarting worker to load it`);
2572
- (this.opts.selfUpgrade?.restart ?? (() => process.exit(0)))();
2573
- }
2574
- else if (!result.ok) {
2575
- this.opts.log("! self-upgrade did not complete — will not retry for an hour; run `pantheon upgrade` by hand to see why");
3185
+ finally {
3186
+ this.giveSlot(slot.token);
2576
3187
  }
2577
- return;
2578
3188
  }
2579
3189
  if (mayAsk(policy, "self_upgrade")) {
2580
3190
  try {