@join-pantheon/cli 0.31.0 → 0.31.2-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/agent-outage.js +87 -0
- package/dist/agents.js +59 -20
- package/dist/claude-confine.js +21 -4
- package/dist/cli.js +260 -57
- package/dist/client.js +3 -3
- package/dist/config.js +1 -0
- package/dist/dbus.js +418 -0
- package/dist/decision-ui.js +89 -0
- package/dist/doctor.js +3 -3
- package/dist/email.js +21 -0
- package/dist/file-lock.js +3 -3
- package/dist/git-safe.js +274 -0
- package/dist/hooks-install.js +6 -1
- package/dist/install.js +21 -0
- package/dist/kiro-files.js +20 -4
- package/dist/mcp-instructions.js +2 -2
- package/dist/nonces.js +5 -3
- package/dist/notice.js +2 -1
- package/dist/notifier.js +36 -1
- package/dist/onboard-defaults.js +18 -0
- package/dist/persona.js +28 -2
- package/dist/plans.js +12 -3
- package/dist/profile.js +23 -5
- package/dist/relay.js +61 -16
- package/dist/roster-file.js +3 -0
- package/dist/run-failure.js +2 -1
- package/dist/safe-write.js +55 -0
- package/dist/setup-guide.js +9 -1
- package/dist/spend.js +65 -5
- package/dist/tray.js +53 -6
- package/dist/worker.js +890 -280
- package/helper/macos/menubar.swift +375 -112
- package/package.json +1 -1
package/dist/worker.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { spawn as procSpawn } from "node:child_process";
|
|
2
2
|
import { execFileSync } from "node:child_process";
|
|
3
|
-
import { chmodSync, mkdirSync, writeFileSync, existsSync, readdirSync, readFileSync, realpathSync, rmSync } from "node:fs";
|
|
3
|
+
import { chmodSync, lstatSync, mkdirSync, writeFileSync, existsSync, readdirSync, readFileSync, readlinkSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
6
6
|
import { detectClaudeFeatures, describeFeatures, ensureRunTmp, NO_CLAUDE_FEATURES, runTmpDir } from "./claude-confine.js";
|
|
@@ -8,19 +8,22 @@ import { snapshotHome, restoreHome, describeChange, trustWrite, askedHere } from
|
|
|
8
8
|
import { PantheonClient } from "./client.js";
|
|
9
9
|
import { createHash, randomBytes } from "node:crypto";
|
|
10
10
|
import { newKeypair, signDelegate } from "./identity.js";
|
|
11
|
-
import { loadPins, pinScope, savePins, teamPinId } from "./pins.js";
|
|
11
|
+
import { loadPins, pinScope, savePins, teamPinId, writeWhole } from "./pins.js";
|
|
12
12
|
import { answeredIn, markAnswered, noteNonce } from "./nonces.js";
|
|
13
|
-
import { brokenStateFiles, brokenStateFix } from "./file-lock.js";
|
|
13
|
+
import { brokenStateFiles, brokenStateFix, pidAlive, readStateFile, withFileLock } from "./file-lock.js";
|
|
14
|
+
import { mkdirInside, writeInside } from "./safe-write.js";
|
|
15
|
+
import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
|
|
14
16
|
import { trusted } from "./verify-act.js";
|
|
15
|
-
import { headlessSpawn, materializeSpawnFiles, chooseAgent, describeChoice, unattendedRefusal, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
17
|
+
import { headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
18
|
+
import { activeOutages, detectOutage, earliest, whenAgain } from "./agent-outage.js";
|
|
16
19
|
import { agentSpawn, CmdLineUnsafe } from "./which.js";
|
|
17
20
|
import { VERSION } from "./version.js";
|
|
18
|
-
import { applyAnswer, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, } from "./worker-policy.js";
|
|
21
|
+
import { applyAnswer, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, } from "./worker-policy.js";
|
|
19
22
|
import { agentHookStates, repairHook } from "./notify.js";
|
|
20
23
|
import { fence, assess } from "./untrusted.js";
|
|
21
|
-
import { allowed, recordRun } from "./spend.js";
|
|
24
|
+
import { allowed, closeStartedRun, finishRun, recordRun, startRun } from "./spend.js";
|
|
22
25
|
import { handledIn, loadContinued, markContinued, unmarkContinued } from "./continued.js";
|
|
23
|
-
import { loadProfile, reservedDecisionsNote, saveProfile, wakeMode } from "./profile.js";
|
|
26
|
+
import { loadProfile, profileForTeam, reservedDecisionsNote, saveProfile, wakeMode } from "./profile.js";
|
|
24
27
|
import { clip } from "./brief.js";
|
|
25
28
|
import { composeWorker } from "./notice.js";
|
|
26
29
|
import { diagnoseRun, gitProbe, workNeedsCommands } from "./run-failure.js";
|
|
@@ -57,7 +60,7 @@ function configuredSshCommand(cwd) {
|
|
|
57
60
|
return hit;
|
|
58
61
|
let set = false;
|
|
59
62
|
try {
|
|
60
|
-
set = execFileSync("git", ["-C", cwd, "config", "--get", "core.sshCommand"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 10_000 }).trim().length > 0;
|
|
63
|
+
set = execFileSync("git", safeGitArgs(cwd, ["-C", cwd, "config", "--get", "core.sshCommand"]), { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 10_000 }).trim().length > 0;
|
|
61
64
|
}
|
|
62
65
|
catch { }
|
|
63
66
|
sshCommandSeen.set(cwd, set);
|
|
@@ -74,7 +77,7 @@ export function agentBaseEnv(base, proxy) {
|
|
|
74
77
|
const GIT_TIMEOUT_MS = 60_000;
|
|
75
78
|
export const WORKTREE_ADD_TIMEOUT_MS = 15 * 60_000;
|
|
76
79
|
export const FETCH_TIMEOUT_MS = 90_000;
|
|
77
|
-
const git = (cwd, ...args) => execFileSync("git", args, { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: gitEnv(process.env, proxyEnvFor(), cwd), timeout: GIT_TIMEOUT_MS, killSignal: "SIGKILL" }).trim();
|
|
80
|
+
const git = (cwd, ...args) => execFileSync("git", safeGitArgs(cwd, args), { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: gitCallEnv(args, gitEnv(process.env, proxyEnvFor(), cwd)), timeout: GIT_TIMEOUT_MS, killSignal: "SIGKILL" }).trim();
|
|
78
81
|
const gitVersions = new Map();
|
|
79
82
|
export function gitVersion() {
|
|
80
83
|
const key = process.env.PATH ?? "";
|
|
@@ -97,7 +100,7 @@ export function fetchOrigin(project, timeoutMs = FETCH_TIMEOUT_MS) {
|
|
|
97
100
|
return new Promise((res) => {
|
|
98
101
|
let child;
|
|
99
102
|
try {
|
|
100
|
-
child = procSpawn("git", ["fetch", "--quiet", "origin"], { cwd: project, env: gitEnv(process.env, proxyEnvFor(), project), stdio: ["ignore", "ignore", "pipe"] });
|
|
103
|
+
child = procSpawn("git", safeGitArgs(project, ["fetch", "--quiet", "origin"]), { cwd: project, env: gitEnv(process.env, proxyEnvFor(), project), stdio: ["ignore", "ignore", "pipe"] });
|
|
101
104
|
}
|
|
102
105
|
catch (e) {
|
|
103
106
|
res({ ok: false, why: e.message });
|
|
@@ -113,7 +116,7 @@ export function fetchOrigin(project, timeoutMs = FETCH_TIMEOUT_MS) {
|
|
|
113
116
|
}
|
|
114
117
|
function fetchOriginSync(project) {
|
|
115
118
|
try {
|
|
116
|
-
execFileSync("git", ["fetch", "--quiet", "origin"], { cwd: project, stdio: ["ignore", "ignore", "pipe"], env: gitEnv(process.env, proxyEnvFor(), project), timeout: FETCH_TIMEOUT_MS, killSignal: "SIGKILL" });
|
|
119
|
+
execFileSync("git", safeGitArgs(project, ["fetch", "--quiet", "origin"]), { cwd: project, stdio: ["ignore", "ignore", "pipe"], env: gitEnv(process.env, proxyEnvFor(), project), timeout: FETCH_TIMEOUT_MS, killSignal: "SIGKILL" });
|
|
117
120
|
return { ok: true };
|
|
118
121
|
}
|
|
119
122
|
catch (e) {
|
|
@@ -133,7 +136,157 @@ const isRepo = (cwd) => {
|
|
|
133
136
|
return false;
|
|
134
137
|
}
|
|
135
138
|
};
|
|
139
|
+
export const worktreeLabel = (wt) => `${basename(dirname(wt))}/${basename(wt)}`;
|
|
140
|
+
const LINK_SCAN_BUDGET = 1_000_000;
|
|
141
|
+
const GIT_SCAN_MAXBUFFER = 1 << 28;
|
|
142
|
+
const gitScan = (cwd, args) => execFileSync("git", safeGitArgs(cwd, args), { cwd, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: gitCallEnv(args, gitEnv(process.env, proxyEnvFor(), cwd)), timeout: GIT_TIMEOUT_MS, killSignal: "SIGKILL", maxBuffer: GIT_SCAN_MAXBUFFER });
|
|
143
|
+
const ENV_DIRS = new Set([".venv", "venv", "env", ".tox", "__pypackages__", ".direnv"]);
|
|
144
|
+
const AGENT_FILE = /^(CLAUDE.*\.md|AGENTS?.*\.md|GEMINI\.md|CONVENTIONS\.md|\.mcp\.json|\.cursorrules|\.windsurfrules|\.clinerules)$/i;
|
|
145
|
+
const AGENT_DIRS = new Set([".claude", ".cursor", ".github", ".kiro", ".codex", ".vscode", ".agents", ".gemini", ".junie", ".amazonq", ".continue", ".clinerules", ".windsurf"]);
|
|
146
|
+
function inEnvDir(rel) {
|
|
147
|
+
const parts = rel.split("/");
|
|
148
|
+
const base = parts[parts.length - 1];
|
|
149
|
+
if (AGENT_FILE.test(base) || parts.some((x) => AGENT_DIRS.has(x)))
|
|
150
|
+
return false;
|
|
151
|
+
const dirs = parts.slice(0, -1);
|
|
152
|
+
if (dirs.some((x) => ENV_DIRS.has(x)))
|
|
153
|
+
return true;
|
|
154
|
+
const nm = dirs.indexOf("node_modules");
|
|
155
|
+
return nm >= 0 && dirs.slice(nm + 1).includes(".bin");
|
|
156
|
+
}
|
|
157
|
+
function executableFile(abs) {
|
|
158
|
+
try {
|
|
159
|
+
const st = statSync(abs);
|
|
160
|
+
return st.isFile() && (st.mode & 0o111) !== 0;
|
|
161
|
+
}
|
|
162
|
+
catch {
|
|
163
|
+
return false;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
export function outsideLinks(worktree, budgetMax = LINK_SCAN_BUDGET) {
|
|
167
|
+
const root = (() => { try {
|
|
168
|
+
return realpathSync.native(worktree);
|
|
169
|
+
}
|
|
170
|
+
catch {
|
|
171
|
+
return null;
|
|
172
|
+
} })();
|
|
173
|
+
if (!root)
|
|
174
|
+
return { links: [], unreadable: null };
|
|
175
|
+
const inside = (p) => p === root || p.startsWith(root + sep);
|
|
176
|
+
const candidates = new Map();
|
|
177
|
+
let repo = false;
|
|
178
|
+
try {
|
|
179
|
+
repo = git(worktree, "rev-parse", "--is-inside-work-tree") === "true";
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
repo = false;
|
|
183
|
+
}
|
|
184
|
+
if (repo) {
|
|
185
|
+
let tracked;
|
|
186
|
+
try {
|
|
187
|
+
tracked = gitScan(worktree, ["ls-files", "-s", "-z"]);
|
|
188
|
+
}
|
|
189
|
+
catch (e) {
|
|
190
|
+
return { links: [], unreadable: `the repo's tracked files could not be listed (${e.message.split("\n")[0]})` };
|
|
191
|
+
}
|
|
192
|
+
for (const e of tracked.split("\0")) {
|
|
193
|
+
const m = /^120000 ([0-9a-f]+) \d+\t([\s\S]+)$/.exec(e);
|
|
194
|
+
if (m)
|
|
195
|
+
candidates.set(m[2], m[1]);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
let budget = budgetMax;
|
|
199
|
+
const walk = (rel) => {
|
|
200
|
+
let ents;
|
|
201
|
+
try {
|
|
202
|
+
ents = readdirSync(join(root, rel), { withFileTypes: true });
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
return null;
|
|
206
|
+
}
|
|
207
|
+
for (const d of ents) {
|
|
208
|
+
if (--budget < 0)
|
|
209
|
+
return `the checkout has more files than can be checked for links (over ${budgetMax})`;
|
|
210
|
+
if (d.name === ".git")
|
|
211
|
+
continue;
|
|
212
|
+
const r = rel ? `${rel}/${d.name}` : d.name;
|
|
213
|
+
if (d.isSymbolicLink()) {
|
|
214
|
+
if (!candidates.has(r))
|
|
215
|
+
candidates.set(r, null);
|
|
216
|
+
}
|
|
217
|
+
else if (d.isDirectory()) {
|
|
218
|
+
const why = walk(r);
|
|
219
|
+
if (why)
|
|
220
|
+
return why;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
return null;
|
|
224
|
+
};
|
|
225
|
+
const why = walk("");
|
|
226
|
+
if (why)
|
|
227
|
+
return { links: [], unreadable: why };
|
|
228
|
+
const out = [];
|
|
229
|
+
for (const [rel, sha] of candidates) {
|
|
230
|
+
const p = join(root, rel);
|
|
231
|
+
let target = null;
|
|
232
|
+
try {
|
|
233
|
+
target = realpathSync.native(p);
|
|
234
|
+
}
|
|
235
|
+
catch {
|
|
236
|
+
try {
|
|
237
|
+
const dir = realpathSync.native(dirname(p));
|
|
238
|
+
target = resolve(dir, readlinkSync(p));
|
|
239
|
+
}
|
|
240
|
+
catch {
|
|
241
|
+
if (sha) {
|
|
242
|
+
try {
|
|
243
|
+
target = resolve(dirname(p), gitScan(worktree, ["cat-file", "blob", sha]).replace(/\n$/, ""));
|
|
244
|
+
}
|
|
245
|
+
catch { }
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
if (target === null || inside(target))
|
|
250
|
+
continue;
|
|
251
|
+
if (inEnvDir(rel) && executableFile(target))
|
|
252
|
+
continue;
|
|
253
|
+
out.push(rel);
|
|
254
|
+
}
|
|
255
|
+
return { links: out.sort(), unreadable: null };
|
|
256
|
+
}
|
|
257
|
+
export function outsideLinksDiagnosis(member, worktree, scan) {
|
|
258
|
+
const names = scan.links.slice(0, 8).join(", ") + (scan.links.length > 8 ? `, and ${scan.links.length - 8} more` : "");
|
|
259
|
+
const where = worktreeLabel(worktree);
|
|
260
|
+
const says = scan.unreadable
|
|
261
|
+
? `the checkout for this could not be fully checked for links leading out of it (${scan.unreadable}) — a run could read a file outside it as the repo's own, so none was started`
|
|
262
|
+
: `the checkout holds links that lead outside it (${names}) — a run would read whatever they point at as the repo's own, and could send it on, so none was started`;
|
|
263
|
+
return {
|
|
264
|
+
cause: "outside_links",
|
|
265
|
+
says,
|
|
266
|
+
fix: scan.unreadable
|
|
267
|
+
? `Look at this machine's worktree ${where}: a very large or unreadable checkout cannot be vouched for. Take this request in a live session, or clear it out (pantheon stops at that worktree; its branch and commits are kept in the repo).`
|
|
268
|
+
: `On this machine, delete these links from the worktree ${where} — not from the repo's branch, where they are not: ${scan.links.map((l) => `${where}/${l}`).join(", ")}. Or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it.`,
|
|
269
|
+
peer: {
|
|
270
|
+
summary: `${member}'s worker did not start a run: its checkout for this could not be vouched for`,
|
|
271
|
+
body: `${member}'s machine could not confirm its checkout for this holds no link leading outside it, so its worker did not start an unattended run: a run would read whatever such a link points at. It is still owed and waits for ${member}. Nothing was run.`,
|
|
272
|
+
},
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
export function tamperedDiagnosis(member, e) {
|
|
276
|
+
return {
|
|
277
|
+
cause: "worktree_tampered",
|
|
278
|
+
says: `the worktree's .git was changed after Pantheon made it (${e.why}) — git would obey that gitdir's config outside any sandbox, so git is not run in it`,
|
|
279
|
+
fix: /hooks folder/.test(e.why)
|
|
280
|
+
? `Look at the repo's own git hooks folder (.git/hooks in your checkout) and remove anything you did not put there; the next request checks out a fresh worktree.`
|
|
281
|
+
: `Look at what is in that worktree, then remove its folder and run git worktree prune in the repo; the next request checks out a fresh one.`,
|
|
282
|
+
peer: {
|
|
283
|
+
summary: `${member}'s worker stopped: its checkout for this was changed underneath it`,
|
|
284
|
+
body: `The checkout ${member}'s worker uses for this was changed in a way it does not run git in, so it stopped. It is still owed and waits for ${member}.`,
|
|
285
|
+
},
|
|
286
|
+
};
|
|
287
|
+
}
|
|
136
288
|
const CHECKOUT_MARK = "pantheon-checkout-complete";
|
|
289
|
+
function writeMark(mark) { writeInside(dirname(mark), mark, `${new Date().toISOString()}\n`, 0o644); }
|
|
137
290
|
function checkoutMark(worktree) {
|
|
138
291
|
try {
|
|
139
292
|
return join(git(worktree, "rev-parse", "--absolute-git-dir"), CHECKOUT_MARK);
|
|
@@ -142,9 +295,23 @@ function checkoutMark(worktree) {
|
|
|
142
295
|
return null;
|
|
143
296
|
}
|
|
144
297
|
}
|
|
298
|
+
const isLinkAt = (p) => { try {
|
|
299
|
+
return lstatSync(p).isSymbolicLink();
|
|
300
|
+
}
|
|
301
|
+
catch {
|
|
302
|
+
return false;
|
|
303
|
+
} };
|
|
145
304
|
export function settleWorktree(project, worktree, log) {
|
|
146
|
-
if (!existsSync(join(worktree, ".git")))
|
|
305
|
+
if (!existsSync(join(worktree, ".git")) && !isLinkAt(join(worktree, ".git")))
|
|
147
306
|
return false;
|
|
307
|
+
const adopt = adoptWorktree(project, worktree);
|
|
308
|
+
if (adopt?.kind === "tampered")
|
|
309
|
+
throw new WorktreeTampered(worktree, adopt.why);
|
|
310
|
+
if (adopt?.kind === "stale") {
|
|
311
|
+
log(` ! the worktree ${worktree} no longer points at this repo (${adopt.why}) — removing it and checking out again`);
|
|
312
|
+
removeWorktree(project, worktree);
|
|
313
|
+
return false;
|
|
314
|
+
}
|
|
148
315
|
const mark = checkoutMark(worktree);
|
|
149
316
|
if (mark && existsSync(mark))
|
|
150
317
|
return true;
|
|
@@ -159,7 +326,7 @@ export function settleWorktree(project, worktree, log) {
|
|
|
159
326
|
}
|
|
160
327
|
if (whole && mark) {
|
|
161
328
|
try {
|
|
162
|
-
|
|
329
|
+
writeMark(mark);
|
|
163
330
|
}
|
|
164
331
|
catch { }
|
|
165
332
|
return true;
|
|
@@ -242,11 +409,14 @@ function planWorktree(project, scratch, goal, log, branch, base, fetch) {
|
|
|
242
409
|
export function worktreeAddArgs(add, os = process.platform) {
|
|
243
410
|
return os === "win32" ? ["-c", "core.longpaths=true", ...add] : add;
|
|
244
411
|
}
|
|
245
|
-
function finishWorktree(r, log) {
|
|
412
|
+
function finishWorktree(project, r, log) {
|
|
413
|
+
const adopt = adoptWorktree(project, r.worktree);
|
|
414
|
+
if (adopt)
|
|
415
|
+
throw new WorktreeTampered(r.worktree, adopt.why);
|
|
246
416
|
const mark = checkoutMark(r.worktree);
|
|
247
417
|
if (mark)
|
|
248
|
-
|
|
249
|
-
excludeRunFiles(r.worktree);
|
|
418
|
+
writeMark(mark);
|
|
419
|
+
excludeRunFiles(r.worktree, log);
|
|
250
420
|
log(` worktree ${r.worktree} on ${r.branch} (from ${r.baseRef} ${r.baseSha.slice(0, 7)})`);
|
|
251
421
|
return r;
|
|
252
422
|
}
|
|
@@ -259,13 +429,13 @@ export function prepareWorktree(project, scratch, goal, log, branch = `pantheon/
|
|
|
259
429
|
if ("done" in p)
|
|
260
430
|
return p.done;
|
|
261
431
|
try {
|
|
262
|
-
execFileSync("git", p.add, { cwd: project, stdio: ["ignore", "ignore", "pipe"], env: gitEnv(process.env, proxyEnvFor(), project), timeout: opts.addTimeoutMs ?? WORKTREE_ADD_TIMEOUT_MS, killSignal: "SIGKILL" });
|
|
432
|
+
execFileSync("git", safeGitArgs(project, p.add), { cwd: project, stdio: ["ignore", "ignore", "pipe"], env: gitCallEnv(p.add, gitEnv(process.env, proxyEnvFor(), project)), timeout: opts.addTimeoutMs ?? WORKTREE_ADD_TIMEOUT_MS, killSignal: "SIGKILL" });
|
|
263
433
|
}
|
|
264
434
|
catch (e) {
|
|
265
435
|
const err = e;
|
|
266
436
|
throw addFailed(project, p.result.worktree, err.killed || err.signal ? `stopped after ${Math.round((opts.addTimeoutMs ?? WORKTREE_ADD_TIMEOUT_MS) / 1000)}s` : (String(err.stderr ?? "").trim().split("\n")[0] || ((err.message ?? "failed").split("\n")[0] ?? "failed")));
|
|
267
437
|
}
|
|
268
|
-
return finishWorktree(p.result, log);
|
|
438
|
+
return finishWorktree(project, p.result, log);
|
|
269
439
|
}
|
|
270
440
|
export async function prepareWorktreeAsync(project, scratch, goal, log, branch = `pantheon/${goal}`, base = null, opts = {}) {
|
|
271
441
|
const inRepo = isRepo(project) && safeSegment(goal) !== null;
|
|
@@ -280,7 +450,7 @@ export async function prepareWorktreeAsync(project, scratch, goal, log, branch =
|
|
|
280
450
|
const why = await new Promise((res) => {
|
|
281
451
|
let child;
|
|
282
452
|
try {
|
|
283
|
-
child = procSpawn("git", p.add, { cwd: project, env: gitEnv(process.env, proxyEnvFor(), project), stdio: ["ignore", "ignore", "pipe"] });
|
|
453
|
+
child = procSpawn("git", safeGitArgs(project, p.add), { cwd: project, env: gitCallEnv(p.add, gitEnv(process.env, proxyEnvFor(), project)), stdio: ["ignore", "ignore", "pipe"] });
|
|
284
454
|
}
|
|
285
455
|
catch (e) {
|
|
286
456
|
res(e.message);
|
|
@@ -295,7 +465,7 @@ export async function prepareWorktreeAsync(project, scratch, goal, log, branch =
|
|
|
295
465
|
});
|
|
296
466
|
if (why !== null)
|
|
297
467
|
throw addFailed(project, p.result.worktree, why);
|
|
298
|
-
return finishWorktree(p.result, log);
|
|
468
|
+
return finishWorktree(project, p.result, log);
|
|
299
469
|
}
|
|
300
470
|
export function branchName(space, goalId, title) {
|
|
301
471
|
const slug = (title ?? "").toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 40).replace(/-+$/, "");
|
|
@@ -327,17 +497,47 @@ YOUR BRANCH STARTS AT ${baseSha.slice(0, 12)} (${baseRef}, ${fresh}).
|
|
|
327
497
|
const RUN_DROPPINGS = [".pantheon-task.md", ".pantheon-run.log", ".pantheon-mcp.json", ".pantheon/*", "!.pantheon/commands", "!.pantheon/personas/",
|
|
328
498
|
`*${AGENT_EDIT_SUFFIX}`, `*${AGENT_EDIT_SUFFIX}-*`];
|
|
329
499
|
const RUN_DROPPINGS_OLD = [".pantheon/"];
|
|
330
|
-
|
|
500
|
+
const plainFile = (p) => { try {
|
|
501
|
+
const st = lstatSync(p);
|
|
502
|
+
return st.isFile() && st.nlink === 1;
|
|
503
|
+
}
|
|
504
|
+
catch {
|
|
505
|
+
return false;
|
|
506
|
+
} };
|
|
507
|
+
const excludeSaid = new Set();
|
|
508
|
+
export function excludeRunFiles(worktree, log) {
|
|
331
509
|
try {
|
|
332
510
|
const common = git(worktree, "rev-parse", "--git-common-dir");
|
|
333
|
-
const
|
|
334
|
-
const
|
|
511
|
+
const root = resolve(worktree, common);
|
|
512
|
+
const info = join(root, "info");
|
|
513
|
+
const file = join(info, "exclude");
|
|
514
|
+
const infoOk = (() => { try {
|
|
515
|
+
const st = lstatSync(info);
|
|
516
|
+
return st.isDirectory();
|
|
517
|
+
}
|
|
518
|
+
catch {
|
|
519
|
+
return !existsSync(info);
|
|
520
|
+
} })();
|
|
521
|
+
const fileOk = (() => { try {
|
|
522
|
+
lstatSync(file);
|
|
523
|
+
return plainFile(file);
|
|
524
|
+
}
|
|
525
|
+
catch {
|
|
526
|
+
return true;
|
|
527
|
+
} })();
|
|
528
|
+
if (!infoOk || !fileOk) {
|
|
529
|
+
if (log && !excludeSaid.has(file)) {
|
|
530
|
+
excludeSaid.add(file);
|
|
531
|
+
log(` exclude ${file} is not a plain file in a plain folder (a link, or a hard link) — left as it is; the run's own files may show as untracked in this repo`);
|
|
532
|
+
}
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
const had = plainFile(file) ? readFileSync(file, "utf8") : "";
|
|
335
536
|
const cur = had.split("\n").filter((l) => !RUN_DROPPINGS_OLD.includes(l)).join("\n");
|
|
336
537
|
const missing = RUN_DROPPINGS.filter((l) => !cur.split("\n").includes(l));
|
|
337
538
|
if (!missing.length && cur === had)
|
|
338
539
|
return;
|
|
339
|
-
|
|
340
|
-
writeFileSync(file, missing.length ? `${cur.replace(/\n?$/, "\n")}${missing.join("\n")}\n` : cur);
|
|
540
|
+
writeInside(root, file, missing.length ? `${cur.replace(/\n?$/, "\n")}${missing.join("\n")}\n` : cur, 0o644);
|
|
341
541
|
}
|
|
342
542
|
catch { }
|
|
343
543
|
}
|
|
@@ -384,10 +584,11 @@ export function workspaceNote(route, attached, sha, outbox) {
|
|
|
384
584
|
return lines.join("\n");
|
|
385
585
|
}
|
|
386
586
|
export function gitOut(cwd, args, timeoutMs = GIT_TIMEOUT_MS) {
|
|
587
|
+
const argv = safeGitArgs(cwd, args);
|
|
387
588
|
return new Promise((res) => {
|
|
388
589
|
let child;
|
|
389
590
|
try {
|
|
390
|
-
child = procSpawn("git",
|
|
591
|
+
child = procSpawn("git", argv, { cwd, env: gitCallEnv(args, gitEnv(process.env, proxyEnvFor(), cwd)), stdio: ["ignore", "pipe", "ignore"] });
|
|
391
592
|
}
|
|
392
593
|
catch {
|
|
393
594
|
res(null);
|
|
@@ -413,6 +614,7 @@ export async function onRemote(cwd, branch, commit) {
|
|
|
413
614
|
return (await gitOut(cwd, ["merge-base", "--is-ancestor", commit, tip])) !== null;
|
|
414
615
|
}
|
|
415
616
|
export const MASS_DELETE_SHARE = 0.5;
|
|
617
|
+
const ONCE_GIVE_BACK_MS = 120_000;
|
|
416
618
|
export const asksForMassDelete = (text) => /\b(?:delete|remove|wipe|drop|rm)\b[^\n]{0,40}\b(?:all|every|entire|whole|most)\b/i.test(text);
|
|
417
619
|
const GIT_HOOK_NAMES = [
|
|
418
620
|
"applypatch-msg", "pre-applypatch", "post-applypatch", "prepare-commit-msg", "commit-msg", "post-commit",
|
|
@@ -466,10 +668,10 @@ exit 0
|
|
|
466
668
|
`;
|
|
467
669
|
export function massDeleteGuardEnv(runHome, worktree, allow, env = process.env) {
|
|
468
670
|
const dir = join(runHome, "git-hooks");
|
|
469
|
-
|
|
671
|
+
mkdirInside(runHome, dir);
|
|
470
672
|
for (const [name, body] of [["pre-commit", PRE_COMMIT], ["pre-push", PRE_PUSH], ...GIT_HOOK_NAMES.map((n) => [n, passThrough(n)])]) {
|
|
471
673
|
const f = join(dir, name);
|
|
472
|
-
|
|
674
|
+
writeInside(runHome, f, body, 0o755);
|
|
473
675
|
try {
|
|
474
676
|
chmodSync(f, 0o755);
|
|
475
677
|
}
|
|
@@ -477,9 +679,13 @@ export function massDeleteGuardEnv(runHome, worktree, allow, env = process.env)
|
|
|
477
679
|
}
|
|
478
680
|
let orig = "";
|
|
479
681
|
try {
|
|
480
|
-
|
|
682
|
+
const out = execFileSync("git", repoHooksArgs(worktree), { cwd: worktree, encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], env: gitCallEnv(["rev-parse"], gitEnv(process.env, proxyEnvFor(), worktree)), timeout: GIT_TIMEOUT_MS }).trim();
|
|
683
|
+
orig = resolve(worktree, out);
|
|
684
|
+
}
|
|
685
|
+
catch (e) {
|
|
686
|
+
if (e instanceof WorktreeTampered)
|
|
687
|
+
throw e;
|
|
481
688
|
}
|
|
482
|
-
catch { }
|
|
483
689
|
const n = Number.parseInt(env.GIT_CONFIG_COUNT ?? "0", 10) || 0;
|
|
484
690
|
return {
|
|
485
691
|
GIT_CONFIG_COUNT: String(n + 1),
|
|
@@ -491,12 +697,14 @@ export function massDeleteGuardEnv(runHome, worktree, allow, env = process.env)
|
|
|
491
697
|
}
|
|
492
698
|
export function deletedSinceBase(worktree, baseSha) {
|
|
493
699
|
try {
|
|
494
|
-
const total =
|
|
495
|
-
const gone =
|
|
496
|
-
return { gone, total };
|
|
700
|
+
const total = gitScan(worktree, ["ls-tree", "-r", "--name-only", baseSha]).split("\n").filter(Boolean).length;
|
|
701
|
+
const gone = gitScan(worktree, ["diff", "--name-only", "--diff-filter=D", baseSha]).split("\n").filter(Boolean).length;
|
|
702
|
+
return { gone, total, unreadable: null };
|
|
497
703
|
}
|
|
498
|
-
catch {
|
|
499
|
-
|
|
704
|
+
catch (e) {
|
|
705
|
+
if (e instanceof WorktreeTampered)
|
|
706
|
+
throw e;
|
|
707
|
+
return { gone: 0, total: 0, unreadable: e.message.split("\n")[0] || "git failed" };
|
|
500
708
|
}
|
|
501
709
|
}
|
|
502
710
|
export const printable = (s) => typeof s === "string" && /^[A-Za-z0-9_][A-Za-z0-9_:.\-]{0,119}$/.test(s) ? s : "(an id this worker will not print)";
|
|
@@ -597,6 +805,70 @@ Act, reply if a reply is owed, then finish.`;
|
|
|
597
805
|
export const TASK_POINTER = "Your task for this run is in the file .pantheon-task.md in the current directory. Read that file first and follow it: it is the whole brief.";
|
|
598
806
|
let fleetLock = null;
|
|
599
807
|
export const RUN_TIMEOUT_MS = 20 * 60_000;
|
|
808
|
+
export const RUN_SLOT = "run-slot.json";
|
|
809
|
+
export const RUN_LEASE_RENEW_MS = 60_000;
|
|
810
|
+
const RUN_LEASE_FRESH_MS = 10 * 60_000;
|
|
811
|
+
function readRunLease(f) {
|
|
812
|
+
let text;
|
|
813
|
+
try {
|
|
814
|
+
text = readStateFile(f);
|
|
815
|
+
}
|
|
816
|
+
catch {
|
|
817
|
+
rmSync(f, { recursive: true, force: true });
|
|
818
|
+
return null;
|
|
819
|
+
}
|
|
820
|
+
if (text === null)
|
|
821
|
+
return null;
|
|
822
|
+
try {
|
|
823
|
+
const v = JSON.parse(text);
|
|
824
|
+
if (v && typeof v === "object")
|
|
825
|
+
return v;
|
|
826
|
+
}
|
|
827
|
+
catch { }
|
|
828
|
+
rmSync(f, { recursive: true, force: true });
|
|
829
|
+
return null;
|
|
830
|
+
}
|
|
831
|
+
export function takeRunLease(dir, space, now = Date.now()) {
|
|
832
|
+
const f = join(dir, RUN_SLOT);
|
|
833
|
+
return withFileLock(f, () => {
|
|
834
|
+
const cur = readRunLease(f);
|
|
835
|
+
if (cur && Number.isInteger(cur.pid) && cur.pid !== process.pid && typeof cur.at === "number" && Number.isFinite(cur.at)
|
|
836
|
+
&& Math.abs(now - cur.at) <= RUN_LEASE_FRESH_MS && pidAlive(cur.pid)) {
|
|
837
|
+
return { held: typeof cur.space === "string" ? cur.space : "another team" };
|
|
838
|
+
}
|
|
839
|
+
const token = randomBytes(12).toString("hex");
|
|
840
|
+
writeWhole(f, JSON.stringify({ pid: process.pid, token, space, at: now }) + "\n");
|
|
841
|
+
return { token };
|
|
842
|
+
});
|
|
843
|
+
}
|
|
844
|
+
export function renewRunLease(dir, token, now = Date.now()) {
|
|
845
|
+
const f = join(dir, RUN_SLOT);
|
|
846
|
+
try {
|
|
847
|
+
withFileLock(f, () => {
|
|
848
|
+
const cur = readRunLease(f);
|
|
849
|
+
if (cur?.token === token)
|
|
850
|
+
writeWhole(f, JSON.stringify({ ...cur, at: now }) + "\n");
|
|
851
|
+
});
|
|
852
|
+
}
|
|
853
|
+
catch { }
|
|
854
|
+
}
|
|
855
|
+
export function releaseRunLease(dir, token) {
|
|
856
|
+
const f = join(dir, RUN_SLOT);
|
|
857
|
+
try {
|
|
858
|
+
withFileLock(f, () => { if (readRunLease(f)?.token === token)
|
|
859
|
+
rmSync(f, { force: true }); });
|
|
860
|
+
}
|
|
861
|
+
catch { }
|
|
862
|
+
}
|
|
863
|
+
export function repairRunLease(dir) {
|
|
864
|
+
const f = join(dir, RUN_SLOT);
|
|
865
|
+
if (!existsSync(f))
|
|
866
|
+
return;
|
|
867
|
+
try {
|
|
868
|
+
withFileLock(f, () => { readRunLease(f); });
|
|
869
|
+
}
|
|
870
|
+
catch { }
|
|
871
|
+
}
|
|
600
872
|
export function killRunTree(pid, os = process.platform) {
|
|
601
873
|
if (!Number.isInteger(pid) || pid <= 1)
|
|
602
874
|
return;
|
|
@@ -782,6 +1054,10 @@ export class Worker {
|
|
|
782
1054
|
}
|
|
783
1055
|
parkedNotHere = new Set();
|
|
784
1056
|
parkedNoSandbox = new Map();
|
|
1057
|
+
outages = new Map();
|
|
1058
|
+
outageHold = new Map();
|
|
1059
|
+
outageHeldUntil = new Map();
|
|
1060
|
+
outageRetries = new Map();
|
|
785
1061
|
parkedNoSandboxConts = new Map();
|
|
786
1062
|
ranNoSandboxCheckedAt = 0;
|
|
787
1063
|
noSandboxDiagnosis(choice) {
|
|
@@ -809,14 +1085,16 @@ export class Worker {
|
|
|
809
1085
|
if (this.opts.agentCmd)
|
|
810
1086
|
return null;
|
|
811
1087
|
const ok = this.unsandboxedOk();
|
|
812
|
-
const key = `${agent ?? "auto"}|${ok}`;
|
|
1088
|
+
const key = `${agent ?? "auto"}|${ok}|${Object.entries(activeOutages(this.outages)).map(([k, v]) => `${k}@${v.until}`).join(",")}`;
|
|
813
1089
|
const hit = this.sandboxHold.get(key);
|
|
814
1090
|
if (hit && Date.now() - hit.at < 60_000)
|
|
815
1091
|
return hit.d;
|
|
816
1092
|
let d = null;
|
|
817
1093
|
try {
|
|
818
|
-
const c = (this.opts.choose ?? chooseAgent)(agent, { prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: ok });
|
|
819
|
-
if (
|
|
1094
|
+
const c = (this.opts.choose ?? chooseAgent)(agent, { prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: ok, outages: activeOutages(this.outages) });
|
|
1095
|
+
if (c.outage)
|
|
1096
|
+
d = this.outageDiagnosis(c.backend.label, c.outage, []);
|
|
1097
|
+
else if (NOT_FOR_UNATTENDED_RUNS.has(c.backend.id))
|
|
820
1098
|
d = this.noUnattendedAgentDiagnosis(unattendedRefusal(c, { unsandboxedOk: ok }) ?? c.backend.label);
|
|
821
1099
|
else if (!SANDBOXED_FOR_UNATTENDED.has(c.backend.id) && !ok)
|
|
822
1100
|
d = this.noSandboxDiagnosis(c);
|
|
@@ -842,6 +1120,45 @@ export class Worker {
|
|
|
842
1120
|
return false;
|
|
843
1121
|
}
|
|
844
1122
|
}
|
|
1123
|
+
outageDiagnosis(label, out, others, meanwhile) {
|
|
1124
|
+
const at = whenAgain(out.until);
|
|
1125
|
+
const rest = meanwhile ? `; the agent standing in for it cannot take this one (${meanwhile.replace(/\.$/, "")})`
|
|
1126
|
+
: others.length ? `; ${others.join("; ")}` : "; no other agent is installed for runs while you are away";
|
|
1127
|
+
return {
|
|
1128
|
+
cause: "agent_unavailable",
|
|
1129
|
+
says: `${label} is unavailable here until about ${at} — ${out.why}${rest}.`,
|
|
1130
|
+
fix: `It runs by itself after ${at}. Sooner: sign in to another agent (cursor-agent, codex or copilot) — and for an agent without a sandbox, pantheon policy grant unsandboxed-runs — or answer it in a live session.`,
|
|
1131
|
+
peer: {
|
|
1132
|
+
summary: `${this.cfg.member}'s agent is unavailable until about ${at} — this waits`,
|
|
1133
|
+
body: `${this.cfg.member}'s worker could not start a run: ${label} ${out.why}, and no other agent on that machine could take it. It is still owed; the worker tries again after ${at}.`,
|
|
1134
|
+
},
|
|
1135
|
+
};
|
|
1136
|
+
}
|
|
1137
|
+
outagesPassed(c) {
|
|
1138
|
+
const now = activeOutages(this.outages);
|
|
1139
|
+
return c.skipped.flatMap((x) => { const out = now[x.backend.id]; return out ? [{ label: x.backend.label, out }] : []; });
|
|
1140
|
+
}
|
|
1141
|
+
fallbackSaid = new Set();
|
|
1142
|
+
sayFallbackUnsandboxed(c) {
|
|
1143
|
+
if (SANDBOXED_FOR_UNATTENDED.has(c.backend.id) || !this.opts.notify)
|
|
1144
|
+
return;
|
|
1145
|
+
const passed = this.outagesPassed(c).filter((p) => p.label === "Claude Code");
|
|
1146
|
+
const first = passed[0];
|
|
1147
|
+
if (!first)
|
|
1148
|
+
return;
|
|
1149
|
+
const key = `${c.backend.id}|${first.out.until}`;
|
|
1150
|
+
if (this.fallbackSaid.has(key))
|
|
1151
|
+
return;
|
|
1152
|
+
this.fallbackSaid.add(key);
|
|
1153
|
+
try {
|
|
1154
|
+
this.opts.notify({
|
|
1155
|
+
kind: "worker", who: "Your worker", heading: `${c.backend.label} is answering for Claude Code`, context: this.cfg.space,
|
|
1156
|
+
body: `Claude Code is unavailable until about ${whenAgain(first.out.until)} — ${first.out.why}. Until then ${c.backend.label} takes teammates' requests on this machine, without Claude Code's sandbox, as your settings allow. To have them wait instead: pantheon policy revoke unsandboxed-runs`,
|
|
1157
|
+
ttl: 0, sound: false,
|
|
1158
|
+
});
|
|
1159
|
+
}
|
|
1160
|
+
catch { }
|
|
1161
|
+
}
|
|
845
1162
|
claudeSandboxableNow() {
|
|
846
1163
|
if (this.opts.agentCmd)
|
|
847
1164
|
return false;
|
|
@@ -983,6 +1300,12 @@ export class Worker {
|
|
|
983
1300
|
}
|
|
984
1301
|
if (!ok && now - h.at < every)
|
|
985
1302
|
continue;
|
|
1303
|
+
const ou = this.outageHeldUntil.get(id);
|
|
1304
|
+
if (ou !== undefined) {
|
|
1305
|
+
if (ou > now)
|
|
1306
|
+
continue;
|
|
1307
|
+
this.outageHeldUntil.delete(id);
|
|
1308
|
+
}
|
|
986
1309
|
if (this.heldForSandbox(h.agent)) {
|
|
987
1310
|
h.at = now;
|
|
988
1311
|
continue;
|
|
@@ -1208,13 +1531,18 @@ export class Worker {
|
|
|
1208
1531
|
const seen = noteNonce(this.pinTeam, act.from, sig.nonce, act.id, this.opts.home, sig.s);
|
|
1209
1532
|
if (this.opts.home !== undefined) {
|
|
1210
1533
|
const also = noteNonce(this.pinTeam, act.from, sig.nonce, act.id, undefined, sig.s);
|
|
1534
|
+
const forged = [seen, also].some((x) => x.replayOf === act.id && !x.unvouched);
|
|
1535
|
+
const unvouched = !forged && [seen, also].some((x) => x.replayOf === act.id && x.unvouched);
|
|
1211
1536
|
seen.replayOf ??= also.replayOf;
|
|
1212
1537
|
seen.answered ||= also.answered;
|
|
1538
|
+
seen.unvouched = unvouched || undefined;
|
|
1213
1539
|
}
|
|
1540
|
+
if (seen.replayOf === act.id && seen.answered && seen.unvouched)
|
|
1541
|
+
return { why: "this machine already answered that request", trust: "answered", from: act.from };
|
|
1214
1542
|
if (seen.replayOf)
|
|
1215
1543
|
return { why: `a copy of ${seen.replayOf}, which this machine already had — the same signed request under a new id`, trust: "replayed", from: act.from };
|
|
1216
1544
|
if (seen.answered)
|
|
1217
|
-
return { why: "this machine already answered that signed request", trust: "
|
|
1545
|
+
return { why: "this machine already answered that signed request", trust: "answered", from: act.from };
|
|
1218
1546
|
if (sig.role === "run" && sig.cert && seen.firstSeen > sig.cert.expires + 6 * 3600_000) {
|
|
1219
1547
|
return { why: "a run's request that first reached this machine hours after that run ended", trust: "stale", from: act.from };
|
|
1220
1548
|
}
|
|
@@ -1262,8 +1590,11 @@ export class Worker {
|
|
|
1262
1590
|
const pol = loadPolicy(this.opts.home);
|
|
1263
1591
|
if (isApprovedWaker(pol, [this.cfg.spaceId, this.cfg.space], from))
|
|
1264
1592
|
return;
|
|
1593
|
+
const before = { ...(pol.wakeOnce ?? {}) };
|
|
1265
1594
|
if (takeWakeOnce(pol, [this.cfg.spaceId, this.cfg.space], from)) {
|
|
1266
1595
|
savePolicy(pol, this.opts.home);
|
|
1596
|
+
const keys = Object.keys(before).filter((k) => (pol.wakeOnce?.[k] ?? 0) < before[k]);
|
|
1597
|
+
this.spentOnce = { ...this.spentOnce, wake: keys, file: this.policyBytes() };
|
|
1267
1598
|
this.opts.log(` wake ${from} may start this one run (you said "just this once")`);
|
|
1268
1599
|
}
|
|
1269
1600
|
}
|
|
@@ -1378,6 +1709,11 @@ export class Worker {
|
|
|
1378
1709
|
}
|
|
1379
1710
|
openRunHome(cfg = this.cfg, features = []) {
|
|
1380
1711
|
const dir = this.runHomePath();
|
|
1712
|
+
try {
|
|
1713
|
+
if (!lstatSync(dir).isDirectory())
|
|
1714
|
+
rmSync(dir, { recursive: true, force: true });
|
|
1715
|
+
}
|
|
1716
|
+
catch { }
|
|
1381
1717
|
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
1382
1718
|
const file = join(dir, "config.json");
|
|
1383
1719
|
let written = cfg;
|
|
@@ -1396,11 +1732,7 @@ export class Worker {
|
|
|
1396
1732
|
this.opts.log(` sign this relay cannot check a run's own key yet, so woken runs sign as ${cfg.member} (upgrade the relay to 0.31)`);
|
|
1397
1733
|
}
|
|
1398
1734
|
written = { ...written, pinTeam: this.pinTeam };
|
|
1399
|
-
|
|
1400
|
-
try {
|
|
1401
|
-
chmodSync(file, 0o600);
|
|
1402
|
-
}
|
|
1403
|
-
catch { }
|
|
1735
|
+
writeInside(dir, file, JSON.stringify(written, null, 2) + "\n", 0o600);
|
|
1404
1736
|
try {
|
|
1405
1737
|
const p = loadPins(this.opts.home);
|
|
1406
1738
|
savePins(p, dir);
|
|
@@ -1408,6 +1740,14 @@ export class Worker {
|
|
|
1408
1740
|
catch { }
|
|
1409
1741
|
return dir;
|
|
1410
1742
|
}
|
|
1743
|
+
cleanupStep(what, step) {
|
|
1744
|
+
try {
|
|
1745
|
+
step();
|
|
1746
|
+
}
|
|
1747
|
+
catch (e) {
|
|
1748
|
+
this.opts.log(`! ${what} failed: ${e.message.split("\n")[0]}`);
|
|
1749
|
+
}
|
|
1750
|
+
}
|
|
1411
1751
|
sealRunHome(dir) {
|
|
1412
1752
|
try {
|
|
1413
1753
|
for (const a of answeredIn(dir))
|
|
@@ -1421,7 +1761,7 @@ export class Worker {
|
|
|
1421
1761
|
catch { }
|
|
1422
1762
|
const { token: _t, privateKey: _k, pendingAuth: _p, delegate: _d, ...stub } = this.cfg;
|
|
1423
1763
|
try {
|
|
1424
|
-
|
|
1764
|
+
writeInside(dir, join(dir, "config.json"), JSON.stringify({ ...stub, sealed: "the token and key are written here only while a woken run holds them" }, null, 2) + "\n", 0o600);
|
|
1425
1765
|
}
|
|
1426
1766
|
catch {
|
|
1427
1767
|
try {
|
|
@@ -1477,10 +1817,18 @@ export class Worker {
|
|
|
1477
1817
|
});
|
|
1478
1818
|
await client.connect();
|
|
1479
1819
|
this.client = client;
|
|
1820
|
+
try {
|
|
1821
|
+
await this.serve(client);
|
|
1822
|
+
}
|
|
1823
|
+
finally {
|
|
1824
|
+
client.close();
|
|
1825
|
+
}
|
|
1826
|
+
}
|
|
1827
|
+
async serve(client) {
|
|
1480
1828
|
const agentLabel = this.opts.agentCmd
|
|
1481
1829
|
? this.opts.agentCmd
|
|
1482
1830
|
: (() => { try {
|
|
1483
|
-
return describeChoice((this.opts.choose ?? chooseAgent)(this.opts.agent, { prefer: loadProfile(this.cfg).agent.primary_surface }));
|
|
1831
|
+
return describeChoice((this.opts.choose ?? chooseAgent)(this.opts.agent, { prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: this.unsandboxedOk() }));
|
|
1484
1832
|
}
|
|
1485
1833
|
catch {
|
|
1486
1834
|
return "auto";
|
|
@@ -1512,6 +1860,7 @@ export class Worker {
|
|
|
1512
1860
|
for (const w of p.warnings)
|
|
1513
1861
|
this.opts.log(` ! ${p.name}: ${w}`);
|
|
1514
1862
|
}
|
|
1863
|
+
this.recoverInterruptedRun();
|
|
1515
1864
|
if (this.opts.continuations !== false)
|
|
1516
1865
|
await this.restoreHeldContinuations(client);
|
|
1517
1866
|
this.writeState("watching", "waiting for work");
|
|
@@ -1566,8 +1915,11 @@ export class Worker {
|
|
|
1566
1915
|
this.sayKeyChanges(client);
|
|
1567
1916
|
const aged = client
|
|
1568
1917
|
.myWork()
|
|
1569
|
-
.filter((o) => o.kind !== "HUMAN" && !o.claimed_by)
|
|
1918
|
+
.filter((o) => o.kind !== "HUMAN" && o.kind !== "DELIVER" && !o.claimed_by)
|
|
1570
1919
|
.filter((o) => Date.now() - o.created_at >= grace)
|
|
1920
|
+
.filter((o) => { const t = this.outageHold.get(o.id); if (t === undefined)
|
|
1921
|
+
return true; if (t > Date.now())
|
|
1922
|
+
return false; this.outageHold.delete(o.id); return true; })
|
|
1571
1923
|
.filter((o) => {
|
|
1572
1924
|
if (this.failuresOf(o.id) < 2)
|
|
1573
1925
|
return true;
|
|
@@ -1719,111 +2071,251 @@ export class Worker {
|
|
|
1719
2071
|
maxRunsPerHour: this.opts.maxRunsPerHour,
|
|
1720
2072
|
maxRunsPerDay: this.opts.maxRunsPerDay,
|
|
1721
2073
|
});
|
|
1722
|
-
const
|
|
1723
|
-
if (
|
|
1724
|
-
|
|
1725
|
-
this.skipLogged.add(`cap:${candidate?.id ?? continuation.id}`);
|
|
1726
|
-
this.opts.log(`! not spawning: ${gate.why}. Raise it with --max-runs / --max-runs-day${persona ? ` (persona "${persona.name}" may cap it lower)` : ""}, or leave it.`);
|
|
1727
|
-
}
|
|
1728
|
-
this.writeState("capped", gate.why);
|
|
2074
|
+
const slot = this.takeSlot();
|
|
2075
|
+
if ("held" in slot) {
|
|
2076
|
+
this.writeState("watching", `waiting: ${slot.held}`);
|
|
1729
2077
|
if (continuation)
|
|
1730
|
-
this.requeueContinuation(continuation,
|
|
2078
|
+
this.requeueContinuation(continuation, 5_000);
|
|
1731
2079
|
continue;
|
|
1732
2080
|
}
|
|
1733
|
-
|
|
2081
|
+
try {
|
|
2082
|
+
const gate = allowed({ perHour: policy.maxRunsPerHour, perDay: policy.maxRunsPerDay }, this.opts.home);
|
|
2083
|
+
if (!gate.ok) {
|
|
2084
|
+
if (!this.skipLogged.has(`cap:${candidate?.id ?? continuation.id}`)) {
|
|
2085
|
+
this.skipLogged.add(`cap:${candidate?.id ?? continuation.id}`);
|
|
2086
|
+
this.opts.log(gate.broken ? `! not spawning: ${gate.why}`
|
|
2087
|
+
: `! not spawning: ${gate.why}. Raise it with --max-runs / --max-runs-day${persona ? ` (persona "${persona.name}" may cap it lower)` : ""}, or leave it.`);
|
|
2088
|
+
}
|
|
2089
|
+
this.writeState("capped", gate.why);
|
|
2090
|
+
if (continuation)
|
|
2091
|
+
this.requeueContinuation(continuation, 60_000);
|
|
2092
|
+
continue;
|
|
2093
|
+
}
|
|
2094
|
+
if (this.stopped)
|
|
2095
|
+
break;
|
|
2096
|
+
if (continuation) {
|
|
2097
|
+
try {
|
|
2098
|
+
await client.claim(`cont:${continuation.id}`);
|
|
2099
|
+
}
|
|
2100
|
+
catch (e) {
|
|
2101
|
+
const msg = e.message;
|
|
2102
|
+
if (/^DEFER/.test(msg)) {
|
|
2103
|
+
this.requeueContinuation(continuation, 10_000);
|
|
2104
|
+
}
|
|
2105
|
+
else {
|
|
2106
|
+
this.markDone(continuation.id);
|
|
2107
|
+
if (!this.skipLogged.has(`cont:${continuation.id}`)) {
|
|
2108
|
+
this.skipLogged.add(`cont:${continuation.id}`);
|
|
2109
|
+
this.opts.log(`- not continuing from ${continuation.id}: ${msg.split("\n")[0]}`);
|
|
2110
|
+
}
|
|
2111
|
+
}
|
|
2112
|
+
continue;
|
|
2113
|
+
}
|
|
2114
|
+
if (this.stopped) {
|
|
2115
|
+
this.opts.log(`- not continuing from ${continuation.id}: the worker is stopping`);
|
|
2116
|
+
break;
|
|
2117
|
+
}
|
|
2118
|
+
}
|
|
2119
|
+
this.running = true;
|
|
2120
|
+
this.writeState("running", candidate ? `${candidate.kind} ${candidate.id}` : `CONTINUE ${continuation.id}`);
|
|
2121
|
+
let job;
|
|
1734
2122
|
try {
|
|
1735
|
-
|
|
2123
|
+
job = candidate
|
|
2124
|
+
? await this.handle(client, candidate, persona, policy, candidateAct)
|
|
2125
|
+
: await this.handleContinuation(client, continuation, persona, policy);
|
|
1736
2126
|
}
|
|
1737
2127
|
catch (e) {
|
|
1738
|
-
const msg = e
|
|
1739
|
-
|
|
1740
|
-
this.
|
|
2128
|
+
const msg = e?.message?.split("\n")[0] ?? String(e);
|
|
2129
|
+
const ob = candidate ?? {
|
|
2130
|
+
id: `cont:${continuation.id}`, kind: "CONTINUE", owner: this.cfg.member, ref: continuation.id,
|
|
2131
|
+
goal: continuation.goal, created_at: continuation.ts, body: `${continuation.act} from ${continuation.from}`,
|
|
2132
|
+
};
|
|
2133
|
+
if (e instanceof WorktreeTampered) {
|
|
2134
|
+
this.opts.log(`! ${ob.id}: ${msg}`);
|
|
2135
|
+
job = { obligation: ob, branch: "", worktree: e.worktree, ok: false, refused: true, detail: msg, diagnosis: tamperedDiagnosis(this.cfg.member, e) };
|
|
1741
2136
|
}
|
|
1742
2137
|
else {
|
|
1743
|
-
this.
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
2138
|
+
this.opts.log(`! ${ob.id}: the run failed with an error — ${msg}`);
|
|
2139
|
+
job = { obligation: ob, branch: "", worktree: "", ok: false, detail: `the run failed with an error: ${msg}` };
|
|
2140
|
+
}
|
|
2141
|
+
}
|
|
2142
|
+
this.runProject = null;
|
|
2143
|
+
this.done.push(job);
|
|
2144
|
+
this.client?.reportWorkerState("idle");
|
|
2145
|
+
if (candidate && job.ok)
|
|
2146
|
+
this.markDone(`handled:${candidate.ref}`);
|
|
2147
|
+
const ran = candidate ? candidateAct : continuation;
|
|
2148
|
+
if (job.ok && ran?.sig) {
|
|
2149
|
+
try {
|
|
2150
|
+
markAnswered(this.pinTeam, ran.from, ran.sig.nonce, ran.id, this.opts.home, ran.sig.s);
|
|
2151
|
+
if (this.opts.home !== undefined)
|
|
2152
|
+
markAnswered(this.pinTeam, ran.from, ran.sig.nonce, ran.id, undefined, ran.sig.s);
|
|
2153
|
+
}
|
|
2154
|
+
catch { }
|
|
2155
|
+
}
|
|
2156
|
+
{
|
|
2157
|
+
const soon = earliest(activeOutages(this.outages));
|
|
2158
|
+
if (soon && job.refused && job.forOutage && job.diagnosis && !job.waitUntil) {
|
|
2159
|
+
const label = [...this.outages.entries()].find(([, v]) => v === soon)?.[0];
|
|
2160
|
+
Object.assign(job, { waitUntil: soon.until, diagnosis: this.outageDiagnosis(label ? agentById(label)?.label ?? label : "The agent", soon, [], job.diagnosis.says) });
|
|
2161
|
+
}
|
|
2162
|
+
}
|
|
2163
|
+
if (job.agentOut) {
|
|
2164
|
+
if (continuation) {
|
|
2165
|
+
this.contDone.delete(continuation.id);
|
|
2166
|
+
for (const k of this.contKeys())
|
|
2167
|
+
unmarkContinued(k, continuation.id, this.opts.home);
|
|
2168
|
+
this.requeueContinuation(continuation, 0);
|
|
1748
2169
|
}
|
|
1749
2170
|
continue;
|
|
1750
2171
|
}
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
if (job.ok && ran?.sig) {
|
|
1765
|
-
try {
|
|
1766
|
-
markAnswered(this.pinTeam, ran.from, ran.sig.nonce, ran.id, this.opts.home, ran.sig.s);
|
|
1767
|
-
if (this.opts.home !== undefined)
|
|
1768
|
-
markAnswered(this.pinTeam, ran.from, ran.sig.nonce, ran.id, undefined, ran.sig.s);
|
|
2172
|
+
if (job.refused && job.diagnosis?.cause === "agent_unavailable" && job.waitUntil) {
|
|
2173
|
+
if (candidate)
|
|
2174
|
+
this.outageHold.set(candidate.id, job.waitUntil);
|
|
2175
|
+
if (this.outageHold.size > 500)
|
|
2176
|
+
this.outageHold.delete(this.outageHold.keys().next().value);
|
|
2177
|
+
if (continuation) {
|
|
2178
|
+
this.holdContinuation(continuation, policy.agent);
|
|
2179
|
+
this.outageHeldUntil.set(`cont:${continuation.id}`, job.waitUntil);
|
|
2180
|
+
}
|
|
2181
|
+
if (!this.skipLogged.has(`out-told:${job.obligation.id}`)) {
|
|
2182
|
+
this.skipLogged.add(`out-told:${job.obligation.id}`);
|
|
2183
|
+
await this.reportRefused(client, job.obligation, job.diagnosis, !!candidate);
|
|
2184
|
+
}
|
|
1769
2185
|
}
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
|
|
1777
|
-
|
|
2186
|
+
else if (job.refused && job.diagnosis) {
|
|
2187
|
+
if (candidate) {
|
|
2188
|
+
this.failedRuns.set(candidate.id, 2);
|
|
2189
|
+
this.skipLogged.add(`parked:${candidate.id}`);
|
|
2190
|
+
if (job.diagnosis.cause === "repo_not_here")
|
|
2191
|
+
this.parkedNotHere.add(candidate.id);
|
|
2192
|
+
}
|
|
2193
|
+
if (!candidate && continuation && this.parkedNoSandbox.has(job.obligation.id))
|
|
2194
|
+
this.holdContinuation(continuation, policy.agent);
|
|
2195
|
+
await this.reportRefused(client, job.obligation, job.diagnosis, !!candidate);
|
|
2196
|
+
if (job.diagnosis.cause === "outside_links" || job.diagnosis.cause === "worktree_tampered")
|
|
2197
|
+
await this.reportStuck(client, job.obligation, job.diagnosis, 0);
|
|
1778
2198
|
}
|
|
1779
|
-
if (
|
|
1780
|
-
this.
|
|
1781
|
-
|
|
1782
|
-
|
|
1783
|
-
|
|
1784
|
-
|
|
1785
|
-
|
|
1786
|
-
|
|
1787
|
-
this.markDone(`parked
|
|
1788
|
-
|
|
1789
|
-
|
|
1790
|
-
|
|
1791
|
-
|
|
1792
|
-
|
|
1793
|
-
|
|
1794
|
-
|
|
1795
|
-
|
|
1796
|
-
|
|
1797
|
-
this.failedRuns.delete(this.failedRuns.keys().next().value);
|
|
2199
|
+
else if (candidate && !job.ok) {
|
|
2200
|
+
const attempts = this.failuresOf(candidate.id) + 1;
|
|
2201
|
+
const failures = job.diagnosis?.cause === "no_commands" || job.diagnosis?.cause === "no_sandbox_commands" ? 2 : attempts;
|
|
2202
|
+
if (job.diagnosis?.cause === "no_sandbox_commands")
|
|
2203
|
+
this.markDone(`parked-ns:${candidate.id}`);
|
|
2204
|
+
if (this.contDone.has(`unparked-ns:${candidate.id}`))
|
|
2205
|
+
this.markDone(`parked2-ns:${candidate.id}`);
|
|
2206
|
+
this.failedRuns.set(candidate.id, failures);
|
|
2207
|
+
this.markDone(failures >= 2 ? `parked:${candidate.id}` : `failed1:${candidate.id}`);
|
|
2208
|
+
if (this.contDone.has(`unparked:${candidate.id}`))
|
|
2209
|
+
this.markDone(`parked2:${candidate.id}`);
|
|
2210
|
+
else if (job.diagnosis?.cause === "no_commands")
|
|
2211
|
+
this.markDone(`parked-nc:${candidate.id}`);
|
|
2212
|
+
if (this.failedRuns.size > 200) {
|
|
2213
|
+
this.failedRuns.delete(this.failedRuns.keys().next().value);
|
|
2214
|
+
}
|
|
2215
|
+
if (failures >= 2 && job.diagnosis)
|
|
2216
|
+
await this.reportStuck(client, candidate, job.diagnosis, attempts);
|
|
1798
2217
|
}
|
|
1799
|
-
if (failures >= 2 && job.diagnosis)
|
|
1800
|
-
await this.reportStuck(client, candidate, job.diagnosis, attempts);
|
|
1801
2218
|
}
|
|
1802
|
-
|
|
1803
|
-
|
|
1804
|
-
|
|
2219
|
+
finally {
|
|
2220
|
+
this.running = false;
|
|
2221
|
+
this.endAttempt();
|
|
2222
|
+
this.giveSlot(slot.token);
|
|
2223
|
+
}
|
|
1805
2224
|
this.writeState("watching", "waiting for work");
|
|
1806
2225
|
if (this.opts.once)
|
|
1807
2226
|
break;
|
|
1808
2227
|
}
|
|
1809
|
-
client.close();
|
|
1810
2228
|
}
|
|
1811
2229
|
stop() {
|
|
1812
2230
|
this.stopped = true;
|
|
1813
2231
|
for (const pid of this.runPids)
|
|
1814
2232
|
killRunTree(pid);
|
|
1815
2233
|
}
|
|
2234
|
+
takeSlot() {
|
|
2235
|
+
if (fleetLock && fleetLock !== this)
|
|
2236
|
+
return { held: `a run for "${fleetLock.cfg.space}" is in progress` };
|
|
2237
|
+
let lease;
|
|
2238
|
+
try {
|
|
2239
|
+
lease = takeRunLease(this.stateDir(), this.cfg.space);
|
|
2240
|
+
}
|
|
2241
|
+
catch (e) {
|
|
2242
|
+
const why = `${RUN_SLOT} cannot be read and written (${e.message.split("\n")[0]}) — repair or delete it`;
|
|
2243
|
+
if (!this.skipLogged.has(`slot:${why}`)) {
|
|
2244
|
+
this.skipLogged.add(`slot:${why}`);
|
|
2245
|
+
this.opts.log(`! holding unattended runs: ${why}`);
|
|
2246
|
+
}
|
|
2247
|
+
return { held: why };
|
|
2248
|
+
}
|
|
2249
|
+
if ("held" in lease)
|
|
2250
|
+
return { held: `a run for "${lease.held}" is in progress in another pantheon worker on this machine` };
|
|
2251
|
+
fleetLock = this;
|
|
2252
|
+
const dir = this.stateDir(), token = lease.token;
|
|
2253
|
+
this.leaseTimer = setInterval(() => renewRunLease(dir, token), this.opts.runLeaseRenewMs ?? RUN_LEASE_RENEW_MS);
|
|
2254
|
+
this.leaseTimer.unref?.();
|
|
2255
|
+
return lease;
|
|
2256
|
+
}
|
|
2257
|
+
leaseTimer = null;
|
|
2258
|
+
giveSlot(token) {
|
|
2259
|
+
if (this.leaseTimer) {
|
|
2260
|
+
clearInterval(this.leaseTimer);
|
|
2261
|
+
this.leaseTimer = null;
|
|
2262
|
+
}
|
|
2263
|
+
releaseRunLease(this.stateDir(), token);
|
|
2264
|
+
if (fleetLock === this)
|
|
2265
|
+
fleetLock = null;
|
|
2266
|
+
}
|
|
1816
2267
|
runPids = new Set();
|
|
1817
|
-
trackRun(pid) {
|
|
2268
|
+
trackRun(pid, attempt) {
|
|
1818
2269
|
this.runPids.add(pid);
|
|
1819
2270
|
liveRuns.add(pid);
|
|
1820
2271
|
installRunReaper();
|
|
1821
|
-
|
|
2272
|
+
this.attemptPid = pid;
|
|
2273
|
+
writeRunRecord(this.stateDir(), { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, obligation: attempt.obligation, ...(attempt.spend ? { spend: attempt.spend } : {}) });
|
|
1822
2274
|
}
|
|
1823
2275
|
untrackRun(pid) {
|
|
1824
2276
|
this.runPids.delete(pid);
|
|
1825
2277
|
liveRuns.delete(pid);
|
|
1826
|
-
|
|
2278
|
+
}
|
|
2279
|
+
attemptPid = null;
|
|
2280
|
+
endAttempt() {
|
|
2281
|
+
if (this.attemptPid === null)
|
|
2282
|
+
return;
|
|
2283
|
+
clearRunRecord(this.stateDir(), this.attemptPid);
|
|
2284
|
+
this.attemptPid = null;
|
|
2285
|
+
}
|
|
2286
|
+
recoverInterruptedRun() {
|
|
2287
|
+
const dir = this.stateDir(), f = join(dir, RUN_RECORD);
|
|
2288
|
+
let r;
|
|
2289
|
+
try {
|
|
2290
|
+
r = JSON.parse(readFileSync(f, "utf8"));
|
|
2291
|
+
}
|
|
2292
|
+
catch {
|
|
2293
|
+
return;
|
|
2294
|
+
}
|
|
2295
|
+
if (!r || r.space !== this.cfg.space || r.member !== this.cfg.member || typeof r.obligation !== "string")
|
|
2296
|
+
return;
|
|
2297
|
+
if (!Number.isInteger(r.worker) || r.worker === process.pid || pidAlive(r.worker))
|
|
2298
|
+
return;
|
|
2299
|
+
if (Number.isInteger(r.pid) && typeof r.at === "number" && Date.now() - r.at <= RUN_TIMEOUT_MS + 5 * 60_000 && recordedRunAlive(r.pid))
|
|
2300
|
+
killRunTree(r.pid);
|
|
2301
|
+
try {
|
|
2302
|
+
rmSync(f, { force: true });
|
|
2303
|
+
}
|
|
2304
|
+
catch { }
|
|
2305
|
+
if (typeof r.spend === "string") {
|
|
2306
|
+
try {
|
|
2307
|
+
closeStartedRun(r.spend, "the worker went before the run ended", { at: r.at, goal: null, obligation: r.obligation, ok: false, triggeredBy: "unknown", summary: "the worker went before the run ended", space: this.spaceKey(), member: r.member }, this.opts.home);
|
|
2308
|
+
}
|
|
2309
|
+
catch { }
|
|
2310
|
+
}
|
|
2311
|
+
if (r.obligation.startsWith("cont:")) {
|
|
2312
|
+
this.opts.log(`! the last run (${r.obligation}) ended with its worker — counted, not run again`);
|
|
2313
|
+
return;
|
|
2314
|
+
}
|
|
2315
|
+
const failures = this.failuresOf(r.obligation) + 1;
|
|
2316
|
+
this.failedRuns.set(r.obligation, failures);
|
|
2317
|
+
this.markDone(failures >= 2 ? `parked:${r.obligation}` : `failed1:${r.obligation}`);
|
|
2318
|
+
this.opts.log(`! the last run for ${r.obligation} ended with its worker (stopped, restarted or killed mid-run) — counted as a failed attempt${failures >= 2 ? "; parked: it is not run again" : ""}`);
|
|
1827
2319
|
}
|
|
1828
2320
|
stateDir() {
|
|
1829
2321
|
return this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
|
|
@@ -1841,7 +2333,10 @@ export class Worker {
|
|
|
1841
2333
|
}
|
|
1842
2334
|
catch { }
|
|
1843
2335
|
}
|
|
1844
|
-
brokenStateFiles() {
|
|
2336
|
+
brokenStateFiles() {
|
|
2337
|
+
repairRunLease(this.stateDir());
|
|
2338
|
+
return brokenStateFiles(this.stateDir());
|
|
2339
|
+
}
|
|
1845
2340
|
hooksInstalledHere() {
|
|
1846
2341
|
try {
|
|
1847
2342
|
return agentHookStates(this.opts.project).some((s) => s.installed);
|
|
@@ -1877,13 +2372,45 @@ export class Worker {
|
|
|
1877
2372
|
const needs = workNeedsCommands("CONTINUE", trigger, act.act);
|
|
1878
2373
|
return this.runJob(client, o, persona, policy, (branch, canPush) => buildContinuationBrief(act, trigger, branch, persona, canPush), false, needs, trigger, act);
|
|
1879
2374
|
}
|
|
2375
|
+
spentOnce = {};
|
|
2376
|
+
policyBytes() { try {
|
|
2377
|
+
return readFileSync(policyPath(this.opts.home), "utf8");
|
|
2378
|
+
}
|
|
2379
|
+
catch {
|
|
2380
|
+
return null;
|
|
2381
|
+
} }
|
|
2382
|
+
restoreOnces() {
|
|
2383
|
+
const spent = this.spentOnce;
|
|
2384
|
+
this.spentOnce = {};
|
|
2385
|
+
if (!spent.cmd && !spent.wake?.length)
|
|
2386
|
+
return;
|
|
2387
|
+
try {
|
|
2388
|
+
if (this.policyBytes() !== spent.file) {
|
|
2389
|
+
this.opts.log(` grants not giving back the "just this once": your grants changed during the run`);
|
|
2390
|
+
return;
|
|
2391
|
+
}
|
|
2392
|
+
const p = loadPolicy(this.opts.home);
|
|
2393
|
+
if (spent.cmd && !p.grants.run_commands)
|
|
2394
|
+
p.grants.run_commands = spent.cmd;
|
|
2395
|
+
for (const k of spent.wake ?? [])
|
|
2396
|
+
(p.wakeOnce ??= {})[k] = Math.min(99, (p.wakeOnce[k] ?? 0) + 1);
|
|
2397
|
+
savePolicy(p, this.opts.home);
|
|
2398
|
+
this.opts.log(` grants the "just this once" spent on that run is given back for the next agent`);
|
|
2399
|
+
}
|
|
2400
|
+
catch (e) {
|
|
2401
|
+
this.opts.log(`! could not give back a "just this once" grant: ${e.message.split("\n")[0]}`);
|
|
2402
|
+
}
|
|
2403
|
+
}
|
|
1880
2404
|
commandGrant(persona) {
|
|
1881
2405
|
if (persona?.allowBash === false)
|
|
1882
2406
|
return false;
|
|
1883
2407
|
const policy = loadPolicy(this.opts.home);
|
|
2408
|
+
const was = policy.grants.run_commands;
|
|
1884
2409
|
if (!useGrant(policy, "run_commands"))
|
|
1885
2410
|
return false;
|
|
1886
2411
|
savePolicy(policy, this.opts.home);
|
|
2412
|
+
if (was?.mode === "once")
|
|
2413
|
+
this.spentOnce = { ...this.spentOnce, cmd: was, file: this.policyBytes() };
|
|
1887
2414
|
this.opts.log(` bash on for this run — your human granted run_commands`);
|
|
1888
2415
|
return true;
|
|
1889
2416
|
}
|
|
@@ -1928,7 +2455,7 @@ export class Worker {
|
|
|
1928
2455
|
let choice;
|
|
1929
2456
|
if (!this.opts.agentCmd) {
|
|
1930
2457
|
try {
|
|
1931
|
-
choice = (this.opts.choose ?? chooseAgent)(policy.agent ?? this.opts.agent, { prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: this.unsandboxedOk() });
|
|
2458
|
+
choice = (this.opts.choose ?? chooseAgent)(policy.agent ?? this.opts.agent, { prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: this.unsandboxedOk(), outages: activeOutages(this.outages) });
|
|
1932
2459
|
}
|
|
1933
2460
|
catch (e) {
|
|
1934
2461
|
const detail = `could not plan the run: ${e.message}`;
|
|
@@ -1941,6 +2468,14 @@ export class Worker {
|
|
|
1941
2468
|
this.opts.log(` agent ${said}`);
|
|
1942
2469
|
this.lastAgentSaid = said;
|
|
1943
2470
|
}
|
|
2471
|
+
if (choice.outage) {
|
|
2472
|
+
const diagnosis = this.outageDiagnosis(choice.backend.label, choice.outage, choice.skipped.filter((x) => x.backend.id !== choice.backend.id).map((x) => `${x.backend.label}: ${x.why}`));
|
|
2473
|
+
if (!this.skipLogged.has(`out:${o.id}:${choice.outage.until}`)) {
|
|
2474
|
+
this.skipLogged.add(`out:${o.id}:${choice.outage.until}`);
|
|
2475
|
+
this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${diagnosis.says} — it waits (not counted as a run)`);
|
|
2476
|
+
}
|
|
2477
|
+
return { obligation: o, branch, worktree, ok: false, refused: true, detail: diagnosis.says, diagnosis, waitUntil: choice.outage.until };
|
|
2478
|
+
}
|
|
1944
2479
|
const refused = unattendedRefusal(choice, { unsandboxedOk: this.unsandboxedOk() });
|
|
1945
2480
|
if (refused) {
|
|
1946
2481
|
const noSandbox = !SANDBOXED_FOR_UNATTENDED.has(choice.backend.id) && !NOT_FOR_UNATTENDED_RUNS.has(choice.backend.id);
|
|
@@ -1957,7 +2492,7 @@ export class Worker {
|
|
|
1957
2492
|
this.skipLogged.add(`refused:${o.id}`);
|
|
1958
2493
|
this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${refused} — it waits for a live session (not counted as a run)`);
|
|
1959
2494
|
}
|
|
1960
|
-
return { obligation: o, branch, worktree, ok: false, refused: true, detail: refused, diagnosis };
|
|
2495
|
+
return { obligation: o, branch, worktree, ok: false, refused: true, detail: refused, diagnosis, ...(this.outagesPassed(choice).length ? { forOutage: true } : {}) };
|
|
1961
2496
|
}
|
|
1962
2497
|
}
|
|
1963
2498
|
if (this.opts.allowBash && !policy.allowBash && persona) {
|
|
@@ -1989,13 +2524,14 @@ export class Worker {
|
|
|
1989
2524
|
this.skipLogged.add(`refused:${o.id}`);
|
|
1990
2525
|
this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${says}`);
|
|
1991
2526
|
}
|
|
1992
|
-
return { obligation: o, branch, worktree, ok: false, refused: true, detail: says, diagnosis };
|
|
2527
|
+
return { obligation: o, branch, worktree, ok: false, refused: true, detail: says, diagnosis, ...(choice && this.outagesPassed(choice).length ? { forOutage: true } : {}) };
|
|
1993
2528
|
}
|
|
1994
2529
|
const byCli = !!this.opts.agentCmd && !!this.opts.agentCmdFromCli;
|
|
1995
2530
|
const noCommandsHere = features?.unknown ? "unknown"
|
|
1996
2531
|
: features && ((choice?.backend.id === "claude-code" && !this.opts.agentCmd) || byCli) && !claudeSandboxable(features) && !this.unsandboxedOk() ? "unsandboxed"
|
|
1997
2532
|
: null;
|
|
1998
2533
|
const wantedBash = policy.allowBash || (persona?.allowBash !== false && !!loadPolicy(this.opts.home).grants.run_commands);
|
|
2534
|
+
this.spentOnce = {};
|
|
1999
2535
|
let allowBash = noCommandsHere ? false : policy.allowBash || this.commandGrant(persona);
|
|
2000
2536
|
const bashDroppedForSandbox = noCommandsHere === "unsandboxed" && wantedBash;
|
|
2001
2537
|
let baseRef = "", baseSha = "", tipSha = "";
|
|
@@ -2004,11 +2540,21 @@ export class Worker {
|
|
|
2004
2540
|
({ branch, worktree, baseRef, baseSha, tipSha, fetched } = await prepareWorktreeAsync(project, scratch, goal, this.opts.log, branch, this.repoBase(place.fp)));
|
|
2005
2541
|
}
|
|
2006
2542
|
catch (e) {
|
|
2543
|
+
if (e instanceof WorktreeTampered)
|
|
2544
|
+
throw e;
|
|
2007
2545
|
const detail = `could not prepare a worktree: ${e.message}`;
|
|
2008
2546
|
this.opts.log(`! ${detail}`);
|
|
2009
2547
|
return { obligation: o, branch, worktree, ok: false, detail };
|
|
2010
2548
|
}
|
|
2549
|
+
const scan = outsideLinks(worktree);
|
|
2550
|
+
if (scan.links.length || scan.unreadable) {
|
|
2551
|
+
const diagnosis = outsideLinksDiagnosis(this.cfg.member, worktree, scan);
|
|
2552
|
+
this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${diagnosis.says}`);
|
|
2553
|
+
return { obligation: o, branch, worktree, ok: false, refused: true, detail: diagnosis.says, diagnosis };
|
|
2554
|
+
}
|
|
2011
2555
|
await this.spendWakeOnce(client, o);
|
|
2556
|
+
if (choice)
|
|
2557
|
+
this.sayFallbackUnsandboxed(choice);
|
|
2012
2558
|
if (noCommandsHere === "unknown" && wantedBash) {
|
|
2013
2559
|
this.opts.log(` bash off for this run — \`${binary} --help\` did not answer, so its sandbox and deny rules cannot be confirmed`);
|
|
2014
2560
|
}
|
|
@@ -2040,12 +2586,12 @@ export class Worker {
|
|
|
2040
2586
|
}
|
|
2041
2587
|
const multi = this.workspaceList().length > 1 || place.kind === "folder" || route.notHere.length > 0;
|
|
2042
2588
|
const whereNote = multi ? workspaceNote(route, attached, tipSha || baseSha, place.kind === "folder" ? worktree : null) : "";
|
|
2043
|
-
const reserved = reservedDecisionsNote(
|
|
2589
|
+
const reserved = reservedDecisionsNote(profileForTeam(this.cfg));
|
|
2044
2590
|
const brief = makeBrief(branch, canPush) + whereNote + baseNote(baseRef, baseSha, fetched) +
|
|
2045
2591
|
(allowBash ? (agentId === "claude-code" ? commandNote(worktree, tipSha || baseSha, (m) => this.opts.log(m)) : "") : holdRepoConfig ? heldNote() : "") +
|
|
2046
2592
|
(reserved ? `\n\n${reserved}` : "");
|
|
2047
2593
|
const startedAt = Date.now();
|
|
2048
|
-
|
|
2594
|
+
writeInside(worktree, join(worktree, ".pantheon-task.md"), brief, 0o644);
|
|
2049
2595
|
let command;
|
|
2050
2596
|
let args;
|
|
2051
2597
|
let env = agentBaseEnv(process.env, proxyEnvFor());
|
|
@@ -2087,14 +2633,14 @@ export class Worker {
|
|
|
2087
2633
|
backend, binary, prompt: brief, worktree, cfg: runCfg, branch, allowBash, runHome,
|
|
2088
2634
|
...(attached.length ? { readOnlyDirs: attached.map((a) => a.path) } : {}),
|
|
2089
2635
|
...(baseSha ? { base: baseSha } : {}), ...(tipSha ? { commandsRev: tipSha } : {}), goal,
|
|
2090
|
-
...(features ? { claude: { features, pantheonHome: this.stateDir(), home: personHome, userSettings: join(personHome, ".claude", "settings.json") } } : {}),
|
|
2636
|
+
...(features ? { claude: { features, pantheonHome: this.stateDir(), home: personHome, userSettings: join(personHome, ".claude", "settings.json"), ...(() => { const c = adoptedCommon(worktree); return c ? { gitCommon: c } : {}; })() } } : {}),
|
|
2091
2637
|
});
|
|
2092
2638
|
for (const n of plan.notes ?? [])
|
|
2093
2639
|
if (!this.notesSaid.has(n)) {
|
|
2094
2640
|
this.notesSaid.add(n);
|
|
2095
2641
|
this.opts.log(` confine ${n}`);
|
|
2096
2642
|
}
|
|
2097
|
-
materializeSpawnFiles(worktree, plan);
|
|
2643
|
+
materializeSpawnFiles(worktree, plan, runHome);
|
|
2098
2644
|
for (const f of plan.writeFiles ?? []) {
|
|
2099
2645
|
const rel = isAbsolute(f.path) ? relative(worktree, f.path) : f.path;
|
|
2100
2646
|
if (!rel.startsWith("..") && !isAbsolute(rel))
|
|
@@ -2118,10 +2664,10 @@ export class Worker {
|
|
|
2118
2664
|
}
|
|
2119
2665
|
}
|
|
2120
2666
|
catch (e) {
|
|
2121
|
-
if (fenced)
|
|
2667
|
+
this.cleanupStep("putting the repo's agent config back", () => { if (fenced)
|
|
2122
2668
|
for (const l of fenced.restore().lines)
|
|
2123
|
-
this.opts.log(` config ${l}`);
|
|
2124
|
-
this.sealRunHome(runHome);
|
|
2669
|
+
this.opts.log(` config ${l}`); });
|
|
2670
|
+
this.cleanupStep("sealing the run home", () => this.sealRunHome(runHome));
|
|
2125
2671
|
const detail = `could not plan the run: ${e.message}`;
|
|
2126
2672
|
this.opts.log(`! ${detail}`);
|
|
2127
2673
|
recordRun({ at: Date.now(), goal: o.goal, obligation: o.id, ok: false, triggeredBy: (await this.actOf(client, o.ref))?.from ?? "unknown", summary: clip(o.body), space: this.spaceKey(), member: this.cfg.member }, this.opts.home);
|
|
@@ -2130,137 +2676,190 @@ export class Worker {
|
|
|
2130
2676
|
let detail;
|
|
2131
2677
|
let transcript = "";
|
|
2132
2678
|
const homeBefore = snapshotHome(this.stateDir());
|
|
2679
|
+
let spendId = null;
|
|
2133
2680
|
try {
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
|
|
2146
|
-
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
if (head.length < 16_000)
|
|
2150
|
-
head += s.slice(0, 16_000 - head.length);
|
|
2151
|
-
tail = (tail + s).slice(-16_000);
|
|
2152
|
-
};
|
|
2153
|
-
child.stdout.on("data", take);
|
|
2154
|
-
child.stderr.on("data", take);
|
|
2155
|
-
let timedOut = false;
|
|
2156
|
-
let dead = false;
|
|
2157
|
-
const markDead = () => { if (dead)
|
|
2158
|
-
return; dead = true; if (pid)
|
|
2159
|
-
this.untrackRun(pid); };
|
|
2160
|
-
let settled = false;
|
|
2161
|
-
let exitGrace = null;
|
|
2162
|
-
let exitCode = null;
|
|
2163
|
-
const limit = this.opts.runTimeoutMs ?? RUN_TIMEOUT_MS;
|
|
2164
|
-
const finish = (code, signal) => {
|
|
2165
|
-
if (settled)
|
|
2681
|
+
spendId = startRun({ at: Date.now(), goal: o.goal, obligation: o.id, ok: false, triggeredBy: client.recentFrom(o.ref) ?? "unknown", summary: clip(o.body), space: this.spaceKey(), member: this.cfg.member }, this.opts.home);
|
|
2682
|
+
}
|
|
2683
|
+
catch (e) {
|
|
2684
|
+
this.opts.log(`! could not record the run as started: ${e.message.split("\n")[0]}`);
|
|
2685
|
+
}
|
|
2686
|
+
let recorded = false;
|
|
2687
|
+
try {
|
|
2688
|
+
try {
|
|
2689
|
+
detail = await new Promise((res) => {
|
|
2690
|
+
let child;
|
|
2691
|
+
try {
|
|
2692
|
+
child = procSpawn(sp.command, sp.args, { cwd: worktree, stdio: ["ignore", "pipe", "pipe"], env, ...sp.options, detached: process.platform !== "win32" });
|
|
2693
|
+
}
|
|
2694
|
+
catch (e) {
|
|
2695
|
+
res(`could not start "${command}": ${e.message}`);
|
|
2166
2696
|
return;
|
|
2167
|
-
settled = true;
|
|
2168
|
-
clearTimeout(kill);
|
|
2169
|
-
if (exitGrace)
|
|
2170
|
-
clearTimeout(exitGrace);
|
|
2171
|
-
markDead();
|
|
2172
|
-
transcript = `${head}\n${tail}`;
|
|
2173
|
-
res(`exit ${code ?? signal}${timedOut ? ` (stopped after ${Math.round(limit / 1000)}s)` : ""}\n${tail.slice(-4000)}`);
|
|
2174
|
-
};
|
|
2175
|
-
const kill = setTimeout(() => { if (dead)
|
|
2176
|
-
return; timedOut = true; if (pid)
|
|
2177
|
-
killRunTree(pid); }, limit);
|
|
2178
|
-
child.on("exit", (code, signal) => {
|
|
2179
|
-
clearTimeout(kill);
|
|
2180
|
-
if (pid && !dead && killTreeOnExit())
|
|
2181
|
-
killRunTree(pid);
|
|
2182
|
-
markDead();
|
|
2183
|
-
const c = child;
|
|
2184
|
-
exitCode = { code, signal };
|
|
2185
|
-
exitGrace = setTimeout(() => { try {
|
|
2186
|
-
c.stdout.destroy();
|
|
2187
|
-
c.stderr.destroy();
|
|
2188
2697
|
}
|
|
2189
|
-
|
|
2698
|
+
const pid = child.pid;
|
|
2699
|
+
if (pid)
|
|
2700
|
+
this.trackRun(pid, { obligation: o.id, spend: spendId });
|
|
2701
|
+
let head = "", tail = "";
|
|
2702
|
+
const take = (b) => {
|
|
2703
|
+
const s = b.toString();
|
|
2704
|
+
if (head.length < 16_000)
|
|
2705
|
+
head += s.slice(0, 16_000 - head.length);
|
|
2706
|
+
tail = (tail + s).slice(-16_000);
|
|
2707
|
+
};
|
|
2708
|
+
child.stdout.on("data", take);
|
|
2709
|
+
child.stderr.on("data", take);
|
|
2710
|
+
let timedOut = false;
|
|
2711
|
+
let dead = false;
|
|
2712
|
+
const markDead = () => { if (dead)
|
|
2713
|
+
return; dead = true; if (pid)
|
|
2714
|
+
this.untrackRun(pid); };
|
|
2715
|
+
let settled = false;
|
|
2716
|
+
let exitGrace = null;
|
|
2717
|
+
let exitCode = null;
|
|
2718
|
+
const limit = this.opts.runTimeoutMs ?? RUN_TIMEOUT_MS;
|
|
2719
|
+
const finish = (code, signal) => {
|
|
2720
|
+
if (settled)
|
|
2721
|
+
return;
|
|
2722
|
+
settled = true;
|
|
2723
|
+
clearTimeout(kill);
|
|
2724
|
+
if (exitGrace)
|
|
2725
|
+
clearTimeout(exitGrace);
|
|
2726
|
+
markDead();
|
|
2727
|
+
transcript = `${head}\n${tail}`;
|
|
2728
|
+
res(`exit ${code ?? signal}${timedOut ? ` (stopped after ${Math.round(limit / 1000)}s)` : ""}\n${tail.slice(-4000)}`);
|
|
2729
|
+
};
|
|
2730
|
+
const kill = setTimeout(() => { if (dead)
|
|
2731
|
+
return; timedOut = true; if (pid)
|
|
2732
|
+
killRunTree(pid); }, limit);
|
|
2733
|
+
child.on("exit", (code, signal) => {
|
|
2734
|
+
clearTimeout(kill);
|
|
2735
|
+
if (pid && !dead && killTreeOnExit())
|
|
2736
|
+
killRunTree(pid);
|
|
2737
|
+
markDead();
|
|
2738
|
+
const c = child;
|
|
2739
|
+
exitCode = { code, signal };
|
|
2740
|
+
exitGrace = setTimeout(() => { try {
|
|
2741
|
+
c.stdout.destroy();
|
|
2742
|
+
c.stderr.destroy();
|
|
2743
|
+
}
|
|
2744
|
+
catch { } finish(code, signal); }, this.opts.exitGraceMs ?? EXIT_GRACE_MS);
|
|
2745
|
+
});
|
|
2746
|
+
child.on("close", (code, signal) => finish(exitCode?.code ?? code, exitCode?.signal ?? signal));
|
|
2747
|
+
child.on("error", (e) => { if (settled)
|
|
2748
|
+
return; settled = true; clearTimeout(kill); if (exitGrace)
|
|
2749
|
+
clearTimeout(exitGrace); markDead(); res(`could not start "${command}": ${e.message}`); });
|
|
2190
2750
|
});
|
|
2191
|
-
child.on("close", (code, signal) => finish(exitCode?.code ?? code, exitCode?.signal ?? signal));
|
|
2192
|
-
child.on("error", (e) => { if (settled)
|
|
2193
|
-
return; settled = true; clearTimeout(kill); if (exitGrace)
|
|
2194
|
-
clearTimeout(exitGrace); markDead(); res(`could not start "${command}": ${e.message}`); });
|
|
2195
|
-
});
|
|
2196
|
-
}
|
|
2197
|
-
finally {
|
|
2198
|
-
if (fenced) {
|
|
2199
|
-
const r = fenced.restore();
|
|
2200
|
-
for (const l of r.lines)
|
|
2201
|
-
this.opts.log(` config ${l}`);
|
|
2202
2751
|
}
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
const
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
const ok = exited && replied;
|
|
2226
|
-
await this.checkDeliveries(client, sent, { branch, worktree, canPush: canPush && !massDelete, goal: o.goal ?? null });
|
|
2227
|
-
this.runProject = null;
|
|
2228
|
-
if (place.kind === "folder") {
|
|
2229
|
-
let drafts = [];
|
|
2230
|
-
try {
|
|
2231
|
-
drafts = readdirSync(worktree).filter((f) => !f.startsWith(".") && f !== "AGENTS.md" && f !== "CLAUDE.md");
|
|
2752
|
+
finally {
|
|
2753
|
+
this.cleanupStep("putting the repo's agent config back", () => { if (fenced)
|
|
2754
|
+
for (const l of fenced.restore().lines)
|
|
2755
|
+
this.opts.log(` config ${l}`); });
|
|
2756
|
+
this.cleanupStep("sealing the run home", () => this.sealRunHome(runHome));
|
|
2757
|
+
this.cleanupStep("checking Pantheon's own files", () => this.restoreHomeAfterRun(homeBefore, o));
|
|
2758
|
+
}
|
|
2759
|
+
const runLog = join(worktree, ".pantheon-run.log");
|
|
2760
|
+
let extra = "";
|
|
2761
|
+
let massDelete = false;
|
|
2762
|
+
if (allowBash && baseSha) {
|
|
2763
|
+
const d = deletedSinceBase(worktree, baseSha);
|
|
2764
|
+
if (d.unreadable) {
|
|
2765
|
+
massDelete = true;
|
|
2766
|
+
extra = `\n! could not check how many files this branch deletes (${d.unreadable}) — it is not pushed from here; look before merging`;
|
|
2767
|
+
this.opts.log(`! ${o.id}: could not check the branch ${branch} for a mass deletion (${d.unreadable}) — not pushing it`);
|
|
2768
|
+
}
|
|
2769
|
+
else if (d.total >= 4 && d.gone > d.total * MASS_DELETE_SHARE) {
|
|
2770
|
+
massDelete = true;
|
|
2771
|
+
extra = `\n! this branch deletes ${d.gone} of the ${d.total} files tracked at its base — do not merge it without looking`;
|
|
2772
|
+
this.opts.log(`! ${o.id}: the branch ${branch} deletes ${d.gone} of the ${d.total} files tracked at ${baseSha.slice(0, 7)} — do not merge it without looking`);
|
|
2773
|
+
}
|
|
2232
2774
|
}
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2775
|
+
this.writeRunLog(worktree, runLog, `obligation ${o.id}\nbranch ${branch}\n\n${detail}\n${extra}`);
|
|
2776
|
+
const exited = /^exit 0/.test(detail);
|
|
2777
|
+
const sent = client.mineSince(startedAt, o.goal);
|
|
2778
|
+
const repliedWith = sent.find((a) => ["DELIVERED", "ANSWER", "ESCALATE", "ACCEPT", "OBJECT", "PROPOSE", "CONTRACT", "CLOSE", "WITHDRAW"].includes(a.act) || (a.act === "NOTE" && !!a.ref));
|
|
2779
|
+
const replied = requireReply ? client.repliedTo(o) || !!repliedWith : true;
|
|
2780
|
+
if (requireReply && replied && !client.repliedTo(o) && repliedWith) {
|
|
2781
|
+
this.opts.log(` reply ${repliedWith.act} ${repliedWith.id} answers ${o.id}, though the relay still lists it as owed${repliedWith.ref ? "" : " (the act named no ref)"}`);
|
|
2782
|
+
}
|
|
2783
|
+
const ok = exited && replied;
|
|
2784
|
+
await this.checkDeliveries(client, sent, { branch, worktree, canPush: canPush && !massDelete, goal: o.goal ?? null });
|
|
2785
|
+
this.runProject = null;
|
|
2786
|
+
if (place.kind === "folder") {
|
|
2787
|
+
let drafts = [];
|
|
2788
|
+
try {
|
|
2789
|
+
drafts = readdirSync(worktree).filter((f) => !f.startsWith(".") && f !== "AGENTS.md" && f !== "CLAUDE.md");
|
|
2790
|
+
}
|
|
2791
|
+
catch { }
|
|
2792
|
+
if (drafts.length) {
|
|
2793
|
+
this.opts.log(` drafts ${drafts.length} file${drafts.length === 1 ? "" : "s"} for ${place.name} in ${worktree}: ${drafts.slice(0, 5).join(", ")}${drafts.length > 5 ? ", …" : ""}`);
|
|
2794
|
+
if (this.opts.notify) {
|
|
2795
|
+
try {
|
|
2796
|
+
this.opts.notify({ kind: "worker", who: "Your worker", heading: `Drafts for ${place.name}`, context: this.cfg.space, body: `A request from ${(await this.actOf(client, o.ref))?.from ?? "a teammate"} left ${drafts.length} draft file${drafts.length === 1 ? "" : "s"} for you to review in ${worktree}. Nothing in ${place.path} was changed.`, ttl: 0, sound: false });
|
|
2797
|
+
}
|
|
2798
|
+
catch { }
|
|
2239
2799
|
}
|
|
2240
|
-
catch { }
|
|
2241
2800
|
}
|
|
2242
2801
|
}
|
|
2802
|
+
const triggeredBy = (await this.actOf(client, o.ref))?.from ?? "unknown";
|
|
2803
|
+
const summary = clip(o.body);
|
|
2804
|
+
const end = { at: Date.now(), goal: o.goal, obligation: o.id, ok, triggeredBy, summary, space: this.spaceKey(), member: this.cfg.member };
|
|
2805
|
+
if (spendId)
|
|
2806
|
+
finishRun(spendId, end, this.opts.home);
|
|
2807
|
+
else
|
|
2808
|
+
recordRun(end, this.opts.home);
|
|
2809
|
+
recorded = true;
|
|
2810
|
+
const runKeys = client.features.includes("delegate");
|
|
2811
|
+
const sentAnywhere = client.mineSince(startedAt).some((a) => !runKeys || !a.sig || a.sig.role === "run");
|
|
2812
|
+
const outage = !ok && !exited && !sentAnywhere && choice && !/\(stopped after \d+s\)/.test(detail.split("\n")[0] ?? "") ? detectOutage(choice.backend.id, detail) : null;
|
|
2813
|
+
if (outage) {
|
|
2814
|
+
this.outages.set(choice.backend.id, outage);
|
|
2815
|
+
const refusedLog = join(worktree, `.pantheon-run.${choice.backend.id}-refused.log`);
|
|
2816
|
+
this.writeRunLog(worktree, refusedLog, `obligation ${o.id}\nbranch ${branch}\nagent ${choice.backend.label}\n\n${detail}\n`);
|
|
2817
|
+
if (Date.now() - startedAt < ONCE_GIVE_BACK_MS)
|
|
2818
|
+
this.restoreOnces();
|
|
2819
|
+
}
|
|
2820
|
+
if (outage && (this.outageRetries.get(o.id) ?? 0) < 3) {
|
|
2821
|
+
this.outageRetries.set(o.id, (this.outageRetries.get(o.id) ?? 0) + 1);
|
|
2822
|
+
if (this.outageRetries.size > 200)
|
|
2823
|
+
this.outageRetries.delete(this.outageRetries.keys().next().value);
|
|
2824
|
+
this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over until about ${whenAgain(outage.until)}; the request goes to the next agent here (transcript: ${join(worktree, `.pantheon-run.${choice.backend.id}-refused.log`)})`);
|
|
2825
|
+
return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
|
|
2826
|
+
}
|
|
2827
|
+
const quiet = ok && !requireReply && sent.length === 0;
|
|
2828
|
+
const diagnosis = ok ? undefined : outage ? this.outageDiagnosis(choice.backend.label, outage, [], "it was handed to another agent three times already") : diagnoseRun({ exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands, ...(bashDroppedForSandbox ? { commandsNeedSandbox: byCli ? "agent-cmd" : "claude" } : {}), ...(agentId ? { agent: agentId } : {}), ...(place.kind === "folder" ? { folder: true } : {}),
|
|
2829
|
+
...(agentId === "claude-code" && allowBash ? { allowedRules: bashRules } : {}), ...(baseRef ? { baseRef } : {}) }, this.opts.probe ?? gitProbe);
|
|
2830
|
+
if (this.opts.notify && !quiet && (!ok || loadProfile(this.cfg).notifications?.worker_done)) {
|
|
2831
|
+
try {
|
|
2832
|
+
const reply = sent[sent.length - 1] ?? null;
|
|
2833
|
+
this.opts.notify(composeWorker({ ok, triggeredBy, obligationBody: o.body, goal: o.goal ? client.goals.get(o.goal) : undefined, reply, obligationRef: o.ref, ...(diagnosis ? { diagnosis } : {}) }, (() => { const tools = installedTools(); return { tools, preferred: preferredTool(loadProfile(this.cfg), tools, this.opts.home) }; })()));
|
|
2834
|
+
}
|
|
2835
|
+
catch { }
|
|
2836
|
+
}
|
|
2837
|
+
this.opts.log(quiet ? `< read ${o.id} on ${branch} — the thread was already settled, nothing sent`
|
|
2838
|
+
: ok ? `< handled ${o.id} on ${branch}`
|
|
2839
|
+
: `! ${o.id} stopped [${diagnosis.cause}]: ${diagnosis.says} (transcript: ${runLog})`);
|
|
2840
|
+
return { obligation: o, branch, worktree, ok, detail, ...(diagnosis ? { diagnosis } : {}) };
|
|
2243
2841
|
}
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
try {
|
|
2255
|
-
const reply = sent[sent.length - 1] ?? null;
|
|
2256
|
-
this.opts.notify(composeWorker({ ok, triggeredBy, obligationBody: o.body, goal: o.goal ? client.goals.get(o.goal) : undefined, reply, obligationRef: o.ref, ...(diagnosis ? { diagnosis } : {}) }, (() => { const tools = installedTools(); return { tools, preferred: preferredTool(loadProfile(this.cfg), tools, this.opts.home) }; })()));
|
|
2842
|
+
catch (e) {
|
|
2843
|
+
if (!recorded) {
|
|
2844
|
+
try {
|
|
2845
|
+
const end = { at: Date.now(), goal: o.goal, obligation: o.id, ok: false, triggeredBy: client.recentFrom(o.ref) ?? "unknown", summary: clip(o.body), space: this.spaceKey(), member: this.cfg.member };
|
|
2846
|
+
if (spendId)
|
|
2847
|
+
finishRun(spendId, end, this.opts.home);
|
|
2848
|
+
else
|
|
2849
|
+
recordRun(end, this.opts.home);
|
|
2850
|
+
}
|
|
2851
|
+
catch { }
|
|
2257
2852
|
}
|
|
2258
|
-
|
|
2853
|
+
throw e;
|
|
2854
|
+
}
|
|
2855
|
+
}
|
|
2856
|
+
writeRunLog(worktree, f, text) {
|
|
2857
|
+
try {
|
|
2858
|
+
writeInside(worktree, f, text, 0o644);
|
|
2859
|
+
}
|
|
2860
|
+
catch (e) {
|
|
2861
|
+
this.opts.log(`! could not write the run log ${f}: ${e.message.split("\n")[0]}`);
|
|
2259
2862
|
}
|
|
2260
|
-
this.opts.log(quiet ? `< read ${o.id} on ${branch} — the thread was already settled, nothing sent`
|
|
2261
|
-
: ok ? `< handled ${o.id} on ${branch}`
|
|
2262
|
-
: `! ${o.id} stopped [${diagnosis.cause}]: ${diagnosis.says} (transcript: ${runLog})`);
|
|
2263
|
-
return { obligation: o, branch, worktree, ok, detail, ...(diagnosis ? { diagnosis } : {}) };
|
|
2264
2863
|
}
|
|
2265
2864
|
lastFeaturesSaid = "";
|
|
2266
2865
|
notesSaid = new Set();
|
|
@@ -2312,7 +2911,7 @@ export class Worker {
|
|
|
2312
2911
|
const r = await gitOut(run.worktree, ["push", "--quiet", "origin", `refs/heads/${br}:refs/heads/${br}`], FETCH_TIMEOUT_MS);
|
|
2313
2912
|
pushed = r !== null && await onRemote(run.worktree, br, commit);
|
|
2314
2913
|
if (pushed)
|
|
2315
|
-
this.opts.log(`* pushed ${br} for the run: its DELIVERED ${d.id} named ${commit.slice(0, 7)}, which origin did not have`);
|
|
2914
|
+
this.opts.log(`* pushed ${br} for the run: its DELIVERED ${d.id} named ${commit.slice(0, 7)}, which origin did not have — repo hooks not run on Pantheon's push${lfsInUse(run.worktree) ? "; LFS objects uploaded" : ""}`);
|
|
2316
2915
|
}
|
|
2317
2916
|
if (pushed)
|
|
2318
2917
|
continue;
|
|
@@ -2375,7 +2974,8 @@ export class Worker {
|
|
|
2375
2974
|
].join("\n"),
|
|
2376
2975
|
});
|
|
2377
2976
|
markContinued(space, key, this.opts.home);
|
|
2378
|
-
this.opts.log(`* told ${peer ?? "the teammate"} that ${o.id} waits
|
|
2977
|
+
this.opts.log(d.cause === "agent_unavailable" ? `* told ${peer ?? "the teammate"} that ${o.id} waits: ${d.says}`
|
|
2978
|
+
: `* told ${peer ?? "the teammate"} that ${o.id} waits for a live session: ${d.cause === "repo_not_here" ? d.says : "no agent here can take unattended work"}`);
|
|
2379
2979
|
}
|
|
2380
2980
|
catch (e) {
|
|
2381
2981
|
this.opts.log(`! could not tell the teammate that ${o.id} is waiting: ${e.message}`);
|
|
@@ -2390,7 +2990,8 @@ export class Worker {
|
|
|
2390
2990
|
const asking = !!d.capability && mayAsk(policy, d.capability);
|
|
2391
2991
|
const what = clip(o.body, 120);
|
|
2392
2992
|
const lines = [
|
|
2393
|
-
|
|
2993
|
+
attempts === 0 ? `No run was started on this machine for ${o.kind === "CONTINUE" ? "a follow-up" : `the ${o.kind}`} it owes: ${what}`
|
|
2994
|
+
: `${attempts === 1 ? "A run" : `${attempts === 2 ? "Two" : attempts} runs`} on this machine could not finish ${o.kind === "CONTINUE" ? "a follow-up" : `the ${o.kind}`} it owes: ${what}`,
|
|
2394
2995
|
``,
|
|
2395
2996
|
`WHY IT STOPPED. ${d.says}`,
|
|
2396
2997
|
`WHAT UNBLOCKS IT. ${d.fix}`,
|
|
@@ -2542,39 +3143,48 @@ export class Worker {
|
|
|
2542
3143
|
const lastAttempt = policy.attempts.self_upgrade ?? null;
|
|
2543
3144
|
if (!needsSelfUpgrade(VERSION, relayPkg, lastAttempt))
|
|
2544
3145
|
return;
|
|
2545
|
-
if (
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
this.opts.log(`* self-upgrade: relay runs ${relayPkg}, this machine runs ${VERSION} — upgrading (granted by this machine's human)`);
|
|
2550
|
-
let result;
|
|
3146
|
+
if (policy.grants.self_upgrade) {
|
|
3147
|
+
const slot = this.takeSlot();
|
|
3148
|
+
if ("held" in slot)
|
|
3149
|
+
return;
|
|
2551
3150
|
try {
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
catch (e) {
|
|
2555
|
-
result = { ok: false, alreadyCurrent: false, from: VERSION, to: relayPkg ?? "?" };
|
|
2556
|
-
this.opts.log(`! self-upgrade failed: ${e.message}`);
|
|
2557
|
-
}
|
|
2558
|
-
if (result.ok && result.to === result.from) {
|
|
2559
|
-
delete policy.attempts.self_upgrade;
|
|
3151
|
+
useGrant(policy, "self_upgrade");
|
|
3152
|
+
policy.attempts.self_upgrade = Date.now();
|
|
2560
3153
|
savePolicy(policy, home);
|
|
2561
|
-
this.
|
|
3154
|
+
this.writeState("upgrading", `pantheon ${VERSION} -> ${relayPkg} (relay is ahead)`);
|
|
3155
|
+
this.opts.log(`* self-upgrade: relay runs ${relayPkg}, this machine runs ${VERSION} — upgrading (granted by this machine's human)`);
|
|
3156
|
+
let result;
|
|
3157
|
+
try {
|
|
3158
|
+
result = this.opts.selfUpgrade?.run ? this.opts.selfUpgrade.run() : await this.runRealUpgrade(relayPkg ?? undefined);
|
|
3159
|
+
}
|
|
3160
|
+
catch (e) {
|
|
3161
|
+
result = { ok: false, alreadyCurrent: false, from: VERSION, to: relayPkg ?? "?" };
|
|
3162
|
+
this.opts.log(`! self-upgrade failed: ${e.message}`);
|
|
3163
|
+
}
|
|
3164
|
+
if (result.ok && result.to === result.from) {
|
|
3165
|
+
delete policy.attempts.self_upgrade;
|
|
3166
|
+
savePolicy(policy, home);
|
|
3167
|
+
this.opts.log(`* self-upgrade: nothing newer than ${result.to} to install yet (the relay runs ${relayPkg}); will check again`);
|
|
3168
|
+
return;
|
|
3169
|
+
}
|
|
3170
|
+
if (result.ok && !result.alreadyCurrent) {
|
|
3171
|
+
await client.send({
|
|
3172
|
+
act: "NOTE", goal: null, after_seq: Math.max(client.readSeq, client.seq),
|
|
3173
|
+
summary: `self-upgraded pantheon ${result.from} -> ${result.to}; worker restarting on the new build`,
|
|
3174
|
+
body: `This machine's worker upgraded pantheon ${result.from} -> ${result.to} (fast-forward pull + rebuild, granted by its human) and is restarting on the new build. Agent windows on this machine opened on 0.24 or later switch to the new version by themselves; a window opened on an older pantheon needs one reconnect (/mcp → pantheon → Reconnect, or reopen it).`,
|
|
3175
|
+
}).catch((e) => this.opts.log(`! could not report the upgrade on the ledger: ${e.message}`));
|
|
3176
|
+
await this.bringMachineAlong();
|
|
3177
|
+
this.opts.log(`* self-upgrade done: ${result.from} -> ${result.to}; restarting worker to load it`);
|
|
3178
|
+
(this.opts.selfUpgrade?.restart ?? (() => process.exit(0)))();
|
|
3179
|
+
}
|
|
3180
|
+
else if (!result.ok) {
|
|
3181
|
+
this.opts.log("! self-upgrade did not complete — will not retry for an hour; run `pantheon upgrade` by hand to see why");
|
|
3182
|
+
}
|
|
2562
3183
|
return;
|
|
2563
3184
|
}
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
act: "NOTE", goal: null, after_seq: Math.max(client.readSeq, client.seq),
|
|
2567
|
-
summary: `self-upgraded pantheon ${result.from} -> ${result.to}; worker restarting on the new build`,
|
|
2568
|
-
body: `This machine's worker upgraded pantheon ${result.from} -> ${result.to} (fast-forward pull + rebuild, granted by its human) and is restarting on the new build. Agent windows on this machine opened on 0.24 or later switch to the new version by themselves; a window opened on an older pantheon needs one reconnect (/mcp → pantheon → Reconnect, or reopen it).`,
|
|
2569
|
-
}).catch((e) => this.opts.log(`! could not report the upgrade on the ledger: ${e.message}`));
|
|
2570
|
-
await this.bringMachineAlong();
|
|
2571
|
-
this.opts.log(`* self-upgrade done: ${result.from} -> ${result.to}; restarting worker to load it`);
|
|
2572
|
-
(this.opts.selfUpgrade?.restart ?? (() => process.exit(0)))();
|
|
2573
|
-
}
|
|
2574
|
-
else if (!result.ok) {
|
|
2575
|
-
this.opts.log("! self-upgrade did not complete — will not retry for an hour; run `pantheon upgrade` by hand to see why");
|
|
3185
|
+
finally {
|
|
3186
|
+
this.giveSlot(slot.token);
|
|
2576
3187
|
}
|
|
2577
|
-
return;
|
|
2578
3188
|
}
|
|
2579
3189
|
if (mayAsk(policy, "self_upgrade")) {
|
|
2580
3190
|
try {
|