@join-pantheon/cli 0.38.1 → 0.38.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/bin/pantheon.js +16 -1
- package/bin/upgrade-guard.js +530 -0
- package/dist/admin-pages.js +1 -0
- package/dist/agent-start.js +324 -0
- package/dist/agents.js +74 -4
- package/dist/cli.js +17 -1
- package/dist/crash-loop.js +55 -5
- package/dist/doctor.js +78 -6
- package/dist/file-lock.js +19 -4
- package/dist/flow-stats.js +1 -0
- package/dist/home-safety.js +5 -0
- package/dist/ledger.js +6 -0
- package/dist/leftovers.js +6 -2
- package/dist/mcp.js +8 -1
- package/dist/notice.js +2 -1
- package/dist/pickup.js +1 -0
- package/dist/plain.js +1 -0
- package/dist/protocol.js +1 -1
- package/dist/readiness-alert.js +17 -2
- package/dist/relay.js +123 -27
- package/dist/run-failure.js +12 -1
- package/dist/store.js +173 -4
- package/dist/update-check.js +17 -2
- package/dist/upgrade-probation.js +314 -0
- package/dist/upgrade.js +50 -3
- package/dist/version-health.js +8 -1
- package/dist/which.js +1 -1
- package/dist/win-host.js +2 -1
- package/dist/worker.js +246 -28
- package/helper/macos/prebuilt/manifest.json +2 -2
- package/helper/macos/prebuilt/pantheon-hud +0 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/worker.js
CHANGED
|
@@ -16,12 +16,14 @@ import { mkdirInside, writeInside } from "./safe-write.js";
|
|
|
16
16
|
import { corruptHeldFiles, MAX_HELD_PER_RUN, remoteTipIs, takeHeldFrames, updateHeldState, verifyHeldFrame, writeHoldConfig } from "./held-delivery.js";
|
|
17
17
|
import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
|
|
18
18
|
import { trusted } from "./verify-act.js";
|
|
19
|
-
import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
19
|
+
import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, cannotStartWords, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
20
20
|
import { activeOutages, detectOutage, earliest, isSignOut, outagePredicate, whenAgain, SIGNIN_REFUSED } from "./agent-outage.js";
|
|
21
|
+
import { forgetStart, markCannotStart, startFailureOf, startWorked, START_CONFIRM_MS } from "./agent-start.js";
|
|
21
22
|
import { agentSpawn, CmdLineUnsafe } from "./which.js";
|
|
22
23
|
import { VERSION } from "./version.js";
|
|
23
24
|
import { composeUpgradeFailed, fetchRelayPkg, recordRelayPkg, upgradeFailedOnce } from "./version-health.js";
|
|
24
|
-
import {
|
|
25
|
+
import { markRolledBackTold, readRolledBack, rolledBackSentence } from "./upgrade-probation.js";
|
|
26
|
+
import { noteServiceConnected, noteServiceReached, noteServiceTried } from "./crash-loop.js";
|
|
25
27
|
import { applyAnswer, ASK_EXPIRY_MS, policyUnreadable, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, policyHome, describeRunMinutes, parseRunMinutes, currentMode, wakeSettled, effectiveRunModel, describeRunModel, effectiveGoalRuns, claudeRouting, } from "./worker-policy.js";
|
|
26
28
|
import { readinessAlerts } from "./readiness-alert.js";
|
|
27
29
|
import { clearDeclined, markDeclined, personsOwnThread, pruneDeclined } from "./declined.js";
|
|
@@ -145,6 +147,8 @@ export const BASE_RE = /^[A-Za-z0-9._][A-Za-z0-9._\/-]{0,99}$/;
|
|
|
145
147
|
export const validBase = (b) => typeof b === "string" && BASE_RE.test(b) && !b.includes("..") && !b.endsWith("/") && !b.endsWith(".lock");
|
|
146
148
|
export const safeSegment = (s) => typeof s === "string" && /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,99}$/.test(s) ? s : null;
|
|
147
149
|
const CLOCK_SKEW_MS = 5_000;
|
|
150
|
+
export const claimNotAnswered = (message) => /^(RATE_LIMIT|STORAGE_BUSY|STORAGE|NO_REPLY): /.test(message) || /^not connected to relay\b/.test(message);
|
|
151
|
+
export const CLAIM_RETRY_MS = 15_000;
|
|
148
152
|
const isRepo = (cwd) => {
|
|
149
153
|
try {
|
|
150
154
|
return git(cwd, "rev-parse", "--is-inside-work-tree") === "true";
|
|
@@ -703,6 +707,8 @@ export async function onRemote(cwd, branch, commit) {
|
|
|
703
707
|
}
|
|
704
708
|
export const MASS_DELETE_SHARE = 0.5;
|
|
705
709
|
const ONCE_GIVE_BACK_MS = 120_000;
|
|
710
|
+
const REFUSED_NOTE_TRIES = 8;
|
|
711
|
+
export const START_CONFIRM_WAIT_MS = START_CONFIRM_MS + 15_000;
|
|
706
712
|
export const asksForMassDelete = (text) => /\b(?:delete|remove|wipe|drop|rm)\b[^\n]{0,40}\b(?:all|every|entire|whole|most)\b/i.test(text);
|
|
707
713
|
const GIT_HOOK_NAMES = [
|
|
708
714
|
"applypatch-msg", "pre-applypatch", "post-applypatch", "prepare-commit-msg", "commit-msg", "post-commit",
|
|
@@ -1089,6 +1095,8 @@ export function composePickups(items, space) {
|
|
|
1089
1095
|
export const UNSANDBOXED_RUN = `\n\nTHIS RUN HAS NO SANDBOX. Never run code a teammate wrote: their branch, a patch or a script they sent, or a\ntest they changed. Read and review it, then ESCALATE to your human to run it. The repo's own commands on your\nown changes are fine.`;
|
|
1090
1096
|
export const TASK_POINTER = "Your task for this run is in the file .pantheon-task.md in the current directory. Read that file first and follow it: it is the whole brief.";
|
|
1091
1097
|
let fleetLock = null;
|
|
1098
|
+
let rolledBackSaid = null;
|
|
1099
|
+
let rolledBackLookedAt = 0;
|
|
1092
1100
|
export const RUN_TIMEOUT_MS = RUN_MINUTES.default * 60_000;
|
|
1093
1101
|
export const RUN_TIMEOUT_MAX_MS = RUN_MINUTES.max * 60_000;
|
|
1094
1102
|
function recordLimitMs(r) {
|
|
@@ -1771,6 +1779,8 @@ export class Worker {
|
|
|
1771
1779
|
d = this.outageDiagnosis(c.backend.label, c.outage, []);
|
|
1772
1780
|
else if (NOT_FOR_UNATTENDED_RUNS.has(c.backend.id))
|
|
1773
1781
|
d = this.noUnattendedAgentDiagnosis(unattendedRefusal(c, { unsandboxedOk: ok }) ?? c.backend.label);
|
|
1782
|
+
else if (c.cannotStart)
|
|
1783
|
+
d = c.cannotStart.confirmed ? this.cannotStartDiagnosis(c) : null;
|
|
1774
1784
|
else if (!SANDBOXED_FOR_UNATTENDED.has(c.backend.id) && !ok)
|
|
1775
1785
|
d = this.noSandboxDiagnosis(c);
|
|
1776
1786
|
}
|
|
@@ -1780,6 +1790,30 @@ export class Worker {
|
|
|
1780
1790
|
this.sandboxHold.set(key, { at: Date.now(), d });
|
|
1781
1791
|
return d;
|
|
1782
1792
|
}
|
|
1793
|
+
cannotStartDiagnosis(c) {
|
|
1794
|
+
const pinned = c.because === "pinned";
|
|
1795
|
+
const others = c.skipped.filter((x) => x.backend.id !== c.backend.id);
|
|
1796
|
+
const w = cannotStartWords(c.backend, c.cannotStart ?? {}, { pinned, claudeHere: c.backend.id === "claude-code" || others.some((x) => x.backend.id === "claude-code") });
|
|
1797
|
+
const rest = pinned ? ` Woken runs here are pinned to it (a persona's agent: line, or the worker's --agent), so teammates' requests wait.`
|
|
1798
|
+
: others.length ? ` Nothing else here can take teammates' requests (${others.map((x) => `${x.backend.label}: ${x.why}`).join("; ")}), so they wait.`
|
|
1799
|
+
: ` Teammates' requests wait until it does.`;
|
|
1800
|
+
const who = this.cfg.member;
|
|
1801
|
+
return {
|
|
1802
|
+
cause: "agent_not_installed",
|
|
1803
|
+
says: `${w.says}${rest}`,
|
|
1804
|
+
fix: `${w.fix} Requests that waited run by themselves once it starts.`,
|
|
1805
|
+
peer: {
|
|
1806
|
+
code: "cannot_start",
|
|
1807
|
+
summary: `${who}'s agent can't start on their computer — this waits for ${who}`,
|
|
1808
|
+
body: `${who}'s worker could not start a run: ${c.backend.label} cannot start on that computer, and nothing else there could take this — ${who} has to fix that there.\nIt is still owed, and runs by itself once that is fixed. If it can't wait, remind ${who} or ask someone else. Nothing was run.`,
|
|
1809
|
+
},
|
|
1810
|
+
};
|
|
1811
|
+
}
|
|
1812
|
+
cannotStartTold() {
|
|
1813
|
+
const r = this.readiness;
|
|
1814
|
+
return !!r && !r.ok && r.code === "cannot_start" && !!r.certain;
|
|
1815
|
+
}
|
|
1816
|
+
cannotStartHold = new Set();
|
|
1783
1817
|
noUnattendedAgentDiagnosis(refused) {
|
|
1784
1818
|
return {
|
|
1785
1819
|
cause: "no_unattended_agent",
|
|
@@ -1821,6 +1855,8 @@ export class Worker {
|
|
|
1821
1855
|
this.readinessAt = now;
|
|
1822
1856
|
if (force || marked)
|
|
1823
1857
|
forgetMissingBinaries();
|
|
1858
|
+
if (force || marked)
|
|
1859
|
+
forgetStart({ runProven: marked });
|
|
1824
1860
|
if (this.opts.agentCmd || this.opts.housekeeping || this.opts.alertsOnly) {
|
|
1825
1861
|
this.readiness = { ok: true, short: "" };
|
|
1826
1862
|
return;
|
|
@@ -1848,9 +1884,13 @@ export class Worker {
|
|
|
1848
1884
|
if (ok && this.readiness && !this.readiness.ok)
|
|
1849
1885
|
this.opts.log(` agent ready again — teammates are told this machine takes requests`);
|
|
1850
1886
|
const outAgent = signedOut ? [...this.signedOutHere][0] : undefined;
|
|
1887
|
+
const couldNotStart = this.readiness?.code === "cannot_start";
|
|
1851
1888
|
this.readiness = signedOut
|
|
1852
1889
|
? { ok: false, short: "its agent CLI was signed out (a run here was refused)", code: "not_signed_in", fix: held.fix ?? "", signedOut: true, ...(outAgent ? { agent: outAgent } : {}) }
|
|
1853
|
-
: { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {})
|
|
1890
|
+
: { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}),
|
|
1891
|
+
...(r.code === "cannot_start" ? { says: r.says.replace(/ A request while you're away would fail.*$/, ""), ...(r.certain ? { certain: true } : {}) } : {}) };
|
|
1892
|
+
if (couldNotStart !== (this.readiness.code === "cannot_start"))
|
|
1893
|
+
this.sandboxHold.clear();
|
|
1854
1894
|
this.tellPerson(now, { quiet: signedOut });
|
|
1855
1895
|
}
|
|
1856
1896
|
catch {
|
|
@@ -1858,6 +1898,7 @@ export class Worker {
|
|
|
1858
1898
|
}
|
|
1859
1899
|
}
|
|
1860
1900
|
readinessCommandsOn = false;
|
|
1901
|
+
ranOkAt = 0;
|
|
1861
1902
|
lastReadinessKey = null;
|
|
1862
1903
|
readinessKey() {
|
|
1863
1904
|
const commands = this.readinessCommandsOn && process.platform === "linux";
|
|
@@ -1891,7 +1932,7 @@ export class Worker {
|
|
|
1891
1932
|
const key = this.readinessKey();
|
|
1892
1933
|
const replaces = this.lastReadinessKey && this.lastReadinessKey !== key ? this.lastReadinessKey : undefined;
|
|
1893
1934
|
this.lastReadinessKey = key;
|
|
1894
|
-
const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet ? { certain: true } : {}) });
|
|
1935
|
+
const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet || r.certain || (r.ok && now - this.ranOkAt >= 0 && now - this.ranOkAt < 2 * WORKER_READINESS_RETRY_MS) ? { certain: true } : {}) });
|
|
1895
1936
|
this.alertPending = t.pending;
|
|
1896
1937
|
if (!t.notice)
|
|
1897
1938
|
return;
|
|
@@ -2030,6 +2071,22 @@ export class Worker {
|
|
|
2030
2071
|
sayFallbackUnsandboxed(c) {
|
|
2031
2072
|
if (SANDBOXED_FOR_UNATTENDED.has(c.backend.id) || !this.opts.notify)
|
|
2032
2073
|
return;
|
|
2074
|
+
const stuck = c.skipped.find((x) => x.backend.id === "claude-code" && x.cannotStart);
|
|
2075
|
+
if (stuck) {
|
|
2076
|
+
const key = `${c.backend.id}|cannot-start`;
|
|
2077
|
+
if (this.fallbackSaid.has(key))
|
|
2078
|
+
return;
|
|
2079
|
+
this.fallbackSaid.add(key);
|
|
2080
|
+
try {
|
|
2081
|
+
this.opts.notify({
|
|
2082
|
+
kind: "worker", who: "Your worker", heading: `${c.backend.label} is answering for Claude Code`, context: this.cfg.space,
|
|
2083
|
+
body: `Claude Code can't start on this computer (pantheon doctor shows why). Until it does, ${c.backend.label} takes teammates' requests on this machine, without Claude Code's sandbox, as your settings allow. To have them wait instead: pantheon policy revoke unsandboxed-runs`,
|
|
2084
|
+
ttl: 0, sound: false,
|
|
2085
|
+
});
|
|
2086
|
+
}
|
|
2087
|
+
catch { }
|
|
2088
|
+
return;
|
|
2089
|
+
}
|
|
2033
2090
|
const passed = this.outagesPassed(c).filter((p) => p.label === "Claude Code");
|
|
2034
2091
|
const first = passed[0];
|
|
2035
2092
|
if (!first)
|
|
@@ -2199,7 +2256,7 @@ export class Worker {
|
|
|
2199
2256
|
h.at = now;
|
|
2200
2257
|
continue;
|
|
2201
2258
|
}
|
|
2202
|
-
this.releaseHeld(id, 0, ok ? "unsandboxed runs are allowed here now" : "an agent here can take unattended work now");
|
|
2259
|
+
this.releaseHeld(id, 0, h.cannotStart ? "the agent that could not start does now" : ok ? "unsandboxed runs are allowed here now" : "an agent here can take unattended work now");
|
|
2203
2260
|
}
|
|
2204
2261
|
if (!ok && now - this.ranNoSandboxCheckedAt < every)
|
|
2205
2262
|
return;
|
|
@@ -3135,7 +3192,10 @@ export class Worker {
|
|
|
3135
3192
|
async run() {
|
|
3136
3193
|
const upkeep = !!(this.opts.housekeeping || this.opts.alertsOnly);
|
|
3137
3194
|
const fromSeq = upkeep ? 0 : this.readCursor();
|
|
3195
|
+
noteServiceTried(this.cfg.relay);
|
|
3138
3196
|
const client = new PantheonClient(this.cfg, (e) => {
|
|
3197
|
+
if (e.t === "welcome" || e.t === "error" || e.t === "pending")
|
|
3198
|
+
noteServiceReached();
|
|
3139
3199
|
if ((e.t === "welcome" || e.t === "members") && this.client === client)
|
|
3140
3200
|
queueMicrotask(() => this.carryWakers(client));
|
|
3141
3201
|
}, {
|
|
@@ -3233,7 +3293,7 @@ export class Worker {
|
|
|
3233
3293
|
await this.restoreHeldContinuations(client);
|
|
3234
3294
|
await this.restoreQueuedContinuations();
|
|
3235
3295
|
}
|
|
3236
|
-
this.
|
|
3296
|
+
this.writeWaiting();
|
|
3237
3297
|
while (!this.stopped) {
|
|
3238
3298
|
const arrived = await client.next(5000);
|
|
3239
3299
|
if (this.deadClaims.size)
|
|
@@ -3321,27 +3381,41 @@ export class Worker {
|
|
|
3321
3381
|
await this.settleMoot(client);
|
|
3322
3382
|
await this.settleClosedQuestions(client);
|
|
3323
3383
|
await this.settleSafety(client);
|
|
3384
|
+
await this.settleRefused(client);
|
|
3324
3385
|
const aged = client
|
|
3325
3386
|
.myWork()
|
|
3326
3387
|
.filter((o) => o.kind !== "HUMAN" && o.kind !== "DELIVER" && !o.claimed_by)
|
|
3327
3388
|
.filter((o) => Date.now() - o.created_at >= grace)
|
|
3328
|
-
.filter((o) => {
|
|
3329
|
-
|
|
3330
|
-
if (
|
|
3331
|
-
|
|
3332
|
-
|
|
3333
|
-
this.
|
|
3389
|
+
.filter((o) => {
|
|
3390
|
+
const t = this.outageHold.get(o.id);
|
|
3391
|
+
if (t === undefined)
|
|
3392
|
+
return true;
|
|
3393
|
+
if (t > Date.now()) {
|
|
3394
|
+
if (this.cannotStartHold.has(o.id)) {
|
|
3395
|
+
if (!this.tickHold)
|
|
3396
|
+
this.tickHold = { reason: "cannot_start", until: t, detail: "its agent did not start; asking it again" };
|
|
3397
|
+
this.decline(o.ref, "your agent did not start — it is asked again in a minute");
|
|
3398
|
+
return false;
|
|
3399
|
+
}
|
|
3400
|
+
if (this.allOutSignedOut()) {
|
|
3401
|
+
if (!this.tickHold)
|
|
3402
|
+
this.tickHold = { reason: "not_signed_in", detail: "its agent is signed out" };
|
|
3403
|
+
this.decline(o.ref, "your agent is signed out");
|
|
3404
|
+
return false;
|
|
3405
|
+
}
|
|
3406
|
+
if (!this.tickHold || (this.tickHold.reason === "outage" && (this.tickHold.until ?? Infinity) > t))
|
|
3407
|
+
this.tickHold = { reason: "outage", until: t, detail: "its agents are out for now" };
|
|
3408
|
+
this.decline(o.ref, "your agents are out for now");
|
|
3334
3409
|
return false;
|
|
3335
3410
|
}
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
} this.outageHold.delete(o.id); return true; })
|
|
3411
|
+
this.outageHold.delete(o.id);
|
|
3412
|
+
this.cannotStartHold.delete(o.id);
|
|
3413
|
+
return true;
|
|
3414
|
+
})
|
|
3341
3415
|
.filter((o) => {
|
|
3342
3416
|
if (this.failuresOf(o.id) < 2)
|
|
3343
3417
|
return true;
|
|
3344
|
-
this.decline(o.ref, "its run failed twice on this computer");
|
|
3418
|
+
this.decline(o.ref, this.parkedNoSandbox.get(o.id)?.cannotStart ? "your agent cannot start on this computer" : "its run failed twice on this computer");
|
|
3345
3419
|
if (!this.skipLogged.has(`parked:${o.id}`)) {
|
|
3346
3420
|
this.skipLogged.add(`parked:${o.id}`);
|
|
3347
3421
|
this.opts.log(`- parking ${o.kind} ${o.id}: its run failed in a way another run would repeat — it stays owed for a live session or your human (see the last .pantheon-run.log)`);
|
|
@@ -3522,7 +3596,7 @@ export class Worker {
|
|
|
3522
3596
|
}
|
|
3523
3597
|
}
|
|
3524
3598
|
if (!candidate && !continuation) {
|
|
3525
|
-
this.
|
|
3599
|
+
this.writeWaiting();
|
|
3526
3600
|
continue;
|
|
3527
3601
|
}
|
|
3528
3602
|
const busyTtlMs = this.opts.busyTtlMs ?? LIMITS.activityBusyTtlMs;
|
|
@@ -3561,7 +3635,7 @@ export class Worker {
|
|
|
3561
3635
|
this.markDone(continuation.id);
|
|
3562
3636
|
this.forgetQueued(continuation.id);
|
|
3563
3637
|
this.opts.log(`- not continuing from ${continuation.act} ${continuation.id}: the window that asked was shown it, and its turn ended after that`);
|
|
3564
|
-
this.
|
|
3638
|
+
this.writeWaiting();
|
|
3565
3639
|
continue;
|
|
3566
3640
|
}
|
|
3567
3641
|
}
|
|
@@ -3628,6 +3702,13 @@ export class Worker {
|
|
|
3628
3702
|
if (/^DEFER/.test(msg)) {
|
|
3629
3703
|
this.requeueContinuation(continuation, 10_000);
|
|
3630
3704
|
}
|
|
3705
|
+
else if (claimNotAnswered(msg)) {
|
|
3706
|
+
this.requeueContinuation(continuation, CLAIM_RETRY_MS);
|
|
3707
|
+
if (!this.skipLogged.has(`claim-wait:${continuation.id}`)) {
|
|
3708
|
+
this.skipLogged.add(`claim-wait:${continuation.id}`);
|
|
3709
|
+
this.opts.log(`- holding ${continuation.act} ${continuation.id}: the relay could not take its claim just now (${msg.split("\n")[0].slice(0, 120)}) — asking again shortly`);
|
|
3710
|
+
}
|
|
3711
|
+
}
|
|
3631
3712
|
else {
|
|
3632
3713
|
this.markDone(continuation.id);
|
|
3633
3714
|
if (!this.skipLogged.has(`cont:${continuation.id}`)) {
|
|
@@ -3717,7 +3798,7 @@ export class Worker {
|
|
|
3717
3798
|
}
|
|
3718
3799
|
continue;
|
|
3719
3800
|
}
|
|
3720
|
-
if (job.refused && job.diagnosis?.cause === "agent_unavailable" && job.waitUntil) {
|
|
3801
|
+
if (job.refused && (job.diagnosis?.cause === "agent_unavailable" || job.diagnosis?.cause === "agent_not_installed") && job.waitUntil) {
|
|
3721
3802
|
if (candidate)
|
|
3722
3803
|
this.outageHold.set(candidate.id, job.waitUntil);
|
|
3723
3804
|
if (this.outageHold.size > 500)
|
|
@@ -3733,7 +3814,8 @@ export class Worker {
|
|
|
3733
3814
|
}
|
|
3734
3815
|
else if (job.refused && job.diagnosis) {
|
|
3735
3816
|
if (candidate)
|
|
3736
|
-
this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer"
|
|
3817
|
+
this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer"
|
|
3818
|
+
: job.diagnosis.cause === "agent_not_installed" ? "your agent cannot start on this computer" : "nothing on this computer can run it while you're away");
|
|
3737
3819
|
if (candidate) {
|
|
3738
3820
|
this.failedRuns.set(this.oKey(candidate.id), 2);
|
|
3739
3821
|
this.skipLogged.add(`parked:${candidate.id}`);
|
|
@@ -3786,7 +3868,7 @@ export class Worker {
|
|
|
3786
3868
|
if (continuation)
|
|
3787
3869
|
client.forgetClaim(`cont:${continuation.id}`);
|
|
3788
3870
|
}
|
|
3789
|
-
this.
|
|
3871
|
+
this.writeWaiting();
|
|
3790
3872
|
if (this.opts.once)
|
|
3791
3873
|
break;
|
|
3792
3874
|
}
|
|
@@ -4275,6 +4357,26 @@ export class Worker {
|
|
|
4275
4357
|
return { obligation: o, branch, worktree, ok: false, refused: true, detail: diagnosis.says, diagnosis, waitUntil: choice.outage.until };
|
|
4276
4358
|
}
|
|
4277
4359
|
const refusal = unattendedRefusalOf(choice, { unsandboxedOk: this.unsandboxedOk() });
|
|
4360
|
+
if (refusal?.cause === "cannot_start") {
|
|
4361
|
+
const diagnosis = this.cannotStartDiagnosis(choice);
|
|
4362
|
+
if (!choice.cannotStart?.confirmed) {
|
|
4363
|
+
this.cannotStartHold.add(o.id);
|
|
4364
|
+
if (this.cannotStartHold.size > 500)
|
|
4365
|
+
this.cannotStartHold.delete(this.cannotStartHold.values().next().value);
|
|
4366
|
+
if (!this.skipLogged.has(`start1:${o.id}`)) {
|
|
4367
|
+
this.skipLogged.add(`start1:${o.id}`);
|
|
4368
|
+
this.opts.log(`- not starting a run for ${o.kind} ${o.id} yet: ${refusal.says} — it is asked again in about a minute before anyone is told (not counted as a run)`);
|
|
4369
|
+
}
|
|
4370
|
+
return { obligation: o, branch, worktree, ok: false, refused: true, detail: refusal.says, diagnosis: { ...diagnosis, quiet: true }, waitUntil: Date.now() + (this.opts.startConfirmWaitMs ?? START_CONFIRM_WAIT_MS) };
|
|
4371
|
+
}
|
|
4372
|
+
if (!this.parkedNoSandbox.has(o.id))
|
|
4373
|
+
this.parkedNoSandbox.set(o.id, { at: Date.now(), ...(policy.agent ? { agent: policy.agent } : {}), cannotStart: true });
|
|
4374
|
+
if (!this.skipLogged.has(`refused:${o.id}`)) {
|
|
4375
|
+
this.skipLogged.add(`refused:${o.id}`);
|
|
4376
|
+
this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${refusal.says} — it waits until that agent starts (not counted as a run)`);
|
|
4377
|
+
}
|
|
4378
|
+
return { obligation: o, branch, worktree, ok: false, refused: true, detail: refusal.says, diagnosis };
|
|
4379
|
+
}
|
|
4278
4380
|
if (refusal) {
|
|
4279
4381
|
const refused = refusal.says;
|
|
4280
4382
|
const noSandbox = refusal.cause === "unsandboxed_off";
|
|
@@ -4610,6 +4712,8 @@ export class Worker {
|
|
|
4610
4712
|
}
|
|
4611
4713
|
let recorded = false;
|
|
4612
4714
|
let stopBy = null;
|
|
4715
|
+
const spawnedAt = Date.now();
|
|
4716
|
+
let ranMs = 0;
|
|
4613
4717
|
try {
|
|
4614
4718
|
try {
|
|
4615
4719
|
detail = await new Promise((res) => {
|
|
@@ -4689,6 +4793,7 @@ export class Worker {
|
|
|
4689
4793
|
return; settled = true; clearTimeout(kill); if (exitGrace)
|
|
4690
4794
|
clearTimeout(exitGrace); markDead(); res(`could not start "${command}": ${e.message}`); });
|
|
4691
4795
|
});
|
|
4796
|
+
ranMs = Date.now() - spawnedAt;
|
|
4692
4797
|
}
|
|
4693
4798
|
finally {
|
|
4694
4799
|
this.cleanupStep("removing the sandbox's empty mount points", () => {
|
|
@@ -4829,8 +4934,26 @@ export class Worker {
|
|
|
4829
4934
|
this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over ${outage.why === SIGNIN_REFUSED ? "until it is signed in again" : `until about ${whenAgain(outage.until)}`}; the request goes to the next agent here (transcript: ${refusedLog})`);
|
|
4830
4935
|
return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
|
|
4831
4936
|
}
|
|
4937
|
+
const notStarted = !ok && !exited && !outage && !sentAnywhere && choice && !this.opts.agentCmd && existsSync(worktree)
|
|
4938
|
+
? startFailureOf(detail, { agent: choice.backend.id, ranMs, binary: choice.binary }) : null;
|
|
4939
|
+
if (notStarted) {
|
|
4940
|
+
const found = markCannotStart(choice.backend, choice.binary, notStarted);
|
|
4941
|
+
this.sandboxHold.clear();
|
|
4942
|
+
this.readinessAt = 0;
|
|
4943
|
+
this.writeRunLog(logRoot, refusedLog, `obligation ${o.id}\nbranch ${branch}\nagent ${choice.backend.label}\n\n${detail}\n`);
|
|
4944
|
+
if (Date.now() - startedAt < ONCE_GIVE_BACK_MS)
|
|
4945
|
+
this.restoreOnces();
|
|
4946
|
+
if ((this.outageRetries.get(o.id) ?? 0) < 3) {
|
|
4947
|
+
this.outageRetries.set(o.id, (this.outageRetries.get(o.id) ?? 0) + 1);
|
|
4948
|
+
if (this.outageRetries.size > 200)
|
|
4949
|
+
this.outageRetries.delete(this.outageRetries.keys().next().value);
|
|
4950
|
+
this.opts.log(`! ${choice.backend.label} could not be started for ${o.id}: ${found.why ?? "its program did not start"} — it is not started again until it can; the request goes to the next agent here (transcript: ${refusedLog})`);
|
|
4951
|
+
return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
|
|
4952
|
+
}
|
|
4953
|
+
}
|
|
4832
4954
|
const quiet = ok && !requireReply && sent.length === 0;
|
|
4833
|
-
const facts = { exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands,
|
|
4955
|
+
const facts = { exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands, ranMs,
|
|
4956
|
+
...(choice ? { binary: choice.binary } : {}), ...(this.opts.agentCmd ? { agentCmd: true } : {}),
|
|
4834
4957
|
...(agentId === "claude-code" && !this.opts.agentCmd && claudeTokenForRun(this.stateDir())
|
|
4835
4958
|
? { env: { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: undefined }, savedToken: true } : { env: process.env }),
|
|
4836
4959
|
...(agentId === "claude-code" ? { claudeSettings: claudeSettingsAuth({ env: process.env }) } : {}),
|
|
@@ -4851,6 +4974,12 @@ export class Worker {
|
|
|
4851
4974
|
}
|
|
4852
4975
|
if (ok && agentId && !this.opts.agentCmd)
|
|
4853
4976
|
clearSignedOut(agentId, { pantheonHome: this.stateDir() });
|
|
4977
|
+
if (ok && choice && !this.opts.agentCmd) {
|
|
4978
|
+
startWorked(choice.backend, choice.binary);
|
|
4979
|
+
this.ranOkAt = Date.now();
|
|
4980
|
+
if (this.alertPending)
|
|
4981
|
+
this.readinessAt = 0;
|
|
4982
|
+
}
|
|
4854
4983
|
if (!ok && diagnosis?.cause === "not_signed_in" && agentId && !this.opts.agentCmd
|
|
4855
4984
|
&& markSignedOut(agentId, detail, { pantheonHome: this.stateDir(), env: process.env })) {
|
|
4856
4985
|
this.opts.log(`! ${choice?.backend.label ?? agentId} is signed out on this machine: requests wait until it is signed in again${agentId === "claude-code" ? " (pantheon worker token keeps Claude Code signed in for a year)" : ""}`);
|
|
@@ -5288,7 +5417,12 @@ export class Worker {
|
|
|
5288
5417
|
async reportRefused(client, o, d, owed) {
|
|
5289
5418
|
if (d.quiet)
|
|
5290
5419
|
return;
|
|
5291
|
-
if (this.
|
|
5420
|
+
if (d.cause === "agent_not_installed" && !this.cannotStartTold() && !this.refusalsNotified.has(d.says)) {
|
|
5421
|
+
this.readinessAt = 0;
|
|
5422
|
+
this.checkReadiness(true);
|
|
5423
|
+
}
|
|
5424
|
+
const toldByReadiness = d.cause === "agent_not_installed" && this.cannotStartTold();
|
|
5425
|
+
if (this.opts.notify && !toldByReadiness && !this.refusalsNotified.has(d.says)) {
|
|
5292
5426
|
this.refusalsNotified.add(d.says);
|
|
5293
5427
|
try {
|
|
5294
5428
|
this.opts.notify(composeWorker({ ok: false, triggeredBy: client.recentFrom(o.ref) ?? "unknown", obligationBody: o.body, goal: o.goal ? client.goals.get(o.goal) : undefined, reply: null, obligationRef: o.ref, diagnosis: d }, (() => { const tools = installedTools(); return { tools, preferred: preferredTool(loadProfile(this.cfg), tools, this.opts.home) }; })()));
|
|
@@ -5301,8 +5435,10 @@ export class Worker {
|
|
|
5301
5435
|
const key = `refused:${this.oKey(o.id)}`;
|
|
5302
5436
|
const space = this.spaceKey();
|
|
5303
5437
|
const told = loadContinued(space, this.opts.home);
|
|
5304
|
-
if (told.has(key) || told.has(`refused:${o.id}`))
|
|
5438
|
+
if (told.has(key) || told.has(`refused:${o.id}`)) {
|
|
5439
|
+
this.refusedDue.delete(key);
|
|
5305
5440
|
return;
|
|
5441
|
+
}
|
|
5306
5442
|
const peer = client.recentFrom(o.ref);
|
|
5307
5443
|
try {
|
|
5308
5444
|
await client.send({
|
|
@@ -5316,11 +5452,36 @@ export class Worker {
|
|
|
5316
5452
|
meta: { declined: d.peer?.code ?? (d.cause === "not_signed_in" ? "signed_out" : "unavailable") },
|
|
5317
5453
|
});
|
|
5318
5454
|
markContinued(space, key, this.opts.home);
|
|
5319
|
-
this.
|
|
5455
|
+
this.refusedDue.delete(key);
|
|
5456
|
+
this.opts.log(d.cause === "agent_unavailable" || d.cause === "agent_not_installed" ? `* told ${peer ?? "the teammate"} that ${o.id} waits: ${d.says}`
|
|
5320
5457
|
: `* told ${peer ?? "the teammate"} that ${o.id} waits for a live session: ${d.cause === "repo_not_here" || d.cause === "folder_too_large" ? d.says : "no agent here can take unattended work"}`);
|
|
5321
5458
|
}
|
|
5322
5459
|
catch (e) {
|
|
5323
|
-
|
|
5460
|
+
const msg = e.message;
|
|
5461
|
+
const tries = (this.refusedDue.get(key)?.tries ?? 0) + 1;
|
|
5462
|
+
const passing = !/^NO_REPLY\b/.test(msg) && (!/^[A-Z_]+:/.test(msg) || /^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(msg));
|
|
5463
|
+
if (passing && tries <= REFUSED_NOTE_TRIES)
|
|
5464
|
+
this.refusedDue.set(key, { o, d, tries, next: Date.now() + Math.min(60_000, 2_000 * tries) });
|
|
5465
|
+
else
|
|
5466
|
+
this.refusedDue.delete(key);
|
|
5467
|
+
if (this.refusedDue.size > 200)
|
|
5468
|
+
this.refusedDue.delete(this.refusedDue.keys().next().value);
|
|
5469
|
+
this.opts.log(`! could not tell the teammate that ${o.id} is waiting: ${msg.split("\n")[0]}${this.refusedDue.has(key) ? " — it is said at the next look" : ""}`);
|
|
5470
|
+
}
|
|
5471
|
+
}
|
|
5472
|
+
refusedDue = new Map();
|
|
5473
|
+
async settleRefused(client) {
|
|
5474
|
+
if (!this.refusedDue.size || !client.connected)
|
|
5475
|
+
return;
|
|
5476
|
+
for (const [key, r] of [...this.refusedDue]) {
|
|
5477
|
+
if (r.next > Date.now())
|
|
5478
|
+
continue;
|
|
5479
|
+
const now = client.myWork().find((x) => this.oKey(x.id) === this.oKey(r.o.id));
|
|
5480
|
+
if (!now) {
|
|
5481
|
+
this.refusedDue.delete(key);
|
|
5482
|
+
continue;
|
|
5483
|
+
}
|
|
5484
|
+
await this.reportRefused(client, now, r.d, true);
|
|
5324
5485
|
}
|
|
5325
5486
|
}
|
|
5326
5487
|
mootSince = new Map();
|
|
@@ -6029,6 +6190,7 @@ export class Worker {
|
|
|
6029
6190
|
lastHealthCheck = 0;
|
|
6030
6191
|
async maintain(client, arrived) {
|
|
6031
6192
|
const home = this.opts.home;
|
|
6193
|
+
this.sayRolledBack();
|
|
6032
6194
|
{
|
|
6033
6195
|
const policy = loadPolicy(home);
|
|
6034
6196
|
const denies = [];
|
|
@@ -6074,6 +6236,14 @@ export class Worker {
|
|
|
6074
6236
|
const lastAttempt = policy.attempts.self_upgrade ?? null;
|
|
6075
6237
|
if (!needsSelfUpgrade(VERSION, relayPkg, lastAttempt))
|
|
6076
6238
|
return;
|
|
6239
|
+
const hold = (() => { try {
|
|
6240
|
+
return this.opts.selfUpgrade?.rolledBack ? this.opts.selfUpgrade.rolledBack.read() : readRolledBack();
|
|
6241
|
+
}
|
|
6242
|
+
catch {
|
|
6243
|
+
return null;
|
|
6244
|
+
} })();
|
|
6245
|
+
if (hold && hold.version === relayPkg)
|
|
6246
|
+
return;
|
|
6077
6247
|
const nodeNow = this.opts.selfUpgrade?.node ?? (() => runtimeNodeVersion());
|
|
6078
6248
|
const held = policy.upgradeHeld;
|
|
6079
6249
|
if (held && held.version === relayPkg && held.node === nodeNow())
|
|
@@ -6179,6 +6349,42 @@ export class Worker {
|
|
|
6179
6349
|
}
|
|
6180
6350
|
}
|
|
6181
6351
|
}
|
|
6352
|
+
sayRolledBack() {
|
|
6353
|
+
const seam = this.opts.selfUpgrade?.rolledBack;
|
|
6354
|
+
const now = Date.now();
|
|
6355
|
+
if (!seam && now - rolledBackLookedAt < 60_000)
|
|
6356
|
+
return;
|
|
6357
|
+
rolledBackLookedAt = now;
|
|
6358
|
+
let back = null;
|
|
6359
|
+
try {
|
|
6360
|
+
back = seam ? seam.read() : readRolledBack();
|
|
6361
|
+
}
|
|
6362
|
+
catch {
|
|
6363
|
+
return;
|
|
6364
|
+
}
|
|
6365
|
+
if (!back || back.told_at)
|
|
6366
|
+
return;
|
|
6367
|
+
const key = `${back.version}@${back.at}`;
|
|
6368
|
+
const line = rolledBackSentence(back);
|
|
6369
|
+
if (rolledBackSaid !== key) {
|
|
6370
|
+
rolledBackSaid = key;
|
|
6371
|
+
this.opts.log(`! ${line}`);
|
|
6372
|
+
}
|
|
6373
|
+
const canAlert = !!this.opts.notify && decisionAlertsOn() && (() => { try {
|
|
6374
|
+
return !this.opts.canAlert || this.opts.canAlert();
|
|
6375
|
+
}
|
|
6376
|
+
catch {
|
|
6377
|
+
return false;
|
|
6378
|
+
} })();
|
|
6379
|
+
if (!canAlert)
|
|
6380
|
+
return;
|
|
6381
|
+
if (!(seam ? seam.told() : markRolledBackTold()))
|
|
6382
|
+
return;
|
|
6383
|
+
try {
|
|
6384
|
+
this.opts.notify?.({ kind: "worker", who: "Pantheon · this computer", heading: `Pantheon went back to ${back.back_to}`, context: this.cfg.space, body: line, ttl: 0, sound: false });
|
|
6385
|
+
}
|
|
6386
|
+
catch { }
|
|
6387
|
+
}
|
|
6182
6388
|
askAt = new Map();
|
|
6183
6389
|
ledgerAllowSeen = new Set();
|
|
6184
6390
|
lastFullScan = 0;
|
|
@@ -6677,11 +6883,23 @@ export class Worker {
|
|
|
6677
6883
|
this.report("running", detail, undefined, { ...(l?.ref ? { ref: l.ref } : {}), ...(l?.goal ? { goal: l.goal } : {}) });
|
|
6678
6884
|
}
|
|
6679
6885
|
heartbeatAt = 0;
|
|
6886
|
+
writeWaiting() {
|
|
6887
|
+
const r = this.readiness;
|
|
6888
|
+
if (!r || r.ok || r.code !== "cannot_start") {
|
|
6889
|
+
this.writeState("watching", "waiting for work");
|
|
6890
|
+
return;
|
|
6891
|
+
}
|
|
6892
|
+
if (Date.now() - this.lastStateAt >= WORKER_STATE_EVERY_MS)
|
|
6893
|
+
this.lastState = "";
|
|
6894
|
+
this.writeState("held", clip(r.says ?? "its agent cannot start on this computer", 160));
|
|
6895
|
+
}
|
|
6896
|
+
lastStateAt = 0;
|
|
6680
6897
|
writeState(state, detail) {
|
|
6681
6898
|
const line = `${state}|${detail}`;
|
|
6682
6899
|
if (line === this.lastState)
|
|
6683
6900
|
return;
|
|
6684
6901
|
this.lastState = line;
|
|
6902
|
+
this.lastStateAt = Date.now();
|
|
6685
6903
|
try {
|
|
6686
6904
|
const base = this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
|
|
6687
6905
|
mkdirSync(base, { recursive: true });
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"format": 1,
|
|
3
|
-
"built_at": "2026-11-
|
|
3
|
+
"built_at": "2026-11-17T05:58:19.032Z",
|
|
4
4
|
"swiftc": "Apple Swift version 6.4.2 (swiftlang-6.4.2.1.9 clang-2100.4.1.1)",
|
|
5
5
|
"helpers": {
|
|
6
6
|
"pantheon-tray": {
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"pantheon-hud": {
|
|
18
18
|
"source": "main.swift",
|
|
19
19
|
"source_sha256": "4976c00b5d4bd0d8c509d2d284d9c6e2d70a1b3db952de5f11108dfff4951a82",
|
|
20
|
-
"sha256": "
|
|
20
|
+
"sha256": "7ea1f02fcab4d53b1fd081ce7bef33c73bd092fcf310bef1619a10ac8611c808",
|
|
21
21
|
"archs": [
|
|
22
22
|
"arm64",
|
|
23
23
|
"x86_64"
|
|
Binary file
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@join-pantheon/cli",
|
|
3
|
-
"version": "0.38.
|
|
3
|
+
"version": "0.38.3",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@join-pantheon/cli",
|
|
9
|
-
"version": "0.38.
|
|
9
|
+
"version": "0.38.3",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@modelcontextprotocol/sdk": "^1.32.1",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@join-pantheon/cli",
|
|
3
|
-
"version": "0.38.
|
|
3
|
+
"version": "0.38.3",
|
|
4
4
|
"description": "Let your coding agent collaborate directly with your teammates' agents — ask, agree an interface, and deliver, without a human relaying messages.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|