@join-pantheon/cli 0.38.1 → 0.38.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/worker.js CHANGED
@@ -16,12 +16,14 @@ import { mkdirInside, writeInside } from "./safe-write.js";
16
16
  import { corruptHeldFiles, MAX_HELD_PER_RUN, remoteTipIs, takeHeldFrames, updateHeldState, verifyHeldFrame, writeHoldConfig } from "./held-delivery.js";
17
17
  import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
18
18
  import { trusted } from "./verify-act.js";
19
- import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
19
+ import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, cannotStartWords, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
20
20
  import { activeOutages, detectOutage, earliest, isSignOut, outagePredicate, whenAgain, SIGNIN_REFUSED } from "./agent-outage.js";
21
+ import { forgetStart, markCannotStart, startFailureOf, startWorked, START_CONFIRM_MS } from "./agent-start.js";
21
22
  import { agentSpawn, CmdLineUnsafe } from "./which.js";
22
23
  import { VERSION } from "./version.js";
23
24
  import { composeUpgradeFailed, fetchRelayPkg, recordRelayPkg, upgradeFailedOnce } from "./version-health.js";
24
- import { noteServiceConnected } from "./crash-loop.js";
25
+ import { markRolledBackTold, readRolledBack, rolledBackSentence } from "./upgrade-probation.js";
26
+ import { noteServiceConnected, noteServiceReached, noteServiceTried } from "./crash-loop.js";
25
27
  import { applyAnswer, ASK_EXPIRY_MS, policyUnreadable, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, policyHome, describeRunMinutes, parseRunMinutes, currentMode, wakeSettled, effectiveRunModel, describeRunModel, effectiveGoalRuns, claudeRouting, } from "./worker-policy.js";
26
28
  import { readinessAlerts } from "./readiness-alert.js";
27
29
  import { clearDeclined, markDeclined, personsOwnThread, pruneDeclined } from "./declined.js";
@@ -145,6 +147,8 @@ export const BASE_RE = /^[A-Za-z0-9._][A-Za-z0-9._\/-]{0,99}$/;
145
147
  export const validBase = (b) => typeof b === "string" && BASE_RE.test(b) && !b.includes("..") && !b.endsWith("/") && !b.endsWith(".lock");
146
148
  export const safeSegment = (s) => typeof s === "string" && /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,99}$/.test(s) ? s : null;
147
149
  const CLOCK_SKEW_MS = 5_000;
150
+ export const claimNotAnswered = (message) => /^(RATE_LIMIT|STORAGE_BUSY|STORAGE|NO_REPLY): /.test(message) || /^not connected to relay\b/.test(message);
151
+ export const CLAIM_RETRY_MS = 15_000;
148
152
  const isRepo = (cwd) => {
149
153
  try {
150
154
  return git(cwd, "rev-parse", "--is-inside-work-tree") === "true";
@@ -703,6 +707,8 @@ export async function onRemote(cwd, branch, commit) {
703
707
  }
704
708
  export const MASS_DELETE_SHARE = 0.5;
705
709
  const ONCE_GIVE_BACK_MS = 120_000;
710
+ const REFUSED_NOTE_TRIES = 8;
711
+ export const START_CONFIRM_WAIT_MS = START_CONFIRM_MS + 15_000;
706
712
  export const asksForMassDelete = (text) => /\b(?:delete|remove|wipe|drop|rm)\b[^\n]{0,40}\b(?:all|every|entire|whole|most)\b/i.test(text);
707
713
  const GIT_HOOK_NAMES = [
708
714
  "applypatch-msg", "pre-applypatch", "post-applypatch", "prepare-commit-msg", "commit-msg", "post-commit",
@@ -1089,6 +1095,8 @@ export function composePickups(items, space) {
1089
1095
  export const UNSANDBOXED_RUN = `\n\nTHIS RUN HAS NO SANDBOX. Never run code a teammate wrote: their branch, a patch or a script they sent, or a\ntest they changed. Read and review it, then ESCALATE to your human to run it. The repo's own commands on your\nown changes are fine.`;
1090
1096
  export const TASK_POINTER = "Your task for this run is in the file .pantheon-task.md in the current directory. Read that file first and follow it: it is the whole brief.";
1091
1097
  let fleetLock = null;
1098
+ let rolledBackSaid = null;
1099
+ let rolledBackLookedAt = 0;
1092
1100
  export const RUN_TIMEOUT_MS = RUN_MINUTES.default * 60_000;
1093
1101
  export const RUN_TIMEOUT_MAX_MS = RUN_MINUTES.max * 60_000;
1094
1102
  function recordLimitMs(r) {
@@ -1771,6 +1779,8 @@ export class Worker {
1771
1779
  d = this.outageDiagnosis(c.backend.label, c.outage, []);
1772
1780
  else if (NOT_FOR_UNATTENDED_RUNS.has(c.backend.id))
1773
1781
  d = this.noUnattendedAgentDiagnosis(unattendedRefusal(c, { unsandboxedOk: ok }) ?? c.backend.label);
1782
+ else if (c.cannotStart)
1783
+ d = c.cannotStart.confirmed ? this.cannotStartDiagnosis(c) : null;
1774
1784
  else if (!SANDBOXED_FOR_UNATTENDED.has(c.backend.id) && !ok)
1775
1785
  d = this.noSandboxDiagnosis(c);
1776
1786
  }
@@ -1780,6 +1790,30 @@ export class Worker {
1780
1790
  this.sandboxHold.set(key, { at: Date.now(), d });
1781
1791
  return d;
1782
1792
  }
1793
+ cannotStartDiagnosis(c) {
1794
+ const pinned = c.because === "pinned";
1795
+ const others = c.skipped.filter((x) => x.backend.id !== c.backend.id);
1796
+ const w = cannotStartWords(c.backend, c.cannotStart ?? {}, { pinned, claudeHere: c.backend.id === "claude-code" || others.some((x) => x.backend.id === "claude-code") });
1797
+ const rest = pinned ? ` Woken runs here are pinned to it (a persona's agent: line, or the worker's --agent), so teammates' requests wait.`
1798
+ : others.length ? ` Nothing else here can take teammates' requests (${others.map((x) => `${x.backend.label}: ${x.why}`).join("; ")}), so they wait.`
1799
+ : ` Teammates' requests wait until it does.`;
1800
+ const who = this.cfg.member;
1801
+ return {
1802
+ cause: "agent_not_installed",
1803
+ says: `${w.says}${rest}`,
1804
+ fix: `${w.fix} Requests that waited run by themselves once it starts.`,
1805
+ peer: {
1806
+ code: "cannot_start",
1807
+ summary: `${who}'s agent can't start on their computer — this waits for ${who}`,
1808
+ body: `${who}'s worker could not start a run: ${c.backend.label} cannot start on that computer, and nothing else there could take this — ${who} has to fix that there.\nIt is still owed, and runs by itself once that is fixed. If it can't wait, remind ${who} or ask someone else. Nothing was run.`,
1809
+ },
1810
+ };
1811
+ }
1812
+ cannotStartTold() {
1813
+ const r = this.readiness;
1814
+ return !!r && !r.ok && r.code === "cannot_start" && !!r.certain;
1815
+ }
1816
+ cannotStartHold = new Set();
1783
1817
  noUnattendedAgentDiagnosis(refused) {
1784
1818
  return {
1785
1819
  cause: "no_unattended_agent",
@@ -1821,6 +1855,8 @@ export class Worker {
1821
1855
  this.readinessAt = now;
1822
1856
  if (force || marked)
1823
1857
  forgetMissingBinaries();
1858
+ if (force || marked)
1859
+ forgetStart({ runProven: marked });
1824
1860
  if (this.opts.agentCmd || this.opts.housekeeping || this.opts.alertsOnly) {
1825
1861
  this.readiness = { ok: true, short: "" };
1826
1862
  return;
@@ -1848,9 +1884,13 @@ export class Worker {
1848
1884
  if (ok && this.readiness && !this.readiness.ok)
1849
1885
  this.opts.log(` agent ready again — teammates are told this machine takes requests`);
1850
1886
  const outAgent = signedOut ? [...this.signedOutHere][0] : undefined;
1887
+ const couldNotStart = this.readiness?.code === "cannot_start";
1851
1888
  this.readiness = signedOut
1852
1889
  ? { ok: false, short: "its agent CLI was signed out (a run here was refused)", code: "not_signed_in", fix: held.fix ?? "", signedOut: true, ...(outAgent ? { agent: outAgent } : {}) }
1853
- : { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) };
1890
+ : { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}),
1891
+ ...(r.code === "cannot_start" ? { says: r.says.replace(/ A request while you're away would fail.*$/, ""), ...(r.certain ? { certain: true } : {}) } : {}) };
1892
+ if (couldNotStart !== (this.readiness.code === "cannot_start"))
1893
+ this.sandboxHold.clear();
1854
1894
  this.tellPerson(now, { quiet: signedOut });
1855
1895
  }
1856
1896
  catch {
@@ -1858,6 +1898,7 @@ export class Worker {
1858
1898
  }
1859
1899
  }
1860
1900
  readinessCommandsOn = false;
1901
+ ranOkAt = 0;
1861
1902
  lastReadinessKey = null;
1862
1903
  readinessKey() {
1863
1904
  const commands = this.readinessCommandsOn && process.platform === "linux";
@@ -1891,7 +1932,7 @@ export class Worker {
1891
1932
  const key = this.readinessKey();
1892
1933
  const replaces = this.lastReadinessKey && this.lastReadinessKey !== key ? this.lastReadinessKey : undefined;
1893
1934
  this.lastReadinessKey = key;
1894
- const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet ? { certain: true } : {}) });
1935
+ const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet || r.certain || (r.ok && now - this.ranOkAt >= 0 && now - this.ranOkAt < 2 * WORKER_READINESS_RETRY_MS) ? { certain: true } : {}) });
1895
1936
  this.alertPending = t.pending;
1896
1937
  if (!t.notice)
1897
1938
  return;
@@ -2030,6 +2071,22 @@ export class Worker {
2030
2071
  sayFallbackUnsandboxed(c) {
2031
2072
  if (SANDBOXED_FOR_UNATTENDED.has(c.backend.id) || !this.opts.notify)
2032
2073
  return;
2074
+ const stuck = c.skipped.find((x) => x.backend.id === "claude-code" && x.cannotStart);
2075
+ if (stuck) {
2076
+ const key = `${c.backend.id}|cannot-start`;
2077
+ if (this.fallbackSaid.has(key))
2078
+ return;
2079
+ this.fallbackSaid.add(key);
2080
+ try {
2081
+ this.opts.notify({
2082
+ kind: "worker", who: "Your worker", heading: `${c.backend.label} is answering for Claude Code`, context: this.cfg.space,
2083
+ body: `Claude Code can't start on this computer (pantheon doctor shows why). Until it does, ${c.backend.label} takes teammates' requests on this machine, without Claude Code's sandbox, as your settings allow. To have them wait instead: pantheon policy revoke unsandboxed-runs`,
2084
+ ttl: 0, sound: false,
2085
+ });
2086
+ }
2087
+ catch { }
2088
+ return;
2089
+ }
2033
2090
  const passed = this.outagesPassed(c).filter((p) => p.label === "Claude Code");
2034
2091
  const first = passed[0];
2035
2092
  if (!first)
@@ -2199,7 +2256,7 @@ export class Worker {
2199
2256
  h.at = now;
2200
2257
  continue;
2201
2258
  }
2202
- this.releaseHeld(id, 0, ok ? "unsandboxed runs are allowed here now" : "an agent here can take unattended work now");
2259
+ this.releaseHeld(id, 0, h.cannotStart ? "the agent that could not start does now" : ok ? "unsandboxed runs are allowed here now" : "an agent here can take unattended work now");
2203
2260
  }
2204
2261
  if (!ok && now - this.ranNoSandboxCheckedAt < every)
2205
2262
  return;
@@ -3135,7 +3192,10 @@ export class Worker {
3135
3192
  async run() {
3136
3193
  const upkeep = !!(this.opts.housekeeping || this.opts.alertsOnly);
3137
3194
  const fromSeq = upkeep ? 0 : this.readCursor();
3195
+ noteServiceTried(this.cfg.relay);
3138
3196
  const client = new PantheonClient(this.cfg, (e) => {
3197
+ if (e.t === "welcome" || e.t === "error" || e.t === "pending")
3198
+ noteServiceReached();
3139
3199
  if ((e.t === "welcome" || e.t === "members") && this.client === client)
3140
3200
  queueMicrotask(() => this.carryWakers(client));
3141
3201
  }, {
@@ -3233,7 +3293,7 @@ export class Worker {
3233
3293
  await this.restoreHeldContinuations(client);
3234
3294
  await this.restoreQueuedContinuations();
3235
3295
  }
3236
- this.writeState("watching", "waiting for work");
3296
+ this.writeWaiting();
3237
3297
  while (!this.stopped) {
3238
3298
  const arrived = await client.next(5000);
3239
3299
  if (this.deadClaims.size)
@@ -3321,27 +3381,41 @@ export class Worker {
3321
3381
  await this.settleMoot(client);
3322
3382
  await this.settleClosedQuestions(client);
3323
3383
  await this.settleSafety(client);
3384
+ await this.settleRefused(client);
3324
3385
  const aged = client
3325
3386
  .myWork()
3326
3387
  .filter((o) => o.kind !== "HUMAN" && o.kind !== "DELIVER" && !o.claimed_by)
3327
3388
  .filter((o) => Date.now() - o.created_at >= grace)
3328
- .filter((o) => { const t = this.outageHold.get(o.id); if (t === undefined)
3329
- return true; if (t > Date.now()) {
3330
- if (this.allOutSignedOut()) {
3331
- if (!this.tickHold)
3332
- this.tickHold = { reason: "not_signed_in", detail: "its agent is signed out" };
3333
- this.decline(o.ref, "your agent is signed out");
3389
+ .filter((o) => {
3390
+ const t = this.outageHold.get(o.id);
3391
+ if (t === undefined)
3392
+ return true;
3393
+ if (t > Date.now()) {
3394
+ if (this.cannotStartHold.has(o.id)) {
3395
+ if (!this.tickHold)
3396
+ this.tickHold = { reason: "cannot_start", until: t, detail: "its agent did not start; asking it again" };
3397
+ this.decline(o.ref, "your agent did not start — it is asked again in a minute");
3398
+ return false;
3399
+ }
3400
+ if (this.allOutSignedOut()) {
3401
+ if (!this.tickHold)
3402
+ this.tickHold = { reason: "not_signed_in", detail: "its agent is signed out" };
3403
+ this.decline(o.ref, "your agent is signed out");
3404
+ return false;
3405
+ }
3406
+ if (!this.tickHold || (this.tickHold.reason === "outage" && (this.tickHold.until ?? Infinity) > t))
3407
+ this.tickHold = { reason: "outage", until: t, detail: "its agents are out for now" };
3408
+ this.decline(o.ref, "your agents are out for now");
3334
3409
  return false;
3335
3410
  }
3336
- if (!this.tickHold || (this.tickHold.reason === "outage" && (this.tickHold.until ?? Infinity) > t))
3337
- this.tickHold = { reason: "outage", until: t, detail: "its agents are out for now" };
3338
- this.decline(o.ref, "your agents are out for now");
3339
- return false;
3340
- } this.outageHold.delete(o.id); return true; })
3411
+ this.outageHold.delete(o.id);
3412
+ this.cannotStartHold.delete(o.id);
3413
+ return true;
3414
+ })
3341
3415
  .filter((o) => {
3342
3416
  if (this.failuresOf(o.id) < 2)
3343
3417
  return true;
3344
- this.decline(o.ref, "its run failed twice on this computer");
3418
+ this.decline(o.ref, this.parkedNoSandbox.get(o.id)?.cannotStart ? "your agent cannot start on this computer" : "its run failed twice on this computer");
3345
3419
  if (!this.skipLogged.has(`parked:${o.id}`)) {
3346
3420
  this.skipLogged.add(`parked:${o.id}`);
3347
3421
  this.opts.log(`- parking ${o.kind} ${o.id}: its run failed in a way another run would repeat — it stays owed for a live session or your human (see the last .pantheon-run.log)`);
@@ -3522,7 +3596,7 @@ export class Worker {
3522
3596
  }
3523
3597
  }
3524
3598
  if (!candidate && !continuation) {
3525
- this.writeState("watching", "waiting for work");
3599
+ this.writeWaiting();
3526
3600
  continue;
3527
3601
  }
3528
3602
  const busyTtlMs = this.opts.busyTtlMs ?? LIMITS.activityBusyTtlMs;
@@ -3561,7 +3635,7 @@ export class Worker {
3561
3635
  this.markDone(continuation.id);
3562
3636
  this.forgetQueued(continuation.id);
3563
3637
  this.opts.log(`- not continuing from ${continuation.act} ${continuation.id}: the window that asked was shown it, and its turn ended after that`);
3564
- this.writeState("watching", "waiting for work");
3638
+ this.writeWaiting();
3565
3639
  continue;
3566
3640
  }
3567
3641
  }
@@ -3628,6 +3702,13 @@ export class Worker {
3628
3702
  if (/^DEFER/.test(msg)) {
3629
3703
  this.requeueContinuation(continuation, 10_000);
3630
3704
  }
3705
+ else if (claimNotAnswered(msg)) {
3706
+ this.requeueContinuation(continuation, CLAIM_RETRY_MS);
3707
+ if (!this.skipLogged.has(`claim-wait:${continuation.id}`)) {
3708
+ this.skipLogged.add(`claim-wait:${continuation.id}`);
3709
+ this.opts.log(`- holding ${continuation.act} ${continuation.id}: the relay could not take its claim just now (${msg.split("\n")[0].slice(0, 120)}) — asking again shortly`);
3710
+ }
3711
+ }
3631
3712
  else {
3632
3713
  this.markDone(continuation.id);
3633
3714
  if (!this.skipLogged.has(`cont:${continuation.id}`)) {
@@ -3717,7 +3798,7 @@ export class Worker {
3717
3798
  }
3718
3799
  continue;
3719
3800
  }
3720
- if (job.refused && job.diagnosis?.cause === "agent_unavailable" && job.waitUntil) {
3801
+ if (job.refused && (job.diagnosis?.cause === "agent_unavailable" || job.diagnosis?.cause === "agent_not_installed") && job.waitUntil) {
3721
3802
  if (candidate)
3722
3803
  this.outageHold.set(candidate.id, job.waitUntil);
3723
3804
  if (this.outageHold.size > 500)
@@ -3733,7 +3814,8 @@ export class Worker {
3733
3814
  }
3734
3815
  else if (job.refused && job.diagnosis) {
3735
3816
  if (candidate)
3736
- this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer" : "nothing on this computer can run it while you're away");
3817
+ this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer"
3818
+ : job.diagnosis.cause === "agent_not_installed" ? "your agent cannot start on this computer" : "nothing on this computer can run it while you're away");
3737
3819
  if (candidate) {
3738
3820
  this.failedRuns.set(this.oKey(candidate.id), 2);
3739
3821
  this.skipLogged.add(`parked:${candidate.id}`);
@@ -3786,7 +3868,7 @@ export class Worker {
3786
3868
  if (continuation)
3787
3869
  client.forgetClaim(`cont:${continuation.id}`);
3788
3870
  }
3789
- this.writeState("watching", "waiting for work");
3871
+ this.writeWaiting();
3790
3872
  if (this.opts.once)
3791
3873
  break;
3792
3874
  }
@@ -4275,6 +4357,26 @@ export class Worker {
4275
4357
  return { obligation: o, branch, worktree, ok: false, refused: true, detail: diagnosis.says, diagnosis, waitUntil: choice.outage.until };
4276
4358
  }
4277
4359
  const refusal = unattendedRefusalOf(choice, { unsandboxedOk: this.unsandboxedOk() });
4360
+ if (refusal?.cause === "cannot_start") {
4361
+ const diagnosis = this.cannotStartDiagnosis(choice);
4362
+ if (!choice.cannotStart?.confirmed) {
4363
+ this.cannotStartHold.add(o.id);
4364
+ if (this.cannotStartHold.size > 500)
4365
+ this.cannotStartHold.delete(this.cannotStartHold.values().next().value);
4366
+ if (!this.skipLogged.has(`start1:${o.id}`)) {
4367
+ this.skipLogged.add(`start1:${o.id}`);
4368
+ this.opts.log(`- not starting a run for ${o.kind} ${o.id} yet: ${refusal.says} — it is asked again in about a minute before anyone is told (not counted as a run)`);
4369
+ }
4370
+ return { obligation: o, branch, worktree, ok: false, refused: true, detail: refusal.says, diagnosis: { ...diagnosis, quiet: true }, waitUntil: Date.now() + (this.opts.startConfirmWaitMs ?? START_CONFIRM_WAIT_MS) };
4371
+ }
4372
+ if (!this.parkedNoSandbox.has(o.id))
4373
+ this.parkedNoSandbox.set(o.id, { at: Date.now(), ...(policy.agent ? { agent: policy.agent } : {}), cannotStart: true });
4374
+ if (!this.skipLogged.has(`refused:${o.id}`)) {
4375
+ this.skipLogged.add(`refused:${o.id}`);
4376
+ this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${refusal.says} — it waits until that agent starts (not counted as a run)`);
4377
+ }
4378
+ return { obligation: o, branch, worktree, ok: false, refused: true, detail: refusal.says, diagnosis };
4379
+ }
4278
4380
  if (refusal) {
4279
4381
  const refused = refusal.says;
4280
4382
  const noSandbox = refusal.cause === "unsandboxed_off";
@@ -4610,6 +4712,8 @@ export class Worker {
4610
4712
  }
4611
4713
  let recorded = false;
4612
4714
  let stopBy = null;
4715
+ const spawnedAt = Date.now();
4716
+ let ranMs = 0;
4613
4717
  try {
4614
4718
  try {
4615
4719
  detail = await new Promise((res) => {
@@ -4689,6 +4793,7 @@ export class Worker {
4689
4793
  return; settled = true; clearTimeout(kill); if (exitGrace)
4690
4794
  clearTimeout(exitGrace); markDead(); res(`could not start "${command}": ${e.message}`); });
4691
4795
  });
4796
+ ranMs = Date.now() - spawnedAt;
4692
4797
  }
4693
4798
  finally {
4694
4799
  this.cleanupStep("removing the sandbox's empty mount points", () => {
@@ -4829,8 +4934,26 @@ export class Worker {
4829
4934
  this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over ${outage.why === SIGNIN_REFUSED ? "until it is signed in again" : `until about ${whenAgain(outage.until)}`}; the request goes to the next agent here (transcript: ${refusedLog})`);
4830
4935
  return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
4831
4936
  }
4937
+ const notStarted = !ok && !exited && !outage && !sentAnywhere && choice && !this.opts.agentCmd && existsSync(worktree)
4938
+ ? startFailureOf(detail, { agent: choice.backend.id, ranMs, binary: choice.binary }) : null;
4939
+ if (notStarted) {
4940
+ const found = markCannotStart(choice.backend, choice.binary, notStarted);
4941
+ this.sandboxHold.clear();
4942
+ this.readinessAt = 0;
4943
+ this.writeRunLog(logRoot, refusedLog, `obligation ${o.id}\nbranch ${branch}\nagent ${choice.backend.label}\n\n${detail}\n`);
4944
+ if (Date.now() - startedAt < ONCE_GIVE_BACK_MS)
4945
+ this.restoreOnces();
4946
+ if ((this.outageRetries.get(o.id) ?? 0) < 3) {
4947
+ this.outageRetries.set(o.id, (this.outageRetries.get(o.id) ?? 0) + 1);
4948
+ if (this.outageRetries.size > 200)
4949
+ this.outageRetries.delete(this.outageRetries.keys().next().value);
4950
+ this.opts.log(`! ${choice.backend.label} could not be started for ${o.id}: ${found.why ?? "its program did not start"} — it is not started again until it can; the request goes to the next agent here (transcript: ${refusedLog})`);
4951
+ return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
4952
+ }
4953
+ }
4832
4954
  const quiet = ok && !requireReply && sent.length === 0;
4833
- const facts = { exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands,
4955
+ const facts = { exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands, ranMs,
4956
+ ...(choice ? { binary: choice.binary } : {}), ...(this.opts.agentCmd ? { agentCmd: true } : {}),
4834
4957
  ...(agentId === "claude-code" && !this.opts.agentCmd && claudeTokenForRun(this.stateDir())
4835
4958
  ? { env: { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: undefined }, savedToken: true } : { env: process.env }),
4836
4959
  ...(agentId === "claude-code" ? { claudeSettings: claudeSettingsAuth({ env: process.env }) } : {}),
@@ -4851,6 +4974,12 @@ export class Worker {
4851
4974
  }
4852
4975
  if (ok && agentId && !this.opts.agentCmd)
4853
4976
  clearSignedOut(agentId, { pantheonHome: this.stateDir() });
4977
+ if (ok && choice && !this.opts.agentCmd) {
4978
+ startWorked(choice.backend, choice.binary);
4979
+ this.ranOkAt = Date.now();
4980
+ if (this.alertPending)
4981
+ this.readinessAt = 0;
4982
+ }
4854
4983
  if (!ok && diagnosis?.cause === "not_signed_in" && agentId && !this.opts.agentCmd
4855
4984
  && markSignedOut(agentId, detail, { pantheonHome: this.stateDir(), env: process.env })) {
4856
4985
  this.opts.log(`! ${choice?.backend.label ?? agentId} is signed out on this machine: requests wait until it is signed in again${agentId === "claude-code" ? " (pantheon worker token keeps Claude Code signed in for a year)" : ""}`);
@@ -5288,7 +5417,12 @@ export class Worker {
5288
5417
  async reportRefused(client, o, d, owed) {
5289
5418
  if (d.quiet)
5290
5419
  return;
5291
- if (this.opts.notify && !this.refusalsNotified.has(d.says)) {
5420
+ if (d.cause === "agent_not_installed" && !this.cannotStartTold() && !this.refusalsNotified.has(d.says)) {
5421
+ this.readinessAt = 0;
5422
+ this.checkReadiness(true);
5423
+ }
5424
+ const toldByReadiness = d.cause === "agent_not_installed" && this.cannotStartTold();
5425
+ if (this.opts.notify && !toldByReadiness && !this.refusalsNotified.has(d.says)) {
5292
5426
  this.refusalsNotified.add(d.says);
5293
5427
  try {
5294
5428
  this.opts.notify(composeWorker({ ok: false, triggeredBy: client.recentFrom(o.ref) ?? "unknown", obligationBody: o.body, goal: o.goal ? client.goals.get(o.goal) : undefined, reply: null, obligationRef: o.ref, diagnosis: d }, (() => { const tools = installedTools(); return { tools, preferred: preferredTool(loadProfile(this.cfg), tools, this.opts.home) }; })()));
@@ -5301,8 +5435,10 @@ export class Worker {
5301
5435
  const key = `refused:${this.oKey(o.id)}`;
5302
5436
  const space = this.spaceKey();
5303
5437
  const told = loadContinued(space, this.opts.home);
5304
- if (told.has(key) || told.has(`refused:${o.id}`))
5438
+ if (told.has(key) || told.has(`refused:${o.id}`)) {
5439
+ this.refusedDue.delete(key);
5305
5440
  return;
5441
+ }
5306
5442
  const peer = client.recentFrom(o.ref);
5307
5443
  try {
5308
5444
  await client.send({
@@ -5316,11 +5452,36 @@ export class Worker {
5316
5452
  meta: { declined: d.peer?.code ?? (d.cause === "not_signed_in" ? "signed_out" : "unavailable") },
5317
5453
  });
5318
5454
  markContinued(space, key, this.opts.home);
5319
- this.opts.log(d.cause === "agent_unavailable" ? `* told ${peer ?? "the teammate"} that ${o.id} waits: ${d.says}`
5455
+ this.refusedDue.delete(key);
5456
+ this.opts.log(d.cause === "agent_unavailable" || d.cause === "agent_not_installed" ? `* told ${peer ?? "the teammate"} that ${o.id} waits: ${d.says}`
5320
5457
  : `* told ${peer ?? "the teammate"} that ${o.id} waits for a live session: ${d.cause === "repo_not_here" || d.cause === "folder_too_large" ? d.says : "no agent here can take unattended work"}`);
5321
5458
  }
5322
5459
  catch (e) {
5323
- this.opts.log(`! could not tell the teammate that ${o.id} is waiting: ${e.message}`);
5460
+ const msg = e.message;
5461
+ const tries = (this.refusedDue.get(key)?.tries ?? 0) + 1;
5462
+ const passing = !/^NO_REPLY\b/.test(msg) && (!/^[A-Z_]+:/.test(msg) || /^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(msg));
5463
+ if (passing && tries <= REFUSED_NOTE_TRIES)
5464
+ this.refusedDue.set(key, { o, d, tries, next: Date.now() + Math.min(60_000, 2_000 * tries) });
5465
+ else
5466
+ this.refusedDue.delete(key);
5467
+ if (this.refusedDue.size > 200)
5468
+ this.refusedDue.delete(this.refusedDue.keys().next().value);
5469
+ this.opts.log(`! could not tell the teammate that ${o.id} is waiting: ${msg.split("\n")[0]}${this.refusedDue.has(key) ? " — it is said at the next look" : ""}`);
5470
+ }
5471
+ }
5472
+ refusedDue = new Map();
5473
+ async settleRefused(client) {
5474
+ if (!this.refusedDue.size || !client.connected)
5475
+ return;
5476
+ for (const [key, r] of [...this.refusedDue]) {
5477
+ if (r.next > Date.now())
5478
+ continue;
5479
+ const now = client.myWork().find((x) => this.oKey(x.id) === this.oKey(r.o.id));
5480
+ if (!now) {
5481
+ this.refusedDue.delete(key);
5482
+ continue;
5483
+ }
5484
+ await this.reportRefused(client, now, r.d, true);
5324
5485
  }
5325
5486
  }
5326
5487
  mootSince = new Map();
@@ -6029,6 +6190,7 @@ export class Worker {
6029
6190
  lastHealthCheck = 0;
6030
6191
  async maintain(client, arrived) {
6031
6192
  const home = this.opts.home;
6193
+ this.sayRolledBack();
6032
6194
  {
6033
6195
  const policy = loadPolicy(home);
6034
6196
  const denies = [];
@@ -6074,6 +6236,14 @@ export class Worker {
6074
6236
  const lastAttempt = policy.attempts.self_upgrade ?? null;
6075
6237
  if (!needsSelfUpgrade(VERSION, relayPkg, lastAttempt))
6076
6238
  return;
6239
+ const hold = (() => { try {
6240
+ return this.opts.selfUpgrade?.rolledBack ? this.opts.selfUpgrade.rolledBack.read() : readRolledBack();
6241
+ }
6242
+ catch {
6243
+ return null;
6244
+ } })();
6245
+ if (hold && hold.version === relayPkg)
6246
+ return;
6077
6247
  const nodeNow = this.opts.selfUpgrade?.node ?? (() => runtimeNodeVersion());
6078
6248
  const held = policy.upgradeHeld;
6079
6249
  if (held && held.version === relayPkg && held.node === nodeNow())
@@ -6179,6 +6349,42 @@ export class Worker {
6179
6349
  }
6180
6350
  }
6181
6351
  }
6352
+ sayRolledBack() {
6353
+ const seam = this.opts.selfUpgrade?.rolledBack;
6354
+ const now = Date.now();
6355
+ if (!seam && now - rolledBackLookedAt < 60_000)
6356
+ return;
6357
+ rolledBackLookedAt = now;
6358
+ let back = null;
6359
+ try {
6360
+ back = seam ? seam.read() : readRolledBack();
6361
+ }
6362
+ catch {
6363
+ return;
6364
+ }
6365
+ if (!back || back.told_at)
6366
+ return;
6367
+ const key = `${back.version}@${back.at}`;
6368
+ const line = rolledBackSentence(back);
6369
+ if (rolledBackSaid !== key) {
6370
+ rolledBackSaid = key;
6371
+ this.opts.log(`! ${line}`);
6372
+ }
6373
+ const canAlert = !!this.opts.notify && decisionAlertsOn() && (() => { try {
6374
+ return !this.opts.canAlert || this.opts.canAlert();
6375
+ }
6376
+ catch {
6377
+ return false;
6378
+ } })();
6379
+ if (!canAlert)
6380
+ return;
6381
+ if (!(seam ? seam.told() : markRolledBackTold()))
6382
+ return;
6383
+ try {
6384
+ this.opts.notify?.({ kind: "worker", who: "Pantheon · this computer", heading: `Pantheon went back to ${back.back_to}`, context: this.cfg.space, body: line, ttl: 0, sound: false });
6385
+ }
6386
+ catch { }
6387
+ }
6182
6388
  askAt = new Map();
6183
6389
  ledgerAllowSeen = new Set();
6184
6390
  lastFullScan = 0;
@@ -6677,11 +6883,23 @@ export class Worker {
6677
6883
  this.report("running", detail, undefined, { ...(l?.ref ? { ref: l.ref } : {}), ...(l?.goal ? { goal: l.goal } : {}) });
6678
6884
  }
6679
6885
  heartbeatAt = 0;
6886
+ writeWaiting() {
6887
+ const r = this.readiness;
6888
+ if (!r || r.ok || r.code !== "cannot_start") {
6889
+ this.writeState("watching", "waiting for work");
6890
+ return;
6891
+ }
6892
+ if (Date.now() - this.lastStateAt >= WORKER_STATE_EVERY_MS)
6893
+ this.lastState = "";
6894
+ this.writeState("held", clip(r.says ?? "its agent cannot start on this computer", 160));
6895
+ }
6896
+ lastStateAt = 0;
6680
6897
  writeState(state, detail) {
6681
6898
  const line = `${state}|${detail}`;
6682
6899
  if (line === this.lastState)
6683
6900
  return;
6684
6901
  this.lastState = line;
6902
+ this.lastStateAt = Date.now();
6685
6903
  try {
6686
6904
  const base = this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
6687
6905
  mkdirSync(base, { recursive: true });
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "format": 1,
3
- "built_at": "2026-11-13T10:46:28.280Z",
3
+ "built_at": "2026-11-17T05:58:19.032Z",
4
4
  "swiftc": "Apple Swift version 6.4.2 (swiftlang-6.4.2.1.9 clang-2100.4.1.1)",
5
5
  "helpers": {
6
6
  "pantheon-tray": {
@@ -17,7 +17,7 @@
17
17
  "pantheon-hud": {
18
18
  "source": "main.swift",
19
19
  "source_sha256": "4976c00b5d4bd0d8c509d2d284d9c6e2d70a1b3db952de5f11108dfff4951a82",
20
- "sha256": "6b81963a5d075d034224d5e9181038194e362af14d281ad98d74c2bb9fe1dd6f",
20
+ "sha256": "7ea1f02fcab4d53b1fd081ce7bef33c73bd092fcf310bef1619a10ac8611c808",
21
21
  "archs": [
22
22
  "arm64",
23
23
  "x86_64"
Binary file
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "@join-pantheon/cli",
3
- "version": "0.38.1",
3
+ "version": "0.38.3",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "@join-pantheon/cli",
9
- "version": "0.38.1",
9
+ "version": "0.38.3",
10
10
  "license": "MIT",
11
11
  "dependencies": {
12
12
  "@modelcontextprotocol/sdk": "^1.32.1",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@join-pantheon/cli",
3
- "version": "0.38.1",
3
+ "version": "0.38.3",
4
4
  "description": "Let your coding agent collaborate directly with your teammates' agents — ask, agree an interface, and deliver, without a human relaying messages.",
5
5
  "type": "module",
6
6
  "bin": {