@join-pantheon/cli 0.38.2 → 0.38.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/worker.js CHANGED
@@ -16,13 +16,14 @@ import { mkdirInside, writeInside } from "./safe-write.js";
16
16
  import { corruptHeldFiles, MAX_HELD_PER_RUN, remoteTipIs, takeHeldFrames, updateHeldState, verifyHeldFrame, writeHoldConfig } from "./held-delivery.js";
17
17
  import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
18
18
  import { trusted } from "./verify-act.js";
19
- import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
19
+ import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, cannotStartWords, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
20
20
  import { activeOutages, detectOutage, earliest, isSignOut, outagePredicate, whenAgain, SIGNIN_REFUSED } from "./agent-outage.js";
21
+ import { forgetStart, markCannotStart, startFailureOf, startWorked, START_CONFIRM_MS } from "./agent-start.js";
21
22
  import { agentSpawn, CmdLineUnsafe } from "./which.js";
22
23
  import { VERSION } from "./version.js";
23
24
  import { composeUpgradeFailed, fetchRelayPkg, recordRelayPkg, upgradeFailedOnce } from "./version-health.js";
24
- import { heldBack, markRolledBackTold, rolledBackSentence } from "./upgrade-probation.js";
25
- import { noteServiceConnected } from "./crash-loop.js";
25
+ import { markRolledBackTold, readRolledBack, rolledBackSentence } from "./upgrade-probation.js";
26
+ import { noteServiceConnected, noteServiceReached, noteServiceTried } from "./crash-loop.js";
26
27
  import { applyAnswer, ASK_EXPIRY_MS, policyUnreadable, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, policyHome, describeRunMinutes, parseRunMinutes, currentMode, wakeSettled, effectiveRunModel, describeRunModel, effectiveGoalRuns, claudeRouting, } from "./worker-policy.js";
27
28
  import { readinessAlerts } from "./readiness-alert.js";
28
29
  import { clearDeclined, markDeclined, personsOwnThread, pruneDeclined } from "./declined.js";
@@ -146,6 +147,8 @@ export const BASE_RE = /^[A-Za-z0-9._][A-Za-z0-9._\/-]{0,99}$/;
146
147
  export const validBase = (b) => typeof b === "string" && BASE_RE.test(b) && !b.includes("..") && !b.endsWith("/") && !b.endsWith(".lock");
147
148
  export const safeSegment = (s) => typeof s === "string" && /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,99}$/.test(s) ? s : null;
148
149
  const CLOCK_SKEW_MS = 5_000;
150
+ export const claimNotAnswered = (message) => /^(RATE_LIMIT|STORAGE_BUSY|STORAGE|NO_REPLY): /.test(message) || /^not connected to relay\b/.test(message);
151
+ export const CLAIM_RETRY_MS = 15_000;
149
152
  const isRepo = (cwd) => {
150
153
  try {
151
154
  return git(cwd, "rev-parse", "--is-inside-work-tree") === "true";
@@ -704,6 +707,8 @@ export async function onRemote(cwd, branch, commit) {
704
707
  }
705
708
  export const MASS_DELETE_SHARE = 0.5;
706
709
  const ONCE_GIVE_BACK_MS = 120_000;
710
+ const REFUSED_NOTE_TRIES = 8;
711
+ export const START_CONFIRM_WAIT_MS = START_CONFIRM_MS + 15_000;
707
712
  export const asksForMassDelete = (text) => /\b(?:delete|remove|wipe|drop|rm)\b[^\n]{0,40}\b(?:all|every|entire|whole|most)\b/i.test(text);
708
713
  const GIT_HOOK_NAMES = [
709
714
  "applypatch-msg", "pre-applypatch", "post-applypatch", "prepare-commit-msg", "commit-msg", "post-commit",
@@ -1091,6 +1096,7 @@ export const UNSANDBOXED_RUN = `\n\nTHIS RUN HAS NO SANDBOX. Never run code a te
1091
1096
  export const TASK_POINTER = "Your task for this run is in the file .pantheon-task.md in the current directory. Read that file first and follow it: it is the whole brief.";
1092
1097
  let fleetLock = null;
1093
1098
  let rolledBackSaid = null;
1099
+ let rolledBackLookedAt = 0;
1094
1100
  export const RUN_TIMEOUT_MS = RUN_MINUTES.default * 60_000;
1095
1101
  export const RUN_TIMEOUT_MAX_MS = RUN_MINUTES.max * 60_000;
1096
1102
  function recordLimitMs(r) {
@@ -1773,6 +1779,8 @@ export class Worker {
1773
1779
  d = this.outageDiagnosis(c.backend.label, c.outage, []);
1774
1780
  else if (NOT_FOR_UNATTENDED_RUNS.has(c.backend.id))
1775
1781
  d = this.noUnattendedAgentDiagnosis(unattendedRefusal(c, { unsandboxedOk: ok }) ?? c.backend.label);
1782
+ else if (c.cannotStart)
1783
+ d = c.cannotStart.confirmed ? this.cannotStartDiagnosis(c) : null;
1776
1784
  else if (!SANDBOXED_FOR_UNATTENDED.has(c.backend.id) && !ok)
1777
1785
  d = this.noSandboxDiagnosis(c);
1778
1786
  }
@@ -1782,6 +1790,30 @@ export class Worker {
1782
1790
  this.sandboxHold.set(key, { at: Date.now(), d });
1783
1791
  return d;
1784
1792
  }
1793
+ cannotStartDiagnosis(c) {
1794
+ const pinned = c.because === "pinned";
1795
+ const others = c.skipped.filter((x) => x.backend.id !== c.backend.id);
1796
+ const w = cannotStartWords(c.backend, c.cannotStart ?? {}, { pinned, claudeHere: c.backend.id === "claude-code" || others.some((x) => x.backend.id === "claude-code") });
1797
+ const rest = pinned ? ` Woken runs here are pinned to it (a persona's agent: line, or the worker's --agent), so teammates' requests wait.`
1798
+ : others.length ? ` Nothing else here can take teammates' requests (${others.map((x) => `${x.backend.label}: ${x.why}`).join("; ")}), so they wait.`
1799
+ : ` Teammates' requests wait until it does.`;
1800
+ const who = this.cfg.member;
1801
+ return {
1802
+ cause: "agent_not_installed",
1803
+ says: `${w.says}${rest}`,
1804
+ fix: `${w.fix} Requests that waited run by themselves once it starts.`,
1805
+ peer: {
1806
+ code: "cannot_start",
1807
+ summary: `${who}'s agent can't start on their computer — this waits for ${who}`,
1808
+ body: `${who}'s worker could not start a run: ${c.backend.label} cannot start on that computer, and nothing else there could take this — ${who} has to fix that there.\nIt is still owed, and runs by itself once that is fixed. If it can't wait, remind ${who} or ask someone else. Nothing was run.`,
1809
+ },
1810
+ };
1811
+ }
1812
+ cannotStartTold() {
1813
+ const r = this.readiness;
1814
+ return !!r && !r.ok && r.code === "cannot_start" && !!r.certain;
1815
+ }
1816
+ cannotStartHold = new Set();
1785
1817
  noUnattendedAgentDiagnosis(refused) {
1786
1818
  return {
1787
1819
  cause: "no_unattended_agent",
@@ -1823,6 +1855,8 @@ export class Worker {
1823
1855
  this.readinessAt = now;
1824
1856
  if (force || marked)
1825
1857
  forgetMissingBinaries();
1858
+ if (force || marked)
1859
+ forgetStart({ runProven: marked });
1826
1860
  if (this.opts.agentCmd || this.opts.housekeeping || this.opts.alertsOnly) {
1827
1861
  this.readiness = { ok: true, short: "" };
1828
1862
  return;
@@ -1850,9 +1884,13 @@ export class Worker {
1850
1884
  if (ok && this.readiness && !this.readiness.ok)
1851
1885
  this.opts.log(` agent ready again — teammates are told this machine takes requests`);
1852
1886
  const outAgent = signedOut ? [...this.signedOutHere][0] : undefined;
1887
+ const couldNotStart = this.readiness?.code === "cannot_start";
1853
1888
  this.readiness = signedOut
1854
1889
  ? { ok: false, short: "its agent CLI was signed out (a run here was refused)", code: "not_signed_in", fix: held.fix ?? "", signedOut: true, ...(outAgent ? { agent: outAgent } : {}) }
1855
- : { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) };
1890
+ : { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}),
1891
+ ...(r.code === "cannot_start" ? { says: r.says.replace(/ A request while you're away would fail.*$/, ""), ...(r.certain ? { certain: true } : {}) } : {}) };
1892
+ if (couldNotStart !== (this.readiness.code === "cannot_start"))
1893
+ this.sandboxHold.clear();
1856
1894
  this.tellPerson(now, { quiet: signedOut });
1857
1895
  }
1858
1896
  catch {
@@ -1860,6 +1898,7 @@ export class Worker {
1860
1898
  }
1861
1899
  }
1862
1900
  readinessCommandsOn = false;
1901
+ ranOkAt = 0;
1863
1902
  lastReadinessKey = null;
1864
1903
  readinessKey() {
1865
1904
  const commands = this.readinessCommandsOn && process.platform === "linux";
@@ -1893,7 +1932,7 @@ export class Worker {
1893
1932
  const key = this.readinessKey();
1894
1933
  const replaces = this.lastReadinessKey && this.lastReadinessKey !== key ? this.lastReadinessKey : undefined;
1895
1934
  this.lastReadinessKey = key;
1896
- const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet ? { certain: true } : {}) });
1935
+ const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet || r.certain || (r.ok && now - this.ranOkAt >= 0 && now - this.ranOkAt < 2 * WORKER_READINESS_RETRY_MS) ? { certain: true } : {}) });
1897
1936
  this.alertPending = t.pending;
1898
1937
  if (!t.notice)
1899
1938
  return;
@@ -2032,6 +2071,22 @@ export class Worker {
2032
2071
  sayFallbackUnsandboxed(c) {
2033
2072
  if (SANDBOXED_FOR_UNATTENDED.has(c.backend.id) || !this.opts.notify)
2034
2073
  return;
2074
+ const stuck = c.skipped.find((x) => x.backend.id === "claude-code" && x.cannotStart);
2075
+ if (stuck) {
2076
+ const key = `${c.backend.id}|cannot-start`;
2077
+ if (this.fallbackSaid.has(key))
2078
+ return;
2079
+ this.fallbackSaid.add(key);
2080
+ try {
2081
+ this.opts.notify({
2082
+ kind: "worker", who: "Your worker", heading: `${c.backend.label} is answering for Claude Code`, context: this.cfg.space,
2083
+ body: `Claude Code can't start on this computer (pantheon doctor shows why). Until it does, ${c.backend.label} takes teammates' requests on this machine, without Claude Code's sandbox, as your settings allow. To have them wait instead: pantheon policy revoke unsandboxed-runs`,
2084
+ ttl: 0, sound: false,
2085
+ });
2086
+ }
2087
+ catch { }
2088
+ return;
2089
+ }
2035
2090
  const passed = this.outagesPassed(c).filter((p) => p.label === "Claude Code");
2036
2091
  const first = passed[0];
2037
2092
  if (!first)
@@ -2201,7 +2256,7 @@ export class Worker {
2201
2256
  h.at = now;
2202
2257
  continue;
2203
2258
  }
2204
- this.releaseHeld(id, 0, ok ? "unsandboxed runs are allowed here now" : "an agent here can take unattended work now");
2259
+ this.releaseHeld(id, 0, h.cannotStart ? "the agent that could not start does now" : ok ? "unsandboxed runs are allowed here now" : "an agent here can take unattended work now");
2205
2260
  }
2206
2261
  if (!ok && now - this.ranNoSandboxCheckedAt < every)
2207
2262
  return;
@@ -3137,7 +3192,10 @@ export class Worker {
3137
3192
  async run() {
3138
3193
  const upkeep = !!(this.opts.housekeeping || this.opts.alertsOnly);
3139
3194
  const fromSeq = upkeep ? 0 : this.readCursor();
3195
+ noteServiceTried(this.cfg.relay);
3140
3196
  const client = new PantheonClient(this.cfg, (e) => {
3197
+ if (e.t === "welcome" || e.t === "error" || e.t === "pending")
3198
+ noteServiceReached();
3141
3199
  if ((e.t === "welcome" || e.t === "members") && this.client === client)
3142
3200
  queueMicrotask(() => this.carryWakers(client));
3143
3201
  }, {
@@ -3235,7 +3293,7 @@ export class Worker {
3235
3293
  await this.restoreHeldContinuations(client);
3236
3294
  await this.restoreQueuedContinuations();
3237
3295
  }
3238
- this.writeState("watching", "waiting for work");
3296
+ this.writeWaiting();
3239
3297
  while (!this.stopped) {
3240
3298
  const arrived = await client.next(5000);
3241
3299
  if (this.deadClaims.size)
@@ -3323,27 +3381,41 @@ export class Worker {
3323
3381
  await this.settleMoot(client);
3324
3382
  await this.settleClosedQuestions(client);
3325
3383
  await this.settleSafety(client);
3384
+ await this.settleRefused(client);
3326
3385
  const aged = client
3327
3386
  .myWork()
3328
3387
  .filter((o) => o.kind !== "HUMAN" && o.kind !== "DELIVER" && !o.claimed_by)
3329
3388
  .filter((o) => Date.now() - o.created_at >= grace)
3330
- .filter((o) => { const t = this.outageHold.get(o.id); if (t === undefined)
3331
- return true; if (t > Date.now()) {
3332
- if (this.allOutSignedOut()) {
3333
- if (!this.tickHold)
3334
- this.tickHold = { reason: "not_signed_in", detail: "its agent is signed out" };
3335
- this.decline(o.ref, "your agent is signed out");
3389
+ .filter((o) => {
3390
+ const t = this.outageHold.get(o.id);
3391
+ if (t === undefined)
3392
+ return true;
3393
+ if (t > Date.now()) {
3394
+ if (this.cannotStartHold.has(o.id)) {
3395
+ if (!this.tickHold)
3396
+ this.tickHold = { reason: "cannot_start", until: t, detail: "its agent did not start; asking it again" };
3397
+ this.decline(o.ref, "your agent did not start — it is asked again in a minute");
3398
+ return false;
3399
+ }
3400
+ if (this.allOutSignedOut()) {
3401
+ if (!this.tickHold)
3402
+ this.tickHold = { reason: "not_signed_in", detail: "its agent is signed out" };
3403
+ this.decline(o.ref, "your agent is signed out");
3404
+ return false;
3405
+ }
3406
+ if (!this.tickHold || (this.tickHold.reason === "outage" && (this.tickHold.until ?? Infinity) > t))
3407
+ this.tickHold = { reason: "outage", until: t, detail: "its agents are out for now" };
3408
+ this.decline(o.ref, "your agents are out for now");
3336
3409
  return false;
3337
3410
  }
3338
- if (!this.tickHold || (this.tickHold.reason === "outage" && (this.tickHold.until ?? Infinity) > t))
3339
- this.tickHold = { reason: "outage", until: t, detail: "its agents are out for now" };
3340
- this.decline(o.ref, "your agents are out for now");
3341
- return false;
3342
- } this.outageHold.delete(o.id); return true; })
3411
+ this.outageHold.delete(o.id);
3412
+ this.cannotStartHold.delete(o.id);
3413
+ return true;
3414
+ })
3343
3415
  .filter((o) => {
3344
3416
  if (this.failuresOf(o.id) < 2)
3345
3417
  return true;
3346
- this.decline(o.ref, "its run failed twice on this computer");
3418
+ this.decline(o.ref, this.parkedNoSandbox.get(o.id)?.cannotStart ? "your agent cannot start on this computer" : "its run failed twice on this computer");
3347
3419
  if (!this.skipLogged.has(`parked:${o.id}`)) {
3348
3420
  this.skipLogged.add(`parked:${o.id}`);
3349
3421
  this.opts.log(`- parking ${o.kind} ${o.id}: its run failed in a way another run would repeat — it stays owed for a live session or your human (see the last .pantheon-run.log)`);
@@ -3524,7 +3596,7 @@ export class Worker {
3524
3596
  }
3525
3597
  }
3526
3598
  if (!candidate && !continuation) {
3527
- this.writeState("watching", "waiting for work");
3599
+ this.writeWaiting();
3528
3600
  continue;
3529
3601
  }
3530
3602
  const busyTtlMs = this.opts.busyTtlMs ?? LIMITS.activityBusyTtlMs;
@@ -3563,7 +3635,7 @@ export class Worker {
3563
3635
  this.markDone(continuation.id);
3564
3636
  this.forgetQueued(continuation.id);
3565
3637
  this.opts.log(`- not continuing from ${continuation.act} ${continuation.id}: the window that asked was shown it, and its turn ended after that`);
3566
- this.writeState("watching", "waiting for work");
3638
+ this.writeWaiting();
3567
3639
  continue;
3568
3640
  }
3569
3641
  }
@@ -3630,6 +3702,13 @@ export class Worker {
3630
3702
  if (/^DEFER/.test(msg)) {
3631
3703
  this.requeueContinuation(continuation, 10_000);
3632
3704
  }
3705
+ else if (claimNotAnswered(msg)) {
3706
+ this.requeueContinuation(continuation, CLAIM_RETRY_MS);
3707
+ if (!this.skipLogged.has(`claim-wait:${continuation.id}`)) {
3708
+ this.skipLogged.add(`claim-wait:${continuation.id}`);
3709
+ this.opts.log(`- holding ${continuation.act} ${continuation.id}: the relay could not take its claim just now (${msg.split("\n")[0].slice(0, 120)}) — asking again shortly`);
3710
+ }
3711
+ }
3633
3712
  else {
3634
3713
  this.markDone(continuation.id);
3635
3714
  if (!this.skipLogged.has(`cont:${continuation.id}`)) {
@@ -3719,7 +3798,7 @@ export class Worker {
3719
3798
  }
3720
3799
  continue;
3721
3800
  }
3722
- if (job.refused && job.diagnosis?.cause === "agent_unavailable" && job.waitUntil) {
3801
+ if (job.refused && (job.diagnosis?.cause === "agent_unavailable" || job.diagnosis?.cause === "agent_not_installed") && job.waitUntil) {
3723
3802
  if (candidate)
3724
3803
  this.outageHold.set(candidate.id, job.waitUntil);
3725
3804
  if (this.outageHold.size > 500)
@@ -3735,7 +3814,8 @@ export class Worker {
3735
3814
  }
3736
3815
  else if (job.refused && job.diagnosis) {
3737
3816
  if (candidate)
3738
- this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer" : "nothing on this computer can run it while you're away");
3817
+ this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer"
3818
+ : job.diagnosis.cause === "agent_not_installed" ? "your agent cannot start on this computer" : "nothing on this computer can run it while you're away");
3739
3819
  if (candidate) {
3740
3820
  this.failedRuns.set(this.oKey(candidate.id), 2);
3741
3821
  this.skipLogged.add(`parked:${candidate.id}`);
@@ -3788,7 +3868,7 @@ export class Worker {
3788
3868
  if (continuation)
3789
3869
  client.forgetClaim(`cont:${continuation.id}`);
3790
3870
  }
3791
- this.writeState("watching", "waiting for work");
3871
+ this.writeWaiting();
3792
3872
  if (this.opts.once)
3793
3873
  break;
3794
3874
  }
@@ -4277,6 +4357,26 @@ export class Worker {
4277
4357
  return { obligation: o, branch, worktree, ok: false, refused: true, detail: diagnosis.says, diagnosis, waitUntil: choice.outage.until };
4278
4358
  }
4279
4359
  const refusal = unattendedRefusalOf(choice, { unsandboxedOk: this.unsandboxedOk() });
4360
+ if (refusal?.cause === "cannot_start") {
4361
+ const diagnosis = this.cannotStartDiagnosis(choice);
4362
+ if (!choice.cannotStart?.confirmed) {
4363
+ this.cannotStartHold.add(o.id);
4364
+ if (this.cannotStartHold.size > 500)
4365
+ this.cannotStartHold.delete(this.cannotStartHold.values().next().value);
4366
+ if (!this.skipLogged.has(`start1:${o.id}`)) {
4367
+ this.skipLogged.add(`start1:${o.id}`);
4368
+ this.opts.log(`- not starting a run for ${o.kind} ${o.id} yet: ${refusal.says} — it is asked again in about a minute before anyone is told (not counted as a run)`);
4369
+ }
4370
+ return { obligation: o, branch, worktree, ok: false, refused: true, detail: refusal.says, diagnosis: { ...diagnosis, quiet: true }, waitUntil: Date.now() + (this.opts.startConfirmWaitMs ?? START_CONFIRM_WAIT_MS) };
4371
+ }
4372
+ if (!this.parkedNoSandbox.has(o.id))
4373
+ this.parkedNoSandbox.set(o.id, { at: Date.now(), ...(policy.agent ? { agent: policy.agent } : {}), cannotStart: true });
4374
+ if (!this.skipLogged.has(`refused:${o.id}`)) {
4375
+ this.skipLogged.add(`refused:${o.id}`);
4376
+ this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${refusal.says} — it waits until that agent starts (not counted as a run)`);
4377
+ }
4378
+ return { obligation: o, branch, worktree, ok: false, refused: true, detail: refusal.says, diagnosis };
4379
+ }
4280
4380
  if (refusal) {
4281
4381
  const refused = refusal.says;
4282
4382
  const noSandbox = refusal.cause === "unsandboxed_off";
@@ -4612,6 +4712,8 @@ export class Worker {
4612
4712
  }
4613
4713
  let recorded = false;
4614
4714
  let stopBy = null;
4715
+ const spawnedAt = Date.now();
4716
+ let ranMs = 0;
4615
4717
  try {
4616
4718
  try {
4617
4719
  detail = await new Promise((res) => {
@@ -4691,6 +4793,7 @@ export class Worker {
4691
4793
  return; settled = true; clearTimeout(kill); if (exitGrace)
4692
4794
  clearTimeout(exitGrace); markDead(); res(`could not start "${command}": ${e.message}`); });
4693
4795
  });
4796
+ ranMs = Date.now() - spawnedAt;
4694
4797
  }
4695
4798
  finally {
4696
4799
  this.cleanupStep("removing the sandbox's empty mount points", () => {
@@ -4831,8 +4934,26 @@ export class Worker {
4831
4934
  this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over ${outage.why === SIGNIN_REFUSED ? "until it is signed in again" : `until about ${whenAgain(outage.until)}`}; the request goes to the next agent here (transcript: ${refusedLog})`);
4832
4935
  return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
4833
4936
  }
4937
+ const notStarted = !ok && !exited && !outage && !sentAnywhere && choice && !this.opts.agentCmd && existsSync(worktree)
4938
+ ? startFailureOf(detail, { agent: choice.backend.id, ranMs, binary: choice.binary }) : null;
4939
+ if (notStarted) {
4940
+ const found = markCannotStart(choice.backend, choice.binary, notStarted);
4941
+ this.sandboxHold.clear();
4942
+ this.readinessAt = 0;
4943
+ this.writeRunLog(logRoot, refusedLog, `obligation ${o.id}\nbranch ${branch}\nagent ${choice.backend.label}\n\n${detail}\n`);
4944
+ if (Date.now() - startedAt < ONCE_GIVE_BACK_MS)
4945
+ this.restoreOnces();
4946
+ if ((this.outageRetries.get(o.id) ?? 0) < 3) {
4947
+ this.outageRetries.set(o.id, (this.outageRetries.get(o.id) ?? 0) + 1);
4948
+ if (this.outageRetries.size > 200)
4949
+ this.outageRetries.delete(this.outageRetries.keys().next().value);
4950
+ this.opts.log(`! ${choice.backend.label} could not be started for ${o.id}: ${found.why ?? "its program did not start"} — it is not started again until it can; the request goes to the next agent here (transcript: ${refusedLog})`);
4951
+ return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
4952
+ }
4953
+ }
4834
4954
  const quiet = ok && !requireReply && sent.length === 0;
4835
- const facts = { exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands,
4955
+ const facts = { exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands, ranMs,
4956
+ ...(choice ? { binary: choice.binary } : {}), ...(this.opts.agentCmd ? { agentCmd: true } : {}),
4836
4957
  ...(agentId === "claude-code" && !this.opts.agentCmd && claudeTokenForRun(this.stateDir())
4837
4958
  ? { env: { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: undefined }, savedToken: true } : { env: process.env }),
4838
4959
  ...(agentId === "claude-code" ? { claudeSettings: claudeSettingsAuth({ env: process.env }) } : {}),
@@ -4853,6 +4974,12 @@ export class Worker {
4853
4974
  }
4854
4975
  if (ok && agentId && !this.opts.agentCmd)
4855
4976
  clearSignedOut(agentId, { pantheonHome: this.stateDir() });
4977
+ if (ok && choice && !this.opts.agentCmd) {
4978
+ startWorked(choice.backend, choice.binary);
4979
+ this.ranOkAt = Date.now();
4980
+ if (this.alertPending)
4981
+ this.readinessAt = 0;
4982
+ }
4856
4983
  if (!ok && diagnosis?.cause === "not_signed_in" && agentId && !this.opts.agentCmd
4857
4984
  && markSignedOut(agentId, detail, { pantheonHome: this.stateDir(), env: process.env })) {
4858
4985
  this.opts.log(`! ${choice?.backend.label ?? agentId} is signed out on this machine: requests wait until it is signed in again${agentId === "claude-code" ? " (pantheon worker token keeps Claude Code signed in for a year)" : ""}`);
@@ -5290,7 +5417,12 @@ export class Worker {
5290
5417
  async reportRefused(client, o, d, owed) {
5291
5418
  if (d.quiet)
5292
5419
  return;
5293
- if (this.opts.notify && !this.refusalsNotified.has(d.says)) {
5420
+ if (d.cause === "agent_not_installed" && !this.cannotStartTold() && !this.refusalsNotified.has(d.says)) {
5421
+ this.readinessAt = 0;
5422
+ this.checkReadiness(true);
5423
+ }
5424
+ const toldByReadiness = d.cause === "agent_not_installed" && this.cannotStartTold();
5425
+ if (this.opts.notify && !toldByReadiness && !this.refusalsNotified.has(d.says)) {
5294
5426
  this.refusalsNotified.add(d.says);
5295
5427
  try {
5296
5428
  this.opts.notify(composeWorker({ ok: false, triggeredBy: client.recentFrom(o.ref) ?? "unknown", obligationBody: o.body, goal: o.goal ? client.goals.get(o.goal) : undefined, reply: null, obligationRef: o.ref, diagnosis: d }, (() => { const tools = installedTools(); return { tools, preferred: preferredTool(loadProfile(this.cfg), tools, this.opts.home) }; })()));
@@ -5303,8 +5435,10 @@ export class Worker {
5303
5435
  const key = `refused:${this.oKey(o.id)}`;
5304
5436
  const space = this.spaceKey();
5305
5437
  const told = loadContinued(space, this.opts.home);
5306
- if (told.has(key) || told.has(`refused:${o.id}`))
5438
+ if (told.has(key) || told.has(`refused:${o.id}`)) {
5439
+ this.refusedDue.delete(key);
5307
5440
  return;
5441
+ }
5308
5442
  const peer = client.recentFrom(o.ref);
5309
5443
  try {
5310
5444
  await client.send({
@@ -5318,11 +5452,36 @@ export class Worker {
5318
5452
  meta: { declined: d.peer?.code ?? (d.cause === "not_signed_in" ? "signed_out" : "unavailable") },
5319
5453
  });
5320
5454
  markContinued(space, key, this.opts.home);
5321
- this.opts.log(d.cause === "agent_unavailable" ? `* told ${peer ?? "the teammate"} that ${o.id} waits: ${d.says}`
5455
+ this.refusedDue.delete(key);
5456
+ this.opts.log(d.cause === "agent_unavailable" || d.cause === "agent_not_installed" ? `* told ${peer ?? "the teammate"} that ${o.id} waits: ${d.says}`
5322
5457
  : `* told ${peer ?? "the teammate"} that ${o.id} waits for a live session: ${d.cause === "repo_not_here" || d.cause === "folder_too_large" ? d.says : "no agent here can take unattended work"}`);
5323
5458
  }
5324
5459
  catch (e) {
5325
- this.opts.log(`! could not tell the teammate that ${o.id} is waiting: ${e.message}`);
5460
+ const msg = e.message;
5461
+ const tries = (this.refusedDue.get(key)?.tries ?? 0) + 1;
5462
+ const passing = !/^NO_REPLY\b/.test(msg) && (!/^[A-Z_]+:/.test(msg) || /^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(msg));
5463
+ if (passing && tries <= REFUSED_NOTE_TRIES)
5464
+ this.refusedDue.set(key, { o, d, tries, next: Date.now() + Math.min(60_000, 2_000 * tries) });
5465
+ else
5466
+ this.refusedDue.delete(key);
5467
+ if (this.refusedDue.size > 200)
5468
+ this.refusedDue.delete(this.refusedDue.keys().next().value);
5469
+ this.opts.log(`! could not tell the teammate that ${o.id} is waiting: ${msg.split("\n")[0]}${this.refusedDue.has(key) ? " — it is said at the next look" : ""}`);
5470
+ }
5471
+ }
5472
+ refusedDue = new Map();
5473
+ async settleRefused(client) {
5474
+ if (!this.refusedDue.size || !client.connected)
5475
+ return;
5476
+ for (const [key, r] of [...this.refusedDue]) {
5477
+ if (r.next > Date.now())
5478
+ continue;
5479
+ const now = client.myWork().find((x) => this.oKey(x.id) === this.oKey(r.o.id));
5480
+ if (!now) {
5481
+ this.refusedDue.delete(key);
5482
+ continue;
5483
+ }
5484
+ await this.reportRefused(client, now, r.d, true);
5326
5485
  }
5327
5486
  }
5328
5487
  mootSince = new Map();
@@ -6031,6 +6190,7 @@ export class Worker {
6031
6190
  lastHealthCheck = 0;
6032
6191
  async maintain(client, arrived) {
6033
6192
  const home = this.opts.home;
6193
+ this.sayRolledBack();
6034
6194
  {
6035
6195
  const policy = loadPolicy(home);
6036
6196
  const denies = [];
@@ -6076,32 +6236,14 @@ export class Worker {
6076
6236
  const lastAttempt = policy.attempts.self_upgrade ?? null;
6077
6237
  if (!needsSelfUpgrade(VERSION, relayPkg, lastAttempt))
6078
6238
  return;
6079
- const back = this.opts.selfUpgrade?.rolledBack ? this.opts.selfUpgrade.rolledBack.read(relayPkg) : heldBack(relayPkg);
6080
- if (back) {
6081
- const key = `${back.version}@${back.at}`;
6082
- if (!back.told_at && rolledBackSaid !== key) {
6083
- rolledBackSaid = key;
6084
- const line = rolledBackSentence(back);
6085
- this.opts.log(`! ${line}`);
6086
- const canAlert = !!this.opts.notify && decisionAlertsOn() && (() => { try {
6087
- return !this.opts.canAlert || this.opts.canAlert();
6088
- }
6089
- catch {
6090
- return false;
6091
- } })();
6092
- if (canAlert) {
6093
- try {
6094
- this.opts.notify?.({ kind: "worker", who: "Pantheon · this computer", heading: `Pantheon went back to ${back.back_to}`, context: this.cfg.space, body: line, ttl: 0, sound: false });
6095
- }
6096
- catch { }
6097
- }
6098
- if (this.opts.selfUpgrade?.rolledBack)
6099
- this.opts.selfUpgrade.rolledBack.told();
6100
- else
6101
- markRolledBackTold();
6102
- }
6103
- return;
6239
+ const hold = (() => { try {
6240
+ return this.opts.selfUpgrade?.rolledBack ? this.opts.selfUpgrade.rolledBack.read() : readRolledBack();
6104
6241
  }
6242
+ catch {
6243
+ return null;
6244
+ } })();
6245
+ if (hold && hold.version === relayPkg)
6246
+ return;
6105
6247
  const nodeNow = this.opts.selfUpgrade?.node ?? (() => runtimeNodeVersion());
6106
6248
  const held = policy.upgradeHeld;
6107
6249
  if (held && held.version === relayPkg && held.node === nodeNow())
@@ -6207,6 +6349,42 @@ export class Worker {
6207
6349
  }
6208
6350
  }
6209
6351
  }
6352
+ sayRolledBack() {
6353
+ const seam = this.opts.selfUpgrade?.rolledBack;
6354
+ const now = Date.now();
6355
+ if (!seam && now - rolledBackLookedAt < 60_000)
6356
+ return;
6357
+ rolledBackLookedAt = now;
6358
+ let back = null;
6359
+ try {
6360
+ back = seam ? seam.read() : readRolledBack();
6361
+ }
6362
+ catch {
6363
+ return;
6364
+ }
6365
+ if (!back || back.told_at)
6366
+ return;
6367
+ const key = `${back.version}@${back.at}`;
6368
+ const line = rolledBackSentence(back);
6369
+ if (rolledBackSaid !== key) {
6370
+ rolledBackSaid = key;
6371
+ this.opts.log(`! ${line}`);
6372
+ }
6373
+ const canAlert = !!this.opts.notify && decisionAlertsOn() && (() => { try {
6374
+ return !this.opts.canAlert || this.opts.canAlert();
6375
+ }
6376
+ catch {
6377
+ return false;
6378
+ } })();
6379
+ if (!canAlert)
6380
+ return;
6381
+ if (!(seam ? seam.told() : markRolledBackTold()))
6382
+ return;
6383
+ try {
6384
+ this.opts.notify?.({ kind: "worker", who: "Pantheon · this computer", heading: `Pantheon went back to ${back.back_to}`, context: this.cfg.space, body: line, ttl: 0, sound: false });
6385
+ }
6386
+ catch { }
6387
+ }
6210
6388
  askAt = new Map();
6211
6389
  ledgerAllowSeen = new Set();
6212
6390
  lastFullScan = 0;
@@ -6705,11 +6883,23 @@ export class Worker {
6705
6883
  this.report("running", detail, undefined, { ...(l?.ref ? { ref: l.ref } : {}), ...(l?.goal ? { goal: l.goal } : {}) });
6706
6884
  }
6707
6885
  heartbeatAt = 0;
6886
+ writeWaiting() {
6887
+ const r = this.readiness;
6888
+ if (!r || r.ok || r.code !== "cannot_start") {
6889
+ this.writeState("watching", "waiting for work");
6890
+ return;
6891
+ }
6892
+ if (Date.now() - this.lastStateAt >= WORKER_STATE_EVERY_MS)
6893
+ this.lastState = "";
6894
+ this.writeState("held", clip(r.says ?? "its agent cannot start on this computer", 160));
6895
+ }
6896
+ lastStateAt = 0;
6708
6897
  writeState(state, detail) {
6709
6898
  const line = `${state}|${detail}`;
6710
6899
  if (line === this.lastState)
6711
6900
  return;
6712
6901
  this.lastState = line;
6902
+ this.lastStateAt = Date.now();
6713
6903
  try {
6714
6904
  const base = this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
6715
6905
  mkdirSync(base, { recursive: true });
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "format": 1,
3
- "built_at": "2026-11-15T08:35:44.857Z",
3
+ "built_at": "2026-11-17T05:58:19.032Z",
4
4
  "swiftc": "Apple Swift version 6.4.2 (swiftlang-6.4.2.1.9 clang-2100.4.1.1)",
5
5
  "helpers": {
6
6
  "pantheon-tray": {
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "@join-pantheon/cli",
3
- "version": "0.38.2",
3
+ "version": "0.38.3",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "@join-pantheon/cli",
9
- "version": "0.38.2",
9
+ "version": "0.38.3",
10
10
  "license": "MIT",
11
11
  "dependencies": {
12
12
  "@modelcontextprotocol/sdk": "^1.32.1",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@join-pantheon/cli",
3
- "version": "0.38.2",
3
+ "version": "0.38.3",
4
4
  "description": "Let your coding agent collaborate directly with your teammates' agents — ask, agree an interface, and deliver, without a human relaying messages.",
5
5
  "type": "module",
6
6
  "bin": {