@join-pantheon/cli 0.38.2 → 0.38.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -1
- package/bin/upgrade-guard.js +314 -57
- package/dist/admin-pages.js +1 -0
- package/dist/agent-start.js +324 -0
- package/dist/agents.js +74 -4
- package/dist/cli.js +16 -19
- package/dist/crash-loop.js +36 -3
- package/dist/doctor.js +78 -6
- package/dist/ledger.js +6 -0
- package/dist/leftovers.js +5 -1
- package/dist/notice.js +2 -1
- package/dist/pickup.js +1 -0
- package/dist/protocol.js +1 -1
- package/dist/readiness-alert.js +17 -2
- package/dist/relay.js +31 -14
- package/dist/run-failure.js +12 -1
- package/dist/store.js +89 -18
- package/dist/update-check.js +17 -2
- package/dist/upgrade-probation.js +235 -51
- package/dist/upgrade.js +43 -9
- package/dist/which.js +1 -1
- package/dist/worker.js +244 -54
- package/helper/macos/prebuilt/manifest.json +1 -1
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/worker.js
CHANGED
|
@@ -16,13 +16,14 @@ import { mkdirInside, writeInside } from "./safe-write.js";
|
|
|
16
16
|
import { corruptHeldFiles, MAX_HELD_PER_RUN, remoteTipIs, takeHeldFrames, updateHeldState, verifyHeldFrame, writeHoldConfig } from "./held-delivery.js";
|
|
17
17
|
import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
|
|
18
18
|
import { trusted } from "./verify-act.js";
|
|
19
|
-
import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
19
|
+
import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, cannotStartWords, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
20
20
|
import { activeOutages, detectOutage, earliest, isSignOut, outagePredicate, whenAgain, SIGNIN_REFUSED } from "./agent-outage.js";
|
|
21
|
+
import { forgetStart, markCannotStart, startFailureOf, startWorked, START_CONFIRM_MS } from "./agent-start.js";
|
|
21
22
|
import { agentSpawn, CmdLineUnsafe } from "./which.js";
|
|
22
23
|
import { VERSION } from "./version.js";
|
|
23
24
|
import { composeUpgradeFailed, fetchRelayPkg, recordRelayPkg, upgradeFailedOnce } from "./version-health.js";
|
|
24
|
-
import {
|
|
25
|
-
import { noteServiceConnected } from "./crash-loop.js";
|
|
25
|
+
import { markRolledBackTold, readRolledBack, rolledBackSentence } from "./upgrade-probation.js";
|
|
26
|
+
import { noteServiceConnected, noteServiceReached, noteServiceTried } from "./crash-loop.js";
|
|
26
27
|
import { applyAnswer, ASK_EXPIRY_MS, policyUnreadable, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, policyHome, describeRunMinutes, parseRunMinutes, currentMode, wakeSettled, effectiveRunModel, describeRunModel, effectiveGoalRuns, claudeRouting, } from "./worker-policy.js";
|
|
27
28
|
import { readinessAlerts } from "./readiness-alert.js";
|
|
28
29
|
import { clearDeclined, markDeclined, personsOwnThread, pruneDeclined } from "./declined.js";
|
|
@@ -146,6 +147,8 @@ export const BASE_RE = /^[A-Za-z0-9._][A-Za-z0-9._\/-]{0,99}$/;
|
|
|
146
147
|
export const validBase = (b) => typeof b === "string" && BASE_RE.test(b) && !b.includes("..") && !b.endsWith("/") && !b.endsWith(".lock");
|
|
147
148
|
export const safeSegment = (s) => typeof s === "string" && /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,99}$/.test(s) ? s : null;
|
|
148
149
|
const CLOCK_SKEW_MS = 5_000;
|
|
150
|
+
export const claimNotAnswered = (message) => /^(RATE_LIMIT|STORAGE_BUSY|STORAGE|NO_REPLY): /.test(message) || /^not connected to relay\b/.test(message);
|
|
151
|
+
export const CLAIM_RETRY_MS = 15_000;
|
|
149
152
|
const isRepo = (cwd) => {
|
|
150
153
|
try {
|
|
151
154
|
return git(cwd, "rev-parse", "--is-inside-work-tree") === "true";
|
|
@@ -704,6 +707,8 @@ export async function onRemote(cwd, branch, commit) {
|
|
|
704
707
|
}
|
|
705
708
|
export const MASS_DELETE_SHARE = 0.5;
|
|
706
709
|
const ONCE_GIVE_BACK_MS = 120_000;
|
|
710
|
+
const REFUSED_NOTE_TRIES = 8;
|
|
711
|
+
export const START_CONFIRM_WAIT_MS = START_CONFIRM_MS + 15_000;
|
|
707
712
|
export const asksForMassDelete = (text) => /\b(?:delete|remove|wipe|drop|rm)\b[^\n]{0,40}\b(?:all|every|entire|whole|most)\b/i.test(text);
|
|
708
713
|
const GIT_HOOK_NAMES = [
|
|
709
714
|
"applypatch-msg", "pre-applypatch", "post-applypatch", "prepare-commit-msg", "commit-msg", "post-commit",
|
|
@@ -1091,6 +1096,7 @@ export const UNSANDBOXED_RUN = `\n\nTHIS RUN HAS NO SANDBOX. Never run code a te
|
|
|
1091
1096
|
export const TASK_POINTER = "Your task for this run is in the file .pantheon-task.md in the current directory. Read that file first and follow it: it is the whole brief.";
|
|
1092
1097
|
let fleetLock = null;
|
|
1093
1098
|
let rolledBackSaid = null;
|
|
1099
|
+
let rolledBackLookedAt = 0;
|
|
1094
1100
|
export const RUN_TIMEOUT_MS = RUN_MINUTES.default * 60_000;
|
|
1095
1101
|
export const RUN_TIMEOUT_MAX_MS = RUN_MINUTES.max * 60_000;
|
|
1096
1102
|
function recordLimitMs(r) {
|
|
@@ -1773,6 +1779,8 @@ export class Worker {
|
|
|
1773
1779
|
d = this.outageDiagnosis(c.backend.label, c.outage, []);
|
|
1774
1780
|
else if (NOT_FOR_UNATTENDED_RUNS.has(c.backend.id))
|
|
1775
1781
|
d = this.noUnattendedAgentDiagnosis(unattendedRefusal(c, { unsandboxedOk: ok }) ?? c.backend.label);
|
|
1782
|
+
else if (c.cannotStart)
|
|
1783
|
+
d = c.cannotStart.confirmed ? this.cannotStartDiagnosis(c) : null;
|
|
1776
1784
|
else if (!SANDBOXED_FOR_UNATTENDED.has(c.backend.id) && !ok)
|
|
1777
1785
|
d = this.noSandboxDiagnosis(c);
|
|
1778
1786
|
}
|
|
@@ -1782,6 +1790,30 @@ export class Worker {
|
|
|
1782
1790
|
this.sandboxHold.set(key, { at: Date.now(), d });
|
|
1783
1791
|
return d;
|
|
1784
1792
|
}
|
|
1793
|
+
cannotStartDiagnosis(c) {
|
|
1794
|
+
const pinned = c.because === "pinned";
|
|
1795
|
+
const others = c.skipped.filter((x) => x.backend.id !== c.backend.id);
|
|
1796
|
+
const w = cannotStartWords(c.backend, c.cannotStart ?? {}, { pinned, claudeHere: c.backend.id === "claude-code" || others.some((x) => x.backend.id === "claude-code") });
|
|
1797
|
+
const rest = pinned ? ` Woken runs here are pinned to it (a persona's agent: line, or the worker's --agent), so teammates' requests wait.`
|
|
1798
|
+
: others.length ? ` Nothing else here can take teammates' requests (${others.map((x) => `${x.backend.label}: ${x.why}`).join("; ")}), so they wait.`
|
|
1799
|
+
: ` Teammates' requests wait until it does.`;
|
|
1800
|
+
const who = this.cfg.member;
|
|
1801
|
+
return {
|
|
1802
|
+
cause: "agent_not_installed",
|
|
1803
|
+
says: `${w.says}${rest}`,
|
|
1804
|
+
fix: `${w.fix} Requests that waited run by themselves once it starts.`,
|
|
1805
|
+
peer: {
|
|
1806
|
+
code: "cannot_start",
|
|
1807
|
+
summary: `${who}'s agent can't start on their computer — this waits for ${who}`,
|
|
1808
|
+
body: `${who}'s worker could not start a run: ${c.backend.label} cannot start on that computer, and nothing else there could take this — ${who} has to fix that there.\nIt is still owed, and runs by itself once that is fixed. If it can't wait, remind ${who} or ask someone else. Nothing was run.`,
|
|
1809
|
+
},
|
|
1810
|
+
};
|
|
1811
|
+
}
|
|
1812
|
+
cannotStartTold() {
|
|
1813
|
+
const r = this.readiness;
|
|
1814
|
+
return !!r && !r.ok && r.code === "cannot_start" && !!r.certain;
|
|
1815
|
+
}
|
|
1816
|
+
cannotStartHold = new Set();
|
|
1785
1817
|
noUnattendedAgentDiagnosis(refused) {
|
|
1786
1818
|
return {
|
|
1787
1819
|
cause: "no_unattended_agent",
|
|
@@ -1823,6 +1855,8 @@ export class Worker {
|
|
|
1823
1855
|
this.readinessAt = now;
|
|
1824
1856
|
if (force || marked)
|
|
1825
1857
|
forgetMissingBinaries();
|
|
1858
|
+
if (force || marked)
|
|
1859
|
+
forgetStart({ runProven: marked });
|
|
1826
1860
|
if (this.opts.agentCmd || this.opts.housekeeping || this.opts.alertsOnly) {
|
|
1827
1861
|
this.readiness = { ok: true, short: "" };
|
|
1828
1862
|
return;
|
|
@@ -1850,9 +1884,13 @@ export class Worker {
|
|
|
1850
1884
|
if (ok && this.readiness && !this.readiness.ok)
|
|
1851
1885
|
this.opts.log(` agent ready again — teammates are told this machine takes requests`);
|
|
1852
1886
|
const outAgent = signedOut ? [...this.signedOutHere][0] : undefined;
|
|
1887
|
+
const couldNotStart = this.readiness?.code === "cannot_start";
|
|
1853
1888
|
this.readiness = signedOut
|
|
1854
1889
|
? { ok: false, short: "its agent CLI was signed out (a run here was refused)", code: "not_signed_in", fix: held.fix ?? "", signedOut: true, ...(outAgent ? { agent: outAgent } : {}) }
|
|
1855
|
-
: { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {})
|
|
1890
|
+
: { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}),
|
|
1891
|
+
...(r.code === "cannot_start" ? { says: r.says.replace(/ A request while you're away would fail.*$/, ""), ...(r.certain ? { certain: true } : {}) } : {}) };
|
|
1892
|
+
if (couldNotStart !== (this.readiness.code === "cannot_start"))
|
|
1893
|
+
this.sandboxHold.clear();
|
|
1856
1894
|
this.tellPerson(now, { quiet: signedOut });
|
|
1857
1895
|
}
|
|
1858
1896
|
catch {
|
|
@@ -1860,6 +1898,7 @@ export class Worker {
|
|
|
1860
1898
|
}
|
|
1861
1899
|
}
|
|
1862
1900
|
readinessCommandsOn = false;
|
|
1901
|
+
ranOkAt = 0;
|
|
1863
1902
|
lastReadinessKey = null;
|
|
1864
1903
|
readinessKey() {
|
|
1865
1904
|
const commands = this.readinessCommandsOn && process.platform === "linux";
|
|
@@ -1893,7 +1932,7 @@ export class Worker {
|
|
|
1893
1932
|
const key = this.readinessKey();
|
|
1894
1933
|
const replaces = this.lastReadinessKey && this.lastReadinessKey !== key ? this.lastReadinessKey : undefined;
|
|
1895
1934
|
this.lastReadinessKey = key;
|
|
1896
|
-
const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet ? { certain: true } : {}) });
|
|
1935
|
+
const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet || r.certain || (r.ok && now - this.ranOkAt >= 0 && now - this.ranOkAt < 2 * WORKER_READINESS_RETRY_MS) ? { certain: true } : {}) });
|
|
1897
1936
|
this.alertPending = t.pending;
|
|
1898
1937
|
if (!t.notice)
|
|
1899
1938
|
return;
|
|
@@ -2032,6 +2071,22 @@ export class Worker {
|
|
|
2032
2071
|
sayFallbackUnsandboxed(c) {
|
|
2033
2072
|
if (SANDBOXED_FOR_UNATTENDED.has(c.backend.id) || !this.opts.notify)
|
|
2034
2073
|
return;
|
|
2074
|
+
const stuck = c.skipped.find((x) => x.backend.id === "claude-code" && x.cannotStart);
|
|
2075
|
+
if (stuck) {
|
|
2076
|
+
const key = `${c.backend.id}|cannot-start`;
|
|
2077
|
+
if (this.fallbackSaid.has(key))
|
|
2078
|
+
return;
|
|
2079
|
+
this.fallbackSaid.add(key);
|
|
2080
|
+
try {
|
|
2081
|
+
this.opts.notify({
|
|
2082
|
+
kind: "worker", who: "Your worker", heading: `${c.backend.label} is answering for Claude Code`, context: this.cfg.space,
|
|
2083
|
+
body: `Claude Code can't start on this computer (pantheon doctor shows why). Until it does, ${c.backend.label} takes teammates' requests on this machine, without Claude Code's sandbox, as your settings allow. To have them wait instead: pantheon policy revoke unsandboxed-runs`,
|
|
2084
|
+
ttl: 0, sound: false,
|
|
2085
|
+
});
|
|
2086
|
+
}
|
|
2087
|
+
catch { }
|
|
2088
|
+
return;
|
|
2089
|
+
}
|
|
2035
2090
|
const passed = this.outagesPassed(c).filter((p) => p.label === "Claude Code");
|
|
2036
2091
|
const first = passed[0];
|
|
2037
2092
|
if (!first)
|
|
@@ -2201,7 +2256,7 @@ export class Worker {
|
|
|
2201
2256
|
h.at = now;
|
|
2202
2257
|
continue;
|
|
2203
2258
|
}
|
|
2204
|
-
this.releaseHeld(id, 0, ok ? "unsandboxed runs are allowed here now" : "an agent here can take unattended work now");
|
|
2259
|
+
this.releaseHeld(id, 0, h.cannotStart ? "the agent that could not start does now" : ok ? "unsandboxed runs are allowed here now" : "an agent here can take unattended work now");
|
|
2205
2260
|
}
|
|
2206
2261
|
if (!ok && now - this.ranNoSandboxCheckedAt < every)
|
|
2207
2262
|
return;
|
|
@@ -3137,7 +3192,10 @@ export class Worker {
|
|
|
3137
3192
|
async run() {
|
|
3138
3193
|
const upkeep = !!(this.opts.housekeeping || this.opts.alertsOnly);
|
|
3139
3194
|
const fromSeq = upkeep ? 0 : this.readCursor();
|
|
3195
|
+
noteServiceTried(this.cfg.relay);
|
|
3140
3196
|
const client = new PantheonClient(this.cfg, (e) => {
|
|
3197
|
+
if (e.t === "welcome" || e.t === "error" || e.t === "pending")
|
|
3198
|
+
noteServiceReached();
|
|
3141
3199
|
if ((e.t === "welcome" || e.t === "members") && this.client === client)
|
|
3142
3200
|
queueMicrotask(() => this.carryWakers(client));
|
|
3143
3201
|
}, {
|
|
@@ -3235,7 +3293,7 @@ export class Worker {
|
|
|
3235
3293
|
await this.restoreHeldContinuations(client);
|
|
3236
3294
|
await this.restoreQueuedContinuations();
|
|
3237
3295
|
}
|
|
3238
|
-
this.
|
|
3296
|
+
this.writeWaiting();
|
|
3239
3297
|
while (!this.stopped) {
|
|
3240
3298
|
const arrived = await client.next(5000);
|
|
3241
3299
|
if (this.deadClaims.size)
|
|
@@ -3323,27 +3381,41 @@ export class Worker {
|
|
|
3323
3381
|
await this.settleMoot(client);
|
|
3324
3382
|
await this.settleClosedQuestions(client);
|
|
3325
3383
|
await this.settleSafety(client);
|
|
3384
|
+
await this.settleRefused(client);
|
|
3326
3385
|
const aged = client
|
|
3327
3386
|
.myWork()
|
|
3328
3387
|
.filter((o) => o.kind !== "HUMAN" && o.kind !== "DELIVER" && !o.claimed_by)
|
|
3329
3388
|
.filter((o) => Date.now() - o.created_at >= grace)
|
|
3330
|
-
.filter((o) => {
|
|
3331
|
-
|
|
3332
|
-
if (
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
this.
|
|
3389
|
+
.filter((o) => {
|
|
3390
|
+
const t = this.outageHold.get(o.id);
|
|
3391
|
+
if (t === undefined)
|
|
3392
|
+
return true;
|
|
3393
|
+
if (t > Date.now()) {
|
|
3394
|
+
if (this.cannotStartHold.has(o.id)) {
|
|
3395
|
+
if (!this.tickHold)
|
|
3396
|
+
this.tickHold = { reason: "cannot_start", until: t, detail: "its agent did not start; asking it again" };
|
|
3397
|
+
this.decline(o.ref, "your agent did not start — it is asked again in a minute");
|
|
3398
|
+
return false;
|
|
3399
|
+
}
|
|
3400
|
+
if (this.allOutSignedOut()) {
|
|
3401
|
+
if (!this.tickHold)
|
|
3402
|
+
this.tickHold = { reason: "not_signed_in", detail: "its agent is signed out" };
|
|
3403
|
+
this.decline(o.ref, "your agent is signed out");
|
|
3404
|
+
return false;
|
|
3405
|
+
}
|
|
3406
|
+
if (!this.tickHold || (this.tickHold.reason === "outage" && (this.tickHold.until ?? Infinity) > t))
|
|
3407
|
+
this.tickHold = { reason: "outage", until: t, detail: "its agents are out for now" };
|
|
3408
|
+
this.decline(o.ref, "your agents are out for now");
|
|
3336
3409
|
return false;
|
|
3337
3410
|
}
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
} this.outageHold.delete(o.id); return true; })
|
|
3411
|
+
this.outageHold.delete(o.id);
|
|
3412
|
+
this.cannotStartHold.delete(o.id);
|
|
3413
|
+
return true;
|
|
3414
|
+
})
|
|
3343
3415
|
.filter((o) => {
|
|
3344
3416
|
if (this.failuresOf(o.id) < 2)
|
|
3345
3417
|
return true;
|
|
3346
|
-
this.decline(o.ref, "its run failed twice on this computer");
|
|
3418
|
+
this.decline(o.ref, this.parkedNoSandbox.get(o.id)?.cannotStart ? "your agent cannot start on this computer" : "its run failed twice on this computer");
|
|
3347
3419
|
if (!this.skipLogged.has(`parked:${o.id}`)) {
|
|
3348
3420
|
this.skipLogged.add(`parked:${o.id}`);
|
|
3349
3421
|
this.opts.log(`- parking ${o.kind} ${o.id}: its run failed in a way another run would repeat — it stays owed for a live session or your human (see the last .pantheon-run.log)`);
|
|
@@ -3524,7 +3596,7 @@ export class Worker {
|
|
|
3524
3596
|
}
|
|
3525
3597
|
}
|
|
3526
3598
|
if (!candidate && !continuation) {
|
|
3527
|
-
this.
|
|
3599
|
+
this.writeWaiting();
|
|
3528
3600
|
continue;
|
|
3529
3601
|
}
|
|
3530
3602
|
const busyTtlMs = this.opts.busyTtlMs ?? LIMITS.activityBusyTtlMs;
|
|
@@ -3563,7 +3635,7 @@ export class Worker {
|
|
|
3563
3635
|
this.markDone(continuation.id);
|
|
3564
3636
|
this.forgetQueued(continuation.id);
|
|
3565
3637
|
this.opts.log(`- not continuing from ${continuation.act} ${continuation.id}: the window that asked was shown it, and its turn ended after that`);
|
|
3566
|
-
this.
|
|
3638
|
+
this.writeWaiting();
|
|
3567
3639
|
continue;
|
|
3568
3640
|
}
|
|
3569
3641
|
}
|
|
@@ -3630,6 +3702,13 @@ export class Worker {
|
|
|
3630
3702
|
if (/^DEFER/.test(msg)) {
|
|
3631
3703
|
this.requeueContinuation(continuation, 10_000);
|
|
3632
3704
|
}
|
|
3705
|
+
else if (claimNotAnswered(msg)) {
|
|
3706
|
+
this.requeueContinuation(continuation, CLAIM_RETRY_MS);
|
|
3707
|
+
if (!this.skipLogged.has(`claim-wait:${continuation.id}`)) {
|
|
3708
|
+
this.skipLogged.add(`claim-wait:${continuation.id}`);
|
|
3709
|
+
this.opts.log(`- holding ${continuation.act} ${continuation.id}: the relay could not take its claim just now (${msg.split("\n")[0].slice(0, 120)}) — asking again shortly`);
|
|
3710
|
+
}
|
|
3711
|
+
}
|
|
3633
3712
|
else {
|
|
3634
3713
|
this.markDone(continuation.id);
|
|
3635
3714
|
if (!this.skipLogged.has(`cont:${continuation.id}`)) {
|
|
@@ -3719,7 +3798,7 @@ export class Worker {
|
|
|
3719
3798
|
}
|
|
3720
3799
|
continue;
|
|
3721
3800
|
}
|
|
3722
|
-
if (job.refused && job.diagnosis?.cause === "agent_unavailable" && job.waitUntil) {
|
|
3801
|
+
if (job.refused && (job.diagnosis?.cause === "agent_unavailable" || job.diagnosis?.cause === "agent_not_installed") && job.waitUntil) {
|
|
3723
3802
|
if (candidate)
|
|
3724
3803
|
this.outageHold.set(candidate.id, job.waitUntil);
|
|
3725
3804
|
if (this.outageHold.size > 500)
|
|
@@ -3735,7 +3814,8 @@ export class Worker {
|
|
|
3735
3814
|
}
|
|
3736
3815
|
else if (job.refused && job.diagnosis) {
|
|
3737
3816
|
if (candidate)
|
|
3738
|
-
this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer"
|
|
3817
|
+
this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer"
|
|
3818
|
+
: job.diagnosis.cause === "agent_not_installed" ? "your agent cannot start on this computer" : "nothing on this computer can run it while you're away");
|
|
3739
3819
|
if (candidate) {
|
|
3740
3820
|
this.failedRuns.set(this.oKey(candidate.id), 2);
|
|
3741
3821
|
this.skipLogged.add(`parked:${candidate.id}`);
|
|
@@ -3788,7 +3868,7 @@ export class Worker {
|
|
|
3788
3868
|
if (continuation)
|
|
3789
3869
|
client.forgetClaim(`cont:${continuation.id}`);
|
|
3790
3870
|
}
|
|
3791
|
-
this.
|
|
3871
|
+
this.writeWaiting();
|
|
3792
3872
|
if (this.opts.once)
|
|
3793
3873
|
break;
|
|
3794
3874
|
}
|
|
@@ -4277,6 +4357,26 @@ export class Worker {
|
|
|
4277
4357
|
return { obligation: o, branch, worktree, ok: false, refused: true, detail: diagnosis.says, diagnosis, waitUntil: choice.outage.until };
|
|
4278
4358
|
}
|
|
4279
4359
|
const refusal = unattendedRefusalOf(choice, { unsandboxedOk: this.unsandboxedOk() });
|
|
4360
|
+
if (refusal?.cause === "cannot_start") {
|
|
4361
|
+
const diagnosis = this.cannotStartDiagnosis(choice);
|
|
4362
|
+
if (!choice.cannotStart?.confirmed) {
|
|
4363
|
+
this.cannotStartHold.add(o.id);
|
|
4364
|
+
if (this.cannotStartHold.size > 500)
|
|
4365
|
+
this.cannotStartHold.delete(this.cannotStartHold.values().next().value);
|
|
4366
|
+
if (!this.skipLogged.has(`start1:${o.id}`)) {
|
|
4367
|
+
this.skipLogged.add(`start1:${o.id}`);
|
|
4368
|
+
this.opts.log(`- not starting a run for ${o.kind} ${o.id} yet: ${refusal.says} — it is asked again in about a minute before anyone is told (not counted as a run)`);
|
|
4369
|
+
}
|
|
4370
|
+
return { obligation: o, branch, worktree, ok: false, refused: true, detail: refusal.says, diagnosis: { ...diagnosis, quiet: true }, waitUntil: Date.now() + (this.opts.startConfirmWaitMs ?? START_CONFIRM_WAIT_MS) };
|
|
4371
|
+
}
|
|
4372
|
+
if (!this.parkedNoSandbox.has(o.id))
|
|
4373
|
+
this.parkedNoSandbox.set(o.id, { at: Date.now(), ...(policy.agent ? { agent: policy.agent } : {}), cannotStart: true });
|
|
4374
|
+
if (!this.skipLogged.has(`refused:${o.id}`)) {
|
|
4375
|
+
this.skipLogged.add(`refused:${o.id}`);
|
|
4376
|
+
this.opts.log(`! not starting a run for ${o.kind} ${o.id}: ${refusal.says} — it waits until that agent starts (not counted as a run)`);
|
|
4377
|
+
}
|
|
4378
|
+
return { obligation: o, branch, worktree, ok: false, refused: true, detail: refusal.says, diagnosis };
|
|
4379
|
+
}
|
|
4280
4380
|
if (refusal) {
|
|
4281
4381
|
const refused = refusal.says;
|
|
4282
4382
|
const noSandbox = refusal.cause === "unsandboxed_off";
|
|
@@ -4612,6 +4712,8 @@ export class Worker {
|
|
|
4612
4712
|
}
|
|
4613
4713
|
let recorded = false;
|
|
4614
4714
|
let stopBy = null;
|
|
4715
|
+
const spawnedAt = Date.now();
|
|
4716
|
+
let ranMs = 0;
|
|
4615
4717
|
try {
|
|
4616
4718
|
try {
|
|
4617
4719
|
detail = await new Promise((res) => {
|
|
@@ -4691,6 +4793,7 @@ export class Worker {
|
|
|
4691
4793
|
return; settled = true; clearTimeout(kill); if (exitGrace)
|
|
4692
4794
|
clearTimeout(exitGrace); markDead(); res(`could not start "${command}": ${e.message}`); });
|
|
4693
4795
|
});
|
|
4796
|
+
ranMs = Date.now() - spawnedAt;
|
|
4694
4797
|
}
|
|
4695
4798
|
finally {
|
|
4696
4799
|
this.cleanupStep("removing the sandbox's empty mount points", () => {
|
|
@@ -4831,8 +4934,26 @@ export class Worker {
|
|
|
4831
4934
|
this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over ${outage.why === SIGNIN_REFUSED ? "until it is signed in again" : `until about ${whenAgain(outage.until)}`}; the request goes to the next agent here (transcript: ${refusedLog})`);
|
|
4832
4935
|
return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
|
|
4833
4936
|
}
|
|
4937
|
+
const notStarted = !ok && !exited && !outage && !sentAnywhere && choice && !this.opts.agentCmd && existsSync(worktree)
|
|
4938
|
+
? startFailureOf(detail, { agent: choice.backend.id, ranMs, binary: choice.binary }) : null;
|
|
4939
|
+
if (notStarted) {
|
|
4940
|
+
const found = markCannotStart(choice.backend, choice.binary, notStarted);
|
|
4941
|
+
this.sandboxHold.clear();
|
|
4942
|
+
this.readinessAt = 0;
|
|
4943
|
+
this.writeRunLog(logRoot, refusedLog, `obligation ${o.id}\nbranch ${branch}\nagent ${choice.backend.label}\n\n${detail}\n`);
|
|
4944
|
+
if (Date.now() - startedAt < ONCE_GIVE_BACK_MS)
|
|
4945
|
+
this.restoreOnces();
|
|
4946
|
+
if ((this.outageRetries.get(o.id) ?? 0) < 3) {
|
|
4947
|
+
this.outageRetries.set(o.id, (this.outageRetries.get(o.id) ?? 0) + 1);
|
|
4948
|
+
if (this.outageRetries.size > 200)
|
|
4949
|
+
this.outageRetries.delete(this.outageRetries.keys().next().value);
|
|
4950
|
+
this.opts.log(`! ${choice.backend.label} could not be started for ${o.id}: ${found.why ?? "its program did not start"} — it is not started again until it can; the request goes to the next agent here (transcript: ${refusedLog})`);
|
|
4951
|
+
return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
|
|
4952
|
+
}
|
|
4953
|
+
}
|
|
4834
4954
|
const quiet = ok && !requireReply && sent.length === 0;
|
|
4835
|
-
const facts = { exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands,
|
|
4955
|
+
const facts = { exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands, ranMs,
|
|
4956
|
+
...(choice ? { binary: choice.binary } : {}), ...(this.opts.agentCmd ? { agentCmd: true } : {}),
|
|
4836
4957
|
...(agentId === "claude-code" && !this.opts.agentCmd && claudeTokenForRun(this.stateDir())
|
|
4837
4958
|
? { env: { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: undefined }, savedToken: true } : { env: process.env }),
|
|
4838
4959
|
...(agentId === "claude-code" ? { claudeSettings: claudeSettingsAuth({ env: process.env }) } : {}),
|
|
@@ -4853,6 +4974,12 @@ export class Worker {
|
|
|
4853
4974
|
}
|
|
4854
4975
|
if (ok && agentId && !this.opts.agentCmd)
|
|
4855
4976
|
clearSignedOut(agentId, { pantheonHome: this.stateDir() });
|
|
4977
|
+
if (ok && choice && !this.opts.agentCmd) {
|
|
4978
|
+
startWorked(choice.backend, choice.binary);
|
|
4979
|
+
this.ranOkAt = Date.now();
|
|
4980
|
+
if (this.alertPending)
|
|
4981
|
+
this.readinessAt = 0;
|
|
4982
|
+
}
|
|
4856
4983
|
if (!ok && diagnosis?.cause === "not_signed_in" && agentId && !this.opts.agentCmd
|
|
4857
4984
|
&& markSignedOut(agentId, detail, { pantheonHome: this.stateDir(), env: process.env })) {
|
|
4858
4985
|
this.opts.log(`! ${choice?.backend.label ?? agentId} is signed out on this machine: requests wait until it is signed in again${agentId === "claude-code" ? " (pantheon worker token keeps Claude Code signed in for a year)" : ""}`);
|
|
@@ -5290,7 +5417,12 @@ export class Worker {
|
|
|
5290
5417
|
async reportRefused(client, o, d, owed) {
|
|
5291
5418
|
if (d.quiet)
|
|
5292
5419
|
return;
|
|
5293
|
-
if (this.
|
|
5420
|
+
if (d.cause === "agent_not_installed" && !this.cannotStartTold() && !this.refusalsNotified.has(d.says)) {
|
|
5421
|
+
this.readinessAt = 0;
|
|
5422
|
+
this.checkReadiness(true);
|
|
5423
|
+
}
|
|
5424
|
+
const toldByReadiness = d.cause === "agent_not_installed" && this.cannotStartTold();
|
|
5425
|
+
if (this.opts.notify && !toldByReadiness && !this.refusalsNotified.has(d.says)) {
|
|
5294
5426
|
this.refusalsNotified.add(d.says);
|
|
5295
5427
|
try {
|
|
5296
5428
|
this.opts.notify(composeWorker({ ok: false, triggeredBy: client.recentFrom(o.ref) ?? "unknown", obligationBody: o.body, goal: o.goal ? client.goals.get(o.goal) : undefined, reply: null, obligationRef: o.ref, diagnosis: d }, (() => { const tools = installedTools(); return { tools, preferred: preferredTool(loadProfile(this.cfg), tools, this.opts.home) }; })()));
|
|
@@ -5303,8 +5435,10 @@ export class Worker {
|
|
|
5303
5435
|
const key = `refused:${this.oKey(o.id)}`;
|
|
5304
5436
|
const space = this.spaceKey();
|
|
5305
5437
|
const told = loadContinued(space, this.opts.home);
|
|
5306
|
-
if (told.has(key) || told.has(`refused:${o.id}`))
|
|
5438
|
+
if (told.has(key) || told.has(`refused:${o.id}`)) {
|
|
5439
|
+
this.refusedDue.delete(key);
|
|
5307
5440
|
return;
|
|
5441
|
+
}
|
|
5308
5442
|
const peer = client.recentFrom(o.ref);
|
|
5309
5443
|
try {
|
|
5310
5444
|
await client.send({
|
|
@@ -5318,11 +5452,36 @@ export class Worker {
|
|
|
5318
5452
|
meta: { declined: d.peer?.code ?? (d.cause === "not_signed_in" ? "signed_out" : "unavailable") },
|
|
5319
5453
|
});
|
|
5320
5454
|
markContinued(space, key, this.opts.home);
|
|
5321
|
-
this.
|
|
5455
|
+
this.refusedDue.delete(key);
|
|
5456
|
+
this.opts.log(d.cause === "agent_unavailable" || d.cause === "agent_not_installed" ? `* told ${peer ?? "the teammate"} that ${o.id} waits: ${d.says}`
|
|
5322
5457
|
: `* told ${peer ?? "the teammate"} that ${o.id} waits for a live session: ${d.cause === "repo_not_here" || d.cause === "folder_too_large" ? d.says : "no agent here can take unattended work"}`);
|
|
5323
5458
|
}
|
|
5324
5459
|
catch (e) {
|
|
5325
|
-
|
|
5460
|
+
const msg = e.message;
|
|
5461
|
+
const tries = (this.refusedDue.get(key)?.tries ?? 0) + 1;
|
|
5462
|
+
const passing = !/^NO_REPLY\b/.test(msg) && (!/^[A-Z_]+:/.test(msg) || /^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(msg));
|
|
5463
|
+
if (passing && tries <= REFUSED_NOTE_TRIES)
|
|
5464
|
+
this.refusedDue.set(key, { o, d, tries, next: Date.now() + Math.min(60_000, 2_000 * tries) });
|
|
5465
|
+
else
|
|
5466
|
+
this.refusedDue.delete(key);
|
|
5467
|
+
if (this.refusedDue.size > 200)
|
|
5468
|
+
this.refusedDue.delete(this.refusedDue.keys().next().value);
|
|
5469
|
+
this.opts.log(`! could not tell the teammate that ${o.id} is waiting: ${msg.split("\n")[0]}${this.refusedDue.has(key) ? " — it is said at the next look" : ""}`);
|
|
5470
|
+
}
|
|
5471
|
+
}
|
|
5472
|
+
refusedDue = new Map();
|
|
5473
|
+
async settleRefused(client) {
|
|
5474
|
+
if (!this.refusedDue.size || !client.connected)
|
|
5475
|
+
return;
|
|
5476
|
+
for (const [key, r] of [...this.refusedDue]) {
|
|
5477
|
+
if (r.next > Date.now())
|
|
5478
|
+
continue;
|
|
5479
|
+
const now = client.myWork().find((x) => this.oKey(x.id) === this.oKey(r.o.id));
|
|
5480
|
+
if (!now) {
|
|
5481
|
+
this.refusedDue.delete(key);
|
|
5482
|
+
continue;
|
|
5483
|
+
}
|
|
5484
|
+
await this.reportRefused(client, now, r.d, true);
|
|
5326
5485
|
}
|
|
5327
5486
|
}
|
|
5328
5487
|
mootSince = new Map();
|
|
@@ -6031,6 +6190,7 @@ export class Worker {
|
|
|
6031
6190
|
lastHealthCheck = 0;
|
|
6032
6191
|
async maintain(client, arrived) {
|
|
6033
6192
|
const home = this.opts.home;
|
|
6193
|
+
this.sayRolledBack();
|
|
6034
6194
|
{
|
|
6035
6195
|
const policy = loadPolicy(home);
|
|
6036
6196
|
const denies = [];
|
|
@@ -6076,32 +6236,14 @@ export class Worker {
|
|
|
6076
6236
|
const lastAttempt = policy.attempts.self_upgrade ?? null;
|
|
6077
6237
|
if (!needsSelfUpgrade(VERSION, relayPkg, lastAttempt))
|
|
6078
6238
|
return;
|
|
6079
|
-
const
|
|
6080
|
-
|
|
6081
|
-
const key = `${back.version}@${back.at}`;
|
|
6082
|
-
if (!back.told_at && rolledBackSaid !== key) {
|
|
6083
|
-
rolledBackSaid = key;
|
|
6084
|
-
const line = rolledBackSentence(back);
|
|
6085
|
-
this.opts.log(`! ${line}`);
|
|
6086
|
-
const canAlert = !!this.opts.notify && decisionAlertsOn() && (() => { try {
|
|
6087
|
-
return !this.opts.canAlert || this.opts.canAlert();
|
|
6088
|
-
}
|
|
6089
|
-
catch {
|
|
6090
|
-
return false;
|
|
6091
|
-
} })();
|
|
6092
|
-
if (canAlert) {
|
|
6093
|
-
try {
|
|
6094
|
-
this.opts.notify?.({ kind: "worker", who: "Pantheon · this computer", heading: `Pantheon went back to ${back.back_to}`, context: this.cfg.space, body: line, ttl: 0, sound: false });
|
|
6095
|
-
}
|
|
6096
|
-
catch { }
|
|
6097
|
-
}
|
|
6098
|
-
if (this.opts.selfUpgrade?.rolledBack)
|
|
6099
|
-
this.opts.selfUpgrade.rolledBack.told();
|
|
6100
|
-
else
|
|
6101
|
-
markRolledBackTold();
|
|
6102
|
-
}
|
|
6103
|
-
return;
|
|
6239
|
+
const hold = (() => { try {
|
|
6240
|
+
return this.opts.selfUpgrade?.rolledBack ? this.opts.selfUpgrade.rolledBack.read() : readRolledBack();
|
|
6104
6241
|
}
|
|
6242
|
+
catch {
|
|
6243
|
+
return null;
|
|
6244
|
+
} })();
|
|
6245
|
+
if (hold && hold.version === relayPkg)
|
|
6246
|
+
return;
|
|
6105
6247
|
const nodeNow = this.opts.selfUpgrade?.node ?? (() => runtimeNodeVersion());
|
|
6106
6248
|
const held = policy.upgradeHeld;
|
|
6107
6249
|
if (held && held.version === relayPkg && held.node === nodeNow())
|
|
@@ -6207,6 +6349,42 @@ export class Worker {
|
|
|
6207
6349
|
}
|
|
6208
6350
|
}
|
|
6209
6351
|
}
|
|
6352
|
+
sayRolledBack() {
|
|
6353
|
+
const seam = this.opts.selfUpgrade?.rolledBack;
|
|
6354
|
+
const now = Date.now();
|
|
6355
|
+
if (!seam && now - rolledBackLookedAt < 60_000)
|
|
6356
|
+
return;
|
|
6357
|
+
rolledBackLookedAt = now;
|
|
6358
|
+
let back = null;
|
|
6359
|
+
try {
|
|
6360
|
+
back = seam ? seam.read() : readRolledBack();
|
|
6361
|
+
}
|
|
6362
|
+
catch {
|
|
6363
|
+
return;
|
|
6364
|
+
}
|
|
6365
|
+
if (!back || back.told_at)
|
|
6366
|
+
return;
|
|
6367
|
+
const key = `${back.version}@${back.at}`;
|
|
6368
|
+
const line = rolledBackSentence(back);
|
|
6369
|
+
if (rolledBackSaid !== key) {
|
|
6370
|
+
rolledBackSaid = key;
|
|
6371
|
+
this.opts.log(`! ${line}`);
|
|
6372
|
+
}
|
|
6373
|
+
const canAlert = !!this.opts.notify && decisionAlertsOn() && (() => { try {
|
|
6374
|
+
return !this.opts.canAlert || this.opts.canAlert();
|
|
6375
|
+
}
|
|
6376
|
+
catch {
|
|
6377
|
+
return false;
|
|
6378
|
+
} })();
|
|
6379
|
+
if (!canAlert)
|
|
6380
|
+
return;
|
|
6381
|
+
if (!(seam ? seam.told() : markRolledBackTold()))
|
|
6382
|
+
return;
|
|
6383
|
+
try {
|
|
6384
|
+
this.opts.notify?.({ kind: "worker", who: "Pantheon · this computer", heading: `Pantheon went back to ${back.back_to}`, context: this.cfg.space, body: line, ttl: 0, sound: false });
|
|
6385
|
+
}
|
|
6386
|
+
catch { }
|
|
6387
|
+
}
|
|
6210
6388
|
askAt = new Map();
|
|
6211
6389
|
ledgerAllowSeen = new Set();
|
|
6212
6390
|
lastFullScan = 0;
|
|
@@ -6705,11 +6883,23 @@ export class Worker {
|
|
|
6705
6883
|
this.report("running", detail, undefined, { ...(l?.ref ? { ref: l.ref } : {}), ...(l?.goal ? { goal: l.goal } : {}) });
|
|
6706
6884
|
}
|
|
6707
6885
|
heartbeatAt = 0;
|
|
6886
|
+
writeWaiting() {
|
|
6887
|
+
const r = this.readiness;
|
|
6888
|
+
if (!r || r.ok || r.code !== "cannot_start") {
|
|
6889
|
+
this.writeState("watching", "waiting for work");
|
|
6890
|
+
return;
|
|
6891
|
+
}
|
|
6892
|
+
if (Date.now() - this.lastStateAt >= WORKER_STATE_EVERY_MS)
|
|
6893
|
+
this.lastState = "";
|
|
6894
|
+
this.writeState("held", clip(r.says ?? "its agent cannot start on this computer", 160));
|
|
6895
|
+
}
|
|
6896
|
+
lastStateAt = 0;
|
|
6708
6897
|
writeState(state, detail) {
|
|
6709
6898
|
const line = `${state}|${detail}`;
|
|
6710
6899
|
if (line === this.lastState)
|
|
6711
6900
|
return;
|
|
6712
6901
|
this.lastState = line;
|
|
6902
|
+
this.lastStateAt = Date.now();
|
|
6713
6903
|
try {
|
|
6714
6904
|
const base = this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
|
|
6715
6905
|
mkdirSync(base, { recursive: true });
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@join-pantheon/cli",
|
|
3
|
-
"version": "0.38.
|
|
3
|
+
"version": "0.38.3",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@join-pantheon/cli",
|
|
9
|
-
"version": "0.38.
|
|
9
|
+
"version": "0.38.3",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@modelcontextprotocol/sdk": "^1.32.1",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@join-pantheon/cli",
|
|
3
|
-
"version": "0.38.
|
|
3
|
+
"version": "0.38.3",
|
|
4
4
|
"description": "Let your coding agent collaborate directly with your teammates' agents — ask, agree an interface, and deliver, without a human relaying messages.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|