@join-pantheon/cli 0.35.0-rc.2 → 0.35.1-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-outage.js +156 -36
- package/dist/agent-token.js +5 -3
- package/dist/approvals-list.js +87 -0
- package/dist/board.js +472 -87
- package/dist/brief.js +17 -0
- package/dist/cli.js +687 -95
- package/dist/client.js +159 -19
- package/dist/config.js +3 -2
- package/dist/continued.js +56 -1
- package/dist/decision-ui.js +5 -3
- package/dist/doctor.js +81 -15
- package/dist/file-lock.js +39 -4
- package/dist/flow-stats.js +18 -1
- package/dist/home-guard.js +317 -54
- package/dist/home-safety.js +2 -0
- package/dist/hud-queue.js +91 -0
- package/dist/install.js +15 -1
- package/dist/mcp-instructions.js +55 -20
- package/dist/mcp.js +114 -24
- package/dist/notice-linux.js +15 -4
- package/dist/notice-windows.js +2 -2
- package/dist/notifier.js +2 -1
- package/dist/notify.js +8 -4
- package/dist/open-in.js +4 -3
- package/dist/pickup.js +17 -0
- package/dist/plain.js +139 -3
- package/dist/profile.js +12 -8
- package/dist/protocol.js +1 -1
- package/dist/question-goal.js +74 -0
- package/dist/readiness-alert.js +24 -3
- package/dist/refusal.js +2 -2
- package/dist/relay.js +142 -28
- package/dist/removed.js +4 -4
- package/dist/run-control.js +4 -3
- package/dist/run-failure.js +67 -10
- package/dist/setup-changes.js +7 -6
- package/dist/signed-out.js +144 -21
- package/dist/spend.js +2 -2
- package/dist/tray.js +24 -6
- package/dist/update-check.js +2 -1
- package/dist/uri-handler.js +10 -2
- package/dist/which.js +2 -0
- package/dist/worker-policy.js +212 -10
- package/dist/worker.js +847 -114
- package/dist/worklist.js +31 -1
- package/helper/macos/main.swift +42 -3
- package/helper/macos/menubar.swift +482 -87
- package/helper/macos/prebuilt/manifest.json +5 -5
- package/helper/macos/prebuilt/pantheon-hud +0 -0
- package/helper/macos/prebuilt/pantheon-tray +0 -0
- package/package.json +1 -1
package/dist/worker.js
CHANGED
|
@@ -5,22 +5,22 @@ import { chmodSync, lstatSync, mkdirSync, writeFileSync, existsSync, readdirSync
|
|
|
5
5
|
import { homedir } from "node:os";
|
|
6
6
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
7
7
|
import { absentDeniedPaths, describeKept, ownWorktreeConfig, detectClaudeFeatures, describeFeatures, ensureRunTmp, leavesSandboxMounts, NO_CLAUDE_FEATURES, removeSandboxLeftovers, runTmpDir } from "./claude-confine.js";
|
|
8
|
-
import { snapshotHome, restoreHome, describeChange, trustWrite, askedHere, persistSnapshot, persistRestored, persistAttempt, loadPersistedGuard, clearPersistedGuard, CHANGES_LOG_FILE } from "./home-guard.js";
|
|
8
|
+
import { snapshotHome, restoreHome, describeChange, trustWrite, askedHere, canon, withdrawnHere, noesHere, adoptOwn, persistOwn, persistPending, resetOwn, persistSnapshot, persistRestored, persistAttempt, loadPersistedGuard, clearPersistedGuard, CHANGES_LOG_FILE, GUARDED_FILES } from "./home-guard.js";
|
|
9
9
|
import { PantheonClient } from "./client.js";
|
|
10
10
|
import { createHash, randomBytes } from "node:crypto";
|
|
11
11
|
import { newKeypair, signDelegate } from "./identity.js";
|
|
12
12
|
import { loadPins, pinScope, savePins, teamPinId, writeWhole } from "./pins.js";
|
|
13
|
-
import { answeredIn, markAnswered, noteNonce } from "./nonces.js";
|
|
13
|
+
import { answeredIn, markAnswered, noteNonce, unmarkAnswered } from "./nonces.js";
|
|
14
14
|
import { brokenStateFiles, brokenStateFix, pidAlive, readStateFile, withFileLock } from "./file-lock.js";
|
|
15
15
|
import { mkdirInside, writeInside } from "./safe-write.js";
|
|
16
16
|
import { corruptHeldFiles, MAX_HELD_PER_RUN, remoteTipIs, takeHeldFrames, updateHeldState, verifyHeldFrame, writeHoldConfig } from "./held-delivery.js";
|
|
17
17
|
import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
|
|
18
18
|
import { trusted } from "./verify-act.js";
|
|
19
|
-
import { headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
19
|
+
import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
20
20
|
import { activeOutages, detectOutage, earliest, whenAgain, SIGNIN_REFUSED } from "./agent-outage.js";
|
|
21
21
|
import { agentSpawn, CmdLineUnsafe } from "./which.js";
|
|
22
22
|
import { VERSION } from "./version.js";
|
|
23
|
-
import { applyAnswer, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, policyHome, describeRunMinutes, parseRunMinutes, currentMode, } from "./worker-policy.js";
|
|
23
|
+
import { applyAnswer, ASK_EXPIRY_MS, policyUnreadable, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, policyHome, describeRunMinutes, parseRunMinutes, currentMode, wakeSettled, } from "./worker-policy.js";
|
|
24
24
|
import { readinessAlerts } from "./readiness-alert.js";
|
|
25
25
|
import { clearDeclined, markDeclined, personsOwnThread, pruneDeclined } from "./declined.js";
|
|
26
26
|
import { loadRunStops, markStopTold, pruneRunStops, requestRunStop, stopFor, stopNote } from "./run-control.js";
|
|
@@ -29,18 +29,18 @@ import { removalWords, removedLine, removedNotice, takeTold } from "./removed.js
|
|
|
29
29
|
import { agentHookStates, repairHook, repairNote } from "./notify.js";
|
|
30
30
|
import { fence, assess } from "./untrusted.js";
|
|
31
31
|
import { allowed, closeStartedRun, finishRun, recordRun, startRun } from "./spend.js";
|
|
32
|
-
import { handledIn, loadContinued, markContinued, unmarkContinued, wakeAlertedKey } from "./continued.js";
|
|
32
|
+
import { handledIn, loadContinued, markContinued, storedIn, storedKey, unmarkContinued, wakeAlertedKey, WORKER_MARK } from "./continued.js";
|
|
33
33
|
import { decisionAlertsOn, loadProfile, profileForTeam, reservedDecisionsNote, saveProfile, wakeMode } from "./profile.js";
|
|
34
34
|
import { clip } from "./brief.js";
|
|
35
35
|
import { composeWakeAsk, composeWorker } from "./notice.js";
|
|
36
|
-
import { diagnoseRun, gitProbe, workNeedsCommands,
|
|
37
|
-
import { clearSignedOut, markSignedOut, retryAfter, signedOutNow, SIGNED_OUT_AGENTS } from "./signed-out.js";
|
|
36
|
+
import { diagnoseRun, gitProbe, refusedWords, workNeedsCommands, signInFixFor } from "./run-failure.js";
|
|
37
|
+
import { claudeSettingsAuth, clearSignedOut, markSignedOut, retryAfter, signedOutChecking, signedOutNow, SIGNED_OUT_AGENTS } from "./signed-out.js";
|
|
38
38
|
import { claudeTokenForRun } from "./agent-token.js";
|
|
39
39
|
import { installedTools, preferredTool } from "./notifier.js";
|
|
40
40
|
import { obligationSignature, StatusNudger } from "./status-nudge.js";
|
|
41
41
|
import { clientState, ConnectionReport } from "./conn-report.js";
|
|
42
42
|
import { loadPersonas, matchPersona, personaByName, personaMatches, effectivePolicy, personaBrief, describePersona, } from "./persona.js";
|
|
43
|
-
import { CONFIG_DIR, CONFIG_PATH, loadAll, repoInfo, updateMembershipToken } from "./config.js";
|
|
43
|
+
import { CONFIG_DIR, CONFIG_PATH, configFor, findMembership, load as loadConfig, loadAll, repoInfo, updateMembershipToken } from "./config.js";
|
|
44
44
|
import { shellArg } from "./team-name.js";
|
|
45
45
|
import { repoName, routeRequest, safeName } from "./repo-route.js";
|
|
46
46
|
import { holdsRepo } from "./repo-find.js";
|
|
@@ -279,7 +279,7 @@ export function outsideLinksDiagnosis(member, worktree, scan) {
|
|
|
279
279
|
says,
|
|
280
280
|
fix: scan.unreadable
|
|
281
281
|
? `Look at this machine's worktree ${where}: a very large or unreadable checkout cannot be vouched for. Take this request in a live session, or clear it out (pantheon stops at that worktree; its branch and commits are kept in the repo).`
|
|
282
|
-
: `On this machine, delete these links from the worktree ${where} — not from the repo's branch, where they are not: ${scan.links.map((l) => `${where}/${l}`).join(", ")}. Or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it.`,
|
|
282
|
+
: `On this machine, delete these links from the worktree ${where} — not from the repo's branch, where they are not: ${scan.links.slice(0, 20).map((l) => `${where}/${l}`).join(", ")}${scan.links.length > 20 ? `, and ${scan.links.length - 20} more in that worktree (find ${where} -type l lists them)` : ""}. Or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it.`,
|
|
283
283
|
peer: {
|
|
284
284
|
summary: `${member}'s worker did not start a run: its checkout for this could not be vouched for`,
|
|
285
285
|
body: `${member}'s machine could not confirm its checkout for this holds no link leading outside it, so its worker did not start an unattended run: a run would read whatever such a link points at. It is still owed and waits for ${member}. Nothing was run.`,
|
|
@@ -780,8 +780,25 @@ export function deliveryStep(branch, canPush, ref) {
|
|
|
780
780
|
for your human to review and push. If the request asked for code, that NOTE is your reply.`;
|
|
781
781
|
}
|
|
782
782
|
const SAFETY_CAUSES = new Set(["outside_links", "worktree_tampered"]);
|
|
783
|
+
const SAFETY_RERAISE_MAX = 3;
|
|
784
|
+
function safetyAgainDiagnosis(cause, member) {
|
|
785
|
+
return cause === "outside_links"
|
|
786
|
+
? { cause: "outside_links", says: "the checkout for this held links that lead outside it — a run would read whatever they point at as the repo's own, so none was started",
|
|
787
|
+
fix: "On this machine, look at this request's worktree and delete any link that leads outside it, or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it." }
|
|
788
|
+
: { cause: "worktree_tampered", says: `the worktree's .git was changed after Pantheon made it — git would obey that gitdir's config outside any sandbox, so git is not run in it`,
|
|
789
|
+
fix: `Look at what is in that worktree, then remove its folder and run git worktree prune in the repo; the next request checks out a fresh one. (${member}'s worker raised this before a restart.)` };
|
|
790
|
+
}
|
|
791
|
+
function decidedBrief(answer) {
|
|
792
|
+
const said = answer.replace(/[\u0000-\u0008\u000b-\u001f\u007f]/g, "").slice(0, 2000);
|
|
793
|
+
return `
|
|
794
|
+
YOUR HUMAN DECIDED — an earlier run here escalated this request to them, and they answered:
|
|
795
|
+
${said.split("\n").map((l) => ` | ${l}`).join("\n")}
|
|
796
|
+
That is their decision. Reply to your teammate with it now (step 4: an ANSWER with the request's ref),
|
|
797
|
+
in your own plain words, and act on it if the request asks for work. Do not escalate the same question again.
|
|
798
|
+
`;
|
|
799
|
+
}
|
|
783
800
|
export const MOOT_GRACE_MS = 2 * 60_000;
|
|
784
|
-
export function buildBrief(o, trigger, branch, persona, canPush = false) {
|
|
801
|
+
export function buildBrief(o, trigger, branch, persona, canPush = false, decided) {
|
|
785
802
|
const placed = branch
|
|
786
803
|
? `in an isolated git worktree on branch ${branch}`
|
|
787
804
|
: `in a scratch directory (your human's workspace is not a git repo, so there is no branch and nothing can be DELIVERED from here)`;
|
|
@@ -797,7 +814,7 @@ ${riskNote(risky, "what was asked")}
|
|
|
797
814
|
WHAT YOU OWE
|
|
798
815
|
${owedLine(o.kind)}${o.goal ? ` — goal ${printable(o.goal)}` : ""}
|
|
799
816
|
obligation id: ${printable(o.id)}
|
|
800
|
-
${persona ? personaBrief(persona) : ""}
|
|
817
|
+
${persona ? personaBrief(persona) : ""}${decided ? decidedBrief(decided) : ""}
|
|
801
818
|
DO THIS
|
|
802
819
|
1. pantheon_claim with obligation id ${printable(o.id)}. If the claim is REFUSED, another session has it —
|
|
803
820
|
stop immediately and say nothing.
|
|
@@ -985,8 +1002,43 @@ function readRunLease(f) {
|
|
|
985
1002
|
rmSync(f, { recursive: true, force: true });
|
|
986
1003
|
return null;
|
|
987
1004
|
}
|
|
1005
|
+
export const PERSON_HOLD = "person-answering.json";
|
|
1006
|
+
export const PERSON_HOLD_TTL_MS = 60_000;
|
|
1007
|
+
export const PERSON_ANSWERING = "your person is answering a question on this computer";
|
|
1008
|
+
export function personHold(dir, now = Date.now()) {
|
|
1009
|
+
try {
|
|
1010
|
+
const v = JSON.parse(readFileSync(join(dir, PERSON_HOLD), "utf8"));
|
|
1011
|
+
const pid = Number(v?.pid), at = Number(v?.at);
|
|
1012
|
+
if (!Number.isInteger(pid) || pid <= 1 || !Number.isFinite(at) || at > now + 5_000 || now - at > PERSON_HOLD_TTL_MS || !pidAlive(pid))
|
|
1013
|
+
return null;
|
|
1014
|
+
return { pid, at };
|
|
1015
|
+
}
|
|
1016
|
+
catch {
|
|
1017
|
+
return null;
|
|
1018
|
+
}
|
|
1019
|
+
}
|
|
1020
|
+
export function writePersonHold(dir, now = Date.now()) {
|
|
1021
|
+
try {
|
|
1022
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
1023
|
+
writeWhole(join(dir, PERSON_HOLD), JSON.stringify({ pid: process.pid, at: now }) + "\n");
|
|
1024
|
+
}
|
|
1025
|
+
catch { }
|
|
1026
|
+
}
|
|
1027
|
+
export function releasePersonHold(dir) {
|
|
1028
|
+
try {
|
|
1029
|
+
const h = personHold(dir);
|
|
1030
|
+
if (!h || h.pid === process.pid)
|
|
1031
|
+
rmSync(join(dir, PERSON_HOLD), { force: true });
|
|
1032
|
+
}
|
|
1033
|
+
catch { }
|
|
1034
|
+
}
|
|
988
1035
|
export function takeRunLease(dir, space, now = Date.now()) {
|
|
989
1036
|
const f = join(dir, RUN_SLOT);
|
|
1037
|
+
{
|
|
1038
|
+
const h = personHold(dir, now);
|
|
1039
|
+
if (h && h.pid !== process.pid)
|
|
1040
|
+
return { held: PERSON_ANSWERING };
|
|
1041
|
+
}
|
|
990
1042
|
return withFileLock(f, () => {
|
|
991
1043
|
const cur = readRunLease(f);
|
|
992
1044
|
if (cur && Number.isInteger(cur.pid) && cur.pid !== process.pid && typeof cur.at === "number" && Number.isFinite(cur.at)
|
|
@@ -1110,7 +1162,30 @@ export function putBackSaid(list) {
|
|
|
1110
1162
|
}
|
|
1111
1163
|
export const GUARD_RESTORE_TRIES = 3;
|
|
1112
1164
|
const GUARD_STALE_MARGIN_MS = 10 * 60_000;
|
|
1113
|
-
export function
|
|
1165
|
+
export async function recoverGuardHere(home) {
|
|
1166
|
+
const g = loadPersistedGuard(home);
|
|
1167
|
+
if (!g)
|
|
1168
|
+
return { outcome: "none" };
|
|
1169
|
+
if (sameProcess(g.worker, g.workerId))
|
|
1170
|
+
return { outcome: "live" };
|
|
1171
|
+
let own;
|
|
1172
|
+
if (g.own && g.snapshot) {
|
|
1173
|
+
const anyCfg = (() => { try {
|
|
1174
|
+
return loadConfig();
|
|
1175
|
+
}
|
|
1176
|
+
catch {
|
|
1177
|
+
return null;
|
|
1178
|
+
} })();
|
|
1179
|
+
if (!anyCfg)
|
|
1180
|
+
return { outcome: "unreachable" };
|
|
1181
|
+
const v = await verifyOwnWrites(g.own, anyCfg.member, (t) => configForTeam(t, anyCfg));
|
|
1182
|
+
if (v === "unreachable")
|
|
1183
|
+
return { outcome: "unreachable" };
|
|
1184
|
+
own = v;
|
|
1185
|
+
}
|
|
1186
|
+
return recoverInterruptedGuard(home, Date.now(), restoreHome, own);
|
|
1187
|
+
}
|
|
1188
|
+
export function recoverInterruptedGuard(home, now = Date.now(), restore = restoreHome, verifiedOwn) {
|
|
1114
1189
|
const g = loadPersistedGuard(home);
|
|
1115
1190
|
if (!g)
|
|
1116
1191
|
return { outcome: "none" };
|
|
@@ -1126,13 +1201,15 @@ export function recoverInterruptedGuard(home, now = Date.now(), restore = restor
|
|
|
1126
1201
|
clearPersistedGuard(home);
|
|
1127
1202
|
return { outcome: "restored", said: g.restored ?? [] };
|
|
1128
1203
|
}
|
|
1129
|
-
|
|
1204
|
+
adoptOwn(verifiedOwn);
|
|
1205
|
+
const pending = !!g.pending?.length;
|
|
1206
|
+
if (!pending && (g.at > now + 60_000 || now - g.at > recordLimitMs({ limit: g.limit }) + GUARD_STALE_MARGIN_MS)) {
|
|
1130
1207
|
clearPersistedGuard(home);
|
|
1131
1208
|
return { outcome: "stale", at: g.at };
|
|
1132
1209
|
}
|
|
1133
1210
|
const lease = readRunLease(join(home, RUN_SLOT));
|
|
1134
1211
|
const wentAt = lease && lease.pid === g.worker && typeof lease.at === "number" && Number.isFinite(lease.at) ? lease.at + RUN_LEASE_RENEW_MS : null;
|
|
1135
|
-
if (wentAt !== null)
|
|
1212
|
+
if (wentAt !== null && !pending)
|
|
1136
1213
|
try {
|
|
1137
1214
|
const mine = readFileSync(join(home, CHANGES_LOG_FILE), "utf8").split("\n").some((l) => {
|
|
1138
1215
|
try {
|
|
@@ -1150,7 +1227,14 @@ export function recoverInterruptedGuard(home, now = Date.now(), restore = restor
|
|
|
1150
1227
|
}
|
|
1151
1228
|
catch { }
|
|
1152
1229
|
try {
|
|
1153
|
-
const
|
|
1230
|
+
const snapshot = pending ? { ...g.snapshot, files: g.snapshot.files.filter((f) => g.pending.includes(f.rel)) } : g.snapshot;
|
|
1231
|
+
const changes = restore(snapshot);
|
|
1232
|
+
const said = putBackList(changes);
|
|
1233
|
+
const rels = changes.filter((c) => c.failed).map((c) => c.rel);
|
|
1234
|
+
if (rels.length) {
|
|
1235
|
+
persistPending(home, rels);
|
|
1236
|
+
return { outcome: "pending", said, rels, snapshot: { ...g.snapshot, files: g.snapshot.files.filter((f) => rels.includes(f.rel)) } };
|
|
1237
|
+
}
|
|
1154
1238
|
clearPersistedGuard(home);
|
|
1155
1239
|
return { outcome: "restored", said };
|
|
1156
1240
|
}
|
|
@@ -1166,6 +1250,8 @@ export function recoverInterruptedGuard(home, now = Date.now(), restore = restor
|
|
|
1166
1250
|
}
|
|
1167
1251
|
export function guardHeld(home, now = Date.now()) {
|
|
1168
1252
|
const g = loadPersistedGuard(home);
|
|
1253
|
+
if (g?.snapshot && g.pending?.length)
|
|
1254
|
+
return true;
|
|
1169
1255
|
if (!g?.snapshot || g.at > now + 60_000 || now - g.at > recordLimitMs({ limit: g.limit }) + GUARD_STALE_MARGIN_MS)
|
|
1170
1256
|
return false;
|
|
1171
1257
|
return sameProcess(g.worker, g.workerId);
|
|
@@ -1287,6 +1373,88 @@ export function answerWindow(askSeq, hint, fullScanDue) {
|
|
|
1287
1373
|
return 0;
|
|
1288
1374
|
return Math.max(0, hint);
|
|
1289
1375
|
}
|
|
1376
|
+
const teamConfigs = new Map();
|
|
1377
|
+
function configForTeam(team, mine) {
|
|
1378
|
+
if (team === mine.spaceId || team === mine.space)
|
|
1379
|
+
return mine;
|
|
1380
|
+
const known = teamConfigs.get(team);
|
|
1381
|
+
if (known)
|
|
1382
|
+
return known;
|
|
1383
|
+
try {
|
|
1384
|
+
const f = loadAll();
|
|
1385
|
+
const m = f ? findMembership(f, team) : undefined;
|
|
1386
|
+
return f && m ? configFor(f, m, null) : null;
|
|
1387
|
+
}
|
|
1388
|
+
catch {
|
|
1389
|
+
return null;
|
|
1390
|
+
}
|
|
1391
|
+
}
|
|
1392
|
+
export async function verifyOwnWrites(o, member, cfgFor) {
|
|
1393
|
+
const out = { wake: [], cap: [], stuck: [], safety: [], withdrawn: [], noes: [], marks: [] };
|
|
1394
|
+
let unreachable = false;
|
|
1395
|
+
const seen = new Map();
|
|
1396
|
+
const act = async (id, team) => {
|
|
1397
|
+
const key = `${team ?? ""}|${id}`;
|
|
1398
|
+
if (seen.has(key))
|
|
1399
|
+
return seen.get(key);
|
|
1400
|
+
const cfgs = team ? [cfgFor(team)].filter((c) => !!c) : [...new Set(teamConfigs.values())];
|
|
1401
|
+
let found = null;
|
|
1402
|
+
for (const cfg of cfgs) {
|
|
1403
|
+
try {
|
|
1404
|
+
const res = await signedRead(cfg, "/act", { id }, { timeoutMs: 3000 });
|
|
1405
|
+
if (res.ok) {
|
|
1406
|
+
found = (await res.json()).act ?? null;
|
|
1407
|
+
if (found)
|
|
1408
|
+
break;
|
|
1409
|
+
}
|
|
1410
|
+
else if (res.status !== 404)
|
|
1411
|
+
unreachable = true;
|
|
1412
|
+
}
|
|
1413
|
+
catch {
|
|
1414
|
+
unreachable = true;
|
|
1415
|
+
}
|
|
1416
|
+
}
|
|
1417
|
+
seen.set(key, found);
|
|
1418
|
+
return found;
|
|
1419
|
+
};
|
|
1420
|
+
const mine = (a) => !!a && a.act === "ESCALATE" && a.from === member && a.sig?.role !== "run";
|
|
1421
|
+
const parse = (v) => { try {
|
|
1422
|
+
const x = JSON.parse(v);
|
|
1423
|
+
return x && typeof x === "object" ? x : {};
|
|
1424
|
+
}
|
|
1425
|
+
catch {
|
|
1426
|
+
return {};
|
|
1427
|
+
} };
|
|
1428
|
+
const refs = new Set(), teamOfAct = new Map();
|
|
1429
|
+
for (const [id, v] of o.stuck) {
|
|
1430
|
+
const rec = parse(v);
|
|
1431
|
+
const team = typeof rec.team === "string" ? rec.team : null;
|
|
1432
|
+
const a = await act(id, team);
|
|
1433
|
+
if (mine(a) && a.ref === rec.ref) {
|
|
1434
|
+
out.stuck.push([id, v]);
|
|
1435
|
+
refs.add(String(rec.ref));
|
|
1436
|
+
if (team)
|
|
1437
|
+
teamOfAct.set(id, team);
|
|
1438
|
+
}
|
|
1439
|
+
}
|
|
1440
|
+
for (const [cap, v] of o.cap) {
|
|
1441
|
+
const id = String(parse(v).act ?? "");
|
|
1442
|
+
if (id && mine(await act(id, teamOfAct.get(id) ?? null)))
|
|
1443
|
+
out.cap.push([cap, v]);
|
|
1444
|
+
}
|
|
1445
|
+
for (const [id, v] of o.wake) {
|
|
1446
|
+
const q = parse(v);
|
|
1447
|
+
const a = await act(id, typeof q.team === "string" ? q.team : null);
|
|
1448
|
+
if (mine(a) && a.meta?.wake_ask === q.member)
|
|
1449
|
+
out.wake.push([id, v]);
|
|
1450
|
+
}
|
|
1451
|
+
for (const m of o.marks) {
|
|
1452
|
+
const r = /:r:[A-Z_]+:(.+)$/.exec(m)?.[1];
|
|
1453
|
+
if (r && refs.has(r))
|
|
1454
|
+
out.marks.push(m);
|
|
1455
|
+
}
|
|
1456
|
+
return unreachable ? "unreachable" : out;
|
|
1457
|
+
}
|
|
1290
1458
|
export class Worker {
|
|
1291
1459
|
cfg;
|
|
1292
1460
|
opts;
|
|
@@ -1328,6 +1496,9 @@ export class Worker {
|
|
|
1328
1496
|
this.cfg = cfg;
|
|
1329
1497
|
this.opts = opts;
|
|
1330
1498
|
this.pinTeam = teamPinId(cfg);
|
|
1499
|
+
for (const k of [cfg.spaceId, cfg.space])
|
|
1500
|
+
if (k)
|
|
1501
|
+
teamConfigs.set(k, cfg);
|
|
1331
1502
|
}
|
|
1332
1503
|
spaceKey() { return this.cfg.spaceId ?? this.cfg.space; }
|
|
1333
1504
|
pinTeam;
|
|
@@ -1344,32 +1515,83 @@ export class Worker {
|
|
|
1344
1515
|
for (const k of this.contKeys())
|
|
1345
1516
|
for (const id of loadContinued(k, this.opts.home))
|
|
1346
1517
|
this.contDone.add(id);
|
|
1518
|
+
for (const id of this.contDone) {
|
|
1519
|
+
const m = /^unparked@([^:]+):(.+)$/.exec(id);
|
|
1520
|
+
if (m)
|
|
1521
|
+
this.unparkedUnder.set(m[2], m[1]);
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
obKeys = new Map();
|
|
1525
|
+
learnRequests(list) {
|
|
1526
|
+
if (this.obKeys.size > 5000)
|
|
1527
|
+
this.obKeys.clear();
|
|
1528
|
+
for (const o of list)
|
|
1529
|
+
if (o.kind !== "CONTINUE" && o.ref)
|
|
1530
|
+
this.obKeys.set(o.id, `r:${o.kind}:${o.ref}`);
|
|
1531
|
+
}
|
|
1532
|
+
oKey(id) { return this.obKeys.get(id) ?? id; }
|
|
1533
|
+
marked(prefix, id) {
|
|
1534
|
+
const k = this.oKey(id);
|
|
1535
|
+
return this.contDone.has(`${prefix}:${k}`) || (k !== id && this.contDone.has(`${prefix}:${id}`));
|
|
1536
|
+
}
|
|
1537
|
+
markFor(prefix, id) { this.markDone(`${prefix}:${this.oKey(id)}`); }
|
|
1538
|
+
unparkedUnder = new Map();
|
|
1539
|
+
grantKey() {
|
|
1540
|
+
const g = loadPolicy(this.opts.home).grants.run_commands;
|
|
1541
|
+
if (this.opts.allowBash)
|
|
1542
|
+
return { key: "bash", once: false };
|
|
1543
|
+
return g ? { key: `${g.mode === "once" ? "o" : "g"}${g.at}`, once: g.mode === "once" } : null;
|
|
1347
1544
|
}
|
|
1348
1545
|
failuresOf(id) {
|
|
1349
|
-
const mem = this.failedRuns.get(id);
|
|
1546
|
+
const mem = this.failedRuns.get(this.oKey(id));
|
|
1350
1547
|
if (mem !== undefined)
|
|
1351
1548
|
return mem;
|
|
1352
1549
|
this.loadContinued();
|
|
1353
|
-
|
|
1550
|
+
const under = this.unparkedUnder.get(this.oKey(id));
|
|
1551
|
+
if (under)
|
|
1552
|
+
return this.contDone.has(`parked2@${under}:${this.oKey(id)}`) ? 2 : 1;
|
|
1553
|
+
if (this.marked("unparked", id) && !this.marked("parked2", id))
|
|
1354
1554
|
return 1;
|
|
1355
|
-
if (this.
|
|
1555
|
+
if (this.marked("unparked-ns", id) && !this.marked("parked2-ns", id))
|
|
1356
1556
|
return 1;
|
|
1357
|
-
return this.
|
|
1557
|
+
return this.marked("parked", id) ? 2 : this.marked("failed1", id) ? 1 : 0;
|
|
1358
1558
|
}
|
|
1359
1559
|
retryParkedForCommands(owed) {
|
|
1360
|
-
const g =
|
|
1361
|
-
if (!g
|
|
1560
|
+
const g = this.grantKey();
|
|
1561
|
+
if (!g)
|
|
1362
1562
|
return;
|
|
1363
1563
|
this.loadContinued();
|
|
1564
|
+
const waiting = (o) => this.marked("parked-nc", o.id) && this.failuresOf(o.id) >= 2;
|
|
1565
|
+
if (g.once && owed.some((o) => this.unparkedUnder.get(this.oKey(o.id)) === g.key && this.failuresOf(o.id) < 2))
|
|
1566
|
+
return;
|
|
1364
1567
|
for (const o of owed) {
|
|
1365
|
-
if (!
|
|
1568
|
+
if (!waiting(o) || this.unparkedUnder.get(this.oKey(o.id)) === g.key)
|
|
1366
1569
|
continue;
|
|
1367
|
-
this.failedRuns.set(o.id, 1);
|
|
1570
|
+
this.failedRuns.set(this.oKey(o.id), 1);
|
|
1368
1571
|
this.skipLogged.delete(`parked:${o.id}`);
|
|
1369
|
-
this.markDone(`unparked:${o.id}`);
|
|
1572
|
+
this.markDone(`unparked@${g.key}:${this.oKey(o.id)}`);
|
|
1573
|
+
this.unparkedUnder.set(this.oKey(o.id), g.key);
|
|
1370
1574
|
this.opts.log(`* commands are allowed now — running ${o.kind} ${o.id} again (it was parked because it needed them)`);
|
|
1575
|
+
if (g.once)
|
|
1576
|
+
break;
|
|
1371
1577
|
}
|
|
1372
1578
|
}
|
|
1579
|
+
async askAgainForParked(client) {
|
|
1580
|
+
if (this.grantKey())
|
|
1581
|
+
return;
|
|
1582
|
+
this.loadContinued();
|
|
1583
|
+
const recent = (k) => /^o\d+$/.test(k) && Date.now() - Number(k.slice(1)) < ASK_EXPIRY_MS;
|
|
1584
|
+
const spent = new Set([...this.unparkedUnder.values()].filter(recent));
|
|
1585
|
+
if (!spent.size)
|
|
1586
|
+
return;
|
|
1587
|
+
if (!mayAsk(loadPolicy(this.opts.home), "run_commands"))
|
|
1588
|
+
return;
|
|
1589
|
+
const o = client.myWork().find((x) => this.marked("parked-nc", x.id) && this.failuresOf(x.id) >= 2 && !spent.has(this.unparkedUnder.get(this.oKey(x.id)) ?? ""));
|
|
1590
|
+
if (!o)
|
|
1591
|
+
return;
|
|
1592
|
+
await this.reportStuck(client, o, { cause: "no_commands", says: "The agent may only edit files on this computer, so it couldn't run tests, commit or push.",
|
|
1593
|
+
fix: "To let it deliver work while you're away: pantheon setup --commands yes", capability: "run_commands" }, 2, 1);
|
|
1594
|
+
}
|
|
1373
1595
|
parkedNotHere = new Set();
|
|
1374
1596
|
parkedNoSandbox = new Map();
|
|
1375
1597
|
outages = new Map();
|
|
@@ -1483,20 +1705,19 @@ export class Worker {
|
|
|
1483
1705
|
this.readinessCommandsOn = commandsOn;
|
|
1484
1706
|
const r = this.opts.readinessCheck ? this.opts.readinessCheck() : unattendedReadiness({ prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: this.unsandboxedOk(), pinned: this.opts.agent ?? "auto", ...(this.opts.choose ? { choose: this.opts.choose } : {}),
|
|
1485
1707
|
commandsOn, inContainer: hostEnv().kind === "container", ...(this.opts.sandboxProbe ? { sandboxProbe: this.opts.sandboxProbe } : {}) });
|
|
1486
|
-
if (!r.ok && (this.readiness?.ok ?? true))
|
|
1487
|
-
this.opts.log(`! teammates are told this machine can't take requests yet: ${r.short} — ${r.fix}`);
|
|
1488
|
-
if (r.ok && this.readiness && !this.readiness.ok)
|
|
1489
|
-
this.opts.log(` agent ready again — teammates are told this machine takes requests`);
|
|
1490
|
-
this.readiness = { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) };
|
|
1491
1708
|
this.syncSignedOut();
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
this.
|
|
1709
|
+
const held = r.ok && this.signedOutHere.size && !this.opts.readinessCheck ? this.heldForSandbox() : null;
|
|
1710
|
+
const signedOut = held?.cause === "agent_unavailable";
|
|
1711
|
+
const ok = r.ok && !signedOut;
|
|
1712
|
+
if (!ok && (this.readiness?.ok ?? true))
|
|
1713
|
+
this.opts.log(`! teammates are told this machine can't take requests yet: ${signedOut ? "its agent CLI was signed out (a run here was refused)" : `${r.short} — ${r.fix}`}`);
|
|
1714
|
+
if (ok && this.readiness && !this.readiness.ok)
|
|
1715
|
+
this.opts.log(` agent ready again — teammates are told this machine takes requests`);
|
|
1716
|
+
const outAgent = signedOut ? [...this.signedOutHere][0] : undefined;
|
|
1717
|
+
this.readiness = signedOut
|
|
1718
|
+
? { ok: false, short: "its agent CLI was signed out (a run here was refused)", code: "not_signed_in", fix: held.fix ?? "", signedOut: true, ...(outAgent ? { agent: outAgent } : {}) }
|
|
1719
|
+
: { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) };
|
|
1720
|
+
this.tellPerson(now, { quiet: signedOut });
|
|
1500
1721
|
}
|
|
1501
1722
|
catch {
|
|
1502
1723
|
this.readiness = null;
|
|
@@ -1509,12 +1730,12 @@ export class Worker {
|
|
|
1509
1730
|
const who = createHash("sha256").update(`${resolve(this.opts.project)}|${this.opts.agent ?? "auto"}|${commands ? "commands" : "-"}`).digest("hex").slice(0, 12);
|
|
1510
1731
|
return `${this.spaceKey()}#${who}@${process.pid}`;
|
|
1511
1732
|
}
|
|
1512
|
-
tellPerson(now) {
|
|
1733
|
+
tellPerson(now, o = {}) {
|
|
1513
1734
|
const r = this.readiness;
|
|
1514
1735
|
if (!r)
|
|
1515
1736
|
return;
|
|
1516
1737
|
const alerts = () => {
|
|
1517
|
-
if (!this.opts.notify || !decisionAlertsOn())
|
|
1738
|
+
if (o.quiet || !this.opts.notify || !decisionAlertsOn())
|
|
1518
1739
|
return false;
|
|
1519
1740
|
try {
|
|
1520
1741
|
return !this.opts.canAlert || this.opts.canAlert();
|
|
@@ -1536,7 +1757,7 @@ export class Worker {
|
|
|
1536
1757
|
const key = this.readinessKey();
|
|
1537
1758
|
const replaces = this.lastReadinessKey && this.lastReadinessKey !== key ? this.lastReadinessKey : undefined;
|
|
1538
1759
|
this.lastReadinessKey = key;
|
|
1539
|
-
const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}) });
|
|
1760
|
+
const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet ? { certain: true } : {}) });
|
|
1540
1761
|
this.alertPending = t.pending;
|
|
1541
1762
|
if (!t.notice)
|
|
1542
1763
|
return;
|
|
@@ -1580,6 +1801,28 @@ export class Worker {
|
|
|
1580
1801
|
}
|
|
1581
1802
|
}
|
|
1582
1803
|
outageDiagnosis(label, out, others, meanwhile) {
|
|
1804
|
+
const id = AGENTS.find((a) => a.label === label)?.id;
|
|
1805
|
+
if (out.why === SIGNIN_REFUSED || (id && this.signedOutHere.has(id))) {
|
|
1806
|
+
let why = (id && this.signedOutHere.has(id) && this.outages.get(id)?.why) || undefined;
|
|
1807
|
+
if (why === SIGNIN_REFUSED)
|
|
1808
|
+
why = undefined;
|
|
1809
|
+
if (!why && id) {
|
|
1810
|
+
const m = signedOutNow(id, { pantheonHome: this.stateDir() }) ?? signedOutChecking(id, { pantheonHome: this.stateDir() });
|
|
1811
|
+
if (m)
|
|
1812
|
+
why = this.signedOutWhy(id, m);
|
|
1813
|
+
}
|
|
1814
|
+
why ??= `is signed out on this machine — ${signInFixFor(id)}`;
|
|
1815
|
+
const rest = meanwhile ? `; the agent standing in for it cannot take this one (${meanwhile.replace(/\.$/, "")})` : others.length ? `; ${others.join("; ")}` : "";
|
|
1816
|
+
return {
|
|
1817
|
+
cause: "agent_unavailable",
|
|
1818
|
+
says: `${label} ${why}${rest}.`,
|
|
1819
|
+
fix: "It runs as soon as you sign in again — or answer it in a live session.",
|
|
1820
|
+
peer: {
|
|
1821
|
+
summary: `${this.cfg.member}'s agent is signed out — this waits until ${this.cfg.member} signs in`,
|
|
1822
|
+
body: `${this.cfg.member}'s worker could not start a run: ${label} is signed out on that machine. It is still owed; it runs once ${this.cfg.member} signs in again.`,
|
|
1823
|
+
},
|
|
1824
|
+
};
|
|
1825
|
+
}
|
|
1583
1826
|
const at = whenAgain(out.until);
|
|
1584
1827
|
const rest = meanwhile ? `; the agent standing in for it cannot take this one (${meanwhile.replace(/\.$/, "")})`
|
|
1585
1828
|
: others.length ? `; ${others.join("; ")}` : "; no other agent is installed for runs while you are away";
|
|
@@ -1588,22 +1831,26 @@ export class Worker {
|
|
|
1588
1831
|
says: `${label} is unavailable here until about ${at} — ${out.why}${rest}.`,
|
|
1589
1832
|
fix: `It runs by itself after ${at}. Sooner: sign in to another agent (cursor-agent, codex or copilot) — and for an agent without a sandbox, pantheon policy grant unsandboxed-runs — or answer it in a live session.`,
|
|
1590
1833
|
peer: {
|
|
1591
|
-
summary: `${this.cfg.member}'s agent is unavailable
|
|
1592
|
-
body: `${this.cfg.member}'s worker could not start a run: ${label} ${out.why}, and no other agent on that machine could take it. It is still owed; the worker tries again
|
|
1834
|
+
summary: `${this.cfg.member}'s agent is unavailable for ${forAbout(out.until)} — this waits`,
|
|
1835
|
+
body: `${this.cfg.member}'s worker could not start a run: ${label} ${out.why}, and no other agent on that machine could take it. It is still owed; the worker tries again in ${forAbout(out.until)}.`,
|
|
1593
1836
|
},
|
|
1594
1837
|
};
|
|
1595
1838
|
}
|
|
1596
1839
|
signedOutHere = new Set();
|
|
1840
|
+
allOutSignedOut() {
|
|
1841
|
+
const out = Object.keys(activeOutages(this.outages));
|
|
1842
|
+
return out.length > 0 && out.every((id) => this.signedOutHere.has(id));
|
|
1843
|
+
}
|
|
1597
1844
|
syncSignedOut() {
|
|
1598
1845
|
if (this.opts.agentCmd)
|
|
1599
1846
|
return;
|
|
1600
1847
|
for (const id of SIGNED_OUT_AGENTS) {
|
|
1601
|
-
const mark = signedOutNow(id, { pantheonHome: this.stateDir(), lift: true });
|
|
1848
|
+
const mark = signedOutNow(id, { pantheonHome: this.stateDir(), lift: true, env: process.env });
|
|
1602
1849
|
if (mark) {
|
|
1603
1850
|
const until = mark.at + retryAfter(mark);
|
|
1604
1851
|
const cur = this.outages.get(id);
|
|
1605
1852
|
if (!cur || cur.until < until)
|
|
1606
|
-
this.outages.set(id, { why: this.signedOutWhy(id, mark
|
|
1853
|
+
this.outages.set(id, { why: this.signedOutWhy(id, mark), until });
|
|
1607
1854
|
this.signedOutHere.add(id);
|
|
1608
1855
|
}
|
|
1609
1856
|
else if (this.signedOutHere.has(id)) {
|
|
@@ -1623,10 +1870,14 @@ export class Worker {
|
|
|
1623
1870
|
}
|
|
1624
1871
|
}
|
|
1625
1872
|
}
|
|
1626
|
-
signedOutWhy(id,
|
|
1873
|
+
signedOutWhy(id, mark) {
|
|
1874
|
+
const says = mark.says;
|
|
1875
|
+
const w = refusedWords(id, { line: says, ...(mark.via ? { env: mark.via } : {}), ...(mark.basis === "settings" ? { settings: true } : {}), ...(mark.helper ? { helper: true } : {}), ...(mark.basis === "token" ? { savedToken: true } : {}) });
|
|
1876
|
+
if (w)
|
|
1877
|
+
return `refused ${w.what} ("${says}") — ${w.fix}`;
|
|
1627
1878
|
if (id === "claude-code" && claudeTokenForRun(this.stateDir()))
|
|
1628
1879
|
return `refused the token saved for this worker ("${says}") — make a new one (claude setup-token, then pantheon worker token), or remove it to use your own sign-in: pantheon worker token --remove`;
|
|
1629
|
-
const how =
|
|
1880
|
+
const how = signInFixFor(id);
|
|
1630
1881
|
return `is signed out on this machine ("${says}") — ${how}${id === "claude-code" ? ", or keep it signed in for a year: claude setup-token, then pantheon worker token" : ""}`;
|
|
1631
1882
|
}
|
|
1632
1883
|
outagesPassed(c) {
|
|
@@ -1762,7 +2013,7 @@ export class Worker {
|
|
|
1762
2013
|
}
|
|
1763
2014
|
releaseHeld(id, failures, why) {
|
|
1764
2015
|
this.parkedNoSandbox.delete(id);
|
|
1765
|
-
this.failedRuns.set(id, failures);
|
|
2016
|
+
this.failedRuns.set(this.oKey(id), failures);
|
|
1766
2017
|
this.skipLogged.delete(`parked:${id}`);
|
|
1767
2018
|
this.skipLogged.delete(`refused:${id}`);
|
|
1768
2019
|
const act = this.parkedNoSandboxConts.get(id);
|
|
@@ -1789,7 +2040,7 @@ export class Worker {
|
|
|
1789
2040
|
}
|
|
1790
2041
|
if (!act && owedIds && !owedIds.has(id)) {
|
|
1791
2042
|
this.parkedNoSandbox.delete(id);
|
|
1792
|
-
this.failedRuns.delete(id);
|
|
2043
|
+
this.failedRuns.delete(this.oKey(id));
|
|
1793
2044
|
this.skipLogged.delete(`parked:${id}`);
|
|
1794
2045
|
continue;
|
|
1795
2046
|
}
|
|
@@ -1812,11 +2063,11 @@ export class Worker {
|
|
|
1812
2063
|
if (!ok)
|
|
1813
2064
|
this.ranNoSandboxCheckedAt = now;
|
|
1814
2065
|
this.loadContinued();
|
|
1815
|
-
const ran = (owed ?? []).filter((o) => this.
|
|
2066
|
+
const ran = (owed ?? []).filter((o) => this.marked("parked-ns", o.id) && !this.marked("unparked-ns", o.id));
|
|
1816
2067
|
if (!ran.length || (!ok && !this.claudeSandboxableNow()))
|
|
1817
2068
|
return;
|
|
1818
2069
|
for (const o of ran) {
|
|
1819
|
-
this.
|
|
2070
|
+
this.markFor("unparked-ns", o.id);
|
|
1820
2071
|
this.releaseHeld(o.id, 1, ok ? "unsandboxed runs are allowed here now" : "this Claude Code can run commands in its sandbox now");
|
|
1821
2072
|
}
|
|
1822
2073
|
}
|
|
@@ -1828,7 +2079,7 @@ export class Worker {
|
|
|
1828
2079
|
if (!changed)
|
|
1829
2080
|
return;
|
|
1830
2081
|
for (const id of this.parkedNotHere) {
|
|
1831
|
-
this.failedRuns.delete(id);
|
|
2082
|
+
this.failedRuns.delete(this.oKey(id));
|
|
1832
2083
|
this.skipLogged.delete(`parked:${id}`);
|
|
1833
2084
|
this.skipLogged.delete(`route:${id}`);
|
|
1834
2085
|
this.opts.log(`* this machine's folders changed — trying ${id} again (it was waiting for a repo that was not here)`);
|
|
@@ -1897,17 +2148,27 @@ export class Worker {
|
|
|
1897
2148
|
return;
|
|
1898
2149
|
}
|
|
1899
2150
|
this.loadContinued();
|
|
2151
|
+
const told = new Set();
|
|
1900
2152
|
for (const team of new Set([this.cfg.spaceId, this.cfg.space].filter((t) => !!t))) {
|
|
1901
2153
|
for (const [k, at] of Object.entries(no)) {
|
|
1902
2154
|
if (!k.startsWith(`${team}|`))
|
|
1903
2155
|
continue;
|
|
1904
2156
|
const member = k.slice(team.length + 1);
|
|
2157
|
+
if (told.has(member)) {
|
|
2158
|
+
this.markDone(`wake-no-said:${k}:${at}`);
|
|
2159
|
+
continue;
|
|
2160
|
+
}
|
|
1905
2161
|
if (!this.askedMembers.has(member))
|
|
1906
2162
|
continue;
|
|
1907
2163
|
const mark = `wake-no-said:${k}:${at}`;
|
|
1908
|
-
if (this.contDone.has(mark)
|
|
2164
|
+
if (this.contDone.has(mark)) {
|
|
2165
|
+
told.add(member);
|
|
2166
|
+
continue;
|
|
2167
|
+
}
|
|
2168
|
+
if (Date.now() - at > 24 * 3600_000)
|
|
1909
2169
|
continue;
|
|
1910
2170
|
this.markDone(mark);
|
|
2171
|
+
told.add(member);
|
|
1911
2172
|
await client.send({
|
|
1912
2173
|
act: "NOTE", goal: null, to: [member], after_seq: Math.max(client.readSeq, client.seq),
|
|
1913
2174
|
summary: `${this.cfg.member}'s machine will not start work for you while they are away`,
|
|
@@ -1916,9 +2177,130 @@ export class Worker {
|
|
|
1916
2177
|
}
|
|
1917
2178
|
}
|
|
1918
2179
|
}
|
|
2180
|
+
answeredHere(ref) {
|
|
2181
|
+
const k = `handled:${ref}`;
|
|
2182
|
+
return this.contDone.has(k) || this.continuedHas(k);
|
|
2183
|
+
}
|
|
2184
|
+
escalationAnswer(client, o, acts = client.mineSince(0)) {
|
|
2185
|
+
const signs = client.memberSigns(this.cfg.member);
|
|
2186
|
+
const as = (a, want) => {
|
|
2187
|
+
const t = client.trustOf(a).trust;
|
|
2188
|
+
if (t === want)
|
|
2189
|
+
return true;
|
|
2190
|
+
return !a.sig && !signs && t === "unsigned" && (want === "verified-run") === (a.role === "worker");
|
|
2191
|
+
};
|
|
2192
|
+
const byId = new Map(acts.map((a) => [a.id, a]));
|
|
2193
|
+
for (let i = acts.length - 1; i >= 0; i--) {
|
|
2194
|
+
const r = acts[i];
|
|
2195
|
+
if (r.act !== "ANSWER" || r.from !== this.cfg.member || !r.ref)
|
|
2196
|
+
continue;
|
|
2197
|
+
const e = byId.get(r.ref) ?? client.seenAct(r.ref);
|
|
2198
|
+
if (!e || e.act !== "ESCALATE" || e.from !== this.cfg.member)
|
|
2199
|
+
continue;
|
|
2200
|
+
if (e.ref ? e.ref !== o.ref : !o.goal || e.goal !== o.goal || e.ts < o.created_at)
|
|
2201
|
+
continue;
|
|
2202
|
+
if (!as(e, "verified-run") || !as(r, "verified"))
|
|
2203
|
+
continue;
|
|
2204
|
+
return { escalation: e, answer: r };
|
|
2205
|
+
}
|
|
2206
|
+
return null;
|
|
2207
|
+
}
|
|
2208
|
+
decidedLookedUp = new Set();
|
|
2209
|
+
async reopenDecided(client, o) {
|
|
2210
|
+
let d = this.escalationAnswer(client, o);
|
|
2211
|
+
if (!d && !this.decidedLookedUp.has(o.ref)) {
|
|
2212
|
+
this.decidedLookedUp.add(o.ref);
|
|
2213
|
+
if (this.decidedLookedUp.size > 500)
|
|
2214
|
+
this.decidedLookedUp.delete(this.decidedLookedUp.values().next().value);
|
|
2215
|
+
const acts = await this.recentActs();
|
|
2216
|
+
d = this.escalationAnswer(client, o, acts.filter((a) => a.from === this.cfg.member).sort((x, y) => x.seq - y.seq));
|
|
2217
|
+
}
|
|
2218
|
+
if (!d)
|
|
2219
|
+
return;
|
|
2220
|
+
const mark = `decided:${d.answer.id}`;
|
|
2221
|
+
if (this.contDone.has(mark) || this.continuedHas(mark))
|
|
2222
|
+
return;
|
|
2223
|
+
this.markDone(mark);
|
|
2224
|
+
this.decided.set(o.ref, d.answer.body);
|
|
2225
|
+
if (this.decided.size > 200)
|
|
2226
|
+
this.decided.delete(this.decided.keys().next().value);
|
|
2227
|
+
const handled = `handled:${o.ref}`;
|
|
2228
|
+
this.contDone.delete(handled);
|
|
2229
|
+
const homes = this.opts.home !== undefined ? [this.opts.home, undefined] : [undefined];
|
|
2230
|
+
for (const h of homes) {
|
|
2231
|
+
for (const k of this.contKeys())
|
|
2232
|
+
unmarkContinued(k, handled, h);
|
|
2233
|
+
try {
|
|
2234
|
+
unmarkAnswered(this.pinTeam, o.ref, h);
|
|
2235
|
+
}
|
|
2236
|
+
catch { }
|
|
2237
|
+
}
|
|
2238
|
+
this.skipLogged.delete(`handled:${o.id}`);
|
|
2239
|
+
this.opts.log(`> ${owedLabel(o.kind)} ${o.id}: its person answered the escalation (${d.answer.id}) — a run takes it again to pass that on`);
|
|
2240
|
+
}
|
|
2241
|
+
decided = new Map();
|
|
2242
|
+
async recentActs() {
|
|
2243
|
+
try {
|
|
2244
|
+
const res = await signedRead(this.cfg, "/history", {}, { timeoutMs: 5000 });
|
|
2245
|
+
if (!res.ok)
|
|
2246
|
+
return [];
|
|
2247
|
+
return (await res.json()).acts ?? [];
|
|
2248
|
+
}
|
|
2249
|
+
catch {
|
|
2250
|
+
return [];
|
|
2251
|
+
}
|
|
2252
|
+
}
|
|
1919
2253
|
markDone(id) {
|
|
1920
2254
|
this.contDone.add(id);
|
|
1921
2255
|
markContinued(this.pinTeam, id, this.opts.home);
|
|
2256
|
+
if (WORKER_MARK.test(id))
|
|
2257
|
+
this.keepOwn();
|
|
2258
|
+
}
|
|
2259
|
+
async verifyOwn(o) {
|
|
2260
|
+
return verifyOwnWrites(o, this.cfg.member, (t) => configForTeam(t, this.cfg));
|
|
2261
|
+
}
|
|
2262
|
+
keepOwn() { try {
|
|
2263
|
+
persistOwn(this.stateDir());
|
|
2264
|
+
}
|
|
2265
|
+
catch { } }
|
|
2266
|
+
pendingRestore = null;
|
|
2267
|
+
pendingSaid = "";
|
|
2268
|
+
settlePendingRestore() {
|
|
2269
|
+
const p = this.pendingRestore;
|
|
2270
|
+
if (!p)
|
|
2271
|
+
return false;
|
|
2272
|
+
let changes = [];
|
|
2273
|
+
const only = { ...p.snap, files: p.snap.files.filter((f) => p.rels.includes(f.rel)) };
|
|
2274
|
+
try {
|
|
2275
|
+
changes = (this.opts.restore ?? restoreHome)(only);
|
|
2276
|
+
}
|
|
2277
|
+
catch {
|
|
2278
|
+
changes = p.rels.map((rel) => ({ rel, what: "reported", keys: [], failed: true }));
|
|
2279
|
+
}
|
|
2280
|
+
const left = changes.filter((c) => c.failed).map((c) => c.rel);
|
|
2281
|
+
if (!left.length) {
|
|
2282
|
+
this.pendingRestore = null;
|
|
2283
|
+
this.pendingSaid = "";
|
|
2284
|
+
clearPersistedGuard(this.stateDir());
|
|
2285
|
+
this.opts.log(`* put back what a teammate's request had locked: ${p.rels.join(", ")} — unattended runs start again`);
|
|
2286
|
+
try {
|
|
2287
|
+
this.opts.notify?.({ kind: "worker", who: "Your worker", heading: "Settings put back — runs start again", context: this.cfg.space, body: `What a teammate's request changed in ${p.rels.join(", ")} is put back now. Unattended runs start again.`, ttl: 0, sound: false });
|
|
2288
|
+
}
|
|
2289
|
+
catch { }
|
|
2290
|
+
return false;
|
|
2291
|
+
}
|
|
2292
|
+
p.rels = left;
|
|
2293
|
+
const key = left.join(",");
|
|
2294
|
+
if (this.pendingSaid !== key) {
|
|
2295
|
+
this.pendingSaid = key;
|
|
2296
|
+
const what = `A teammate's request locked ${left.join(", ")} in ${this.stateDir()} while it ran, so Pantheon could not put back what it changed there. No unattended run starts until it can; your worker tries again every few seconds.`;
|
|
2297
|
+
this.opts.log(`! holding every unattended run: ${what}`);
|
|
2298
|
+
try {
|
|
2299
|
+
this.opts.notify?.({ kind: "worker", who: "Your worker", heading: "Unattended runs are on hold", context: this.cfg.space, body: `${what} To fix: ${brokenStateFix(left)}`, ttl: 0, sound: true });
|
|
2300
|
+
}
|
|
2301
|
+
catch { }
|
|
2302
|
+
}
|
|
2303
|
+
return true;
|
|
1922
2304
|
}
|
|
1923
2305
|
async bringMachineAlong() {
|
|
1924
2306
|
if (this.opts.selfUpgrade?.run)
|
|
@@ -2126,6 +2508,7 @@ export class Worker {
|
|
|
2126
2508
|
if (open) {
|
|
2127
2509
|
if (open[1].refs && !open[1].refs.includes(o.ref)) {
|
|
2128
2510
|
open[1].refs.push(o.ref);
|
|
2511
|
+
askedHere.wake.set(open[0], canon(open[1]));
|
|
2129
2512
|
savePolicy(pol, this.opts.home);
|
|
2130
2513
|
}
|
|
2131
2514
|
this.alertWakeAsk(member, what, open[0]);
|
|
@@ -2165,9 +2548,10 @@ export class Worker {
|
|
|
2165
2548
|
const me2 = client.members.find((m) => m.member === member);
|
|
2166
2549
|
const teamBlocked = me2?.admitted === true && !me2.revoked_by && me2.may_wake === false;
|
|
2167
2550
|
(p.wakeAsks ??= {})[sent.id] = { team: this.cfg.spaceId ?? this.cfg.space, member, at: Date.now(), refs: [o.ref], ...(teamBlocked ? { teamBlocked: true } : {}) };
|
|
2168
|
-
askedHere.
|
|
2551
|
+
askedHere.wake.set(sent.id, canon(p.wakeAsks[sent.id]));
|
|
2169
2552
|
this.askedMembers.add(member);
|
|
2170
2553
|
savePolicy(p, this.opts.home);
|
|
2554
|
+
this.keepOwn();
|
|
2171
2555
|
this.markDone(`wake-asked:${sent.id}`);
|
|
2172
2556
|
this.opts.log(`* asked you whether ${member} may start work on this machine (their ${owedLabel(o.kind)} ${o.id} waits for the answer)`);
|
|
2173
2557
|
this.alertWakeAsk(member, what, sent.id);
|
|
@@ -2312,6 +2696,9 @@ export class Worker {
|
|
|
2312
2696
|
this.contDone.add(id);
|
|
2313
2697
|
markContinued(this.pinTeam, id, this.opts.home);
|
|
2314
2698
|
}
|
|
2699
|
+
for (const x of storedIn(dir))
|
|
2700
|
+
if (x.team === this.pinTeam)
|
|
2701
|
+
markContinued(storedKey(this.pinTeam), x.ref, this.opts.home);
|
|
2315
2702
|
}
|
|
2316
2703
|
catch { }
|
|
2317
2704
|
const { token: _t, privateKey: _k, pendingAuth: _p, delegate: _d, ...stub } = this.cfg;
|
|
@@ -2377,7 +2764,7 @@ export class Worker {
|
|
|
2377
2764
|
report.observe("down");
|
|
2378
2765
|
client.onChange = () => { nudger.observe(obligationSignature(client.obligations, client.connected)); report.observe(clientState(client)); };
|
|
2379
2766
|
try {
|
|
2380
|
-
this.recoverGuard();
|
|
2767
|
+
await this.recoverGuard();
|
|
2381
2768
|
await client.connect();
|
|
2382
2769
|
this.client = client;
|
|
2383
2770
|
await (this.opts.alertsOnly ? this.sayKeyChanges(client) : this.opts.housekeeping ? this.keepHouse(client) : this.serve(client));
|
|
@@ -2453,7 +2840,7 @@ export class Worker {
|
|
|
2453
2840
|
for (const w of p.warnings)
|
|
2454
2841
|
this.opts.log(` ! ${p.name}: ${w}`);
|
|
2455
2842
|
}
|
|
2456
|
-
this.recoverInterruptedRun();
|
|
2843
|
+
await this.recoverInterruptedRun(client);
|
|
2457
2844
|
await this.settleLeftHeld(client);
|
|
2458
2845
|
this.reportStanding(true);
|
|
2459
2846
|
if (this.opts.continuations !== false)
|
|
@@ -2461,11 +2848,26 @@ export class Worker {
|
|
|
2461
2848
|
this.writeState("watching", "waiting for work");
|
|
2462
2849
|
while (!this.stopped) {
|
|
2463
2850
|
const arrived = await client.next(5000);
|
|
2851
|
+
if (this.deadClaims.size)
|
|
2852
|
+
await this.releaseDeadClaims(client);
|
|
2464
2853
|
{
|
|
2465
2854
|
const held = this.tickHold;
|
|
2466
2855
|
this.tickHold = null;
|
|
2467
2856
|
this.reportStanding(false, held);
|
|
2468
2857
|
}
|
|
2858
|
+
if (this.guardWaiting) {
|
|
2859
|
+
if (Date.now() >= this.guardRetryAt)
|
|
2860
|
+
await this.recoverGuard();
|
|
2861
|
+
if (this.guardWaiting) {
|
|
2862
|
+
this.writeState("held", "putting back after an interrupted run waits for the relay");
|
|
2863
|
+
continue;
|
|
2864
|
+
}
|
|
2865
|
+
}
|
|
2866
|
+
if (policyUnreadable(loadPolicy(this.opts.home))) {
|
|
2867
|
+
this.sayBrokenState(["worker-policy.json"]);
|
|
2868
|
+
this.writeState("held", "worker-policy.json cannot be read");
|
|
2869
|
+
continue;
|
|
2870
|
+
}
|
|
2469
2871
|
if (client.refused && !this.refusedSaid) {
|
|
2470
2872
|
this.refusedSaid = true;
|
|
2471
2873
|
if (client.removedFrom)
|
|
@@ -2490,6 +2892,10 @@ export class Worker {
|
|
|
2490
2892
|
this.tickHold = { reason: "busy_elsewhere", detail: "another team's run is in progress on this machine" };
|
|
2491
2893
|
continue;
|
|
2492
2894
|
}
|
|
2895
|
+
if (this.settlePendingRestore()) {
|
|
2896
|
+
this.writeState("held", `${this.pendingRestore.rels.join(", ")} could not be put back after a run`);
|
|
2897
|
+
continue;
|
|
2898
|
+
}
|
|
2493
2899
|
const broken = this.brokenStateFiles();
|
|
2494
2900
|
if (broken.length) {
|
|
2495
2901
|
this.sayBrokenState(broken);
|
|
@@ -2506,7 +2912,9 @@ export class Worker {
|
|
|
2506
2912
|
this.busyObservedSince ??= Date.now();
|
|
2507
2913
|
else
|
|
2508
2914
|
this.busyObservedSince = null;
|
|
2915
|
+
this.learnRequests(client.myWork());
|
|
2509
2916
|
this.retryParkedForCommands(client.myWork());
|
|
2917
|
+
await this.askAgainForParked(client);
|
|
2510
2918
|
this.retryParkedForWorkspaces();
|
|
2511
2919
|
if (this.heldRestorePending && Date.now() - this.heldRestoreAt >= (this.opts.sandboxRecheckMs ?? 5 * 60_000))
|
|
2512
2920
|
await this.restoreHeldContinuations(client);
|
|
@@ -2520,12 +2928,20 @@ export class Worker {
|
|
|
2520
2928
|
this.sayKeyChanges(client);
|
|
2521
2929
|
this.syncSignedOut();
|
|
2522
2930
|
await this.settleMoot(client);
|
|
2931
|
+
await this.settleClosedQuestions(client);
|
|
2932
|
+
await this.settleSafety(client);
|
|
2523
2933
|
const aged = client
|
|
2524
2934
|
.myWork()
|
|
2525
2935
|
.filter((o) => o.kind !== "HUMAN" && o.kind !== "DELIVER" && !o.claimed_by)
|
|
2526
2936
|
.filter((o) => Date.now() - o.created_at >= grace)
|
|
2527
2937
|
.filter((o) => { const t = this.outageHold.get(o.id); if (t === undefined)
|
|
2528
2938
|
return true; if (t > Date.now()) {
|
|
2939
|
+
if (this.allOutSignedOut()) {
|
|
2940
|
+
if (!this.tickHold)
|
|
2941
|
+
this.tickHold = { reason: "not_signed_in", detail: "its agent is signed out" };
|
|
2942
|
+
this.decline(o.ref, "your agent is signed out");
|
|
2943
|
+
return false;
|
|
2944
|
+
}
|
|
2529
2945
|
if (!this.tickHold || (this.tickHold.reason === "outage" && (this.tickHold.until ?? Infinity) > t))
|
|
2530
2946
|
this.tickHold = { reason: "outage", until: t, detail: "its agents are out for now" };
|
|
2531
2947
|
this.decline(o.ref, "your agents are out for now");
|
|
@@ -2551,7 +2967,9 @@ export class Worker {
|
|
|
2551
2967
|
for (let i = start; i < aged.length; i++) {
|
|
2552
2968
|
const o = aged[i];
|
|
2553
2969
|
this.loadContinued();
|
|
2554
|
-
if (this.
|
|
2970
|
+
if (this.answeredHere(o.ref))
|
|
2971
|
+
await this.reopenDecided(client, o);
|
|
2972
|
+
if (this.answeredHere(o.ref)) {
|
|
2555
2973
|
if (!this.skipLogged.has(`handled:${o.id}`)) {
|
|
2556
2974
|
this.skipLogged.add(`handled:${o.id}`);
|
|
2557
2975
|
this.opts.log(`- not taking ${owedLabel(o.kind)} ${o.id}: this machine already answered the request it points at (${o.ref})`);
|
|
@@ -2734,7 +3152,9 @@ export class Worker {
|
|
|
2734
3152
|
const slot = this.takeSlot();
|
|
2735
3153
|
if ("held" in slot) {
|
|
2736
3154
|
this.writeState("watching", `waiting: ${slot.held}`);
|
|
2737
|
-
this.tickHold =
|
|
3155
|
+
this.tickHold = slot.held === PERSON_ANSWERING
|
|
3156
|
+
? { reason: "recently_active", detail: PERSON_ANSWERING, until: Date.now() + PERSON_HOLD_TTL_MS }
|
|
3157
|
+
: { reason: "busy_elsewhere", detail: "another run is in progress on this machine" };
|
|
2738
3158
|
if (continuation)
|
|
2739
3159
|
this.requeueContinuation(continuation, 5_000);
|
|
2740
3160
|
continue;
|
|
@@ -2870,7 +3290,7 @@ export class Worker {
|
|
|
2870
3290
|
if (candidate)
|
|
2871
3291
|
this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer" : "nothing on this computer can run it while you're away");
|
|
2872
3292
|
if (candidate) {
|
|
2873
|
-
this.failedRuns.set(candidate.id, 2);
|
|
3293
|
+
this.failedRuns.set(this.oKey(candidate.id), 2);
|
|
2874
3294
|
this.skipLogged.add(`parked:${candidate.id}`);
|
|
2875
3295
|
if (job.diagnosis.cause === "repo_not_here")
|
|
2876
3296
|
this.parkedNotHere.add(candidate.id);
|
|
@@ -2885,15 +3305,18 @@ export class Worker {
|
|
|
2885
3305
|
const attempts = this.failuresOf(candidate.id) + 1;
|
|
2886
3306
|
const failures = job.diagnosis?.cause === "no_commands" || job.diagnosis?.cause === "no_sandbox_commands" ? 2 : attempts;
|
|
2887
3307
|
if (job.diagnosis?.cause === "no_sandbox_commands")
|
|
2888
|
-
this.
|
|
2889
|
-
if (this.
|
|
2890
|
-
this.
|
|
2891
|
-
this.failedRuns.set(candidate.id, failures);
|
|
2892
|
-
this.
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
3308
|
+
this.markFor("parked-ns", candidate.id);
|
|
3309
|
+
if (this.marked("unparked-ns", candidate.id))
|
|
3310
|
+
this.markFor("parked2-ns", candidate.id);
|
|
3311
|
+
this.failedRuns.set(this.oKey(candidate.id), failures);
|
|
3312
|
+
this.markFor(failures >= 2 ? "parked" : "failed1", candidate.id);
|
|
3313
|
+
const under = this.unparkedUnder.get(this.oKey(candidate.id));
|
|
3314
|
+
if (under)
|
|
3315
|
+
this.markDone(`parked2@${under}:${this.oKey(candidate.id)}`);
|
|
3316
|
+
else if (this.marked("unparked", candidate.id))
|
|
3317
|
+
this.markFor("parked2", candidate.id);
|
|
3318
|
+
if (job.diagnosis?.cause === "no_commands")
|
|
3319
|
+
this.markFor("parked-nc", candidate.id);
|
|
2897
3320
|
if (this.failedRuns.size > 200) {
|
|
2898
3321
|
this.failedRuns.delete(this.failedRuns.keys().next().value);
|
|
2899
3322
|
}
|
|
@@ -2905,6 +3328,22 @@ export class Worker {
|
|
|
2905
3328
|
}
|
|
2906
3329
|
finally {
|
|
2907
3330
|
this.running = false;
|
|
3331
|
+
if (this.stopped) {
|
|
3332
|
+
if (continuation) {
|
|
3333
|
+
try {
|
|
3334
|
+
await client.releaseClaim(`cont:${continuation.id}`);
|
|
3335
|
+
}
|
|
3336
|
+
catch { }
|
|
3337
|
+
}
|
|
3338
|
+
else if (candidate) {
|
|
3339
|
+
this.deadClaims.set(candidate.id, { session: null, until: Date.now() + LIMITS.claimTtlMs + 60_000 });
|
|
3340
|
+
for (let i = 0; i < 12 && this.deadClaims.has(candidate.id); i++) {
|
|
3341
|
+
await this.releaseDeadClaims(client);
|
|
3342
|
+
if (this.deadClaims.has(candidate.id))
|
|
3343
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
3344
|
+
}
|
|
3345
|
+
}
|
|
3346
|
+
}
|
|
2908
3347
|
this.endAttempt();
|
|
2909
3348
|
this.giveSlot(slot.token);
|
|
2910
3349
|
if (continuation)
|
|
@@ -2936,7 +3375,7 @@ export class Worker {
|
|
|
2936
3375
|
return { held: why };
|
|
2937
3376
|
}
|
|
2938
3377
|
if ("held" in lease)
|
|
2939
|
-
return { held: `a run for "${lease.held}" is in progress in another pantheon worker on this machine` };
|
|
3378
|
+
return { held: lease.held === PERSON_ANSWERING ? PERSON_ANSWERING : `a run for "${lease.held}" is in progress in another pantheon worker on this machine` };
|
|
2940
3379
|
fleetLock = this;
|
|
2941
3380
|
const dir = this.stateDir(), token = lease.token;
|
|
2942
3381
|
this.heartbeatAt = Date.now();
|
|
@@ -2966,7 +3405,7 @@ export class Worker {
|
|
|
2966
3405
|
liveRuns.add(pid);
|
|
2967
3406
|
installRunReaper();
|
|
2968
3407
|
this.attemptPid = pid;
|
|
2969
|
-
this.runRecord = { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, ...(workerIdentity() ? { worker_id: workerIdentity() } : {}), obligation: attempt.obligation, ...(attempt.ref ? { ref: attempt.ref } : {}), ...(attempt.spend ? { spend: attempt.spend } : {}), ...(attempt.limit ? { limit: attempt.limit } : {}), ...(attempt.sandbox ? { sandbox: attempt.sandbox } : {}) };
|
|
3408
|
+
this.runRecord = { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, ...(workerIdentity() ? { worker_id: workerIdentity() } : {}), obligation: attempt.obligation, ...(this.oKey(attempt.obligation) !== attempt.obligation ? { okey: this.oKey(attempt.obligation) } : {}), ...(attempt.ref ? { ref: attempt.ref } : {}), ...(attempt.spend ? { spend: attempt.spend } : {}), ...(attempt.limit ? { limit: attempt.limit } : {}), ...(attempt.sandbox ? { sandbox: attempt.sandbox } : {}) };
|
|
2970
3409
|
writeRunRecord(this.stateDir(), this.runRecord);
|
|
2971
3410
|
}
|
|
2972
3411
|
runRecord = null;
|
|
@@ -3005,8 +3444,20 @@ export class Worker {
|
|
|
3005
3444
|
this.attemptPid = null;
|
|
3006
3445
|
this.runRecord = null;
|
|
3007
3446
|
}
|
|
3008
|
-
|
|
3009
|
-
|
|
3447
|
+
guardWaiting = false;
|
|
3448
|
+
guardRetryAt = 0;
|
|
3449
|
+
async recoverGuard() {
|
|
3450
|
+
const g = loadPersistedGuard(this.stateDir());
|
|
3451
|
+
const v = g?.own && g.snapshot && !sameProcess(g.worker, g.workerId) ? await this.verifyOwn(g.own) : undefined;
|
|
3452
|
+
if (v === "unreachable") {
|
|
3453
|
+
if (!this.guardWaiting)
|
|
3454
|
+
this.opts.log(" guard a run was interrupted with its worker; putting Pantheon's files back waits until the relay can be asked — unattended runs wait meanwhile");
|
|
3455
|
+
this.guardWaiting = true;
|
|
3456
|
+
this.guardRetryAt = Date.now() + 30_000;
|
|
3457
|
+
return;
|
|
3458
|
+
}
|
|
3459
|
+
this.guardWaiting = false;
|
|
3460
|
+
const r = recoverInterruptedGuard(this.stateDir(), Date.now(), restoreHome, v);
|
|
3010
3461
|
if (r.outcome === "none" || r.outcome === "live")
|
|
3011
3462
|
return;
|
|
3012
3463
|
const tell = (heading, body) => {
|
|
@@ -3021,6 +3472,12 @@ export class Worker {
|
|
|
3021
3472
|
this.opts.log(` guard a snapshot from a run on ${new Date(r.at).toLocaleString()} was not restored — ${r.outcome === "stale" ? "too old to be that run's" : "you have changed your setup since"}; dropped`);
|
|
3022
3473
|
return;
|
|
3023
3474
|
}
|
|
3475
|
+
if (r.outcome === "pending") {
|
|
3476
|
+
for (const l of r.said)
|
|
3477
|
+
this.opts.log(`! the run interrupted with its worker: put back ${l}`);
|
|
3478
|
+
this.pendingRestore = { snap: r.snapshot, rels: r.rels };
|
|
3479
|
+
return;
|
|
3480
|
+
}
|
|
3024
3481
|
if (r.outcome === "failed") {
|
|
3025
3482
|
this.opts.log(`! could not put Pantheon's own files back after the interrupted run (try ${r.attempts} of ${GUARD_RESTORE_TRIES}): ${r.why}`);
|
|
3026
3483
|
if (r.dropped)
|
|
@@ -3035,7 +3492,7 @@ export class Worker {
|
|
|
3035
3492
|
this.opts.log(`! the run interrupted with its worker: put back ${l}`);
|
|
3036
3493
|
tell("Your worker stopped mid-run — settings put back", `${putBackSaid(r.said)} If you made that change yourself, make it again.`);
|
|
3037
3494
|
}
|
|
3038
|
-
recoverInterruptedRun() {
|
|
3495
|
+
async recoverInterruptedRun(client) {
|
|
3039
3496
|
const dir = this.stateDir(), f = join(dir, RUN_RECORD);
|
|
3040
3497
|
let r;
|
|
3041
3498
|
try {
|
|
@@ -3059,21 +3516,81 @@ export class Worker {
|
|
|
3059
3516
|
rmSync(f, { force: true });
|
|
3060
3517
|
}
|
|
3061
3518
|
catch { }
|
|
3519
|
+
const replied = r.obligation.startsWith("cont:") ? null : await this.runReplied(client, r);
|
|
3062
3520
|
if (typeof r.spend === "string") {
|
|
3521
|
+
const summary = replied ? "answered (its worker went before the run ended)" : "the worker went before the run ended";
|
|
3522
|
+
try {
|
|
3523
|
+
closeStartedRun(r.spend, summary, { at: r.at, goal: null, obligation: r.obligation, ok: !!replied, triggeredBy: "unknown", summary, space: this.spaceKey(), member: r.member }, this.opts.home, !!replied);
|
|
3524
|
+
}
|
|
3525
|
+
catch { }
|
|
3526
|
+
}
|
|
3527
|
+
if (r.obligation.startsWith("cont:")) {
|
|
3063
3528
|
try {
|
|
3064
|
-
|
|
3529
|
+
await client.releaseClaim(r.obligation);
|
|
3065
3530
|
}
|
|
3066
3531
|
catch { }
|
|
3067
3532
|
}
|
|
3533
|
+
else {
|
|
3534
|
+
const held = client.obligations.find((o) => o.id === r.obligation || (typeof r.okey === "string" && `r:${o.kind}:${o.ref}` === r.okey));
|
|
3535
|
+
if (held) {
|
|
3536
|
+
this.deadClaims.set(held.id, { session: null, until: Date.now() + LIMITS.claimTtlMs + 60_000 });
|
|
3537
|
+
await this.releaseDeadClaims(client);
|
|
3538
|
+
}
|
|
3539
|
+
}
|
|
3068
3540
|
if (r.obligation.startsWith("cont:")) {
|
|
3069
3541
|
this.opts.log(`! the last run (${r.obligation}) ended with its worker — counted, not run again`);
|
|
3070
3542
|
return;
|
|
3071
3543
|
}
|
|
3544
|
+
if (replied) {
|
|
3545
|
+
if (r.ref)
|
|
3546
|
+
this.markDone(`handled:${r.ref}`);
|
|
3547
|
+
this.opts.log(` the last run for ${r.obligation} sent its ${replied} before its worker went — counted as handled`);
|
|
3548
|
+
return;
|
|
3549
|
+
}
|
|
3550
|
+
if (typeof r.okey === "string" && r.okey.startsWith("r:"))
|
|
3551
|
+
this.obKeys.set(r.obligation, r.okey);
|
|
3072
3552
|
const failures = this.failuresOf(r.obligation) + 1;
|
|
3073
|
-
this.failedRuns.set(r.obligation, failures);
|
|
3074
|
-
this.
|
|
3553
|
+
this.failedRuns.set(this.oKey(r.obligation), failures);
|
|
3554
|
+
this.markFor(failures >= 2 ? "parked" : "failed1", r.obligation);
|
|
3075
3555
|
this.opts.log(`! the last run for ${r.obligation} ended with its worker (stopped, restarted or killed mid-run) — counted as a failed attempt${failures >= 2 ? "; parked: it is not run again" : ""}`);
|
|
3076
3556
|
}
|
|
3557
|
+
async runReplied(client, r) {
|
|
3558
|
+
const ref = r.ref, at = r.at;
|
|
3559
|
+
if (typeof ref !== "string" || typeof at !== "number")
|
|
3560
|
+
return null;
|
|
3561
|
+
if (client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" && (o.id === r.obligation || o.ref === ref)))
|
|
3562
|
+
return null;
|
|
3563
|
+
try {
|
|
3564
|
+
const res = await signedRead(this.cfg, "/history", {}, { timeoutMs: 5000 });
|
|
3565
|
+
if (!res.ok)
|
|
3566
|
+
return null;
|
|
3567
|
+
const j = await res.json();
|
|
3568
|
+
const a = (j.acts ?? []).find((x) => x.ref === ref && x.from === this.cfg.member && x.ts >= at && x.sig?.role === "run" && x.act !== "NOTE" && x.act !== "RETRACT");
|
|
3569
|
+
return a ? a.act : null;
|
|
3570
|
+
}
|
|
3571
|
+
catch {
|
|
3572
|
+
return null;
|
|
3573
|
+
}
|
|
3574
|
+
}
|
|
3575
|
+
deadClaims = new Map();
|
|
3576
|
+
async releaseDeadClaims(client) {
|
|
3577
|
+
for (const [id, d] of [...this.deadClaims]) {
|
|
3578
|
+
const by = client.obligations.find((o) => o.id === id)?.claimed_by;
|
|
3579
|
+
if (Date.now() > d.until || !by || by.role !== "worker" || by.session === client.session || (d.session !== null && by.session !== d.session)) {
|
|
3580
|
+
this.deadClaims.delete(id);
|
|
3581
|
+
continue;
|
|
3582
|
+
}
|
|
3583
|
+
d.session = by.session;
|
|
3584
|
+
if (client.roster.some((m) => m.sessions.some((s) => s.id === by.session)))
|
|
3585
|
+
continue;
|
|
3586
|
+
this.deadClaims.delete(id);
|
|
3587
|
+
try {
|
|
3588
|
+
if (await client.releaseClaim(id))
|
|
3589
|
+
this.opts.log(` let go of ${id}: the run that held it went with its worker — it is picked up again now, not in ${Math.round(LIMITS.claimTtlMs / 60_000)} minutes`);
|
|
3590
|
+
}
|
|
3591
|
+
catch { }
|
|
3592
|
+
}
|
|
3593
|
+
}
|
|
3077
3594
|
stateDir() {
|
|
3078
3595
|
return this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
|
|
3079
3596
|
}
|
|
@@ -3107,7 +3624,7 @@ export class Worker {
|
|
|
3107
3624
|
}
|
|
3108
3625
|
brokenStateFiles() {
|
|
3109
3626
|
repairRunLease(this.stateDir());
|
|
3110
|
-
return brokenStateFiles(this.stateDir());
|
|
3627
|
+
return brokenStateFiles(this.stateDir(), GUARDED_FILES);
|
|
3111
3628
|
}
|
|
3112
3629
|
hooksInstalledHere() {
|
|
3113
3630
|
try {
|
|
@@ -3128,7 +3645,8 @@ export class Worker {
|
|
|
3128
3645
|
: `(the triggering act ${printable(o.ref)} is not in this session's buffer; the relay's one-line summary of it follows)\n${o.body}`;
|
|
3129
3646
|
const act = (base ?? undefined);
|
|
3130
3647
|
const needs = workNeedsCommands(o.kind, trigger, o.kind === "REVISE" && act?.ref ? client.seenAct(act.ref)?.act : act?.act);
|
|
3131
|
-
|
|
3648
|
+
const decided = this.decided.get(o.ref) ?? this.escalationAnswer(client, o)?.answer.body;
|
|
3649
|
+
return this.runJob(client, o, persona, policy, (branch, canPush) => buildBrief(o, trigger, branch, persona, canPush, decided), true, needs, trigger, act);
|
|
3132
3650
|
}
|
|
3133
3651
|
async handleContinuation(client, act, persona, policy) {
|
|
3134
3652
|
this.markDone(act.id);
|
|
@@ -3543,6 +4061,7 @@ export class Worker {
|
|
|
3543
4061
|
}
|
|
3544
4062
|
let detail;
|
|
3545
4063
|
let transcript = "";
|
|
4064
|
+
resetOwn();
|
|
3546
4065
|
const homeBefore = snapshotHome(this.stateDir());
|
|
3547
4066
|
const guardHome = this.stateDir();
|
|
3548
4067
|
try {
|
|
@@ -3658,7 +4177,7 @@ export class Worker {
|
|
|
3658
4177
|
this.cleanupStep("checking Pantheon's own files", () => {
|
|
3659
4178
|
const ok = this.restoreHomeAfterRun(homeBefore, o);
|
|
3660
4179
|
pendingGuards.delete(guardHome);
|
|
3661
|
-
if (ok)
|
|
4180
|
+
if (ok && !this.pendingRestore)
|
|
3662
4181
|
clearPersistedGuard(guardHome);
|
|
3663
4182
|
this.markRunRestored();
|
|
3664
4183
|
});
|
|
@@ -3748,9 +4267,10 @@ export class Worker {
|
|
|
3748
4267
|
const sentAnywhere = client.mineSince(startedAt).some((a) => !runKeys || !a.sig || a.sig.role === "run");
|
|
3749
4268
|
const outage = !ok && !exited && !sentAnywhere && choice && !/\(stopped after \d+s\)/.test(detail.split("\n")[0] ?? "") ? detectOutage(choice.backend.id, detail) : null;
|
|
3750
4269
|
const refusedLog = snap ? join(scratch, `${basename(worktree)}.${choice?.backend.id}-refused.run.log`) : join(worktree, `.pantheon-run.${choice?.backend.id}-refused.log`);
|
|
4270
|
+
const hintLed = outage?.why === SIGNIN_REFUSED && !this.opts.agentCmd && signedOutChecking(choice.backend.id, { pantheonHome: this.stateDir() })?.lifted === "hint";
|
|
3751
4271
|
if (outage) {
|
|
3752
4272
|
this.outages.set(choice.backend.id, outage);
|
|
3753
|
-
if (outage.why === SIGNIN_REFUSED && !this.opts.agentCmd && markSignedOut(choice.backend.id, detail, { pantheonHome: this.stateDir() })) {
|
|
4273
|
+
if (outage.why === SIGNIN_REFUSED && !this.opts.agentCmd && markSignedOut(choice.backend.id, detail, { pantheonHome: this.stateDir(), env: process.env })) {
|
|
3754
4274
|
this.syncSignedOut();
|
|
3755
4275
|
this.readinessAt = 0;
|
|
3756
4276
|
}
|
|
@@ -3758,22 +4278,26 @@ export class Worker {
|
|
|
3758
4278
|
if (Date.now() - startedAt < ONCE_GIVE_BACK_MS)
|
|
3759
4279
|
this.restoreOnces();
|
|
3760
4280
|
}
|
|
3761
|
-
if (outage && (this.outageRetries.get(o.id) ?? 0) < 3) {
|
|
3762
|
-
|
|
4281
|
+
if (outage && (hintLed || (this.outageRetries.get(o.id) ?? 0) < 3)) {
|
|
4282
|
+
if (!hintLed)
|
|
4283
|
+
this.outageRetries.set(o.id, (this.outageRetries.get(o.id) ?? 0) + 1);
|
|
3763
4284
|
if (this.outageRetries.size > 200)
|
|
3764
4285
|
this.outageRetries.delete(this.outageRetries.keys().next().value);
|
|
3765
|
-
this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over until about ${whenAgain(outage.until)}; the request goes to the next agent here (transcript: ${refusedLog})`);
|
|
4286
|
+
this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over ${outage.why === SIGNIN_REFUSED ? "until it is signed in again" : `until about ${whenAgain(outage.until)}`}; the request goes to the next agent here (transcript: ${refusedLog})`);
|
|
3766
4287
|
return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
|
|
3767
4288
|
}
|
|
3768
4289
|
const quiet = ok && !requireReply && sent.length === 0;
|
|
3769
4290
|
const diagnosis = ok ? undefined : outage ? this.outageDiagnosis(choice.backend.label, outage, [], "it was handed to another agent three times already") : diagnoseRun({ exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands,
|
|
4291
|
+
...(agentId === "claude-code" && !this.opts.agentCmd && claudeTokenForRun(this.stateDir())
|
|
4292
|
+
? { env: { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: undefined }, savedToken: true } : { env: process.env }),
|
|
4293
|
+
...(agentId === "claude-code" ? { claudeSettings: claudeSettingsAuth({ env: process.env }) } : {}),
|
|
3770
4294
|
...(/\(stopped after \d+s\)/.test(detail.split("\n")[0] ?? "") ? { timedOut: { minutes: runLimit.minutes, from: runLimit.from } } : {}),
|
|
3771
4295
|
...(bashDroppedForSandbox ? { commandsNeedSandbox: byCli ? "agent-cmd" : "claude" } : {}), ...(agentId ? { agent: agentId } : {}), ...(place.kind === "folder" ? { folder: true } : {}),
|
|
3772
4296
|
...(agentId === "claude-code" && allowBash ? { allowedRules: bashRules } : {}), ...(baseRef ? { baseRef } : {}) }, this.opts.probe ?? gitProbe);
|
|
3773
4297
|
if (ok && agentId && !this.opts.agentCmd)
|
|
3774
4298
|
clearSignedOut(agentId, { pantheonHome: this.stateDir() });
|
|
3775
4299
|
if (!ok && diagnosis?.cause === "not_signed_in" && agentId && !this.opts.agentCmd
|
|
3776
|
-
&& markSignedOut(agentId, detail, { pantheonHome: this.stateDir() })) {
|
|
4300
|
+
&& markSignedOut(agentId, detail, { pantheonHome: this.stateDir(), env: process.env })) {
|
|
3777
4301
|
this.opts.log(`! ${choice?.backend.label ?? agentId} is signed out on this machine: requests wait until it is signed in again${agentId === "claude-code" ? " (pantheon worker token keeps Claude Code signed in for a year)" : ""}`);
|
|
3778
4302
|
this.syncSignedOut();
|
|
3779
4303
|
this.readinessAt = 0;
|
|
@@ -3817,7 +4341,7 @@ export class Worker {
|
|
|
3817
4341
|
restoreHomeAfterRun(before, o) {
|
|
3818
4342
|
let changes = [];
|
|
3819
4343
|
try {
|
|
3820
|
-
changes = restoreHome(before);
|
|
4344
|
+
changes = (this.opts.restore ?? restoreHome)(before);
|
|
3821
4345
|
}
|
|
3822
4346
|
catch (e) {
|
|
3823
4347
|
this.opts.log(`! could not check Pantheon's own files after the run: ${e.message}`);
|
|
@@ -3825,6 +4349,14 @@ export class Worker {
|
|
|
3825
4349
|
}
|
|
3826
4350
|
if (!changes.length)
|
|
3827
4351
|
return true;
|
|
4352
|
+
const failed = changes.filter((c) => c.failed).map((c) => c.rel);
|
|
4353
|
+
if (failed.length) {
|
|
4354
|
+
this.pendingRestore = { snap: { ...before, files: before.files.filter((f) => failed.includes(f.rel)) }, rels: failed };
|
|
4355
|
+
try {
|
|
4356
|
+
persistPending(this.stateDir(), failed);
|
|
4357
|
+
}
|
|
4358
|
+
catch { }
|
|
4359
|
+
}
|
|
3828
4360
|
for (const c of changes)
|
|
3829
4361
|
this.opts.log(`${c.what === "reported" || c.quiet ? " " : "!"} ${o.id}: ${describeChange(c)}`);
|
|
3830
4362
|
const taken = changes.filter((c) => c.what !== "reported" && !c.quiet);
|
|
@@ -4197,9 +4729,11 @@ export class Worker {
|
|
|
4197
4729
|
}
|
|
4198
4730
|
if (!owed)
|
|
4199
4731
|
return;
|
|
4200
|
-
|
|
4732
|
+
this.learnRequests([o]);
|
|
4733
|
+
const key = `refused:${this.oKey(o.id)}`;
|
|
4201
4734
|
const space = this.spaceKey();
|
|
4202
|
-
|
|
4735
|
+
const told = loadContinued(space, this.opts.home);
|
|
4736
|
+
if (told.has(key) || told.has(`refused:${o.id}`))
|
|
4203
4737
|
return;
|
|
4204
4738
|
const peer = client.recentFrom(o.ref);
|
|
4205
4739
|
try {
|
|
@@ -4226,7 +4760,7 @@ export class Worker {
|
|
|
4226
4760
|
if (!client.connected)
|
|
4227
4761
|
return;
|
|
4228
4762
|
const policy = loadPolicy(this.opts.home);
|
|
4229
|
-
const teams = [this.cfg.spaceId
|
|
4763
|
+
const teams = this.cfg.spaceId ? [this.cfg.spaceId] : [this.cfg.space];
|
|
4230
4764
|
const reports = [
|
|
4231
4765
|
...Object.entries(policy.stuckReports ?? {}).filter(([, r]) => teams.includes(r.team))
|
|
4232
4766
|
.map(([act, r]) => [act, [r.ref], "stuck"]),
|
|
@@ -4240,31 +4774,43 @@ export class Worker {
|
|
|
4240
4774
|
return;
|
|
4241
4775
|
const now = Date.now();
|
|
4242
4776
|
const done = [];
|
|
4777
|
+
const waitingHere = (ref) => client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" && o.ref === ref);
|
|
4778
|
+
const stillNeeds = (cap, except) => Object.entries(policy.stuckReports ?? {}).some(([a, r]) => a !== except && r.cap === cap && (teams.includes(r.team) ? waitingHere(r.ref) : true)) ||
|
|
4779
|
+
(cap === "run_commands" && client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" &&
|
|
4780
|
+
!(policy.stuckReports?.[except]?.ref === o.ref) && this.marked("parked-nc", o.id) && this.failuresOf(o.id) >= 2));
|
|
4243
4781
|
for (const [act, refs, kind] of reports) {
|
|
4244
4782
|
const member = kind === "wake" ? policy.wakeAsks?.[act]?.member : undefined;
|
|
4245
4783
|
const owed = client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" &&
|
|
4246
4784
|
(refs.includes(o.ref) || (!!member && (client.seenAct(o.ref)?.from ?? /^([a-z0-9._-]+)\s/i.exec(o.body)?.[1]) === member)));
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4785
|
+
const asksFor = kind === "stuck" ? Object.entries(policy.asks).find(([, a]) => a?.act === act)?.[0] : undefined;
|
|
4786
|
+
const settled = member ? wakeSettled(policy, [this.cfg.spaceId, this.cfg.space], member) : null;
|
|
4787
|
+
if (!settled) {
|
|
4788
|
+
if (owed || (asksFor && stillNeeds(asksFor, act))) {
|
|
4789
|
+
this.mootSince.delete(act);
|
|
4790
|
+
continue;
|
|
4791
|
+
}
|
|
4792
|
+
const since = this.mootSince.get(act);
|
|
4793
|
+
if (since === undefined) {
|
|
4794
|
+
this.mootSince.set(act, now);
|
|
4795
|
+
continue;
|
|
4796
|
+
}
|
|
4797
|
+
if (now - since < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
|
|
4798
|
+
continue;
|
|
4255
4799
|
}
|
|
4256
|
-
if (now - since < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
|
|
4257
|
-
continue;
|
|
4258
4800
|
if ((this.mootRetryAt.get(act) ?? 0) > now)
|
|
4259
4801
|
continue;
|
|
4260
4802
|
const open = client.obligations.some((o) => o.kind === "HUMAN" && o.ref === act);
|
|
4261
4803
|
if (open) {
|
|
4262
4804
|
try {
|
|
4263
4805
|
await client.send({ act: "RETRACT", goal: null, ref: act, after_seq: Math.max(client.readSeq, client.seq),
|
|
4264
|
-
body:
|
|
4265
|
-
?
|
|
4266
|
-
:
|
|
4267
|
-
|
|
4806
|
+
body: settled
|
|
4807
|
+
? `No longer needed: settled on this computer — ${settled === "yes" ? `${member} may start work here (pantheon wake allow ${member})` : `no to unattended runs from ${member} here (pantheon wake deny ${member})`}.`
|
|
4808
|
+
: kind === "wake"
|
|
4809
|
+
? "No longer needed: the request this was about isn't waiting here any more (it was answered, withdrawn or closed). Nothing was changed on this computer — you'll be asked again if they need it."
|
|
4810
|
+
: "No longer needed: the request this was about isn't waiting here any more (it was answered, withdrawn or closed). Nothing was changed on this computer." });
|
|
4811
|
+
this.opts.log(settled
|
|
4812
|
+
? `* withdrew ${act}: you settled it here (pantheon wake ${settled === "yes" ? "allow" : "deny"} ${member})`
|
|
4813
|
+
: `* withdrew ${act}: the request${refs.length > 1 ? "s" : ""} it was about (${refs.join(", ")}) ${refs.length > 1 ? "are" : "is"} finished`);
|
|
4268
4814
|
}
|
|
4269
4815
|
catch (e) {
|
|
4270
4816
|
this.mootRetryAt.set(act, now + 5 * 60_000);
|
|
@@ -4280,6 +4826,7 @@ export class Worker {
|
|
|
4280
4826
|
return;
|
|
4281
4827
|
const fresh = loadPolicy(this.opts.home);
|
|
4282
4828
|
for (const [act, kind] of done) {
|
|
4829
|
+
withdrawnHere.add(act);
|
|
4283
4830
|
if (kind === "wake")
|
|
4284
4831
|
delete fresh.wakeAsks?.[act];
|
|
4285
4832
|
else {
|
|
@@ -4291,11 +4838,23 @@ export class Worker {
|
|
|
4291
4838
|
}
|
|
4292
4839
|
savePolicy(fresh, this.opts.home);
|
|
4293
4840
|
}
|
|
4294
|
-
|
|
4295
|
-
|
|
4841
|
+
safetyDue = new Map();
|
|
4842
|
+
safetyRaised = new Map();
|
|
4843
|
+
async reportStuck(client, o, d, attempts, again = 0) {
|
|
4844
|
+
this.learnRequests([o]);
|
|
4845
|
+
const safety = SAFETY_CAUSES.has(d.cause);
|
|
4846
|
+
const key = safety ? `stuck-safety:${this.oKey(o.id)}:${d.cause}` : `stuck:${this.oKey(o.id)}`;
|
|
4296
4847
|
const space = this.spaceKey();
|
|
4297
|
-
|
|
4848
|
+
const said = loadContinued(space, this.opts.home);
|
|
4849
|
+
if (!again && (said.has(key) || (!safety && said.has(`stuck:${o.id}`))))
|
|
4298
4850
|
return;
|
|
4851
|
+
if (safety && !again) {
|
|
4852
|
+
this.safetyDue.set(key, { o, d });
|
|
4853
|
+
try {
|
|
4854
|
+
markContinued(space, `stuck-safety-due:${this.oKey(o.id)}:${d.cause}`, this.opts.home);
|
|
4855
|
+
}
|
|
4856
|
+
catch { }
|
|
4857
|
+
}
|
|
4299
4858
|
const policy = loadPolicy(this.opts.home);
|
|
4300
4859
|
const asking = !!d.capability && mayAsk(policy, d.capability);
|
|
4301
4860
|
const what = clip(plainRequest(o.body, this.cfg.member), 160);
|
|
@@ -4309,27 +4868,187 @@ export class Worker {
|
|
|
4309
4868
|
...(d.dir && d.lines?.length ? [``, `On this computer, in ${d.dir}:`, ...d.lines.map((l) => ` ${l}`)] : []),
|
|
4310
4869
|
``,
|
|
4311
4870
|
`The request is still open, and the teammate who asked hasn't been told. It won't be retried on its own.`,
|
|
4871
|
+
...(again ? [``, safety ? `Raised again: the earlier warning about this was cleared without your answer.`
|
|
4872
|
+
: `Asked again: the "just this once" went to another request, and this one still needs commands.`] : []),
|
|
4312
4873
|
];
|
|
4313
4874
|
if (asking) {
|
|
4314
4875
|
lines.push(``, `Also: may ${this.cfg.member}'s agent run commands on this computer (tests, commit, push)? Answer "allow always", "allow once", or "deny" — in the menu bar, or with pantheon approvals.`, `Today it may only edit files, which is why it couldn't commit or push. Allowing it means: ${COMMAND_GRANT_SCOPE}`, `Answer on this computer: an answer from anywhere else settles this but grants nothing.`);
|
|
4315
4876
|
}
|
|
4316
4877
|
try {
|
|
4317
|
-
const
|
|
4318
|
-
|
|
4878
|
+
const fit = (t) => t.length <= LIMITS.bodyChars - 100 ? t : `${t.slice(0, LIMITS.bodyChars - 300)}\n\n(cut here: too long for one message — the rest is on this computer)`;
|
|
4879
|
+
const frame = (goal, short = false, ref = o.ref || null) => ({
|
|
4880
|
+
act: "ESCALATE", goal, ref, after_seq: Math.max(client.readSeq, client.seq),
|
|
4319
4881
|
summary: clip(this.ledgerPaths(`${this.cfg.member}'s agent is stuck: ${d.says}`), 190),
|
|
4320
|
-
body: this.ledgerPaths(lines.join("\n")),
|
|
4882
|
+
body: fit(this.ledgerPaths(short ? [lines[0], ``, `Why: ${clip(d.says, 600)}`, `What to do: ${clip(d.fix, 600)}`].join("\n") : lines.join("\n"))),
|
|
4883
|
+
...(safety ? { meta: { safety_stop: d.cause } } : {}),
|
|
4884
|
+
});
|
|
4885
|
+
const passing = (m) => !/^[A-Z_]+:/.test(m) || /^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(m);
|
|
4886
|
+
const sent = await client.send(frame(o.goal)).catch((e) => {
|
|
4887
|
+
if (!safety || passing(e.message))
|
|
4888
|
+
return Promise.reject(e);
|
|
4889
|
+
return client.send(frame(null, false, null)).catch((e2) => passing(e2.message) ? Promise.reject(e2) : client.send(frame(null, true, null)));
|
|
4321
4890
|
});
|
|
4322
4891
|
markContinued(space, key, this.opts.home);
|
|
4892
|
+
if (safety) {
|
|
4893
|
+
this.safetyDue.delete(key);
|
|
4894
|
+
this.safetyRaised.set(sent.id, { o, d, seq: sent.seq, again });
|
|
4895
|
+
}
|
|
4323
4896
|
const fresh = loadPolicy(this.opts.home);
|
|
4324
|
-
if (asking && d.capability)
|
|
4897
|
+
if (asking && d.capability) {
|
|
4325
4898
|
recordAsk(fresh, d.capability, sent.id);
|
|
4326
|
-
|
|
4327
|
-
|
|
4899
|
+
askedHere.cap.set(d.capability, canon(fresh.asks[d.capability]));
|
|
4900
|
+
}
|
|
4901
|
+
if (o.kind !== "CONTINUE" && !safety) {
|
|
4902
|
+
const rec = { ref: o.ref, team: this.cfg.spaceId ?? this.cfg.space, at: Date.now(), ...(d.capability ? { cap: d.capability } : {}) };
|
|
4903
|
+
(fresh.stuckReports ??= {})[sent.id] = rec;
|
|
4904
|
+
askedHere.stuck.set(sent.id, canon(rec));
|
|
4905
|
+
}
|
|
4906
|
+
this.keepOwn();
|
|
4328
4907
|
savePolicy(fresh, this.opts.home);
|
|
4329
4908
|
this.opts.log(`* escalated ${o.id}: ${d.cause}${asking ? ` (and asked for ${d.capability})` : ""}`);
|
|
4330
4909
|
}
|
|
4331
4910
|
catch (e) {
|
|
4332
|
-
|
|
4911
|
+
const msg = e.message;
|
|
4912
|
+
const lasting = safety && /^[A-Z_]+:/.test(msg) && !/^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(msg);
|
|
4913
|
+
if (lasting) {
|
|
4914
|
+
markContinued(space, key, this.opts.home);
|
|
4915
|
+
this.safetyDue.delete(key);
|
|
4916
|
+
try {
|
|
4917
|
+
const note = { says: clip(d.says, 400), fix: clip(d.fix, 500), at: Date.now(), team: this.cfg.spaceId ?? this.cfg.space };
|
|
4918
|
+
const fresh = loadPolicy(this.opts.home);
|
|
4919
|
+
(fresh.safetyNotes ??= {})[key] = note;
|
|
4920
|
+
askedHere.safety.set(key, canon(note));
|
|
4921
|
+
savePolicy(fresh, this.opts.home);
|
|
4922
|
+
this.keepOwn();
|
|
4923
|
+
}
|
|
4924
|
+
catch { }
|
|
4925
|
+
try {
|
|
4926
|
+
this.opts.notify?.({ kind: "worker", who: "Your worker", heading: "A safety warning could not reach your board", context: this.cfg.space,
|
|
4927
|
+
body: `${clip(d.says, 300)}\nWhat to do: ${clip(d.fix, 400)}`, ttl: 0, sound: true });
|
|
4928
|
+
}
|
|
4929
|
+
catch { }
|
|
4930
|
+
}
|
|
4931
|
+
this.opts.log(`! could not report the stuck obligation on the ledger: ${msg}${safety ? lasting ? " — a safety warning: shown on this computer instead" : " — a safety warning: tried again until it is said" : ""}`);
|
|
4932
|
+
}
|
|
4933
|
+
}
|
|
4934
|
+
async settleSafety(client) {
|
|
4935
|
+
if (!client.connected)
|
|
4936
|
+
return;
|
|
4937
|
+
for (const [key, { o, d }] of [...this.safetyDue]) {
|
|
4938
|
+
if (loadContinued(this.spaceKey(), this.opts.home).has(key)) {
|
|
4939
|
+
this.safetyDue.delete(key);
|
|
4940
|
+
continue;
|
|
4941
|
+
}
|
|
4942
|
+
const now = client.myWork().find((x) => this.oKey(x.id) === this.oKey(o.id)) ?? o;
|
|
4943
|
+
await this.reportStuck(client, now, d, 0);
|
|
4944
|
+
}
|
|
4945
|
+
if (!this.safetyDiskRead) {
|
|
4946
|
+
this.safetyDiskRead = true;
|
|
4947
|
+
const marks = loadContinued(this.spaceKey(), this.opts.home);
|
|
4948
|
+
for (const m of marks) {
|
|
4949
|
+
if (!m.startsWith("stuck-safety-due:"))
|
|
4950
|
+
continue;
|
|
4951
|
+
const rest = m.slice("stuck-safety-due:".length);
|
|
4952
|
+
const cause = rest.slice(rest.lastIndexOf(":") + 1), okey = rest.slice(0, rest.lastIndexOf(":"));
|
|
4953
|
+
if (!SAFETY_CAUSES.has(cause) || marks.has(`stuck-safety:${okey}:${cause}`))
|
|
4954
|
+
continue;
|
|
4955
|
+
const o = client.myWork().find((x) => this.oKey(x.id) === okey)
|
|
4956
|
+
?? { id: `gone:${okey}`, kind: "ANSWER", owner: this.cfg.member, ref: "", goal: null, body: "a request that is no longer waiting", created_at: 0 };
|
|
4957
|
+
if (o.id.startsWith("gone:"))
|
|
4958
|
+
this.obKeys.set(o.id, okey);
|
|
4959
|
+
this.safetyDue.set(`stuck-safety:${okey}:${cause}`, { o, d: safetyAgainDiagnosis(cause, this.cfg.member) });
|
|
4960
|
+
}
|
|
4961
|
+
}
|
|
4962
|
+
for (const [act, r] of [...this.safetyRaised]) {
|
|
4963
|
+
if (client.obligations.some((x) => x.kind === "HUMAN" && x.ref === act)) {
|
|
4964
|
+
r.goneAt = undefined;
|
|
4965
|
+
continue;
|
|
4966
|
+
}
|
|
4967
|
+
const now = Date.now();
|
|
4968
|
+
if (r.goneAt === undefined) {
|
|
4969
|
+
r.goneAt = now;
|
|
4970
|
+
continue;
|
|
4971
|
+
}
|
|
4972
|
+
if (now - r.goneAt < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
|
|
4973
|
+
continue;
|
|
4974
|
+
if ((r.nextAt ?? 0) > now)
|
|
4975
|
+
continue;
|
|
4976
|
+
const settled = await this.personSettled(act, r.seq);
|
|
4977
|
+
if (settled === null) {
|
|
4978
|
+
r.wait = Math.min((r.wait ?? 30_000) * 2, 30 * 60_000);
|
|
4979
|
+
r.nextAt = now + r.wait;
|
|
4980
|
+
continue;
|
|
4981
|
+
}
|
|
4982
|
+
this.safetyRaised.delete(act);
|
|
4983
|
+
if (settled || r.again >= SAFETY_RERAISE_MAX)
|
|
4984
|
+
continue;
|
|
4985
|
+
const o = client.myWork().find((x) => this.oKey(x.id) === this.oKey(r.o.id)) ?? r.o;
|
|
4986
|
+
this.opts.log(`! the safety warning ${act} left the board without your answer — raising it again`);
|
|
4987
|
+
await this.reportStuck(client, o, r.d, 0, r.again + 1);
|
|
4988
|
+
}
|
|
4989
|
+
}
|
|
4990
|
+
safetyDiskRead = false;
|
|
4991
|
+
closedSince = new Map();
|
|
4992
|
+
async settleClosedQuestions(client) {
|
|
4993
|
+
if (!client.connected)
|
|
4994
|
+
return;
|
|
4995
|
+
const p = loadPolicy(this.opts.home);
|
|
4996
|
+
const teams = this.cfg.spaceId ? [this.cfg.spaceId] : [this.cfg.space];
|
|
4997
|
+
const mine = [
|
|
4998
|
+
...Object.entries(p.asks).filter(([, a]) => !!a?.act && teams.includes(p.stuckReports?.[a.act]?.team ?? ""))
|
|
4999
|
+
.map(([cap, a]) => [a.act, "cap", cap]),
|
|
5000
|
+
...Object.entries(p.wakeAsks ?? {}).filter(([, a]) => teams.includes(a.team)).map(([id]) => [id, "wake", id]),
|
|
5001
|
+
];
|
|
5002
|
+
const now = Date.now();
|
|
5003
|
+
const gone = [];
|
|
5004
|
+
for (const [act, kind, key] of mine) {
|
|
5005
|
+
if (client.obligations.some((o) => o.kind === "HUMAN" && o.ref === act)) {
|
|
5006
|
+
this.closedSince.delete(act);
|
|
5007
|
+
continue;
|
|
5008
|
+
}
|
|
5009
|
+
const since = this.closedSince.get(act);
|
|
5010
|
+
if (since === undefined) {
|
|
5011
|
+
this.closedSince.set(act, now);
|
|
5012
|
+
continue;
|
|
5013
|
+
}
|
|
5014
|
+
if (now - since < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
|
|
5015
|
+
continue;
|
|
5016
|
+
gone.push([act, kind, key]);
|
|
5017
|
+
}
|
|
5018
|
+
if (!gone.length)
|
|
5019
|
+
return;
|
|
5020
|
+
const fresh = loadPolicy(this.opts.home);
|
|
5021
|
+
for (const [act, kind, key] of gone) {
|
|
5022
|
+
this.closedSince.delete(act);
|
|
5023
|
+
withdrawnHere.add(act);
|
|
5024
|
+
if (kind === "wake")
|
|
5025
|
+
delete fresh.wakeAsks?.[act];
|
|
5026
|
+
else if (fresh.asks[key]?.act === act)
|
|
5027
|
+
delete fresh.asks[key];
|
|
5028
|
+
this.opts.log(`* ${act}: settled on the record — no longer waiting for an answer here`);
|
|
5029
|
+
}
|
|
5030
|
+
savePolicy(fresh, this.opts.home);
|
|
5031
|
+
}
|
|
5032
|
+
async personSettled(act, seq) {
|
|
5033
|
+
let since = Math.max(0, seq - 1);
|
|
5034
|
+
try {
|
|
5035
|
+
for (let page = 0; page < 40; page++) {
|
|
5036
|
+
const res = await signedRead(this.cfg, "/history", { since: String(since) }, { timeoutMs: 5000 });
|
|
5037
|
+
if (!res.ok)
|
|
5038
|
+
return null;
|
|
5039
|
+
const j = await res.json();
|
|
5040
|
+
const acts = j.acts ?? [];
|
|
5041
|
+
if (acts.some((a) => a.ref === act && (a.act === "ANSWER" || a.act === "RETRACT") && a.sig?.role !== "run" && a.from === this.cfg.member))
|
|
5042
|
+
return true;
|
|
5043
|
+
const last = acts.length ? acts[acts.length - 1].seq : since;
|
|
5044
|
+
if (!acts.length || last <= since || (typeof j.head === "number" && last >= j.head))
|
|
5045
|
+
return false;
|
|
5046
|
+
since = last;
|
|
5047
|
+
}
|
|
5048
|
+
return null;
|
|
5049
|
+
}
|
|
5050
|
+
catch {
|
|
5051
|
+
return null;
|
|
4333
5052
|
}
|
|
4334
5053
|
}
|
|
4335
5054
|
ledgerPaths(text) {
|
|
@@ -4499,8 +5218,12 @@ export class Worker {
|
|
|
4499
5218
|
}
|
|
4500
5219
|
if (denies.length) {
|
|
4501
5220
|
const fresh = loadPolicy(home);
|
|
4502
|
-
for (const [act, body] of denies)
|
|
4503
|
-
applyAnswer(fresh, act, body);
|
|
5221
|
+
for (const [act, body] of denies) {
|
|
5222
|
+
const r = applyAnswer(fresh, act, body);
|
|
5223
|
+
withdrawnHere.add(act);
|
|
5224
|
+
if (r)
|
|
5225
|
+
noesHere.add(`denied:${r.cap}`);
|
|
5226
|
+
}
|
|
4504
5227
|
savePolicy(fresh, home);
|
|
4505
5228
|
}
|
|
4506
5229
|
}
|
|
@@ -4929,7 +5652,7 @@ export class Worker {
|
|
|
4929
5652
|
return false;
|
|
4930
5653
|
}
|
|
4931
5654
|
catch { }
|
|
4932
|
-
if (this.
|
|
5655
|
+
if (this.answeredHere(ref))
|
|
4933
5656
|
return false;
|
|
4934
5657
|
const c = this.client;
|
|
4935
5658
|
if (c && !ref.startsWith("cont:")) {
|
|
@@ -5066,4 +5789,14 @@ export class Worker {
|
|
|
5066
5789
|
catch { }
|
|
5067
5790
|
}
|
|
5068
5791
|
}
|
|
5792
|
+
export function forAbout(until, now = Date.now()) {
|
|
5793
|
+
const min = Math.max(1, Math.round((until - now) / 60_000));
|
|
5794
|
+
if (min < 60)
|
|
5795
|
+
return `about ${min} minute${min === 1 ? "" : "s"}`;
|
|
5796
|
+
const h = Math.round(min / 60);
|
|
5797
|
+
if (h < 48)
|
|
5798
|
+
return `about ${h} hour${h === 1 ? "" : "s"}`;
|
|
5799
|
+
const d = Math.round(h / 24);
|
|
5800
|
+
return `about ${d} days`;
|
|
5801
|
+
}
|
|
5069
5802
|
//# sourceMappingURL=worker.js.map
|