@join-pantheon/cli 0.35.0 → 0.35.1-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/dist/agent-outage.js +165 -35
  2. package/dist/agent-token.js +5 -3
  3. package/dist/agents.js +6 -4
  4. package/dist/approvals-list.js +87 -0
  5. package/dist/board.js +491 -88
  6. package/dist/brief.js +17 -0
  7. package/dist/cli.js +687 -95
  8. package/dist/client.js +159 -19
  9. package/dist/config.js +3 -2
  10. package/dist/continued.js +56 -1
  11. package/dist/decision-ui.js +5 -3
  12. package/dist/doctor.js +84 -16
  13. package/dist/file-lock.js +40 -5
  14. package/dist/flow-stats.js +18 -1
  15. package/dist/home-guard.js +336 -54
  16. package/dist/home-safety.js +2 -0
  17. package/dist/hud-queue.js +91 -0
  18. package/dist/install.js +15 -1
  19. package/dist/mcp-instructions.js +67 -28
  20. package/dist/mcp.js +129 -24
  21. package/dist/notice-linux.js +15 -4
  22. package/dist/notice-windows.js +2 -2
  23. package/dist/notifier.js +2 -1
  24. package/dist/notify.js +28 -9
  25. package/dist/open-in.js +4 -3
  26. package/dist/pickup.js +17 -0
  27. package/dist/plain.js +139 -3
  28. package/dist/profile.js +12 -8
  29. package/dist/protocol.js +1 -1
  30. package/dist/question-goal.js +74 -0
  31. package/dist/readiness-alert.js +24 -3
  32. package/dist/refusal.js +2 -2
  33. package/dist/relay.js +143 -28
  34. package/dist/removed.js +4 -4
  35. package/dist/run-control.js +4 -3
  36. package/dist/run-failure.js +67 -10
  37. package/dist/setup-changes.js +7 -6
  38. package/dist/signed-out.js +144 -21
  39. package/dist/spend.js +5 -4
  40. package/dist/tray.js +24 -6
  41. package/dist/update-check.js +2 -1
  42. package/dist/uri-handler.js +10 -2
  43. package/dist/which.js +2 -0
  44. package/dist/worker-policy.js +212 -10
  45. package/dist/worker.js +942 -133
  46. package/dist/worklist.js +31 -1
  47. package/helper/macos/main.swift +42 -3
  48. package/helper/macos/menubar.swift +482 -87
  49. package/helper/macos/prebuilt/manifest.json +5 -5
  50. package/helper/macos/prebuilt/pantheon-hud +0 -0
  51. package/helper/macos/prebuilt/pantheon-tray +0 -0
  52. package/package.json +1 -1
package/dist/worker.js CHANGED
@@ -5,22 +5,22 @@ import { chmodSync, lstatSync, mkdirSync, writeFileSync, existsSync, readdirSync
5
5
  import { homedir } from "node:os";
6
6
  import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
7
7
  import { absentDeniedPaths, describeKept, ownWorktreeConfig, detectClaudeFeatures, describeFeatures, ensureRunTmp, leavesSandboxMounts, NO_CLAUDE_FEATURES, removeSandboxLeftovers, runTmpDir } from "./claude-confine.js";
8
- import { snapshotHome, restoreHome, describeChange, trustWrite, askedHere, persistSnapshot, persistRestored, persistAttempt, loadPersistedGuard, clearPersistedGuard, CHANGES_LOG_FILE } from "./home-guard.js";
9
- import { PantheonClient } from "./client.js";
8
+ import { snapshotHome, restoreHome, describeChange, trustWrite, askedHere, canon, withdrawnHere, noesHere, adoptOwn, persistOwn, persistPending, resetOwn, persistSnapshot, persistRestored, persistAttempt, loadPersistedGuard, clearPersistedGuard, CHANGES_LOG_FILE, GUARDED_FILES } from "./home-guard.js";
9
+ import { PantheonClient, failedActId } from "./client.js";
10
10
  import { createHash, randomBytes } from "node:crypto";
11
11
  import { newKeypair, signDelegate } from "./identity.js";
12
12
  import { loadPins, pinScope, savePins, teamPinId, writeWhole } from "./pins.js";
13
- import { answeredIn, markAnswered, noteNonce } from "./nonces.js";
14
- import { brokenStateFiles, brokenStateFix, pidAlive, readStateFile, withFileLock } from "./file-lock.js";
13
+ import { answeredIn, markAnswered, noteNonce, unmarkAnswered } from "./nonces.js";
14
+ import { brokenStateFiles, brokenStateFix, pidAlive, readStateFile, readStateText, withFileLock } from "./file-lock.js";
15
15
  import { mkdirInside, writeInside } from "./safe-write.js";
16
16
  import { corruptHeldFiles, MAX_HELD_PER_RUN, remoteTipIs, takeHeldFrames, updateHeldState, verifyHeldFrame, writeHoldConfig } from "./held-delivery.js";
17
17
  import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
18
18
  import { trusted } from "./verify-act.js";
19
- import { headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
20
- import { activeOutages, detectOutage, earliest, whenAgain, SIGNIN_REFUSED } from "./agent-outage.js";
19
+ import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
20
+ import { activeOutages, detectOutage, earliest, isSignOut, outagePredicate, whenAgain, SIGNIN_REFUSED } from "./agent-outage.js";
21
21
  import { agentSpawn, CmdLineUnsafe } from "./which.js";
22
22
  import { VERSION } from "./version.js";
23
- import { applyAnswer, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, policyHome, describeRunMinutes, parseRunMinutes, currentMode, } from "./worker-policy.js";
23
+ import { applyAnswer, ASK_EXPIRY_MS, policyUnreadable, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, policyHome, describeRunMinutes, parseRunMinutes, currentMode, wakeSettled, } from "./worker-policy.js";
24
24
  import { readinessAlerts } from "./readiness-alert.js";
25
25
  import { clearDeclined, markDeclined, personsOwnThread, pruneDeclined } from "./declined.js";
26
26
  import { loadRunStops, markStopTold, pruneRunStops, requestRunStop, stopFor, stopNote } from "./run-control.js";
@@ -29,18 +29,18 @@ import { removalWords, removedLine, removedNotice, takeTold } from "./removed.js
29
29
  import { agentHookStates, repairHook, repairNote } from "./notify.js";
30
30
  import { fence, assess } from "./untrusted.js";
31
31
  import { allowed, closeStartedRun, finishRun, recordRun, startRun } from "./spend.js";
32
- import { handledIn, loadContinued, markContinued, unmarkContinued, wakeAlertedKey } from "./continued.js";
32
+ import { handledIn, loadContinued, markContinued, storedIn, storedKey, unmarkContinued, wakeAlertedKey, WORKER_MARK } from "./continued.js";
33
33
  import { decisionAlertsOn, loadProfile, profileForTeam, reservedDecisionsNote, saveProfile, wakeMode } from "./profile.js";
34
34
  import { clip } from "./brief.js";
35
35
  import { composeWakeAsk, composeWorker } from "./notice.js";
36
- import { diagnoseRun, gitProbe, workNeedsCommands, SIGN_IN_FIX } from "./run-failure.js";
37
- import { clearSignedOut, markSignedOut, retryAfter, signedOutNow, SIGNED_OUT_AGENTS } from "./signed-out.js";
36
+ import { diagnoseRun, gitProbe, refusedWords, workNeedsCommands, signInFixFor } from "./run-failure.js";
37
+ import { claudeSettingsAuth, clearSignedOut, markSignedOut, retryAfter, signedOutChecking, signedOutNow, SIGNED_OUT_AGENTS } from "./signed-out.js";
38
38
  import { claudeTokenForRun } from "./agent-token.js";
39
39
  import { installedTools, preferredTool } from "./notifier.js";
40
40
  import { obligationSignature, StatusNudger } from "./status-nudge.js";
41
41
  import { clientState, ConnectionReport } from "./conn-report.js";
42
42
  import { loadPersonas, matchPersona, personaByName, personaMatches, effectivePolicy, personaBrief, describePersona, } from "./persona.js";
43
- import { CONFIG_DIR, CONFIG_PATH, loadAll, repoInfo, updateMembershipToken } from "./config.js";
43
+ import { CONFIG_DIR, CONFIG_PATH, configFor, findMembership, load as loadConfig, loadAll, repoInfo, updateMembershipToken } from "./config.js";
44
44
  import { shellArg } from "./team-name.js";
45
45
  import { repoName, routeRequest, safeName } from "./repo-route.js";
46
46
  import { holdsRepo } from "./repo-find.js";
@@ -279,7 +279,7 @@ export function outsideLinksDiagnosis(member, worktree, scan) {
279
279
  says,
280
280
  fix: scan.unreadable
281
281
  ? `Look at this machine's worktree ${where}: a very large or unreadable checkout cannot be vouched for. Take this request in a live session, or clear it out (pantheon stops at that worktree; its branch and commits are kept in the repo).`
282
- : `On this machine, delete these links from the worktree ${where} — not from the repo's branch, where they are not: ${scan.links.map((l) => `${where}/${l}`).join(", ")}. Or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it.`,
282
+ : `On this machine, delete these links from the worktree ${where} — not from the repo's branch, where they are not: ${scan.links.slice(0, 20).map((l) => `${where}/${l}`).join(", ")}${scan.links.length > 20 ? `, and ${scan.links.length - 20} more in that worktree (find ${where} -type l lists them)` : ""}. Or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it.`,
283
283
  peer: {
284
284
  summary: `${member}'s worker did not start a run: its checkout for this could not be vouched for`,
285
285
  body: `${member}'s machine could not confirm its checkout for this holds no link leading outside it, so its worker did not start an unattended run: a run would read whatever such a link points at. It is still owed and waits for ${member}. Nothing was run.`,
@@ -780,8 +780,27 @@ export function deliveryStep(branch, canPush, ref) {
780
780
  for your human to review and push. If the request asked for code, that NOTE is your reply.`;
781
781
  }
782
782
  const SAFETY_CAUSES = new Set(["outside_links", "worktree_tampered"]);
783
+ const SAFETY_RERAISE_MAX = 3;
784
+ function safetyAgainDiagnosis(cause, member) {
785
+ return cause === "outside_links"
786
+ ? { cause: "outside_links", says: "the checkout for this held links that lead outside it — a run would read whatever they point at as the repo's own, so none was started",
787
+ fix: "On this machine, look at this request's worktree and delete any link that leads outside it, or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it." }
788
+ : { cause: "worktree_tampered", says: `the worktree's .git was changed after Pantheon made it — git would obey that gitdir's config outside any sandbox, so git is not run in it`,
789
+ fix: `Look at what is in that worktree, then remove its folder and run git worktree prune in the repo; the next request checks out a fresh one. (${member}'s worker raised this before a restart.)` };
790
+ }
791
+ function decidedBrief(answer) {
792
+ const said = answer.replace(/[\u0000-\u0008\u000b-\u001f\u007f]/g, "").slice(0, 2000);
793
+ return `
794
+ YOUR HUMAN DECIDED — an earlier run here escalated this request to them, and they answered:
795
+ ${said.split("\n").map((l) => ` | ${l}`).join("\n")}
796
+ That is their decision. Reply to your teammate with it now (step 4: an ANSWER with the request's ref),
797
+ in your own plain words, and act on it if the request asks for work. Do not escalate the same question again.
798
+ `;
799
+ }
800
+ const ESCALATED_FOR = "esc-for:";
801
+ const DECIDED_FOR = "decided-for:";
783
802
  export const MOOT_GRACE_MS = 2 * 60_000;
784
- export function buildBrief(o, trigger, branch, persona, canPush = false) {
803
+ export function buildBrief(o, trigger, branch, persona, canPush = false, decided) {
785
804
  const placed = branch
786
805
  ? `in an isolated git worktree on branch ${branch}`
787
806
  : `in a scratch directory (your human's workspace is not a git repo, so there is no branch and nothing can be DELIVERED from here)`;
@@ -797,7 +816,7 @@ ${riskNote(risky, "what was asked")}
797
816
  WHAT YOU OWE
798
817
  ${owedLine(o.kind)}${o.goal ? ` — goal ${printable(o.goal)}` : ""}
799
818
  obligation id: ${printable(o.id)}
800
- ${persona ? personaBrief(persona) : ""}
819
+ ${persona ? personaBrief(persona) : ""}${decided ? decidedBrief(decided) : ""}
801
820
  DO THIS
802
821
  1. pantheon_claim with obligation id ${printable(o.id)}. If the claim is REFUSED, another session has it —
803
822
  stop immediately and say nothing.
@@ -985,8 +1004,43 @@ function readRunLease(f) {
985
1004
  rmSync(f, { recursive: true, force: true });
986
1005
  return null;
987
1006
  }
1007
+ export const PERSON_HOLD = "person-answering.json";
1008
+ export const PERSON_HOLD_TTL_MS = 60_000;
1009
+ export const PERSON_ANSWERING = "your person is answering a question on this computer";
1010
+ export function personHold(dir, now = Date.now()) {
1011
+ try {
1012
+ const v = JSON.parse(readStateText(join(dir, PERSON_HOLD)));
1013
+ const pid = Number(v?.pid), at = Number(v?.at);
1014
+ if (!Number.isInteger(pid) || pid <= 1 || !Number.isFinite(at) || at > now + 5_000 || now - at > PERSON_HOLD_TTL_MS || !pidAlive(pid))
1015
+ return null;
1016
+ return { pid, at };
1017
+ }
1018
+ catch {
1019
+ return null;
1020
+ }
1021
+ }
1022
+ export function writePersonHold(dir, now = Date.now()) {
1023
+ try {
1024
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
1025
+ writeWhole(join(dir, PERSON_HOLD), JSON.stringify({ pid: process.pid, at: now }) + "\n");
1026
+ }
1027
+ catch { }
1028
+ }
1029
+ export function releasePersonHold(dir) {
1030
+ try {
1031
+ const h = personHold(dir);
1032
+ if (!h || h.pid === process.pid)
1033
+ rmSync(join(dir, PERSON_HOLD), { force: true });
1034
+ }
1035
+ catch { }
1036
+ }
988
1037
  export function takeRunLease(dir, space, now = Date.now()) {
989
1038
  const f = join(dir, RUN_SLOT);
1039
+ {
1040
+ const h = personHold(dir, now);
1041
+ if (h && h.pid !== process.pid)
1042
+ return { held: PERSON_ANSWERING };
1043
+ }
990
1044
  return withFileLock(f, () => {
991
1045
  const cur = readRunLease(f);
992
1046
  if (cur && Number.isInteger(cur.pid) && cur.pid !== process.pid && typeof cur.at === "number" && Number.isFinite(cur.at)
@@ -1110,14 +1164,37 @@ export function putBackSaid(list) {
1110
1164
  }
1111
1165
  export const GUARD_RESTORE_TRIES = 3;
1112
1166
  const GUARD_STALE_MARGIN_MS = 10 * 60_000;
1113
- export function recoverInterruptedGuard(home, now = Date.now(), restore = restoreHome) {
1167
+ export async function recoverGuardHere(home) {
1168
+ const g = loadPersistedGuard(home);
1169
+ if (!g)
1170
+ return { outcome: "none" };
1171
+ if (sameProcess(g.worker, g.workerId))
1172
+ return { outcome: "live" };
1173
+ let own;
1174
+ if (g.own && g.snapshot) {
1175
+ const anyCfg = (() => { try {
1176
+ return loadConfig();
1177
+ }
1178
+ catch {
1179
+ return null;
1180
+ } })();
1181
+ if (!anyCfg)
1182
+ return { outcome: "unreachable" };
1183
+ const v = await verifyOwnWrites(g.own, anyCfg.member, (t) => configForTeam(t, anyCfg), refusedSaid((l) => console.error(l)));
1184
+ if (v === "unreachable")
1185
+ return { outcome: "unreachable" };
1186
+ own = v;
1187
+ }
1188
+ return recoverInterruptedGuard(home, Date.now(), restoreHome, own);
1189
+ }
1190
+ export function recoverInterruptedGuard(home, now = Date.now(), restore = restoreHome, verifiedOwn) {
1114
1191
  const g = loadPersistedGuard(home);
1115
1192
  if (!g)
1116
1193
  return { outcome: "none" };
1117
1194
  if (sameProcess(g.worker, g.workerId))
1118
1195
  return { outcome: "live" };
1119
1196
  try {
1120
- const r = JSON.parse(readFileSync(join(home, RUN_RECORD), "utf8"));
1197
+ const r = JSON.parse(readStateText(join(home, RUN_RECORD)));
1121
1198
  if (r && r.worker === g.worker && Number.isInteger(r.pid) && r.pid > 1 && typeof r.at === "number" && now - r.at <= recordLimitMs(r) && recordedRunAlive(r.pid))
1122
1199
  killRunTree(r.pid);
1123
1200
  }
@@ -1126,15 +1203,17 @@ export function recoverInterruptedGuard(home, now = Date.now(), restore = restor
1126
1203
  clearPersistedGuard(home);
1127
1204
  return { outcome: "restored", said: g.restored ?? [] };
1128
1205
  }
1129
- if (g.at > now + 60_000 || now - g.at > recordLimitMs({ limit: g.limit }) + GUARD_STALE_MARGIN_MS) {
1206
+ adoptOwn(verifiedOwn);
1207
+ const pending = !!g.pending?.length;
1208
+ if (!pending && (g.at > now + 60_000 || now - g.at > recordLimitMs({ limit: g.limit }) + GUARD_STALE_MARGIN_MS)) {
1130
1209
  clearPersistedGuard(home);
1131
1210
  return { outcome: "stale", at: g.at };
1132
1211
  }
1133
1212
  const lease = readRunLease(join(home, RUN_SLOT));
1134
1213
  const wentAt = lease && lease.pid === g.worker && typeof lease.at === "number" && Number.isFinite(lease.at) ? lease.at + RUN_LEASE_RENEW_MS : null;
1135
- if (wentAt !== null)
1214
+ if (wentAt !== null && !pending)
1136
1215
  try {
1137
- const mine = readFileSync(join(home, CHANGES_LOG_FILE), "utf8").split("\n").some((l) => {
1216
+ const mine = readStateText(join(home, CHANGES_LOG_FILE)).split("\n").some((l) => {
1138
1217
  try {
1139
1218
  const e = JSON.parse(l);
1140
1219
  return typeof e.at === "number" && e.at > Math.max(g.at, wentAt) && e.setting !== "first-setup" && e.asked_by === "terminal" && (e.outcome === "applied" || e.outcome === "approved" || e.outcome === "kept");
@@ -1150,7 +1229,14 @@ export function recoverInterruptedGuard(home, now = Date.now(), restore = restor
1150
1229
  }
1151
1230
  catch { }
1152
1231
  try {
1153
- const said = putBackList(restore(g.snapshot));
1232
+ const snapshot = pending ? { ...g.snapshot, files: g.snapshot.files.filter((f) => g.pending.includes(f.rel)), queueDone: true } : g.snapshot;
1233
+ const changes = restore(snapshot);
1234
+ const said = putBackList(changes);
1235
+ const rels = changes.filter((c) => c.failed).map((c) => c.rel);
1236
+ if (rels.length) {
1237
+ persistPending(home, rels);
1238
+ return { outcome: "pending", said, rels, snapshot: { ...g.snapshot, files: g.snapshot.files.filter((f) => rels.includes(f.rel)) } };
1239
+ }
1154
1240
  clearPersistedGuard(home);
1155
1241
  return { outcome: "restored", said };
1156
1242
  }
@@ -1166,6 +1252,8 @@ export function recoverInterruptedGuard(home, now = Date.now(), restore = restor
1166
1252
  }
1167
1253
  export function guardHeld(home, now = Date.now()) {
1168
1254
  const g = loadPersistedGuard(home);
1255
+ if (g?.snapshot && g.pending?.length)
1256
+ return true;
1169
1257
  if (!g?.snapshot || g.at > now + 60_000 || now - g.at > recordLimitMs({ limit: g.limit }) + GUARD_STALE_MARGIN_MS)
1170
1258
  return false;
1171
1259
  return sameProcess(g.worker, g.workerId);
@@ -1187,7 +1275,7 @@ function writeRunRecord(dir, r) {
1187
1275
  function clearRunRecord(dir, pid) {
1188
1276
  try {
1189
1277
  const f = join(dir, RUN_RECORD);
1190
- const r = JSON.parse(readFileSync(f, "utf8"));
1278
+ const r = JSON.parse(readStateText(f));
1191
1279
  if (r.pid === pid)
1192
1280
  rmSync(f, { force: true });
1193
1281
  }
@@ -1197,7 +1285,7 @@ export function killRecordedRun(dir, now = Date.now()) {
1197
1285
  const f = join(dir, RUN_RECORD);
1198
1286
  let r;
1199
1287
  try {
1200
- r = JSON.parse(readFileSync(f, "utf8"));
1288
+ r = JSON.parse(readStateText(f));
1201
1289
  }
1202
1290
  catch {
1203
1291
  return null;
@@ -1237,7 +1325,7 @@ export function parentOf(pid) {
1237
1325
  export function runInFlight(dir, now = Date.now(), o = {}) {
1238
1326
  let r;
1239
1327
  try {
1240
- r = JSON.parse(readFileSync(join(dir, RUN_RECORD), "utf8"));
1328
+ r = JSON.parse(readStateText(join(dir, RUN_RECORD)));
1241
1329
  }
1242
1330
  catch {
1243
1331
  return null;
@@ -1264,7 +1352,7 @@ export function runMinutesLeft(dir, now = Date.now()) {
1264
1352
  if (!runInFlight(dir, now))
1265
1353
  return null;
1266
1354
  try {
1267
- const r = JSON.parse(readFileSync(join(dir, RUN_RECORD), "utf8"));
1355
+ const r = JSON.parse(readStateText(join(dir, RUN_RECORD)));
1268
1356
  if (typeof r.exited_at === "number")
1269
1357
  return 1;
1270
1358
  if (typeof r.limit !== "number" || !Number.isFinite(r.limit))
@@ -1287,6 +1375,99 @@ export function answerWindow(askSeq, hint, fullScanDue) {
1287
1375
  return 0;
1288
1376
  return Math.max(0, hint);
1289
1377
  }
1378
+ const teamConfigs = new Map();
1379
+ const refusedSaid = (log) => {
1380
+ const said = new Set();
1381
+ return (team, status) => {
1382
+ if (said.has(team))
1383
+ return;
1384
+ said.add(team);
1385
+ log(` guard the relay of "${team}" refused this machine (${status === 410 ? "the team was deleted" : status === 401 ? "it needs a newer pantheon" : "removed from the team, or signed out"}) — what the interrupted run's worker had asked there is not kept; nothing waits on it`);
1386
+ };
1387
+ };
1388
+ function configForTeam(team, mine) {
1389
+ if (team === mine.spaceId || team === mine.space)
1390
+ return mine;
1391
+ const known = teamConfigs.get(team);
1392
+ if (known)
1393
+ return known;
1394
+ try {
1395
+ const f = loadAll();
1396
+ const m = f ? findMembership(f, team) : undefined;
1397
+ return f && m ? configFor(f, m, null) : null;
1398
+ }
1399
+ catch {
1400
+ return null;
1401
+ }
1402
+ }
1403
+ export async function verifyOwnWrites(o, member, cfgFor, refused) {
1404
+ const out = { wake: [], cap: [], stuck: [], safety: [], withdrawn: [], noes: [], marks: [] };
1405
+ let unreachable = false;
1406
+ const seen = new Map();
1407
+ const act = async (id, team) => {
1408
+ const key = `${team ?? ""}|${id}`;
1409
+ if (seen.has(key))
1410
+ return seen.get(key);
1411
+ const cfgs = team ? [cfgFor(team)].filter((c) => !!c) : [...new Set(teamConfigs.values())];
1412
+ let found = null;
1413
+ for (const cfg of cfgs) {
1414
+ try {
1415
+ const res = await signedRead(cfg, "/act", { id }, { timeoutMs: 3000 });
1416
+ if (res.ok) {
1417
+ found = (await res.json()).act ?? null;
1418
+ if (found)
1419
+ break;
1420
+ }
1421
+ else if (res.status === 401 || res.status === 403 || res.status === 410)
1422
+ refused?.(cfg.space, res.status);
1423
+ else if (res.status !== 404)
1424
+ unreachable = true;
1425
+ }
1426
+ catch {
1427
+ unreachable = true;
1428
+ }
1429
+ }
1430
+ seen.set(key, found);
1431
+ return found;
1432
+ };
1433
+ const mine = (a) => !!a && a.act === "ESCALATE" && a.from === member && a.sig?.role !== "run";
1434
+ const parse = (v) => { try {
1435
+ const x = JSON.parse(v);
1436
+ return x && typeof x === "object" ? x : {};
1437
+ }
1438
+ catch {
1439
+ return {};
1440
+ } };
1441
+ const refs = new Set(), teamOfAct = new Map();
1442
+ for (const [id, v] of o.stuck) {
1443
+ const rec = parse(v);
1444
+ const team = typeof rec.team === "string" ? rec.team : null;
1445
+ const a = await act(id, team);
1446
+ if (mine(a) && a.ref === rec.ref) {
1447
+ out.stuck.push([id, v]);
1448
+ refs.add(String(rec.ref));
1449
+ if (team)
1450
+ teamOfAct.set(id, team);
1451
+ }
1452
+ }
1453
+ for (const [cap, v] of o.cap) {
1454
+ const id = String(parse(v).act ?? "");
1455
+ if (id && mine(await act(id, teamOfAct.get(id) ?? null)))
1456
+ out.cap.push([cap, v]);
1457
+ }
1458
+ for (const [id, v] of o.wake) {
1459
+ const q = parse(v);
1460
+ const a = await act(id, typeof q.team === "string" ? q.team : null);
1461
+ if (mine(a) && a.meta?.wake_ask === q.member)
1462
+ out.wake.push([id, v]);
1463
+ }
1464
+ for (const m of o.marks) {
1465
+ const r = /:r:[A-Z_]+:(.+)$/.exec(m)?.[1];
1466
+ if (r && refs.has(r))
1467
+ out.marks.push(m);
1468
+ }
1469
+ return unreachable ? "unreachable" : out;
1470
+ }
1290
1471
  export class Worker {
1291
1472
  cfg;
1292
1473
  opts;
@@ -1328,6 +1509,9 @@ export class Worker {
1328
1509
  this.cfg = cfg;
1329
1510
  this.opts = opts;
1330
1511
  this.pinTeam = teamPinId(cfg);
1512
+ for (const k of [cfg.spaceId, cfg.space])
1513
+ if (k)
1514
+ teamConfigs.set(k, cfg);
1331
1515
  }
1332
1516
  spaceKey() { return this.cfg.spaceId ?? this.cfg.space; }
1333
1517
  pinTeam;
@@ -1344,32 +1528,83 @@ export class Worker {
1344
1528
  for (const k of this.contKeys())
1345
1529
  for (const id of loadContinued(k, this.opts.home))
1346
1530
  this.contDone.add(id);
1531
+ for (const id of this.contDone) {
1532
+ const m = /^unparked@([^:]+):(.+)$/.exec(id);
1533
+ if (m)
1534
+ this.unparkedUnder.set(m[2], m[1]);
1535
+ }
1536
+ }
1537
+ obKeys = new Map();
1538
+ learnRequests(list) {
1539
+ if (this.obKeys.size > 5000)
1540
+ this.obKeys.clear();
1541
+ for (const o of list)
1542
+ if (o.kind !== "CONTINUE" && o.ref)
1543
+ this.obKeys.set(o.id, `r:${o.kind}:${o.ref}`);
1544
+ }
1545
+ oKey(id) { return this.obKeys.get(id) ?? id; }
1546
+ marked(prefix, id) {
1547
+ const k = this.oKey(id);
1548
+ return this.contDone.has(`${prefix}:${k}`) || (k !== id && this.contDone.has(`${prefix}:${id}`));
1549
+ }
1550
+ markFor(prefix, id) { this.markDone(`${prefix}:${this.oKey(id)}`); }
1551
+ unparkedUnder = new Map();
1552
+ grantKey() {
1553
+ const g = loadPolicy(this.opts.home).grants.run_commands;
1554
+ if (this.opts.allowBash)
1555
+ return { key: "bash", once: false };
1556
+ return g ? { key: `${g.mode === "once" ? "o" : "g"}${g.at}`, once: g.mode === "once" } : null;
1347
1557
  }
1348
1558
  failuresOf(id) {
1349
- const mem = this.failedRuns.get(id);
1559
+ const mem = this.failedRuns.get(this.oKey(id));
1350
1560
  if (mem !== undefined)
1351
1561
  return mem;
1352
1562
  this.loadContinued();
1353
- if (this.contDone.has(`unparked:${id}`) && !this.contDone.has(`parked2:${id}`))
1563
+ const under = this.unparkedUnder.get(this.oKey(id));
1564
+ if (under)
1565
+ return this.contDone.has(`parked2@${under}:${this.oKey(id)}`) ? 2 : 1;
1566
+ if (this.marked("unparked", id) && !this.marked("parked2", id))
1354
1567
  return 1;
1355
- if (this.contDone.has(`unparked-ns:${id}`) && !this.contDone.has(`parked2-ns:${id}`))
1568
+ if (this.marked("unparked-ns", id) && !this.marked("parked2-ns", id))
1356
1569
  return 1;
1357
- return this.contDone.has(`parked:${id}`) ? 2 : this.contDone.has(`failed1:${id}`) ? 1 : 0;
1570
+ return this.marked("parked", id) ? 2 : this.marked("failed1", id) ? 1 : 0;
1358
1571
  }
1359
1572
  retryParkedForCommands(owed) {
1360
- const g = loadPolicy(this.opts.home).grants.run_commands;
1361
- if (!g && !this.opts.allowBash)
1573
+ const g = this.grantKey();
1574
+ if (!g)
1362
1575
  return;
1363
1576
  this.loadContinued();
1577
+ const waiting = (o) => this.marked("parked-nc", o.id) && this.failuresOf(o.id) >= 2;
1578
+ if (g.once && owed.some((o) => this.unparkedUnder.get(this.oKey(o.id)) === g.key && this.failuresOf(o.id) < 2))
1579
+ return;
1364
1580
  for (const o of owed) {
1365
- if (!this.contDone.has(`parked-nc:${o.id}`) || this.contDone.has(`unparked:${o.id}`))
1581
+ if (!waiting(o) || this.unparkedUnder.get(this.oKey(o.id)) === g.key)
1366
1582
  continue;
1367
- this.failedRuns.set(o.id, 1);
1583
+ this.failedRuns.set(this.oKey(o.id), 1);
1368
1584
  this.skipLogged.delete(`parked:${o.id}`);
1369
- this.markDone(`unparked:${o.id}`);
1585
+ this.markDone(`unparked@${g.key}:${this.oKey(o.id)}`);
1586
+ this.unparkedUnder.set(this.oKey(o.id), g.key);
1370
1587
  this.opts.log(`* commands are allowed now — running ${o.kind} ${o.id} again (it was parked because it needed them)`);
1588
+ if (g.once)
1589
+ break;
1371
1590
  }
1372
1591
  }
1592
+ async askAgainForParked(client) {
1593
+ if (this.grantKey())
1594
+ return;
1595
+ this.loadContinued();
1596
+ const recent = (k) => /^o\d+$/.test(k) && Date.now() - Number(k.slice(1)) < ASK_EXPIRY_MS;
1597
+ const spent = new Set([...this.unparkedUnder.values()].filter(recent));
1598
+ if (!spent.size)
1599
+ return;
1600
+ if (!mayAsk(loadPolicy(this.opts.home), "run_commands"))
1601
+ return;
1602
+ const o = client.myWork().find((x) => this.marked("parked-nc", x.id) && this.failuresOf(x.id) >= 2 && !spent.has(this.unparkedUnder.get(this.oKey(x.id)) ?? ""));
1603
+ if (!o)
1604
+ return;
1605
+ await this.reportStuck(client, o, { cause: "no_commands", says: "The agent may only edit files on this computer, so it couldn't run tests, commit or push.",
1606
+ fix: "To let it deliver work while you're away: pantheon setup --commands yes", capability: "run_commands" }, 2, 1);
1607
+ }
1373
1608
  parkedNotHere = new Set();
1374
1609
  parkedNoSandbox = new Map();
1375
1610
  outages = new Map();
@@ -1483,20 +1718,19 @@ export class Worker {
1483
1718
  this.readinessCommandsOn = commandsOn;
1484
1719
  const r = this.opts.readinessCheck ? this.opts.readinessCheck() : unattendedReadiness({ prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: this.unsandboxedOk(), pinned: this.opts.agent ?? "auto", ...(this.opts.choose ? { choose: this.opts.choose } : {}),
1485
1720
  commandsOn, inContainer: hostEnv().kind === "container", ...(this.opts.sandboxProbe ? { sandboxProbe: this.opts.sandboxProbe } : {}) });
1486
- if (!r.ok && (this.readiness?.ok ?? true))
1487
- this.opts.log(`! teammates are told this machine can't take requests yet: ${r.short} — ${r.fix}`);
1488
- if (r.ok && this.readiness && !this.readiness.ok)
1489
- this.opts.log(` agent ready again — teammates are told this machine takes requests`);
1490
- this.readiness = { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) };
1491
1721
  this.syncSignedOut();
1492
- if (r.ok && this.signedOutHere.size && !this.opts.readinessCheck) {
1493
- const held = this.heldForSandbox();
1494
- if (held?.cause === "agent_unavailable") {
1495
- this.readiness = { ok: false, short: "its agent CLI was signed out (a run here was refused)", code: "not_signed_in", fix: held.fix ?? "", signedOut: true };
1496
- return;
1497
- }
1498
- }
1499
- this.tellPerson(now);
1722
+ const held = r.ok && this.signedOutHere.size && !this.opts.readinessCheck ? this.heldForSandbox() : null;
1723
+ const signedOut = held?.cause === "agent_unavailable";
1724
+ const ok = r.ok && !signedOut;
1725
+ if (!ok && (this.readiness?.ok ?? true))
1726
+ this.opts.log(`! teammates are told this machine can't take requests yet: ${signedOut ? "its agent CLI was signed out (a run here was refused)" : `${r.short} — ${r.fix}`}`);
1727
+ if (ok && this.readiness && !this.readiness.ok)
1728
+ this.opts.log(` agent ready again — teammates are told this machine takes requests`);
1729
+ const outAgent = signedOut ? [...this.signedOutHere][0] : undefined;
1730
+ this.readiness = signedOut
1731
+ ? { ok: false, short: "its agent CLI was signed out (a run here was refused)", code: "not_signed_in", fix: held.fix ?? "", signedOut: true, ...(outAgent ? { agent: outAgent } : {}) }
1732
+ : { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) };
1733
+ this.tellPerson(now, { quiet: signedOut });
1500
1734
  }
1501
1735
  catch {
1502
1736
  this.readiness = null;
@@ -1509,12 +1743,12 @@ export class Worker {
1509
1743
  const who = createHash("sha256").update(`${resolve(this.opts.project)}|${this.opts.agent ?? "auto"}|${commands ? "commands" : "-"}`).digest("hex").slice(0, 12);
1510
1744
  return `${this.spaceKey()}#${who}@${process.pid}`;
1511
1745
  }
1512
- tellPerson(now) {
1746
+ tellPerson(now, o = {}) {
1513
1747
  const r = this.readiness;
1514
1748
  if (!r)
1515
1749
  return;
1516
1750
  const alerts = () => {
1517
- if (!this.opts.notify || !decisionAlertsOn())
1751
+ if (o.quiet || !this.opts.notify || !decisionAlertsOn())
1518
1752
  return false;
1519
1753
  try {
1520
1754
  return !this.opts.canAlert || this.opts.canAlert();
@@ -1536,7 +1770,7 @@ export class Worker {
1536
1770
  const key = this.readinessKey();
1537
1771
  const replaces = this.lastReadinessKey && this.lastReadinessKey !== key ? this.lastReadinessKey : undefined;
1538
1772
  this.lastReadinessKey = key;
1539
- const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}) });
1773
+ const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet ? { certain: true } : {}) });
1540
1774
  this.alertPending = t.pending;
1541
1775
  if (!t.notice)
1542
1776
  return;
@@ -1580,30 +1814,56 @@ export class Worker {
1580
1814
  }
1581
1815
  }
1582
1816
  outageDiagnosis(label, out, others, meanwhile) {
1817
+ const id = AGENTS.find((a) => a.label === label)?.id;
1818
+ if (isSignOut(out) || (id && this.signedOutHere.has(id))) {
1819
+ let why = (id && this.signedOutHere.has(id) && this.outages.get(id)?.why) || undefined;
1820
+ if (why === SIGNIN_REFUSED)
1821
+ why = undefined;
1822
+ if (!why && id) {
1823
+ const m = signedOutNow(id, { pantheonHome: this.stateDir() }) ?? signedOutChecking(id, { pantheonHome: this.stateDir() });
1824
+ if (m)
1825
+ why = this.signedOutWhy(id, m);
1826
+ }
1827
+ why ??= `is signed out on this machine — ${signInFixFor(id)}`;
1828
+ const rest = meanwhile ? `; the agent standing in for it cannot take this one (${meanwhile.replace(/\.$/, "")})` : others.length ? `; ${others.join("; ")}` : "";
1829
+ return {
1830
+ cause: "agent_unavailable",
1831
+ says: `${label} ${why}${rest}.`,
1832
+ fix: "It runs as soon as you sign in again — or answer it in a live session.",
1833
+ peer: {
1834
+ summary: `${this.cfg.member}'s agent is signed out — this waits until ${this.cfg.member} signs in`,
1835
+ body: `${this.cfg.member}'s worker could not start a run: ${label} is signed out on that machine. It is still owed; it runs once ${this.cfg.member} signs in again.`,
1836
+ },
1837
+ };
1838
+ }
1583
1839
  const at = whenAgain(out.until);
1584
1840
  const rest = meanwhile ? `; the agent standing in for it cannot take this one (${meanwhile.replace(/\.$/, "")})`
1585
1841
  : others.length ? `; ${others.join("; ")}` : "; no other agent is installed for runs while you are away";
1586
1842
  return {
1587
1843
  cause: "agent_unavailable",
1588
- says: `${label} is unavailable here until about ${at} — ${out.why}${rest}.`,
1844
+ says: `${label} ${outagePredicate(out.why)}, so it is unavailable here until about ${at}${rest}.`,
1589
1845
  fix: `It runs by itself after ${at}. Sooner: sign in to another agent (cursor-agent, codex or copilot) — and for an agent without a sandbox, pantheon policy grant unsandboxed-runs — or answer it in a live session.`,
1590
1846
  peer: {
1591
- summary: `${this.cfg.member}'s agent is unavailable until about ${at} — this waits`,
1592
- body: `${this.cfg.member}'s worker could not start a run: ${label} ${out.why}, and no other agent on that machine could take it. It is still owed; the worker tries again after ${at}.`,
1847
+ summary: `${this.cfg.member}'s agent is unavailable for ${forAbout(out.until)} — this waits`,
1848
+ body: `${this.cfg.member}'s worker could not start a run: ${label} ${outagePredicate(out.why)}, and no other agent on that machine could take it. It is still owed; the worker tries again in ${forAbout(out.until)}.`,
1593
1849
  },
1594
1850
  };
1595
1851
  }
1596
1852
  signedOutHere = new Set();
1853
+ allOutSignedOut() {
1854
+ const out = Object.keys(activeOutages(this.outages));
1855
+ return out.length > 0 && out.every((id) => this.signedOutHere.has(id));
1856
+ }
1597
1857
  syncSignedOut() {
1598
1858
  if (this.opts.agentCmd)
1599
1859
  return;
1600
1860
  for (const id of SIGNED_OUT_AGENTS) {
1601
- const mark = signedOutNow(id, { pantheonHome: this.stateDir(), lift: true });
1861
+ const mark = signedOutNow(id, { pantheonHome: this.stateDir(), lift: true, env: process.env });
1602
1862
  if (mark) {
1603
1863
  const until = mark.at + retryAfter(mark);
1604
1864
  const cur = this.outages.get(id);
1605
1865
  if (!cur || cur.until < until)
1606
- this.outages.set(id, { why: this.signedOutWhy(id, mark.says), until });
1866
+ this.outages.set(id, { why: this.signedOutWhy(id, mark), until, signIn: true });
1607
1867
  this.signedOutHere.add(id);
1608
1868
  }
1609
1869
  else if (this.signedOutHere.has(id)) {
@@ -1623,10 +1883,14 @@ export class Worker {
1623
1883
  }
1624
1884
  }
1625
1885
  }
1626
- signedOutWhy(id, says) {
1886
+ signedOutWhy(id, mark) {
1887
+ const says = mark.says;
1888
+ const w = refusedWords(id, { line: says, ...(mark.via ? { env: mark.via } : {}), ...(mark.basis === "settings" ? { settings: true } : {}), ...(mark.helper ? { helper: true } : {}), ...(mark.basis === "token" ? { savedToken: true } : {}) });
1889
+ if (w)
1890
+ return `refused ${w.what} ("${says}") — ${w.fix}`;
1627
1891
  if (id === "claude-code" && claudeTokenForRun(this.stateDir()))
1628
1892
  return `refused the token saved for this worker ("${says}") — make a new one (claude setup-token, then pantheon worker token), or remove it to use your own sign-in: pantheon worker token --remove`;
1629
- const how = SIGN_IN_FIX[id] ?? "sign in to it again on this machine";
1893
+ const how = signInFixFor(id);
1630
1894
  return `is signed out on this machine ("${says}") — ${how}${id === "claude-code" ? ", or keep it signed in for a year: claude setup-token, then pantheon worker token" : ""}`;
1631
1895
  }
1632
1896
  outagesPassed(c) {
@@ -1648,7 +1912,7 @@ export class Worker {
1648
1912
  try {
1649
1913
  this.opts.notify({
1650
1914
  kind: "worker", who: "Your worker", heading: `${c.backend.label} is answering for Claude Code`, context: this.cfg.space,
1651
- body: `Claude Code is unavailable until about ${whenAgain(first.out.until)} — ${first.out.why}. Until then ${c.backend.label} takes teammates' requests on this machine, without Claude Code's sandbox, as your settings allow. To have them wait instead: pantheon policy revoke unsandboxed-runs`,
1915
+ body: `${isSignOut(first.out) ? `Claude Code ${outagePredicate(first.out.why === SIGNIN_REFUSED ? "is signed out on this machine" : first.out.why)}. Until it is signed in again` : `Claude Code ${outagePredicate(first.out.why)}, so it is unavailable until about ${whenAgain(first.out.until)}. Until then`} ${c.backend.label} takes teammates' requests on this machine, without Claude Code's sandbox, as your settings allow. To have them wait instead: pantheon policy revoke unsandboxed-runs`,
1652
1916
  ttl: 0, sound: false,
1653
1917
  });
1654
1918
  }
@@ -1762,7 +2026,7 @@ export class Worker {
1762
2026
  }
1763
2027
  releaseHeld(id, failures, why) {
1764
2028
  this.parkedNoSandbox.delete(id);
1765
- this.failedRuns.set(id, failures);
2029
+ this.failedRuns.set(this.oKey(id), failures);
1766
2030
  this.skipLogged.delete(`parked:${id}`);
1767
2031
  this.skipLogged.delete(`refused:${id}`);
1768
2032
  const act = this.parkedNoSandboxConts.get(id);
@@ -1789,7 +2053,7 @@ export class Worker {
1789
2053
  }
1790
2054
  if (!act && owedIds && !owedIds.has(id)) {
1791
2055
  this.parkedNoSandbox.delete(id);
1792
- this.failedRuns.delete(id);
2056
+ this.failedRuns.delete(this.oKey(id));
1793
2057
  this.skipLogged.delete(`parked:${id}`);
1794
2058
  continue;
1795
2059
  }
@@ -1812,11 +2076,11 @@ export class Worker {
1812
2076
  if (!ok)
1813
2077
  this.ranNoSandboxCheckedAt = now;
1814
2078
  this.loadContinued();
1815
- const ran = (owed ?? []).filter((o) => this.contDone.has(`parked-ns:${o.id}`) && !this.contDone.has(`unparked-ns:${o.id}`));
2079
+ const ran = (owed ?? []).filter((o) => this.marked("parked-ns", o.id) && !this.marked("unparked-ns", o.id));
1816
2080
  if (!ran.length || (!ok && !this.claudeSandboxableNow()))
1817
2081
  return;
1818
2082
  for (const o of ran) {
1819
- this.markDone(`unparked-ns:${o.id}`);
2083
+ this.markFor("unparked-ns", o.id);
1820
2084
  this.releaseHeld(o.id, 1, ok ? "unsandboxed runs are allowed here now" : "this Claude Code can run commands in its sandbox now");
1821
2085
  }
1822
2086
  }
@@ -1828,7 +2092,7 @@ export class Worker {
1828
2092
  if (!changed)
1829
2093
  return;
1830
2094
  for (const id of this.parkedNotHere) {
1831
- this.failedRuns.delete(id);
2095
+ this.failedRuns.delete(this.oKey(id));
1832
2096
  this.skipLogged.delete(`parked:${id}`);
1833
2097
  this.skipLogged.delete(`route:${id}`);
1834
2098
  this.opts.log(`* this machine's folders changed — trying ${id} again (it was waiting for a repo that was not here)`);
@@ -1897,17 +2161,27 @@ export class Worker {
1897
2161
  return;
1898
2162
  }
1899
2163
  this.loadContinued();
2164
+ const told = new Set();
1900
2165
  for (const team of new Set([this.cfg.spaceId, this.cfg.space].filter((t) => !!t))) {
1901
2166
  for (const [k, at] of Object.entries(no)) {
1902
2167
  if (!k.startsWith(`${team}|`))
1903
2168
  continue;
1904
2169
  const member = k.slice(team.length + 1);
2170
+ if (told.has(member)) {
2171
+ this.markDone(`wake-no-said:${k}:${at}`);
2172
+ continue;
2173
+ }
1905
2174
  if (!this.askedMembers.has(member))
1906
2175
  continue;
1907
2176
  const mark = `wake-no-said:${k}:${at}`;
1908
- if (this.contDone.has(mark) || Date.now() - at > 24 * 3600_000)
2177
+ if (this.contDone.has(mark)) {
2178
+ told.add(member);
2179
+ continue;
2180
+ }
2181
+ if (Date.now() - at > 24 * 3600_000)
1909
2182
  continue;
1910
2183
  this.markDone(mark);
2184
+ told.add(member);
1911
2185
  await client.send({
1912
2186
  act: "NOTE", goal: null, to: [member], after_seq: Math.max(client.readSeq, client.seq),
1913
2187
  summary: `${this.cfg.member}'s machine will not start work for you while they are away`,
@@ -1916,9 +2190,151 @@ export class Worker {
1916
2190
  }
1917
2191
  }
1918
2192
  }
2193
+ answeredHere(ref) {
2194
+ const k = `handled:${ref}`;
2195
+ return this.contDone.has(k) || this.continuedHas(k);
2196
+ }
2197
+ escalationAnswer(client, o, acts = client.mineSince(0)) {
2198
+ const signs = client.memberSigns(this.cfg.member);
2199
+ const as = (a, want) => {
2200
+ const t = client.trustOf(a).trust;
2201
+ if (t === want)
2202
+ return true;
2203
+ return !a.sig && !signs && t === "unsigned" && (want === "verified-run") === (a.role === "worker");
2204
+ };
2205
+ const byId = new Map(acts.map((a) => [a.id, a]));
2206
+ for (let i = acts.length - 1; i >= 0; i--) {
2207
+ const r = acts[i];
2208
+ if (r.act !== "ANSWER" || r.from !== this.cfg.member || !r.ref)
2209
+ continue;
2210
+ const e = byId.get(r.ref) ?? client.seenAct(r.ref);
2211
+ if (!e || e.act !== "ESCALATE" || e.from !== this.cfg.member)
2212
+ continue;
2213
+ if (e.ref ? e.ref !== o.ref : !this.hasMark(`${ESCALATED_FOR}${e.id}:${o.ref}`))
2214
+ continue;
2215
+ if (!as(e, "verified-run") || !as(r, "verified"))
2216
+ continue;
2217
+ return { escalation: e, answer: r };
2218
+ }
2219
+ return null;
2220
+ }
2221
+ decidedLookedUp = new Set();
2222
+ async reopenDecided(client, o) {
2223
+ let d = this.escalationAnswer(client, o);
2224
+ if (!d && !this.decidedLookedUp.has(o.ref)) {
2225
+ this.decidedLookedUp.add(o.ref);
2226
+ if (this.decidedLookedUp.size > 500)
2227
+ this.decidedLookedUp.delete(this.decidedLookedUp.values().next().value);
2228
+ const acts = await this.recentActs();
2229
+ d = this.escalationAnswer(client, o, acts.filter((a) => a.from === this.cfg.member).sort((x, y) => x.seq - y.seq));
2230
+ }
2231
+ if (!d)
2232
+ return;
2233
+ const mark = `decided:${d.answer.id}`;
2234
+ if (this.contDone.has(mark) || this.continuedHas(mark))
2235
+ return;
2236
+ this.markDone(mark);
2237
+ this.markDone(`${DECIDED_FOR}${o.ref}:${d.answer.id}`);
2238
+ this.decided.set(o.ref, d.answer.body);
2239
+ if (this.decided.size > 200)
2240
+ this.decided.delete(this.decided.keys().next().value);
2241
+ const handled = `handled:${o.ref}`;
2242
+ this.contDone.delete(handled);
2243
+ const homes = this.opts.home !== undefined ? [this.opts.home, undefined] : [undefined];
2244
+ for (const h of homes) {
2245
+ for (const k of this.contKeys())
2246
+ unmarkContinued(k, handled, h);
2247
+ try {
2248
+ unmarkAnswered(this.pinTeam, o.ref, h);
2249
+ }
2250
+ catch { }
2251
+ }
2252
+ this.skipLogged.delete(`handled:${o.id}`);
2253
+ this.opts.log(`> ${owedLabel(o.kind)} ${o.id}: its person answered the escalation (${d.answer.id}) — a run takes it again to pass that on`);
2254
+ }
2255
+ decided = new Map();
2256
+ hasMark(id) { return this.contDone.has(id) || this.continuedHas(id); }
2257
+ async decisionFor(client, o) {
2258
+ const kept = this.decided.get(o.ref);
2259
+ if (kept !== undefined)
2260
+ return kept;
2261
+ const seen = this.escalationAnswer(client, o)?.answer.body;
2262
+ if (seen !== undefined)
2263
+ return seen;
2264
+ this.loadContinued();
2265
+ const prefix = `${DECIDED_FOR}${o.ref}:`;
2266
+ const ids = [...this.contDone].filter((k) => k.startsWith(prefix)).map((k) => k.slice(prefix.length));
2267
+ const id = ids.at(-1);
2268
+ if (!id)
2269
+ return undefined;
2270
+ const answer = (await this.fetchActById(id));
2271
+ const escalation = answer?.ref ? (await this.fetchActById(answer.ref)) : null;
2272
+ if (!answer || !escalation)
2273
+ return undefined;
2274
+ return this.escalationAnswer(client, o, [escalation, answer])?.answer.body;
2275
+ }
2276
+ async recentActs() {
2277
+ try {
2278
+ const res = await signedRead(this.cfg, "/history", {}, { timeoutMs: 5000 });
2279
+ if (!res.ok)
2280
+ return [];
2281
+ return (await res.json()).acts ?? [];
2282
+ }
2283
+ catch {
2284
+ return [];
2285
+ }
2286
+ }
1919
2287
  markDone(id) {
1920
2288
  this.contDone.add(id);
1921
2289
  markContinued(this.pinTeam, id, this.opts.home);
2290
+ if (WORKER_MARK.test(id))
2291
+ this.keepOwn();
2292
+ }
2293
+ async verifyOwn(o) {
2294
+ return verifyOwnWrites(o, this.cfg.member, (t) => configForTeam(t, this.cfg), refusedSaid(this.opts.log));
2295
+ }
2296
+ keepOwn() { try {
2297
+ persistOwn(this.stateDir());
2298
+ }
2299
+ catch { } }
2300
+ pendingRestore = null;
2301
+ pendingSaid = "";
2302
+ settlePendingRestore() {
2303
+ const p = this.pendingRestore;
2304
+ if (!p)
2305
+ return false;
2306
+ let changes = [];
2307
+ const only = { ...p.snap, files: p.snap.files.filter((f) => p.rels.includes(f.rel)), queueDone: true };
2308
+ try {
2309
+ changes = (this.opts.restore ?? restoreHome)(only);
2310
+ }
2311
+ catch {
2312
+ changes = p.rels.map((rel) => ({ rel, what: "reported", keys: [], failed: true }));
2313
+ }
2314
+ const left = changes.filter((c) => c.failed).map((c) => c.rel);
2315
+ if (!left.length) {
2316
+ this.pendingRestore = null;
2317
+ this.pendingSaid = "";
2318
+ clearPersistedGuard(this.stateDir());
2319
+ this.opts.log(`* put back what a teammate's request had locked: ${p.rels.join(", ")} — unattended runs start again`);
2320
+ try {
2321
+ this.opts.notify?.({ kind: "worker", who: "Your worker", heading: "Settings put back — runs start again", context: this.cfg.space, body: `What a teammate's request changed in ${p.rels.join(", ")} is put back now. Unattended runs start again.`, ttl: 0, sound: false });
2322
+ }
2323
+ catch { }
2324
+ return false;
2325
+ }
2326
+ p.rels = left;
2327
+ const key = left.join(",");
2328
+ if (this.pendingSaid !== key) {
2329
+ this.pendingSaid = key;
2330
+ const what = `A teammate's request locked ${left.join(", ")} in ${this.stateDir()} while it ran, so Pantheon could not put back what it changed there. No unattended run starts until it can; your worker tries again every few seconds.`;
2331
+ this.opts.log(`! holding every unattended run: ${what}`);
2332
+ try {
2333
+ this.opts.notify?.({ kind: "worker", who: "Your worker", heading: "Unattended runs are on hold", context: this.cfg.space, body: `${what} To fix: ${brokenStateFix(left)}`, ttl: 0, sound: true });
2334
+ }
2335
+ catch { }
2336
+ }
2337
+ return true;
1922
2338
  }
1923
2339
  async bringMachineAlong() {
1924
2340
  if (this.opts.selfUpgrade?.run)
@@ -2126,6 +2542,7 @@ export class Worker {
2126
2542
  if (open) {
2127
2543
  if (open[1].refs && !open[1].refs.includes(o.ref)) {
2128
2544
  open[1].refs.push(o.ref);
2545
+ askedHere.wake.set(open[0], canon(open[1]));
2129
2546
  savePolicy(pol, this.opts.home);
2130
2547
  }
2131
2548
  this.alertWakeAsk(member, what, open[0]);
@@ -2165,9 +2582,10 @@ export class Worker {
2165
2582
  const me2 = client.members.find((m) => m.member === member);
2166
2583
  const teamBlocked = me2?.admitted === true && !me2.revoked_by && me2.may_wake === false;
2167
2584
  (p.wakeAsks ??= {})[sent.id] = { team: this.cfg.spaceId ?? this.cfg.space, member, at: Date.now(), refs: [o.ref], ...(teamBlocked ? { teamBlocked: true } : {}) };
2168
- askedHere.add(sent.id);
2585
+ askedHere.wake.set(sent.id, canon(p.wakeAsks[sent.id]));
2169
2586
  this.askedMembers.add(member);
2170
2587
  savePolicy(p, this.opts.home);
2588
+ this.keepOwn();
2171
2589
  this.markDone(`wake-asked:${sent.id}`);
2172
2590
  this.opts.log(`* asked you whether ${member} may start work on this machine (their ${owedLabel(o.kind)} ${o.id} waits for the answer)`);
2173
2591
  this.alertWakeAsk(member, what, sent.id);
@@ -2312,6 +2730,9 @@ export class Worker {
2312
2730
  this.contDone.add(id);
2313
2731
  markContinued(this.pinTeam, id, this.opts.home);
2314
2732
  }
2733
+ for (const x of storedIn(dir))
2734
+ if (x.team === this.pinTeam)
2735
+ markContinued(storedKey(this.pinTeam), x.ref, this.opts.home);
2315
2736
  }
2316
2737
  catch { }
2317
2738
  const { token: _t, privateKey: _k, pendingAuth: _p, delegate: _d, ...stub } = this.cfg;
@@ -2377,7 +2798,7 @@ export class Worker {
2377
2798
  report.observe("down");
2378
2799
  client.onChange = () => { nudger.observe(obligationSignature(client.obligations, client.connected)); report.observe(clientState(client)); };
2379
2800
  try {
2380
- this.recoverGuard();
2801
+ await this.recoverGuard();
2381
2802
  await client.connect();
2382
2803
  this.client = client;
2383
2804
  await (this.opts.alertsOnly ? this.sayKeyChanges(client) : this.opts.housekeeping ? this.keepHouse(client) : this.serve(client));
@@ -2453,7 +2874,7 @@ export class Worker {
2453
2874
  for (const w of p.warnings)
2454
2875
  this.opts.log(` ! ${p.name}: ${w}`);
2455
2876
  }
2456
- this.recoverInterruptedRun();
2877
+ await this.recoverInterruptedRun(client);
2457
2878
  await this.settleLeftHeld(client);
2458
2879
  this.reportStanding(true);
2459
2880
  if (this.opts.continuations !== false)
@@ -2461,11 +2882,26 @@ export class Worker {
2461
2882
  this.writeState("watching", "waiting for work");
2462
2883
  while (!this.stopped) {
2463
2884
  const arrived = await client.next(5000);
2885
+ if (this.deadClaims.size)
2886
+ await this.releaseDeadClaims(client);
2464
2887
  {
2465
2888
  const held = this.tickHold;
2466
2889
  this.tickHold = null;
2467
2890
  this.reportStanding(false, held);
2468
2891
  }
2892
+ if (this.guardWaiting) {
2893
+ if (Date.now() >= this.guardRetryAt)
2894
+ await this.recoverGuard();
2895
+ if (this.guardWaiting) {
2896
+ this.writeState("held", "putting back after an interrupted run waits for the relay");
2897
+ continue;
2898
+ }
2899
+ }
2900
+ if (policyUnreadable(loadPolicy(this.opts.home))) {
2901
+ this.sayBrokenState(["worker-policy.json"]);
2902
+ this.writeState("held", "worker-policy.json cannot be read");
2903
+ continue;
2904
+ }
2469
2905
  if (client.refused && !this.refusedSaid) {
2470
2906
  this.refusedSaid = true;
2471
2907
  if (client.removedFrom)
@@ -2490,6 +2926,10 @@ export class Worker {
2490
2926
  this.tickHold = { reason: "busy_elsewhere", detail: "another team's run is in progress on this machine" };
2491
2927
  continue;
2492
2928
  }
2929
+ if (this.settlePendingRestore()) {
2930
+ this.writeState("held", `${this.pendingRestore.rels.join(", ")} could not be put back after a run`);
2931
+ continue;
2932
+ }
2493
2933
  const broken = this.brokenStateFiles();
2494
2934
  if (broken.length) {
2495
2935
  this.sayBrokenState(broken);
@@ -2506,7 +2946,9 @@ export class Worker {
2506
2946
  this.busyObservedSince ??= Date.now();
2507
2947
  else
2508
2948
  this.busyObservedSince = null;
2949
+ this.learnRequests(client.myWork());
2509
2950
  this.retryParkedForCommands(client.myWork());
2951
+ await this.askAgainForParked(client);
2510
2952
  this.retryParkedForWorkspaces();
2511
2953
  if (this.heldRestorePending && Date.now() - this.heldRestoreAt >= (this.opts.sandboxRecheckMs ?? 5 * 60_000))
2512
2954
  await this.restoreHeldContinuations(client);
@@ -2520,12 +2962,20 @@ export class Worker {
2520
2962
  this.sayKeyChanges(client);
2521
2963
  this.syncSignedOut();
2522
2964
  await this.settleMoot(client);
2965
+ await this.settleClosedQuestions(client);
2966
+ await this.settleSafety(client);
2523
2967
  const aged = client
2524
2968
  .myWork()
2525
2969
  .filter((o) => o.kind !== "HUMAN" && o.kind !== "DELIVER" && !o.claimed_by)
2526
2970
  .filter((o) => Date.now() - o.created_at >= grace)
2527
2971
  .filter((o) => { const t = this.outageHold.get(o.id); if (t === undefined)
2528
2972
  return true; if (t > Date.now()) {
2973
+ if (this.allOutSignedOut()) {
2974
+ if (!this.tickHold)
2975
+ this.tickHold = { reason: "not_signed_in", detail: "its agent is signed out" };
2976
+ this.decline(o.ref, "your agent is signed out");
2977
+ return false;
2978
+ }
2529
2979
  if (!this.tickHold || (this.tickHold.reason === "outage" && (this.tickHold.until ?? Infinity) > t))
2530
2980
  this.tickHold = { reason: "outage", until: t, detail: "its agents are out for now" };
2531
2981
  this.decline(o.ref, "your agents are out for now");
@@ -2551,7 +3001,9 @@ export class Worker {
2551
3001
  for (let i = start; i < aged.length; i++) {
2552
3002
  const o = aged[i];
2553
3003
  this.loadContinued();
2554
- if (this.contDone.has(`handled:${o.ref}`) || this.continuedHas(`handled:${o.ref}`)) {
3004
+ if (this.answeredHere(o.ref))
3005
+ await this.reopenDecided(client, o);
3006
+ if (this.answeredHere(o.ref)) {
2555
3007
  if (!this.skipLogged.has(`handled:${o.id}`)) {
2556
3008
  this.skipLogged.add(`handled:${o.id}`);
2557
3009
  this.opts.log(`- not taking ${owedLabel(o.kind)} ${o.id}: this machine already answered the request it points at (${o.ref})`);
@@ -2734,7 +3186,9 @@ export class Worker {
2734
3186
  const slot = this.takeSlot();
2735
3187
  if ("held" in slot) {
2736
3188
  this.writeState("watching", `waiting: ${slot.held}`);
2737
- this.tickHold = { reason: "busy_elsewhere", detail: "another run is in progress on this machine" };
3189
+ this.tickHold = slot.held === PERSON_ANSWERING
3190
+ ? { reason: "recently_active", detail: PERSON_ANSWERING, until: Date.now() + PERSON_HOLD_TTL_MS }
3191
+ : { reason: "busy_elsewhere", detail: "another run is in progress on this machine" };
2738
3192
  if (continuation)
2739
3193
  this.requeueContinuation(continuation, 5_000);
2740
3194
  continue;
@@ -2820,6 +3274,9 @@ export class Worker {
2820
3274
  this.report("idle");
2821
3275
  if (candidate && job.ok)
2822
3276
  this.markDone(`handled:${candidate.ref}`);
3277
+ if (candidate)
3278
+ for (const id of job.escalated ?? [])
3279
+ this.markDone(`${ESCALATED_FOR}${id}:${candidate.ref}`);
2823
3280
  const ran = candidate ? candidateAct : continuation;
2824
3281
  if (job.ok && ran?.sig) {
2825
3282
  try {
@@ -2870,7 +3327,7 @@ export class Worker {
2870
3327
  if (candidate)
2871
3328
  this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer" : "nothing on this computer can run it while you're away");
2872
3329
  if (candidate) {
2873
- this.failedRuns.set(candidate.id, 2);
3330
+ this.failedRuns.set(this.oKey(candidate.id), 2);
2874
3331
  this.skipLogged.add(`parked:${candidate.id}`);
2875
3332
  if (job.diagnosis.cause === "repo_not_here")
2876
3333
  this.parkedNotHere.add(candidate.id);
@@ -2885,15 +3342,18 @@ export class Worker {
2885
3342
  const attempts = this.failuresOf(candidate.id) + 1;
2886
3343
  const failures = job.diagnosis?.cause === "no_commands" || job.diagnosis?.cause === "no_sandbox_commands" ? 2 : attempts;
2887
3344
  if (job.diagnosis?.cause === "no_sandbox_commands")
2888
- this.markDone(`parked-ns:${candidate.id}`);
2889
- if (this.contDone.has(`unparked-ns:${candidate.id}`))
2890
- this.markDone(`parked2-ns:${candidate.id}`);
2891
- this.failedRuns.set(candidate.id, failures);
2892
- this.markDone(failures >= 2 ? `parked:${candidate.id}` : `failed1:${candidate.id}`);
2893
- if (this.contDone.has(`unparked:${candidate.id}`))
2894
- this.markDone(`parked2:${candidate.id}`);
2895
- else if (job.diagnosis?.cause === "no_commands")
2896
- this.markDone(`parked-nc:${candidate.id}`);
3345
+ this.markFor("parked-ns", candidate.id);
3346
+ if (this.marked("unparked-ns", candidate.id))
3347
+ this.markFor("parked2-ns", candidate.id);
3348
+ this.failedRuns.set(this.oKey(candidate.id), failures);
3349
+ this.markFor(failures >= 2 ? "parked" : "failed1", candidate.id);
3350
+ const under = this.unparkedUnder.get(this.oKey(candidate.id));
3351
+ if (under)
3352
+ this.markDone(`parked2@${under}:${this.oKey(candidate.id)}`);
3353
+ else if (this.marked("unparked", candidate.id))
3354
+ this.markFor("parked2", candidate.id);
3355
+ if (job.diagnosis?.cause === "no_commands")
3356
+ this.markFor("parked-nc", candidate.id);
2897
3357
  if (this.failedRuns.size > 200) {
2898
3358
  this.failedRuns.delete(this.failedRuns.keys().next().value);
2899
3359
  }
@@ -2905,6 +3365,22 @@ export class Worker {
2905
3365
  }
2906
3366
  finally {
2907
3367
  this.running = false;
3368
+ if (this.stopped) {
3369
+ if (continuation) {
3370
+ try {
3371
+ await client.releaseClaim(`cont:${continuation.id}`);
3372
+ }
3373
+ catch { }
3374
+ }
3375
+ else if (candidate) {
3376
+ this.deadClaims.set(candidate.id, { session: null, until: Date.now() + LIMITS.claimTtlMs + 60_000 });
3377
+ for (let i = 0; i < 12 && this.deadClaims.has(candidate.id); i++) {
3378
+ await this.releaseDeadClaims(client);
3379
+ if (this.deadClaims.has(candidate.id))
3380
+ await new Promise((r) => setTimeout(r, 250));
3381
+ }
3382
+ }
3383
+ }
2908
3384
  this.endAttempt();
2909
3385
  this.giveSlot(slot.token);
2910
3386
  if (continuation)
@@ -2936,7 +3412,7 @@ export class Worker {
2936
3412
  return { held: why };
2937
3413
  }
2938
3414
  if ("held" in lease)
2939
- return { held: `a run for "${lease.held}" is in progress in another pantheon worker on this machine` };
3415
+ return { held: lease.held === PERSON_ANSWERING ? PERSON_ANSWERING : `a run for "${lease.held}" is in progress in another pantheon worker on this machine` };
2940
3416
  fleetLock = this;
2941
3417
  const dir = this.stateDir(), token = lease.token;
2942
3418
  this.heartbeatAt = Date.now();
@@ -2966,13 +3442,13 @@ export class Worker {
2966
3442
  liveRuns.add(pid);
2967
3443
  installRunReaper();
2968
3444
  this.attemptPid = pid;
2969
- this.runRecord = { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, ...(workerIdentity() ? { worker_id: workerIdentity() } : {}), obligation: attempt.obligation, ...(attempt.ref ? { ref: attempt.ref } : {}), ...(attempt.spend ? { spend: attempt.spend } : {}), ...(attempt.limit ? { limit: attempt.limit } : {}), ...(attempt.sandbox ? { sandbox: attempt.sandbox } : {}) };
3445
+ this.runRecord = { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, ...(workerIdentity() ? { worker_id: workerIdentity() } : {}), obligation: attempt.obligation, ...(this.oKey(attempt.obligation) !== attempt.obligation ? { okey: this.oKey(attempt.obligation) } : {}), ...(attempt.ref ? { ref: attempt.ref } : {}), ...(attempt.spend ? { spend: attempt.spend } : {}), ...(attempt.limit ? { limit: attempt.limit } : {}), ...(attempt.sandbox ? { sandbox: attempt.sandbox } : {}) };
2970
3446
  writeRunRecord(this.stateDir(), this.runRecord);
2971
3447
  }
2972
3448
  runRecord = null;
2973
3449
  recordChanged = false;
2974
3450
  recordOnDisk() { try {
2975
- return readFileSync(join(this.stateDir(), RUN_RECORD), "utf8");
3451
+ return readStateText(join(this.stateDir(), RUN_RECORD));
2976
3452
  }
2977
3453
  catch {
2978
3454
  return null;
@@ -3005,8 +3481,20 @@ export class Worker {
3005
3481
  this.attemptPid = null;
3006
3482
  this.runRecord = null;
3007
3483
  }
3008
- recoverGuard() {
3009
- const r = recoverInterruptedGuard(this.stateDir());
3484
+ guardWaiting = false;
3485
+ guardRetryAt = 0;
3486
+ async recoverGuard() {
3487
+ const g = loadPersistedGuard(this.stateDir());
3488
+ const v = g?.own && g.snapshot && !sameProcess(g.worker, g.workerId) ? await this.verifyOwn(g.own) : undefined;
3489
+ if (v === "unreachable") {
3490
+ if (!this.guardWaiting)
3491
+ this.opts.log(" guard a run was interrupted with its worker; putting Pantheon's files back waits until the relay can be asked — unattended runs wait meanwhile");
3492
+ this.guardWaiting = true;
3493
+ this.guardRetryAt = Date.now() + 30_000;
3494
+ return;
3495
+ }
3496
+ this.guardWaiting = false;
3497
+ const r = recoverInterruptedGuard(this.stateDir(), Date.now(), restoreHome, v);
3010
3498
  if (r.outcome === "none" || r.outcome === "live")
3011
3499
  return;
3012
3500
  const tell = (heading, body) => {
@@ -3021,6 +3509,12 @@ export class Worker {
3021
3509
  this.opts.log(` guard a snapshot from a run on ${new Date(r.at).toLocaleString()} was not restored — ${r.outcome === "stale" ? "too old to be that run's" : "you have changed your setup since"}; dropped`);
3022
3510
  return;
3023
3511
  }
3512
+ if (r.outcome === "pending") {
3513
+ for (const l of r.said)
3514
+ this.opts.log(`! the run interrupted with its worker: put back ${l}`);
3515
+ this.pendingRestore = { snap: r.snapshot, rels: r.rels };
3516
+ return;
3517
+ }
3024
3518
  if (r.outcome === "failed") {
3025
3519
  this.opts.log(`! could not put Pantheon's own files back after the interrupted run (try ${r.attempts} of ${GUARD_RESTORE_TRIES}): ${r.why}`);
3026
3520
  if (r.dropped)
@@ -3035,11 +3529,11 @@ export class Worker {
3035
3529
  this.opts.log(`! the run interrupted with its worker: put back ${l}`);
3036
3530
  tell("Your worker stopped mid-run — settings put back", `${putBackSaid(r.said)} If you made that change yourself, make it again.`);
3037
3531
  }
3038
- recoverInterruptedRun() {
3532
+ async recoverInterruptedRun(client) {
3039
3533
  const dir = this.stateDir(), f = join(dir, RUN_RECORD);
3040
3534
  let r;
3041
3535
  try {
3042
- r = JSON.parse(readFileSync(f, "utf8"));
3536
+ r = JSON.parse(readStateText(f));
3043
3537
  }
3044
3538
  catch {
3045
3539
  return;
@@ -3059,21 +3553,81 @@ export class Worker {
3059
3553
  rmSync(f, { force: true });
3060
3554
  }
3061
3555
  catch { }
3556
+ const replied = r.obligation.startsWith("cont:") ? null : await this.runReplied(client, r);
3062
3557
  if (typeof r.spend === "string") {
3558
+ const summary = replied ? "answered (its worker went before the run ended)" : "the worker went before the run ended";
3559
+ try {
3560
+ closeStartedRun(r.spend, summary, { at: r.at, goal: null, obligation: r.obligation, ok: !!replied, triggeredBy: "unknown", summary, space: this.spaceKey(), member: r.member }, this.opts.home, !!replied);
3561
+ }
3562
+ catch { }
3563
+ }
3564
+ if (r.obligation.startsWith("cont:")) {
3063
3565
  try {
3064
- closeStartedRun(r.spend, "the worker went before the run ended", { at: r.at, goal: null, obligation: r.obligation, ok: false, triggeredBy: "unknown", summary: "the worker went before the run ended", space: this.spaceKey(), member: r.member }, this.opts.home);
3566
+ await client.releaseClaim(r.obligation);
3065
3567
  }
3066
3568
  catch { }
3067
3569
  }
3570
+ else {
3571
+ const held = client.obligations.find((o) => o.id === r.obligation || (typeof r.okey === "string" && `r:${o.kind}:${o.ref}` === r.okey));
3572
+ if (held) {
3573
+ this.deadClaims.set(held.id, { session: null, until: Date.now() + LIMITS.claimTtlMs + 60_000 });
3574
+ await this.releaseDeadClaims(client);
3575
+ }
3576
+ }
3068
3577
  if (r.obligation.startsWith("cont:")) {
3069
3578
  this.opts.log(`! the last run (${r.obligation}) ended with its worker — counted, not run again`);
3070
3579
  return;
3071
3580
  }
3581
+ if (replied) {
3582
+ if (r.ref)
3583
+ this.markDone(`handled:${r.ref}`);
3584
+ this.opts.log(` the last run for ${r.obligation} sent its ${replied} before its worker went — counted as handled`);
3585
+ return;
3586
+ }
3587
+ if (typeof r.okey === "string" && r.okey.startsWith("r:"))
3588
+ this.obKeys.set(r.obligation, r.okey);
3072
3589
  const failures = this.failuresOf(r.obligation) + 1;
3073
- this.failedRuns.set(r.obligation, failures);
3074
- this.markDone(failures >= 2 ? `parked:${r.obligation}` : `failed1:${r.obligation}`);
3590
+ this.failedRuns.set(this.oKey(r.obligation), failures);
3591
+ this.markFor(failures >= 2 ? "parked" : "failed1", r.obligation);
3075
3592
  this.opts.log(`! the last run for ${r.obligation} ended with its worker (stopped, restarted or killed mid-run) — counted as a failed attempt${failures >= 2 ? "; parked: it is not run again" : ""}`);
3076
3593
  }
3594
+ async runReplied(client, r) {
3595
+ const ref = r.ref, at = r.at;
3596
+ if (typeof ref !== "string" || typeof at !== "number")
3597
+ return null;
3598
+ if (client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" && (o.id === r.obligation || o.ref === ref)))
3599
+ return null;
3600
+ try {
3601
+ const res = await signedRead(this.cfg, "/history", {}, { timeoutMs: 5000 });
3602
+ if (!res.ok)
3603
+ return null;
3604
+ const j = await res.json();
3605
+ const a = (j.acts ?? []).find((x) => x.ref === ref && x.from === this.cfg.member && x.ts >= at && x.sig?.role === "run" && x.act !== "NOTE" && x.act !== "RETRACT");
3606
+ return a ? a.act : null;
3607
+ }
3608
+ catch {
3609
+ return null;
3610
+ }
3611
+ }
3612
+ deadClaims = new Map();
3613
+ async releaseDeadClaims(client) {
3614
+ for (const [id, d] of [...this.deadClaims]) {
3615
+ const by = client.obligations.find((o) => o.id === id)?.claimed_by;
3616
+ if (Date.now() > d.until || !by || by.role !== "worker" || by.session === client.session || (d.session !== null && by.session !== d.session)) {
3617
+ this.deadClaims.delete(id);
3618
+ continue;
3619
+ }
3620
+ d.session = by.session;
3621
+ if (client.roster.some((m) => m.sessions.some((s) => s.id === by.session)))
3622
+ continue;
3623
+ this.deadClaims.delete(id);
3624
+ try {
3625
+ if (await client.releaseClaim(id))
3626
+ this.opts.log(` let go of ${id}: the run that held it went with its worker — it is picked up again now, not in ${Math.round(LIMITS.claimTtlMs / 60_000)} minutes`);
3627
+ }
3628
+ catch { }
3629
+ }
3630
+ }
3077
3631
  stateDir() {
3078
3632
  return this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
3079
3633
  }
@@ -3107,7 +3661,7 @@ export class Worker {
3107
3661
  }
3108
3662
  brokenStateFiles() {
3109
3663
  repairRunLease(this.stateDir());
3110
- return brokenStateFiles(this.stateDir());
3664
+ return brokenStateFiles(this.stateDir(), GUARDED_FILES);
3111
3665
  }
3112
3666
  hooksInstalledHere() {
3113
3667
  try {
@@ -3128,7 +3682,8 @@ export class Worker {
3128
3682
  : `(the triggering act ${printable(o.ref)} is not in this session's buffer; the relay's one-line summary of it follows)\n${o.body}`;
3129
3683
  const act = (base ?? undefined);
3130
3684
  const needs = workNeedsCommands(o.kind, trigger, o.kind === "REVISE" && act?.ref ? client.seenAct(act.ref)?.act : act?.act);
3131
- return this.runJob(client, o, persona, policy, (branch, canPush) => buildBrief(o, trigger, branch, persona, canPush), true, needs, trigger, act);
3685
+ const decided = await this.decisionFor(client, o);
3686
+ return this.runJob(client, o, persona, policy, (branch, canPush) => buildBrief(o, trigger, branch, persona, canPush, decided), true, needs, trigger, act);
3132
3687
  }
3133
3688
  async handleContinuation(client, act, persona, policy) {
3134
3689
  this.markDone(act.id);
@@ -3146,7 +3701,7 @@ export class Worker {
3146
3701
  }
3147
3702
  spentOnce = {};
3148
3703
  policyBytes() { try {
3149
- return readFileSync(policyPath(this.opts.home), "utf8");
3704
+ return readStateText(policyPath(this.opts.home));
3150
3705
  }
3151
3706
  catch {
3152
3707
  return null;
@@ -3543,6 +4098,7 @@ export class Worker {
3543
4098
  }
3544
4099
  let detail;
3545
4100
  let transcript = "";
4101
+ resetOwn();
3546
4102
  const homeBefore = snapshotHome(this.stateDir());
3547
4103
  const guardHome = this.stateDir();
3548
4104
  try {
@@ -3658,7 +4214,7 @@ export class Worker {
3658
4214
  this.cleanupStep("checking Pantheon's own files", () => {
3659
4215
  const ok = this.restoreHomeAfterRun(homeBefore, o);
3660
4216
  pendingGuards.delete(guardHome);
3661
- if (ok)
4217
+ if (ok && !this.pendingRestore)
3662
4218
  clearPersistedGuard(guardHome);
3663
4219
  this.markRunRestored();
3664
4220
  });
@@ -3701,7 +4257,10 @@ export class Worker {
3701
4257
  const held = await this.settleHeld(client, heldRun ? { run: heldRun, canPush: canPush && !massDelete, massDelete } : null);
3702
4258
  const settled = [...held.sent, ...held.waiting];
3703
4259
  const settledIds = new Set(settled.map((a) => a.id));
3704
- const sent = [...client.mineSince(startedAt, o.goal).filter((a) => !settledIds.has(a.id)), ...settled.filter((a) => o.goal == null || a.goal === o.goal)];
4260
+ const runId = this.runKey?.runId;
4261
+ const elsewhere = o.goal == null ? [] : client.mineSince(startedAt).filter((a) => a.goal !== o.goal && !settledIds.has(a.id)
4262
+ && ((!!o.ref && a.ref === o.ref) || (!!runId && a.goal == null && a.sig?.role === "run" && a.sig.cert?.run === runId)));
4263
+ const sent = [...client.mineSince(startedAt, o.goal).filter((a) => !settledIds.has(a.id)), ...elsewhere, ...settled.filter((a) => o.goal == null || a.goal === o.goal)];
3705
4264
  const repliedWith = sent.find((a) => ["DELIVERED", "ANSWER", "ESCALATE", "ACCEPT", "OBJECT", "PROPOSE", "CONTRACT", "CLOSE", "WITHDRAW"].includes(a.act) || (a.act === "NOTE" && !!a.ref));
3706
4265
  const replied = requireReply ? client.repliedTo(o) || !!repliedWith : true;
3707
4266
  if (requireReply && replied && !client.repliedTo(o) && repliedWith) {
@@ -3748,9 +4307,10 @@ export class Worker {
3748
4307
  const sentAnywhere = client.mineSince(startedAt).some((a) => !runKeys || !a.sig || a.sig.role === "run");
3749
4308
  const outage = !ok && !exited && !sentAnywhere && choice && !/\(stopped after \d+s\)/.test(detail.split("\n")[0] ?? "") ? detectOutage(choice.backend.id, detail) : null;
3750
4309
  const refusedLog = snap ? join(scratch, `${basename(worktree)}.${choice?.backend.id}-refused.run.log`) : join(worktree, `.pantheon-run.${choice?.backend.id}-refused.log`);
4310
+ const hintLed = outage?.why === SIGNIN_REFUSED && !this.opts.agentCmd && signedOutChecking(choice.backend.id, { pantheonHome: this.stateDir() })?.lifted === "hint";
3751
4311
  if (outage) {
3752
4312
  this.outages.set(choice.backend.id, outage);
3753
- if (outage.why === SIGNIN_REFUSED && !this.opts.agentCmd && markSignedOut(choice.backend.id, detail, { pantheonHome: this.stateDir() })) {
4313
+ if (outage.why === SIGNIN_REFUSED && !this.opts.agentCmd && markSignedOut(choice.backend.id, detail, { pantheonHome: this.stateDir(), env: process.env })) {
3754
4314
  this.syncSignedOut();
3755
4315
  this.readinessAt = 0;
3756
4316
  }
@@ -3758,22 +4318,26 @@ export class Worker {
3758
4318
  if (Date.now() - startedAt < ONCE_GIVE_BACK_MS)
3759
4319
  this.restoreOnces();
3760
4320
  }
3761
- if (outage && (this.outageRetries.get(o.id) ?? 0) < 3) {
3762
- this.outageRetries.set(o.id, (this.outageRetries.get(o.id) ?? 0) + 1);
4321
+ if (outage && (hintLed || (this.outageRetries.get(o.id) ?? 0) < 3)) {
4322
+ if (!hintLed)
4323
+ this.outageRetries.set(o.id, (this.outageRetries.get(o.id) ?? 0) + 1);
3763
4324
  if (this.outageRetries.size > 200)
3764
4325
  this.outageRetries.delete(this.outageRetries.keys().next().value);
3765
- this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over until about ${whenAgain(outage.until)}; the request goes to the next agent here (transcript: ${refusedLog})`);
4326
+ this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over ${outage.why === SIGNIN_REFUSED ? "until it is signed in again" : `until about ${whenAgain(outage.until)}`}; the request goes to the next agent here (transcript: ${refusedLog})`);
3766
4327
  return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
3767
4328
  }
3768
4329
  const quiet = ok && !requireReply && sent.length === 0;
3769
4330
  const diagnosis = ok ? undefined : outage ? this.outageDiagnosis(choice.backend.label, outage, [], "it was handed to another agent three times already") : diagnoseRun({ exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands,
4331
+ ...(agentId === "claude-code" && !this.opts.agentCmd && claudeTokenForRun(this.stateDir())
4332
+ ? { env: { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: undefined }, savedToken: true } : { env: process.env }),
4333
+ ...(agentId === "claude-code" ? { claudeSettings: claudeSettingsAuth({ env: process.env }) } : {}),
3770
4334
  ...(/\(stopped after \d+s\)/.test(detail.split("\n")[0] ?? "") ? { timedOut: { minutes: runLimit.minutes, from: runLimit.from } } : {}),
3771
4335
  ...(bashDroppedForSandbox ? { commandsNeedSandbox: byCli ? "agent-cmd" : "claude" } : {}), ...(agentId ? { agent: agentId } : {}), ...(place.kind === "folder" ? { folder: true } : {}),
3772
4336
  ...(agentId === "claude-code" && allowBash ? { allowedRules: bashRules } : {}), ...(baseRef ? { baseRef } : {}) }, this.opts.probe ?? gitProbe);
3773
4337
  if (ok && agentId && !this.opts.agentCmd)
3774
4338
  clearSignedOut(agentId, { pantheonHome: this.stateDir() });
3775
4339
  if (!ok && diagnosis?.cause === "not_signed_in" && agentId && !this.opts.agentCmd
3776
- && markSignedOut(agentId, detail, { pantheonHome: this.stateDir() })) {
4340
+ && markSignedOut(agentId, detail, { pantheonHome: this.stateDir(), env: process.env })) {
3777
4341
  this.opts.log(`! ${choice?.backend.label ?? agentId} is signed out on this machine: requests wait until it is signed in again${agentId === "claude-code" ? " (pantheon worker token keeps Claude Code signed in for a year)" : ""}`);
3778
4342
  this.syncSignedOut();
3779
4343
  this.readinessAt = 0;
@@ -3788,7 +4352,8 @@ export class Worker {
3788
4352
  this.opts.log(quiet ? `< read ${o.id} on ${branch} — the thread was already settled, nothing sent`
3789
4353
  : ok ? `< handled ${o.id} on ${branch}`
3790
4354
  : `! ${o.id} stopped [${diagnosis.cause}]: ${diagnosis.says} (transcript: ${runLog})`);
3791
- return { obligation: o, branch, worktree, ok, detail, ...(diagnosis ? { diagnosis } : {}) };
4355
+ const escalated = sent.filter((a) => a.act === "ESCALATE" && !a.ref && a.from === this.cfg.member && (a.sig?.role === "run" || a.role === "worker")).map((a) => a.id);
4356
+ return { obligation: o, branch, worktree, ok, detail, ...(diagnosis ? { diagnosis } : {}), ...(escalated.length ? { escalated } : {}) };
3792
4357
  }
3793
4358
  catch (e) {
3794
4359
  if (!recorded) {
@@ -3817,7 +4382,7 @@ export class Worker {
3817
4382
  restoreHomeAfterRun(before, o) {
3818
4383
  let changes = [];
3819
4384
  try {
3820
- changes = restoreHome(before);
4385
+ changes = (this.opts.restore ?? restoreHome)(before);
3821
4386
  }
3822
4387
  catch (e) {
3823
4388
  this.opts.log(`! could not check Pantheon's own files after the run: ${e.message}`);
@@ -3825,6 +4390,14 @@ export class Worker {
3825
4390
  }
3826
4391
  if (!changes.length)
3827
4392
  return true;
4393
+ const failed = changes.filter((c) => c.failed).map((c) => c.rel);
4394
+ if (failed.length) {
4395
+ this.pendingRestore = { snap: { ...before, files: before.files.filter((f) => failed.includes(f.rel)) }, rels: failed };
4396
+ try {
4397
+ persistPending(this.stateDir(), failed);
4398
+ }
4399
+ catch { }
4400
+ }
3828
4401
  for (const c of changes)
3829
4402
  this.opts.log(`${c.what === "reported" || c.quiet ? " " : "!"} ${o.id}: ${describeChange(c)}`);
3830
4403
  const taken = changes.filter((c) => c.what !== "reported" && !c.quiet);
@@ -4120,7 +4693,7 @@ export class Worker {
4120
4693
  if (this.running || !client.connected)
4121
4694
  return;
4122
4695
  try {
4123
- const r = JSON.parse(readFileSync(join(this.stateDir(), RUN_RECORD), "utf8"));
4696
+ const r = JSON.parse(readStateText(join(this.stateDir(), RUN_RECORD)));
4124
4697
  if (r && r.space === this.cfg.space && Number.isInteger(r.worker) && r.worker !== process.pid && pidAlive(r.worker))
4125
4698
  return;
4126
4699
  }
@@ -4197,9 +4770,11 @@ export class Worker {
4197
4770
  }
4198
4771
  if (!owed)
4199
4772
  return;
4200
- const key = `refused:${o.id}`;
4773
+ this.learnRequests([o]);
4774
+ const key = `refused:${this.oKey(o.id)}`;
4201
4775
  const space = this.spaceKey();
4202
- if (loadContinued(space, this.opts.home).has(key))
4776
+ const told = loadContinued(space, this.opts.home);
4777
+ if (told.has(key) || told.has(`refused:${o.id}`))
4203
4778
  return;
4204
4779
  const peer = client.recentFrom(o.ref);
4205
4780
  try {
@@ -4226,7 +4801,7 @@ export class Worker {
4226
4801
  if (!client.connected)
4227
4802
  return;
4228
4803
  const policy = loadPolicy(this.opts.home);
4229
- const teams = [this.cfg.spaceId, this.cfg.space].filter((t) => !!t);
4804
+ const teams = this.cfg.spaceId ? [this.cfg.spaceId] : [this.cfg.space];
4230
4805
  const reports = [
4231
4806
  ...Object.entries(policy.stuckReports ?? {}).filter(([, r]) => teams.includes(r.team))
4232
4807
  .map(([act, r]) => [act, [r.ref], "stuck"]),
@@ -4240,31 +4815,43 @@ export class Worker {
4240
4815
  return;
4241
4816
  const now = Date.now();
4242
4817
  const done = [];
4818
+ const waitingHere = (ref) => client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" && o.ref === ref);
4819
+ const stillNeeds = (cap, except) => Object.entries(policy.stuckReports ?? {}).some(([a, r]) => a !== except && r.cap === cap && (teams.includes(r.team) ? waitingHere(r.ref) : true)) ||
4820
+ (cap === "run_commands" && client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" &&
4821
+ !(policy.stuckReports?.[except]?.ref === o.ref) && this.marked("parked-nc", o.id) && this.failuresOf(o.id) >= 2));
4243
4822
  for (const [act, refs, kind] of reports) {
4244
4823
  const member = kind === "wake" ? policy.wakeAsks?.[act]?.member : undefined;
4245
4824
  const owed = client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" &&
4246
4825
  (refs.includes(o.ref) || (!!member && (client.seenAct(o.ref)?.from ?? /^([a-z0-9._-]+)\s/i.exec(o.body)?.[1]) === member)));
4247
- if (owed) {
4248
- this.mootSince.delete(act);
4249
- continue;
4250
- }
4251
- const since = this.mootSince.get(act);
4252
- if (since === undefined) {
4253
- this.mootSince.set(act, now);
4254
- continue;
4826
+ const asksFor = kind === "stuck" ? Object.entries(policy.asks).find(([, a]) => a?.act === act)?.[0] : undefined;
4827
+ const settled = member ? wakeSettled(policy, [this.cfg.spaceId, this.cfg.space], member) : null;
4828
+ if (!settled) {
4829
+ if (owed || (asksFor && stillNeeds(asksFor, act))) {
4830
+ this.mootSince.delete(act);
4831
+ continue;
4832
+ }
4833
+ const since = this.mootSince.get(act);
4834
+ if (since === undefined) {
4835
+ this.mootSince.set(act, now);
4836
+ continue;
4837
+ }
4838
+ if (now - since < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
4839
+ continue;
4255
4840
  }
4256
- if (now - since < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
4257
- continue;
4258
4841
  if ((this.mootRetryAt.get(act) ?? 0) > now)
4259
4842
  continue;
4260
4843
  const open = client.obligations.some((o) => o.kind === "HUMAN" && o.ref === act);
4261
4844
  if (open) {
4262
4845
  try {
4263
4846
  await client.send({ act: "RETRACT", goal: null, ref: act, after_seq: Math.max(client.readSeq, client.seq),
4264
- body: kind === "wake"
4265
- ? "No longer needed: the request this was about isn't waiting here any more (it was answered, withdrawn or closed). Nothing was changed on this computer — you'll be asked again if they need it."
4266
- : "No longer needed: the request this was about isn't waiting here any more (it was answered, withdrawn or closed). Nothing was changed on this computer." });
4267
- this.opts.log(`* withdrew ${act}: the request${refs.length > 1 ? "s" : ""} it was about (${refs.join(", ")}) ${refs.length > 1 ? "are" : "is"} finished`);
4847
+ body: settled
4848
+ ? `No longer needed: settled on this computer — ${settled === "yes" ? `${member} may start work here (pantheon wake allow ${member})` : `no to unattended runs from ${member} here (pantheon wake deny ${member})`}.`
4849
+ : kind === "wake"
4850
+ ? "No longer needed: the request this was about isn't waiting here any more (it was answered, withdrawn or closed). Nothing was changed on this computer — you'll be asked again if they need it."
4851
+ : "No longer needed: the request this was about isn't waiting here any more (it was answered, withdrawn or closed). Nothing was changed on this computer." });
4852
+ this.opts.log(settled
4853
+ ? `* withdrew ${act}: you settled it here (pantheon wake ${settled === "yes" ? "allow" : "deny"} ${member})`
4854
+ : `* withdrew ${act}: the request${refs.length > 1 ? "s" : ""} it was about (${refs.join(", ")}) ${refs.length > 1 ? "are" : "is"} finished`);
4268
4855
  }
4269
4856
  catch (e) {
4270
4857
  this.mootRetryAt.set(act, now + 5 * 60_000);
@@ -4280,6 +4867,7 @@ export class Worker {
4280
4867
  return;
4281
4868
  const fresh = loadPolicy(this.opts.home);
4282
4869
  for (const [act, kind] of done) {
4870
+ withdrawnHere.add(act);
4283
4871
  if (kind === "wake")
4284
4872
  delete fresh.wakeAsks?.[act];
4285
4873
  else {
@@ -4291,11 +4879,28 @@ export class Worker {
4291
4879
  }
4292
4880
  savePolicy(fresh, this.opts.home);
4293
4881
  }
4294
- async reportStuck(client, o, d, attempts) {
4295
- const key = `stuck:${o.id}`;
4882
+ safetyDue = new Map();
4883
+ stuckDue = new Map();
4884
+ async ownOnRecord(client, id, waitMs) {
4885
+ const r = (await client.seenOwn(id, waitMs).catch(() => null)) ?? (await client.storedAct(id).catch(() => null));
4886
+ return r ? { id: r.id, seq: r.seq } : null;
4887
+ }
4888
+ safetyRaised = new Map();
4889
+ async reportStuck(client, o, d, attempts, again = 0, known) {
4890
+ this.learnRequests([o]);
4891
+ const safety = SAFETY_CAUSES.has(d.cause);
4892
+ const key = safety ? `stuck-safety:${this.oKey(o.id)}:${d.cause}` : `stuck:${this.oKey(o.id)}`;
4296
4893
  const space = this.spaceKey();
4297
- if (loadContinued(space, this.opts.home).has(key))
4894
+ const said = loadContinued(space, this.opts.home);
4895
+ if (!again && (said.has(key) || (!safety && said.has(`stuck:${o.id}`))))
4298
4896
  return;
4897
+ if (safety && !again) {
4898
+ this.safetyDue.set(key, { o, d });
4899
+ try {
4900
+ markContinued(space, `stuck-safety-due:${this.oKey(o.id)}:${d.cause}`, this.opts.home);
4901
+ }
4902
+ catch { }
4903
+ }
4299
4904
  const policy = loadPolicy(this.opts.home);
4300
4905
  const asking = !!d.capability && mayAsk(policy, d.capability);
4301
4906
  const what = clip(plainRequest(o.body, this.cfg.member), 160);
@@ -4309,27 +4914,217 @@ export class Worker {
4309
4914
  ...(d.dir && d.lines?.length ? [``, `On this computer, in ${d.dir}:`, ...d.lines.map((l) => ` ${l}`)] : []),
4310
4915
  ``,
4311
4916
  `The request is still open, and the teammate who asked hasn't been told. It won't be retried on its own.`,
4917
+ ...(again ? [``, safety ? `Raised again: the earlier warning about this was cleared without your answer.`
4918
+ : `Asked again: the "just this once" went to another request, and this one still needs commands.`] : []),
4312
4919
  ];
4313
4920
  if (asking) {
4314
4921
  lines.push(``, `Also: may ${this.cfg.member}'s agent run commands on this computer (tests, commit, push)? Answer "allow always", "allow once", or "deny" — in the menu bar, or with pantheon approvals.`, `Today it may only edit files, which is why it couldn't commit or push. Allowing it means: ${COMMAND_GRANT_SCOPE}`, `Answer on this computer: an answer from anywhere else settles this but grants nothing.`);
4315
4922
  }
4316
4923
  try {
4317
- const sent = await client.send({
4318
- act: "ESCALATE", goal: o.goal, ref: o.ref, after_seq: Math.max(client.readSeq, client.seq),
4924
+ const fit = (t) => t.length <= LIMITS.bodyChars - 100 ? t : `${t.slice(0, LIMITS.bodyChars - 300)}\n\n(cut here: too long for one message — the rest is on this computer)`;
4925
+ const frame = (goal, short = false, ref = o.ref || null) => ({
4926
+ act: "ESCALATE", goal, ref, after_seq: Math.max(client.readSeq, client.seq),
4319
4927
  summary: clip(this.ledgerPaths(`${this.cfg.member}'s agent is stuck: ${d.says}`), 190),
4320
- body: this.ledgerPaths(lines.join("\n")),
4928
+ body: fit(this.ledgerPaths(short ? [lines[0], ``, `Why: ${clip(d.says, 600)}`, `What to do: ${clip(d.fix, 600)}`].join("\n") : lines.join("\n"))),
4929
+ ...(safety ? { meta: { safety_stop: d.cause } } : {}),
4930
+ });
4931
+ const passing = (m) => !/^[A-Z_]+:/.test(m) || /^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(m);
4932
+ const before = known ? await this.ownOnRecord(client, known, 500) : null;
4933
+ const sent = before ?? await client.send(frame(o.goal)).catch((e) => {
4934
+ if (!safety || passing(e.message))
4935
+ return Promise.reject(e);
4936
+ return client.send(frame(null, false, null)).catch((e2) => passing(e2.message) ? Promise.reject(e2) : client.send(frame(null, true, null)));
4321
4937
  });
4322
4938
  markContinued(space, key, this.opts.home);
4939
+ this.stuckDue.delete(key);
4940
+ if (safety) {
4941
+ this.safetyDue.delete(key);
4942
+ this.safetyRaised.set(sent.id, { o, d, seq: sent.seq, again });
4943
+ }
4323
4944
  const fresh = loadPolicy(this.opts.home);
4324
- if (asking && d.capability)
4945
+ if (asking && d.capability) {
4325
4946
  recordAsk(fresh, d.capability, sent.id);
4326
- if (o.kind !== "CONTINUE" && !SAFETY_CAUSES.has(d.cause))
4327
- (fresh.stuckReports ??= {})[sent.id] = { ref: o.ref, team: this.cfg.spaceId ?? this.cfg.space, at: Date.now() };
4947
+ askedHere.cap.set(d.capability, canon(fresh.asks[d.capability]));
4948
+ }
4949
+ if (o.kind !== "CONTINUE" && !safety) {
4950
+ const rec = { ref: o.ref, team: this.cfg.spaceId ?? this.cfg.space, at: Date.now(), ...(d.capability ? { cap: d.capability } : {}) };
4951
+ (fresh.stuckReports ??= {})[sent.id] = rec;
4952
+ askedHere.stuck.set(sent.id, canon(rec));
4953
+ }
4954
+ this.keepOwn();
4328
4955
  savePolicy(fresh, this.opts.home);
4329
4956
  this.opts.log(`* escalated ${o.id}: ${d.cause}${asking ? ` (and asked for ${d.capability})` : ""}`);
4330
4957
  }
4331
4958
  catch (e) {
4332
- this.opts.log(`! could not report the stuck obligation on the ledger: ${e.message}`);
4959
+ const msg = e.message;
4960
+ const lasting = safety && /^[A-Z_]+:/.test(msg) && !/^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(msg);
4961
+ if (lasting) {
4962
+ markContinued(space, key, this.opts.home);
4963
+ this.safetyDue.delete(key);
4964
+ try {
4965
+ const note = { says: clip(d.says, 400), fix: clip(d.fix, 500), at: Date.now(), team: this.cfg.spaceId ?? this.cfg.space };
4966
+ const fresh = loadPolicy(this.opts.home);
4967
+ (fresh.safetyNotes ??= {})[key] = note;
4968
+ askedHere.safety.set(key, canon(note));
4969
+ savePolicy(fresh, this.opts.home);
4970
+ this.keepOwn();
4971
+ }
4972
+ catch { }
4973
+ try {
4974
+ this.opts.notify?.({ kind: "worker", who: "Your worker", heading: "A safety warning could not reach your board", context: this.cfg.space,
4975
+ body: `${clip(d.says, 300)}\nWhat to do: ${clip(d.fix, 400)}`, ttl: 0, sound: true });
4976
+ }
4977
+ catch { }
4978
+ }
4979
+ const unanswered = !safety && /^NO_REPLY\b/.test(msg) ? failedActId(e) : null;
4980
+ if (unanswered) {
4981
+ const on = await this.ownOnRecord(client, unanswered, 2_500);
4982
+ if (on) {
4983
+ await this.reportStuck(client, o, d, attempts, again, on.id);
4984
+ return;
4985
+ }
4986
+ }
4987
+ const retry = !safety && (!!unanswered || !/^[A-Z_]+:/.test(msg) || /^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(msg));
4988
+ if (retry)
4989
+ this.stuckDue.set(key, { o, d, attempts, again, next: Date.now() + 60_000, ...(unanswered ? { known: unanswered } : known ? { known } : {}) });
4990
+ else if (!safety)
4991
+ this.stuckDue.delete(key);
4992
+ this.opts.log(`! could not report the stuck obligation on the ledger: ${msg}${safety ? lasting ? " — a safety warning: shown on this computer instead" : " — a safety warning: tried again until it is said" : retry ? " — tried again in a minute" : ""}`);
4993
+ }
4994
+ }
4995
+ async settleSafety(client) {
4996
+ if (!client.connected)
4997
+ return;
4998
+ for (const [key, s] of [...this.stuckDue]) {
4999
+ if (loadContinued(this.spaceKey(), this.opts.home).has(key)) {
5000
+ this.stuckDue.delete(key);
5001
+ continue;
5002
+ }
5003
+ const now = client.myWork().find((x) => this.oKey(x.id) === this.oKey(s.o.id));
5004
+ if (!now) {
5005
+ this.stuckDue.delete(key);
5006
+ continue;
5007
+ }
5008
+ if (s.next > Date.now())
5009
+ continue;
5010
+ s.next = Date.now() + 60_000;
5011
+ await this.reportStuck(client, now, s.d, s.attempts, s.again, s.known);
5012
+ }
5013
+ for (const [key, { o, d }] of [...this.safetyDue]) {
5014
+ if (loadContinued(this.spaceKey(), this.opts.home).has(key)) {
5015
+ this.safetyDue.delete(key);
5016
+ continue;
5017
+ }
5018
+ const now = client.myWork().find((x) => this.oKey(x.id) === this.oKey(o.id)) ?? o;
5019
+ await this.reportStuck(client, now, d, 0);
5020
+ }
5021
+ if (!this.safetyDiskRead) {
5022
+ this.safetyDiskRead = true;
5023
+ const marks = loadContinued(this.spaceKey(), this.opts.home);
5024
+ for (const m of marks) {
5025
+ if (!m.startsWith("stuck-safety-due:"))
5026
+ continue;
5027
+ const rest = m.slice("stuck-safety-due:".length);
5028
+ const cause = rest.slice(rest.lastIndexOf(":") + 1), okey = rest.slice(0, rest.lastIndexOf(":"));
5029
+ if (!SAFETY_CAUSES.has(cause) || marks.has(`stuck-safety:${okey}:${cause}`))
5030
+ continue;
5031
+ const o = client.myWork().find((x) => this.oKey(x.id) === okey)
5032
+ ?? { id: `gone:${okey}`, kind: "ANSWER", owner: this.cfg.member, ref: "", goal: null, body: "a request that is no longer waiting", created_at: 0 };
5033
+ if (o.id.startsWith("gone:"))
5034
+ this.obKeys.set(o.id, okey);
5035
+ this.safetyDue.set(`stuck-safety:${okey}:${cause}`, { o, d: safetyAgainDiagnosis(cause, this.cfg.member) });
5036
+ }
5037
+ }
5038
+ for (const [act, r] of [...this.safetyRaised]) {
5039
+ if (client.obligations.some((x) => x.kind === "HUMAN" && x.ref === act)) {
5040
+ r.goneAt = undefined;
5041
+ continue;
5042
+ }
5043
+ const now = Date.now();
5044
+ if (r.goneAt === undefined) {
5045
+ r.goneAt = now;
5046
+ continue;
5047
+ }
5048
+ if (now - r.goneAt < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
5049
+ continue;
5050
+ if ((r.nextAt ?? 0) > now)
5051
+ continue;
5052
+ const settled = await this.personSettled(act, r.seq);
5053
+ if (settled === null) {
5054
+ r.wait = Math.min((r.wait ?? 30_000) * 2, 30 * 60_000);
5055
+ r.nextAt = now + r.wait;
5056
+ continue;
5057
+ }
5058
+ this.safetyRaised.delete(act);
5059
+ if (settled || r.again >= SAFETY_RERAISE_MAX)
5060
+ continue;
5061
+ const o = client.myWork().find((x) => this.oKey(x.id) === this.oKey(r.o.id)) ?? r.o;
5062
+ this.opts.log(`! the safety warning ${act} left the board without your answer — raising it again`);
5063
+ await this.reportStuck(client, o, r.d, 0, r.again + 1);
5064
+ }
5065
+ }
5066
+ safetyDiskRead = false;
5067
+ closedSince = new Map();
5068
+ async settleClosedQuestions(client) {
5069
+ if (!client.connected)
5070
+ return;
5071
+ const p = loadPolicy(this.opts.home);
5072
+ const teams = this.cfg.spaceId ? [this.cfg.spaceId] : [this.cfg.space];
5073
+ const mine = [
5074
+ ...Object.entries(p.asks).filter(([, a]) => !!a?.act && teams.includes(p.stuckReports?.[a.act]?.team ?? ""))
5075
+ .map(([cap, a]) => [a.act, "cap", cap]),
5076
+ ...Object.entries(p.wakeAsks ?? {}).filter(([, a]) => teams.includes(a.team)).map(([id]) => [id, "wake", id]),
5077
+ ];
5078
+ const now = Date.now();
5079
+ const gone = [];
5080
+ for (const [act, kind, key] of mine) {
5081
+ if (client.obligations.some((o) => o.kind === "HUMAN" && o.ref === act)) {
5082
+ this.closedSince.delete(act);
5083
+ continue;
5084
+ }
5085
+ const since = this.closedSince.get(act);
5086
+ if (since === undefined) {
5087
+ this.closedSince.set(act, now);
5088
+ continue;
5089
+ }
5090
+ if (now - since < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
5091
+ continue;
5092
+ gone.push([act, kind, key]);
5093
+ }
5094
+ if (!gone.length)
5095
+ return;
5096
+ const fresh = loadPolicy(this.opts.home);
5097
+ for (const [act, kind, key] of gone) {
5098
+ this.closedSince.delete(act);
5099
+ withdrawnHere.add(act);
5100
+ if (kind === "wake")
5101
+ delete fresh.wakeAsks?.[act];
5102
+ else if (fresh.asks[key]?.act === act)
5103
+ delete fresh.asks[key];
5104
+ this.opts.log(`* ${act}: settled on the record — no longer waiting for an answer here`);
5105
+ }
5106
+ savePolicy(fresh, this.opts.home);
5107
+ }
5108
+ async personSettled(act, seq) {
5109
+ let since = Math.max(0, seq - 1);
5110
+ try {
5111
+ for (let page = 0; page < 40; page++) {
5112
+ const res = await signedRead(this.cfg, "/history", { since: String(since) }, { timeoutMs: 5000 });
5113
+ if (!res.ok)
5114
+ return null;
5115
+ const j = await res.json();
5116
+ const acts = j.acts ?? [];
5117
+ if (acts.some((a) => a.ref === act && (a.act === "ANSWER" || a.act === "RETRACT") && a.sig?.role !== "run" && a.from === this.cfg.member))
5118
+ return true;
5119
+ const last = acts.length ? acts[acts.length - 1].seq : since;
5120
+ if (!acts.length || last <= since || (typeof j.head === "number" && last >= j.head))
5121
+ return false;
5122
+ since = last;
5123
+ }
5124
+ return null;
5125
+ }
5126
+ catch {
5127
+ return null;
4333
5128
  }
4334
5129
  }
4335
5130
  ledgerPaths(text) {
@@ -4499,8 +5294,12 @@ export class Worker {
4499
5294
  }
4500
5295
  if (denies.length) {
4501
5296
  const fresh = loadPolicy(home);
4502
- for (const [act, body] of denies)
4503
- applyAnswer(fresh, act, body);
5297
+ for (const [act, body] of denies) {
5298
+ const r = applyAnswer(fresh, act, body);
5299
+ withdrawnHere.add(act);
5300
+ if (r)
5301
+ noesHere.add(`denied:${r.cap}`);
5302
+ }
4504
5303
  savePolicy(fresh, home);
4505
5304
  }
4506
5305
  }
@@ -4889,7 +5688,7 @@ export class Worker {
4889
5688
  cursorSaid = 0;
4890
5689
  readCursor() {
4891
5690
  try {
4892
- const j = JSON.parse(readFileSync(this.cursorFile(), "utf8"));
5691
+ const j = JSON.parse(readStateText(this.cursorFile()));
4893
5692
  const v = j[this.spaceKey()] ?? j[this.cfg.space];
4894
5693
  return typeof v === "number" && v > 0 ? v : 0;
4895
5694
  }
@@ -4904,7 +5703,7 @@ export class Worker {
4904
5703
  try {
4905
5704
  let j = {};
4906
5705
  try {
4907
- j = JSON.parse(readFileSync(this.cursorFile(), "utf8"));
5706
+ j = JSON.parse(readStateText(this.cursorFile()));
4908
5707
  }
4909
5708
  catch { }
4910
5709
  j[this.spaceKey()] = seq;
@@ -4929,7 +5728,7 @@ export class Worker {
4929
5728
  return false;
4930
5729
  }
4931
5730
  catch { }
4932
- if (this.contDone.has(`handled:${ref}`))
5731
+ if (this.answeredHere(ref))
4933
5732
  return false;
4934
5733
  const c = this.client;
4935
5734
  if (c && !ref.startsWith("cont:")) {
@@ -5066,4 +5865,14 @@ export class Worker {
5066
5865
  catch { }
5067
5866
  }
5068
5867
  }
5868
+ export function forAbout(until, now = Date.now()) {
5869
+ const min = Math.max(1, Math.round((until - now) / 60_000));
5870
+ if (min < 60)
5871
+ return `about ${min} minute${min === 1 ? "" : "s"}`;
5872
+ const h = Math.round(min / 60);
5873
+ if (h < 48)
5874
+ return `about ${h} hour${h === 1 ? "" : "s"}`;
5875
+ const d = Math.round(h / 24);
5876
+ return `about ${d} days`;
5877
+ }
5069
5878
  //# sourceMappingURL=worker.js.map