@join-pantheon/cli 0.35.0 → 0.35.1-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-outage.js +165 -35
- package/dist/agent-token.js +5 -3
- package/dist/agents.js +6 -4
- package/dist/approvals-list.js +87 -0
- package/dist/board.js +491 -88
- package/dist/brief.js +17 -0
- package/dist/cli.js +687 -95
- package/dist/client.js +159 -19
- package/dist/config.js +3 -2
- package/dist/continued.js +56 -1
- package/dist/decision-ui.js +5 -3
- package/dist/doctor.js +84 -16
- package/dist/file-lock.js +40 -5
- package/dist/flow-stats.js +18 -1
- package/dist/home-guard.js +336 -54
- package/dist/home-safety.js +2 -0
- package/dist/hud-queue.js +91 -0
- package/dist/install.js +15 -1
- package/dist/mcp-instructions.js +67 -28
- package/dist/mcp.js +129 -24
- package/dist/notice-linux.js +15 -4
- package/dist/notice-windows.js +2 -2
- package/dist/notifier.js +2 -1
- package/dist/notify.js +28 -9
- package/dist/open-in.js +4 -3
- package/dist/pickup.js +17 -0
- package/dist/plain.js +139 -3
- package/dist/profile.js +12 -8
- package/dist/protocol.js +1 -1
- package/dist/question-goal.js +74 -0
- package/dist/readiness-alert.js +24 -3
- package/dist/refusal.js +2 -2
- package/dist/relay.js +143 -28
- package/dist/removed.js +4 -4
- package/dist/run-control.js +4 -3
- package/dist/run-failure.js +67 -10
- package/dist/setup-changes.js +7 -6
- package/dist/signed-out.js +144 -21
- package/dist/spend.js +5 -4
- package/dist/tray.js +24 -6
- package/dist/update-check.js +2 -1
- package/dist/uri-handler.js +10 -2
- package/dist/which.js +2 -0
- package/dist/worker-policy.js +212 -10
- package/dist/worker.js +942 -133
- package/dist/worklist.js +31 -1
- package/helper/macos/main.swift +42 -3
- package/helper/macos/menubar.swift +482 -87
- package/helper/macos/prebuilt/manifest.json +5 -5
- package/helper/macos/prebuilt/pantheon-hud +0 -0
- package/helper/macos/prebuilt/pantheon-tray +0 -0
- package/package.json +1 -1
package/dist/worker.js
CHANGED
|
@@ -5,22 +5,22 @@ import { chmodSync, lstatSync, mkdirSync, writeFileSync, existsSync, readdirSync
|
|
|
5
5
|
import { homedir } from "node:os";
|
|
6
6
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
7
7
|
import { absentDeniedPaths, describeKept, ownWorktreeConfig, detectClaudeFeatures, describeFeatures, ensureRunTmp, leavesSandboxMounts, NO_CLAUDE_FEATURES, removeSandboxLeftovers, runTmpDir } from "./claude-confine.js";
|
|
8
|
-
import { snapshotHome, restoreHome, describeChange, trustWrite, askedHere, persistSnapshot, persistRestored, persistAttempt, loadPersistedGuard, clearPersistedGuard, CHANGES_LOG_FILE } from "./home-guard.js";
|
|
9
|
-
import { PantheonClient } from "./client.js";
|
|
8
|
+
import { snapshotHome, restoreHome, describeChange, trustWrite, askedHere, canon, withdrawnHere, noesHere, adoptOwn, persistOwn, persistPending, resetOwn, persistSnapshot, persistRestored, persistAttempt, loadPersistedGuard, clearPersistedGuard, CHANGES_LOG_FILE, GUARDED_FILES } from "./home-guard.js";
|
|
9
|
+
import { PantheonClient, failedActId } from "./client.js";
|
|
10
10
|
import { createHash, randomBytes } from "node:crypto";
|
|
11
11
|
import { newKeypair, signDelegate } from "./identity.js";
|
|
12
12
|
import { loadPins, pinScope, savePins, teamPinId, writeWhole } from "./pins.js";
|
|
13
|
-
import { answeredIn, markAnswered, noteNonce } from "./nonces.js";
|
|
14
|
-
import { brokenStateFiles, brokenStateFix, pidAlive, readStateFile, withFileLock } from "./file-lock.js";
|
|
13
|
+
import { answeredIn, markAnswered, noteNonce, unmarkAnswered } from "./nonces.js";
|
|
14
|
+
import { brokenStateFiles, brokenStateFix, pidAlive, readStateFile, readStateText, withFileLock } from "./file-lock.js";
|
|
15
15
|
import { mkdirInside, writeInside } from "./safe-write.js";
|
|
16
16
|
import { corruptHeldFiles, MAX_HELD_PER_RUN, remoteTipIs, takeHeldFrames, updateHeldState, verifyHeldFrame, writeHoldConfig } from "./held-delivery.js";
|
|
17
17
|
import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
|
|
18
18
|
import { trusted } from "./verify-act.js";
|
|
19
|
-
import { headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
20
|
-
import { activeOutages, detectOutage, earliest, whenAgain, SIGNIN_REFUSED } from "./agent-outage.js";
|
|
19
|
+
import { AGENTS, headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, gitMissing, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
20
|
+
import { activeOutages, detectOutage, earliest, isSignOut, outagePredicate, whenAgain, SIGNIN_REFUSED } from "./agent-outage.js";
|
|
21
21
|
import { agentSpawn, CmdLineUnsafe } from "./which.js";
|
|
22
22
|
import { VERSION } from "./version.js";
|
|
23
|
-
import { applyAnswer, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, policyHome, describeRunMinutes, parseRunMinutes, currentMode, } from "./worker-policy.js";
|
|
23
|
+
import { applyAnswer, ASK_EXPIRY_MS, policyUnreadable, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, policyHome, describeRunMinutes, parseRunMinutes, currentMode, wakeSettled, } from "./worker-policy.js";
|
|
24
24
|
import { readinessAlerts } from "./readiness-alert.js";
|
|
25
25
|
import { clearDeclined, markDeclined, personsOwnThread, pruneDeclined } from "./declined.js";
|
|
26
26
|
import { loadRunStops, markStopTold, pruneRunStops, requestRunStop, stopFor, stopNote } from "./run-control.js";
|
|
@@ -29,18 +29,18 @@ import { removalWords, removedLine, removedNotice, takeTold } from "./removed.js
|
|
|
29
29
|
import { agentHookStates, repairHook, repairNote } from "./notify.js";
|
|
30
30
|
import { fence, assess } from "./untrusted.js";
|
|
31
31
|
import { allowed, closeStartedRun, finishRun, recordRun, startRun } from "./spend.js";
|
|
32
|
-
import { handledIn, loadContinued, markContinued, unmarkContinued, wakeAlertedKey } from "./continued.js";
|
|
32
|
+
import { handledIn, loadContinued, markContinued, storedIn, storedKey, unmarkContinued, wakeAlertedKey, WORKER_MARK } from "./continued.js";
|
|
33
33
|
import { decisionAlertsOn, loadProfile, profileForTeam, reservedDecisionsNote, saveProfile, wakeMode } from "./profile.js";
|
|
34
34
|
import { clip } from "./brief.js";
|
|
35
35
|
import { composeWakeAsk, composeWorker } from "./notice.js";
|
|
36
|
-
import { diagnoseRun, gitProbe, workNeedsCommands,
|
|
37
|
-
import { clearSignedOut, markSignedOut, retryAfter, signedOutNow, SIGNED_OUT_AGENTS } from "./signed-out.js";
|
|
36
|
+
import { diagnoseRun, gitProbe, refusedWords, workNeedsCommands, signInFixFor } from "./run-failure.js";
|
|
37
|
+
import { claudeSettingsAuth, clearSignedOut, markSignedOut, retryAfter, signedOutChecking, signedOutNow, SIGNED_OUT_AGENTS } from "./signed-out.js";
|
|
38
38
|
import { claudeTokenForRun } from "./agent-token.js";
|
|
39
39
|
import { installedTools, preferredTool } from "./notifier.js";
|
|
40
40
|
import { obligationSignature, StatusNudger } from "./status-nudge.js";
|
|
41
41
|
import { clientState, ConnectionReport } from "./conn-report.js";
|
|
42
42
|
import { loadPersonas, matchPersona, personaByName, personaMatches, effectivePolicy, personaBrief, describePersona, } from "./persona.js";
|
|
43
|
-
import { CONFIG_DIR, CONFIG_PATH, loadAll, repoInfo, updateMembershipToken } from "./config.js";
|
|
43
|
+
import { CONFIG_DIR, CONFIG_PATH, configFor, findMembership, load as loadConfig, loadAll, repoInfo, updateMembershipToken } from "./config.js";
|
|
44
44
|
import { shellArg } from "./team-name.js";
|
|
45
45
|
import { repoName, routeRequest, safeName } from "./repo-route.js";
|
|
46
46
|
import { holdsRepo } from "./repo-find.js";
|
|
@@ -279,7 +279,7 @@ export function outsideLinksDiagnosis(member, worktree, scan) {
|
|
|
279
279
|
says,
|
|
280
280
|
fix: scan.unreadable
|
|
281
281
|
? `Look at this machine's worktree ${where}: a very large or unreadable checkout cannot be vouched for. Take this request in a live session, or clear it out (pantheon stops at that worktree; its branch and commits are kept in the repo).`
|
|
282
|
-
: `On this machine, delete these links from the worktree ${where} — not from the repo's branch, where they are not: ${scan.links.map((l) => `${where}/${l}`).join(", ")}. Or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it.`,
|
|
282
|
+
: `On this machine, delete these links from the worktree ${where} — not from the repo's branch, where they are not: ${scan.links.slice(0, 20).map((l) => `${where}/${l}`).join(", ")}${scan.links.length > 20 ? `, and ${scan.links.length - 20} more in that worktree (find ${where} -type l lists them)` : ""}. Or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it.`,
|
|
283
283
|
peer: {
|
|
284
284
|
summary: `${member}'s worker did not start a run: its checkout for this could not be vouched for`,
|
|
285
285
|
body: `${member}'s machine could not confirm its checkout for this holds no link leading outside it, so its worker did not start an unattended run: a run would read whatever such a link points at. It is still owed and waits for ${member}. Nothing was run.`,
|
|
@@ -780,8 +780,27 @@ export function deliveryStep(branch, canPush, ref) {
|
|
|
780
780
|
for your human to review and push. If the request asked for code, that NOTE is your reply.`;
|
|
781
781
|
}
|
|
782
782
|
const SAFETY_CAUSES = new Set(["outside_links", "worktree_tampered"]);
|
|
783
|
+
const SAFETY_RERAISE_MAX = 3;
|
|
784
|
+
function safetyAgainDiagnosis(cause, member) {
|
|
785
|
+
return cause === "outside_links"
|
|
786
|
+
? { cause: "outside_links", says: "the checkout for this held links that lead outside it — a run would read whatever they point at as the repo's own, so none was started",
|
|
787
|
+
fix: "On this machine, look at this request's worktree and delete any link that leads outside it, or take this request in a live session. Unattended runs start again once nothing in the checkout leads out of it." }
|
|
788
|
+
: { cause: "worktree_tampered", says: `the worktree's .git was changed after Pantheon made it — git would obey that gitdir's config outside any sandbox, so git is not run in it`,
|
|
789
|
+
fix: `Look at what is in that worktree, then remove its folder and run git worktree prune in the repo; the next request checks out a fresh one. (${member}'s worker raised this before a restart.)` };
|
|
790
|
+
}
|
|
791
|
+
function decidedBrief(answer) {
|
|
792
|
+
const said = answer.replace(/[\u0000-\u0008\u000b-\u001f\u007f]/g, "").slice(0, 2000);
|
|
793
|
+
return `
|
|
794
|
+
YOUR HUMAN DECIDED — an earlier run here escalated this request to them, and they answered:
|
|
795
|
+
${said.split("\n").map((l) => ` | ${l}`).join("\n")}
|
|
796
|
+
That is their decision. Reply to your teammate with it now (step 4: an ANSWER with the request's ref),
|
|
797
|
+
in your own plain words, and act on it if the request asks for work. Do not escalate the same question again.
|
|
798
|
+
`;
|
|
799
|
+
}
|
|
800
|
+
const ESCALATED_FOR = "esc-for:";
|
|
801
|
+
const DECIDED_FOR = "decided-for:";
|
|
783
802
|
export const MOOT_GRACE_MS = 2 * 60_000;
|
|
784
|
-
export function buildBrief(o, trigger, branch, persona, canPush = false) {
|
|
803
|
+
export function buildBrief(o, trigger, branch, persona, canPush = false, decided) {
|
|
785
804
|
const placed = branch
|
|
786
805
|
? `in an isolated git worktree on branch ${branch}`
|
|
787
806
|
: `in a scratch directory (your human's workspace is not a git repo, so there is no branch and nothing can be DELIVERED from here)`;
|
|
@@ -797,7 +816,7 @@ ${riskNote(risky, "what was asked")}
|
|
|
797
816
|
WHAT YOU OWE
|
|
798
817
|
${owedLine(o.kind)}${o.goal ? ` — goal ${printable(o.goal)}` : ""}
|
|
799
818
|
obligation id: ${printable(o.id)}
|
|
800
|
-
${persona ? personaBrief(persona) : ""}
|
|
819
|
+
${persona ? personaBrief(persona) : ""}${decided ? decidedBrief(decided) : ""}
|
|
801
820
|
DO THIS
|
|
802
821
|
1. pantheon_claim with obligation id ${printable(o.id)}. If the claim is REFUSED, another session has it —
|
|
803
822
|
stop immediately and say nothing.
|
|
@@ -985,8 +1004,43 @@ function readRunLease(f) {
|
|
|
985
1004
|
rmSync(f, { recursive: true, force: true });
|
|
986
1005
|
return null;
|
|
987
1006
|
}
|
|
1007
|
+
export const PERSON_HOLD = "person-answering.json";
|
|
1008
|
+
export const PERSON_HOLD_TTL_MS = 60_000;
|
|
1009
|
+
export const PERSON_ANSWERING = "your person is answering a question on this computer";
|
|
1010
|
+
export function personHold(dir, now = Date.now()) {
|
|
1011
|
+
try {
|
|
1012
|
+
const v = JSON.parse(readStateText(join(dir, PERSON_HOLD)));
|
|
1013
|
+
const pid = Number(v?.pid), at = Number(v?.at);
|
|
1014
|
+
if (!Number.isInteger(pid) || pid <= 1 || !Number.isFinite(at) || at > now + 5_000 || now - at > PERSON_HOLD_TTL_MS || !pidAlive(pid))
|
|
1015
|
+
return null;
|
|
1016
|
+
return { pid, at };
|
|
1017
|
+
}
|
|
1018
|
+
catch {
|
|
1019
|
+
return null;
|
|
1020
|
+
}
|
|
1021
|
+
}
|
|
1022
|
+
export function writePersonHold(dir, now = Date.now()) {
|
|
1023
|
+
try {
|
|
1024
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
1025
|
+
writeWhole(join(dir, PERSON_HOLD), JSON.stringify({ pid: process.pid, at: now }) + "\n");
|
|
1026
|
+
}
|
|
1027
|
+
catch { }
|
|
1028
|
+
}
|
|
1029
|
+
export function releasePersonHold(dir) {
|
|
1030
|
+
try {
|
|
1031
|
+
const h = personHold(dir);
|
|
1032
|
+
if (!h || h.pid === process.pid)
|
|
1033
|
+
rmSync(join(dir, PERSON_HOLD), { force: true });
|
|
1034
|
+
}
|
|
1035
|
+
catch { }
|
|
1036
|
+
}
|
|
988
1037
|
export function takeRunLease(dir, space, now = Date.now()) {
|
|
989
1038
|
const f = join(dir, RUN_SLOT);
|
|
1039
|
+
{
|
|
1040
|
+
const h = personHold(dir, now);
|
|
1041
|
+
if (h && h.pid !== process.pid)
|
|
1042
|
+
return { held: PERSON_ANSWERING };
|
|
1043
|
+
}
|
|
990
1044
|
return withFileLock(f, () => {
|
|
991
1045
|
const cur = readRunLease(f);
|
|
992
1046
|
if (cur && Number.isInteger(cur.pid) && cur.pid !== process.pid && typeof cur.at === "number" && Number.isFinite(cur.at)
|
|
@@ -1110,14 +1164,37 @@ export function putBackSaid(list) {
|
|
|
1110
1164
|
}
|
|
1111
1165
|
export const GUARD_RESTORE_TRIES = 3;
|
|
1112
1166
|
const GUARD_STALE_MARGIN_MS = 10 * 60_000;
|
|
1113
|
-
export function
|
|
1167
|
+
export async function recoverGuardHere(home) {
|
|
1168
|
+
const g = loadPersistedGuard(home);
|
|
1169
|
+
if (!g)
|
|
1170
|
+
return { outcome: "none" };
|
|
1171
|
+
if (sameProcess(g.worker, g.workerId))
|
|
1172
|
+
return { outcome: "live" };
|
|
1173
|
+
let own;
|
|
1174
|
+
if (g.own && g.snapshot) {
|
|
1175
|
+
const anyCfg = (() => { try {
|
|
1176
|
+
return loadConfig();
|
|
1177
|
+
}
|
|
1178
|
+
catch {
|
|
1179
|
+
return null;
|
|
1180
|
+
} })();
|
|
1181
|
+
if (!anyCfg)
|
|
1182
|
+
return { outcome: "unreachable" };
|
|
1183
|
+
const v = await verifyOwnWrites(g.own, anyCfg.member, (t) => configForTeam(t, anyCfg), refusedSaid((l) => console.error(l)));
|
|
1184
|
+
if (v === "unreachable")
|
|
1185
|
+
return { outcome: "unreachable" };
|
|
1186
|
+
own = v;
|
|
1187
|
+
}
|
|
1188
|
+
return recoverInterruptedGuard(home, Date.now(), restoreHome, own);
|
|
1189
|
+
}
|
|
1190
|
+
export function recoverInterruptedGuard(home, now = Date.now(), restore = restoreHome, verifiedOwn) {
|
|
1114
1191
|
const g = loadPersistedGuard(home);
|
|
1115
1192
|
if (!g)
|
|
1116
1193
|
return { outcome: "none" };
|
|
1117
1194
|
if (sameProcess(g.worker, g.workerId))
|
|
1118
1195
|
return { outcome: "live" };
|
|
1119
1196
|
try {
|
|
1120
|
-
const r = JSON.parse(
|
|
1197
|
+
const r = JSON.parse(readStateText(join(home, RUN_RECORD)));
|
|
1121
1198
|
if (r && r.worker === g.worker && Number.isInteger(r.pid) && r.pid > 1 && typeof r.at === "number" && now - r.at <= recordLimitMs(r) && recordedRunAlive(r.pid))
|
|
1122
1199
|
killRunTree(r.pid);
|
|
1123
1200
|
}
|
|
@@ -1126,15 +1203,17 @@ export function recoverInterruptedGuard(home, now = Date.now(), restore = restor
|
|
|
1126
1203
|
clearPersistedGuard(home);
|
|
1127
1204
|
return { outcome: "restored", said: g.restored ?? [] };
|
|
1128
1205
|
}
|
|
1129
|
-
|
|
1206
|
+
adoptOwn(verifiedOwn);
|
|
1207
|
+
const pending = !!g.pending?.length;
|
|
1208
|
+
if (!pending && (g.at > now + 60_000 || now - g.at > recordLimitMs({ limit: g.limit }) + GUARD_STALE_MARGIN_MS)) {
|
|
1130
1209
|
clearPersistedGuard(home);
|
|
1131
1210
|
return { outcome: "stale", at: g.at };
|
|
1132
1211
|
}
|
|
1133
1212
|
const lease = readRunLease(join(home, RUN_SLOT));
|
|
1134
1213
|
const wentAt = lease && lease.pid === g.worker && typeof lease.at === "number" && Number.isFinite(lease.at) ? lease.at + RUN_LEASE_RENEW_MS : null;
|
|
1135
|
-
if (wentAt !== null)
|
|
1214
|
+
if (wentAt !== null && !pending)
|
|
1136
1215
|
try {
|
|
1137
|
-
const mine =
|
|
1216
|
+
const mine = readStateText(join(home, CHANGES_LOG_FILE)).split("\n").some((l) => {
|
|
1138
1217
|
try {
|
|
1139
1218
|
const e = JSON.parse(l);
|
|
1140
1219
|
return typeof e.at === "number" && e.at > Math.max(g.at, wentAt) && e.setting !== "first-setup" && e.asked_by === "terminal" && (e.outcome === "applied" || e.outcome === "approved" || e.outcome === "kept");
|
|
@@ -1150,7 +1229,14 @@ export function recoverInterruptedGuard(home, now = Date.now(), restore = restor
|
|
|
1150
1229
|
}
|
|
1151
1230
|
catch { }
|
|
1152
1231
|
try {
|
|
1153
|
-
const
|
|
1232
|
+
const snapshot = pending ? { ...g.snapshot, files: g.snapshot.files.filter((f) => g.pending.includes(f.rel)), queueDone: true } : g.snapshot;
|
|
1233
|
+
const changes = restore(snapshot);
|
|
1234
|
+
const said = putBackList(changes);
|
|
1235
|
+
const rels = changes.filter((c) => c.failed).map((c) => c.rel);
|
|
1236
|
+
if (rels.length) {
|
|
1237
|
+
persistPending(home, rels);
|
|
1238
|
+
return { outcome: "pending", said, rels, snapshot: { ...g.snapshot, files: g.snapshot.files.filter((f) => rels.includes(f.rel)) } };
|
|
1239
|
+
}
|
|
1154
1240
|
clearPersistedGuard(home);
|
|
1155
1241
|
return { outcome: "restored", said };
|
|
1156
1242
|
}
|
|
@@ -1166,6 +1252,8 @@ export function recoverInterruptedGuard(home, now = Date.now(), restore = restor
|
|
|
1166
1252
|
}
|
|
1167
1253
|
export function guardHeld(home, now = Date.now()) {
|
|
1168
1254
|
const g = loadPersistedGuard(home);
|
|
1255
|
+
if (g?.snapshot && g.pending?.length)
|
|
1256
|
+
return true;
|
|
1169
1257
|
if (!g?.snapshot || g.at > now + 60_000 || now - g.at > recordLimitMs({ limit: g.limit }) + GUARD_STALE_MARGIN_MS)
|
|
1170
1258
|
return false;
|
|
1171
1259
|
return sameProcess(g.worker, g.workerId);
|
|
@@ -1187,7 +1275,7 @@ function writeRunRecord(dir, r) {
|
|
|
1187
1275
|
function clearRunRecord(dir, pid) {
|
|
1188
1276
|
try {
|
|
1189
1277
|
const f = join(dir, RUN_RECORD);
|
|
1190
|
-
const r = JSON.parse(
|
|
1278
|
+
const r = JSON.parse(readStateText(f));
|
|
1191
1279
|
if (r.pid === pid)
|
|
1192
1280
|
rmSync(f, { force: true });
|
|
1193
1281
|
}
|
|
@@ -1197,7 +1285,7 @@ export function killRecordedRun(dir, now = Date.now()) {
|
|
|
1197
1285
|
const f = join(dir, RUN_RECORD);
|
|
1198
1286
|
let r;
|
|
1199
1287
|
try {
|
|
1200
|
-
r = JSON.parse(
|
|
1288
|
+
r = JSON.parse(readStateText(f));
|
|
1201
1289
|
}
|
|
1202
1290
|
catch {
|
|
1203
1291
|
return null;
|
|
@@ -1237,7 +1325,7 @@ export function parentOf(pid) {
|
|
|
1237
1325
|
export function runInFlight(dir, now = Date.now(), o = {}) {
|
|
1238
1326
|
let r;
|
|
1239
1327
|
try {
|
|
1240
|
-
r = JSON.parse(
|
|
1328
|
+
r = JSON.parse(readStateText(join(dir, RUN_RECORD)));
|
|
1241
1329
|
}
|
|
1242
1330
|
catch {
|
|
1243
1331
|
return null;
|
|
@@ -1264,7 +1352,7 @@ export function runMinutesLeft(dir, now = Date.now()) {
|
|
|
1264
1352
|
if (!runInFlight(dir, now))
|
|
1265
1353
|
return null;
|
|
1266
1354
|
try {
|
|
1267
|
-
const r = JSON.parse(
|
|
1355
|
+
const r = JSON.parse(readStateText(join(dir, RUN_RECORD)));
|
|
1268
1356
|
if (typeof r.exited_at === "number")
|
|
1269
1357
|
return 1;
|
|
1270
1358
|
if (typeof r.limit !== "number" || !Number.isFinite(r.limit))
|
|
@@ -1287,6 +1375,99 @@ export function answerWindow(askSeq, hint, fullScanDue) {
|
|
|
1287
1375
|
return 0;
|
|
1288
1376
|
return Math.max(0, hint);
|
|
1289
1377
|
}
|
|
1378
|
+
const teamConfigs = new Map();
|
|
1379
|
+
const refusedSaid = (log) => {
|
|
1380
|
+
const said = new Set();
|
|
1381
|
+
return (team, status) => {
|
|
1382
|
+
if (said.has(team))
|
|
1383
|
+
return;
|
|
1384
|
+
said.add(team);
|
|
1385
|
+
log(` guard the relay of "${team}" refused this machine (${status === 410 ? "the team was deleted" : status === 401 ? "it needs a newer pantheon" : "removed from the team, or signed out"}) — what the interrupted run's worker had asked there is not kept; nothing waits on it`);
|
|
1386
|
+
};
|
|
1387
|
+
};
|
|
1388
|
+
function configForTeam(team, mine) {
|
|
1389
|
+
if (team === mine.spaceId || team === mine.space)
|
|
1390
|
+
return mine;
|
|
1391
|
+
const known = teamConfigs.get(team);
|
|
1392
|
+
if (known)
|
|
1393
|
+
return known;
|
|
1394
|
+
try {
|
|
1395
|
+
const f = loadAll();
|
|
1396
|
+
const m = f ? findMembership(f, team) : undefined;
|
|
1397
|
+
return f && m ? configFor(f, m, null) : null;
|
|
1398
|
+
}
|
|
1399
|
+
catch {
|
|
1400
|
+
return null;
|
|
1401
|
+
}
|
|
1402
|
+
}
|
|
1403
|
+
export async function verifyOwnWrites(o, member, cfgFor, refused) {
|
|
1404
|
+
const out = { wake: [], cap: [], stuck: [], safety: [], withdrawn: [], noes: [], marks: [] };
|
|
1405
|
+
let unreachable = false;
|
|
1406
|
+
const seen = new Map();
|
|
1407
|
+
const act = async (id, team) => {
|
|
1408
|
+
const key = `${team ?? ""}|${id}`;
|
|
1409
|
+
if (seen.has(key))
|
|
1410
|
+
return seen.get(key);
|
|
1411
|
+
const cfgs = team ? [cfgFor(team)].filter((c) => !!c) : [...new Set(teamConfigs.values())];
|
|
1412
|
+
let found = null;
|
|
1413
|
+
for (const cfg of cfgs) {
|
|
1414
|
+
try {
|
|
1415
|
+
const res = await signedRead(cfg, "/act", { id }, { timeoutMs: 3000 });
|
|
1416
|
+
if (res.ok) {
|
|
1417
|
+
found = (await res.json()).act ?? null;
|
|
1418
|
+
if (found)
|
|
1419
|
+
break;
|
|
1420
|
+
}
|
|
1421
|
+
else if (res.status === 401 || res.status === 403 || res.status === 410)
|
|
1422
|
+
refused?.(cfg.space, res.status);
|
|
1423
|
+
else if (res.status !== 404)
|
|
1424
|
+
unreachable = true;
|
|
1425
|
+
}
|
|
1426
|
+
catch {
|
|
1427
|
+
unreachable = true;
|
|
1428
|
+
}
|
|
1429
|
+
}
|
|
1430
|
+
seen.set(key, found);
|
|
1431
|
+
return found;
|
|
1432
|
+
};
|
|
1433
|
+
const mine = (a) => !!a && a.act === "ESCALATE" && a.from === member && a.sig?.role !== "run";
|
|
1434
|
+
const parse = (v) => { try {
|
|
1435
|
+
const x = JSON.parse(v);
|
|
1436
|
+
return x && typeof x === "object" ? x : {};
|
|
1437
|
+
}
|
|
1438
|
+
catch {
|
|
1439
|
+
return {};
|
|
1440
|
+
} };
|
|
1441
|
+
const refs = new Set(), teamOfAct = new Map();
|
|
1442
|
+
for (const [id, v] of o.stuck) {
|
|
1443
|
+
const rec = parse(v);
|
|
1444
|
+
const team = typeof rec.team === "string" ? rec.team : null;
|
|
1445
|
+
const a = await act(id, team);
|
|
1446
|
+
if (mine(a) && a.ref === rec.ref) {
|
|
1447
|
+
out.stuck.push([id, v]);
|
|
1448
|
+
refs.add(String(rec.ref));
|
|
1449
|
+
if (team)
|
|
1450
|
+
teamOfAct.set(id, team);
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
for (const [cap, v] of o.cap) {
|
|
1454
|
+
const id = String(parse(v).act ?? "");
|
|
1455
|
+
if (id && mine(await act(id, teamOfAct.get(id) ?? null)))
|
|
1456
|
+
out.cap.push([cap, v]);
|
|
1457
|
+
}
|
|
1458
|
+
for (const [id, v] of o.wake) {
|
|
1459
|
+
const q = parse(v);
|
|
1460
|
+
const a = await act(id, typeof q.team === "string" ? q.team : null);
|
|
1461
|
+
if (mine(a) && a.meta?.wake_ask === q.member)
|
|
1462
|
+
out.wake.push([id, v]);
|
|
1463
|
+
}
|
|
1464
|
+
for (const m of o.marks) {
|
|
1465
|
+
const r = /:r:[A-Z_]+:(.+)$/.exec(m)?.[1];
|
|
1466
|
+
if (r && refs.has(r))
|
|
1467
|
+
out.marks.push(m);
|
|
1468
|
+
}
|
|
1469
|
+
return unreachable ? "unreachable" : out;
|
|
1470
|
+
}
|
|
1290
1471
|
export class Worker {
|
|
1291
1472
|
cfg;
|
|
1292
1473
|
opts;
|
|
@@ -1328,6 +1509,9 @@ export class Worker {
|
|
|
1328
1509
|
this.cfg = cfg;
|
|
1329
1510
|
this.opts = opts;
|
|
1330
1511
|
this.pinTeam = teamPinId(cfg);
|
|
1512
|
+
for (const k of [cfg.spaceId, cfg.space])
|
|
1513
|
+
if (k)
|
|
1514
|
+
teamConfigs.set(k, cfg);
|
|
1331
1515
|
}
|
|
1332
1516
|
spaceKey() { return this.cfg.spaceId ?? this.cfg.space; }
|
|
1333
1517
|
pinTeam;
|
|
@@ -1344,32 +1528,83 @@ export class Worker {
|
|
|
1344
1528
|
for (const k of this.contKeys())
|
|
1345
1529
|
for (const id of loadContinued(k, this.opts.home))
|
|
1346
1530
|
this.contDone.add(id);
|
|
1531
|
+
for (const id of this.contDone) {
|
|
1532
|
+
const m = /^unparked@([^:]+):(.+)$/.exec(id);
|
|
1533
|
+
if (m)
|
|
1534
|
+
this.unparkedUnder.set(m[2], m[1]);
|
|
1535
|
+
}
|
|
1536
|
+
}
|
|
1537
|
+
obKeys = new Map();
|
|
1538
|
+
learnRequests(list) {
|
|
1539
|
+
if (this.obKeys.size > 5000)
|
|
1540
|
+
this.obKeys.clear();
|
|
1541
|
+
for (const o of list)
|
|
1542
|
+
if (o.kind !== "CONTINUE" && o.ref)
|
|
1543
|
+
this.obKeys.set(o.id, `r:${o.kind}:${o.ref}`);
|
|
1544
|
+
}
|
|
1545
|
+
oKey(id) { return this.obKeys.get(id) ?? id; }
|
|
1546
|
+
marked(prefix, id) {
|
|
1547
|
+
const k = this.oKey(id);
|
|
1548
|
+
return this.contDone.has(`${prefix}:${k}`) || (k !== id && this.contDone.has(`${prefix}:${id}`));
|
|
1549
|
+
}
|
|
1550
|
+
markFor(prefix, id) { this.markDone(`${prefix}:${this.oKey(id)}`); }
|
|
1551
|
+
unparkedUnder = new Map();
|
|
1552
|
+
grantKey() {
|
|
1553
|
+
const g = loadPolicy(this.opts.home).grants.run_commands;
|
|
1554
|
+
if (this.opts.allowBash)
|
|
1555
|
+
return { key: "bash", once: false };
|
|
1556
|
+
return g ? { key: `${g.mode === "once" ? "o" : "g"}${g.at}`, once: g.mode === "once" } : null;
|
|
1347
1557
|
}
|
|
1348
1558
|
failuresOf(id) {
|
|
1349
|
-
const mem = this.failedRuns.get(id);
|
|
1559
|
+
const mem = this.failedRuns.get(this.oKey(id));
|
|
1350
1560
|
if (mem !== undefined)
|
|
1351
1561
|
return mem;
|
|
1352
1562
|
this.loadContinued();
|
|
1353
|
-
|
|
1563
|
+
const under = this.unparkedUnder.get(this.oKey(id));
|
|
1564
|
+
if (under)
|
|
1565
|
+
return this.contDone.has(`parked2@${under}:${this.oKey(id)}`) ? 2 : 1;
|
|
1566
|
+
if (this.marked("unparked", id) && !this.marked("parked2", id))
|
|
1354
1567
|
return 1;
|
|
1355
|
-
if (this.
|
|
1568
|
+
if (this.marked("unparked-ns", id) && !this.marked("parked2-ns", id))
|
|
1356
1569
|
return 1;
|
|
1357
|
-
return this.
|
|
1570
|
+
return this.marked("parked", id) ? 2 : this.marked("failed1", id) ? 1 : 0;
|
|
1358
1571
|
}
|
|
1359
1572
|
retryParkedForCommands(owed) {
|
|
1360
|
-
const g =
|
|
1361
|
-
if (!g
|
|
1573
|
+
const g = this.grantKey();
|
|
1574
|
+
if (!g)
|
|
1362
1575
|
return;
|
|
1363
1576
|
this.loadContinued();
|
|
1577
|
+
const waiting = (o) => this.marked("parked-nc", o.id) && this.failuresOf(o.id) >= 2;
|
|
1578
|
+
if (g.once && owed.some((o) => this.unparkedUnder.get(this.oKey(o.id)) === g.key && this.failuresOf(o.id) < 2))
|
|
1579
|
+
return;
|
|
1364
1580
|
for (const o of owed) {
|
|
1365
|
-
if (!
|
|
1581
|
+
if (!waiting(o) || this.unparkedUnder.get(this.oKey(o.id)) === g.key)
|
|
1366
1582
|
continue;
|
|
1367
|
-
this.failedRuns.set(o.id, 1);
|
|
1583
|
+
this.failedRuns.set(this.oKey(o.id), 1);
|
|
1368
1584
|
this.skipLogged.delete(`parked:${o.id}`);
|
|
1369
|
-
this.markDone(`unparked:${o.id}`);
|
|
1585
|
+
this.markDone(`unparked@${g.key}:${this.oKey(o.id)}`);
|
|
1586
|
+
this.unparkedUnder.set(this.oKey(o.id), g.key);
|
|
1370
1587
|
this.opts.log(`* commands are allowed now — running ${o.kind} ${o.id} again (it was parked because it needed them)`);
|
|
1588
|
+
if (g.once)
|
|
1589
|
+
break;
|
|
1371
1590
|
}
|
|
1372
1591
|
}
|
|
1592
|
+
async askAgainForParked(client) {
|
|
1593
|
+
if (this.grantKey())
|
|
1594
|
+
return;
|
|
1595
|
+
this.loadContinued();
|
|
1596
|
+
const recent = (k) => /^o\d+$/.test(k) && Date.now() - Number(k.slice(1)) < ASK_EXPIRY_MS;
|
|
1597
|
+
const spent = new Set([...this.unparkedUnder.values()].filter(recent));
|
|
1598
|
+
if (!spent.size)
|
|
1599
|
+
return;
|
|
1600
|
+
if (!mayAsk(loadPolicy(this.opts.home), "run_commands"))
|
|
1601
|
+
return;
|
|
1602
|
+
const o = client.myWork().find((x) => this.marked("parked-nc", x.id) && this.failuresOf(x.id) >= 2 && !spent.has(this.unparkedUnder.get(this.oKey(x.id)) ?? ""));
|
|
1603
|
+
if (!o)
|
|
1604
|
+
return;
|
|
1605
|
+
await this.reportStuck(client, o, { cause: "no_commands", says: "The agent may only edit files on this computer, so it couldn't run tests, commit or push.",
|
|
1606
|
+
fix: "To let it deliver work while you're away: pantheon setup --commands yes", capability: "run_commands" }, 2, 1);
|
|
1607
|
+
}
|
|
1373
1608
|
parkedNotHere = new Set();
|
|
1374
1609
|
parkedNoSandbox = new Map();
|
|
1375
1610
|
outages = new Map();
|
|
@@ -1483,20 +1718,19 @@ export class Worker {
|
|
|
1483
1718
|
this.readinessCommandsOn = commandsOn;
|
|
1484
1719
|
const r = this.opts.readinessCheck ? this.opts.readinessCheck() : unattendedReadiness({ prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: this.unsandboxedOk(), pinned: this.opts.agent ?? "auto", ...(this.opts.choose ? { choose: this.opts.choose } : {}),
|
|
1485
1720
|
commandsOn, inContainer: hostEnv().kind === "container", ...(this.opts.sandboxProbe ? { sandboxProbe: this.opts.sandboxProbe } : {}) });
|
|
1486
|
-
if (!r.ok && (this.readiness?.ok ?? true))
|
|
1487
|
-
this.opts.log(`! teammates are told this machine can't take requests yet: ${r.short} — ${r.fix}`);
|
|
1488
|
-
if (r.ok && this.readiness && !this.readiness.ok)
|
|
1489
|
-
this.opts.log(` agent ready again — teammates are told this machine takes requests`);
|
|
1490
|
-
this.readiness = { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) };
|
|
1491
1721
|
this.syncSignedOut();
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
this.
|
|
1722
|
+
const held = r.ok && this.signedOutHere.size && !this.opts.readinessCheck ? this.heldForSandbox() : null;
|
|
1723
|
+
const signedOut = held?.cause === "agent_unavailable";
|
|
1724
|
+
const ok = r.ok && !signedOut;
|
|
1725
|
+
if (!ok && (this.readiness?.ok ?? true))
|
|
1726
|
+
this.opts.log(`! teammates are told this machine can't take requests yet: ${signedOut ? "its agent CLI was signed out (a run here was refused)" : `${r.short} — ${r.fix}`}`);
|
|
1727
|
+
if (ok && this.readiness && !this.readiness.ok)
|
|
1728
|
+
this.opts.log(` agent ready again — teammates are told this machine takes requests`);
|
|
1729
|
+
const outAgent = signedOut ? [...this.signedOutHere][0] : undefined;
|
|
1730
|
+
this.readiness = signedOut
|
|
1731
|
+
? { ok: false, short: "its agent CLI was signed out (a run here was refused)", code: "not_signed_in", fix: held.fix ?? "", signedOut: true, ...(outAgent ? { agent: outAgent } : {}) }
|
|
1732
|
+
: { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}), ...(r.agent ? { agent: r.agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) };
|
|
1733
|
+
this.tellPerson(now, { quiet: signedOut });
|
|
1500
1734
|
}
|
|
1501
1735
|
catch {
|
|
1502
1736
|
this.readiness = null;
|
|
@@ -1509,12 +1743,12 @@ export class Worker {
|
|
|
1509
1743
|
const who = createHash("sha256").update(`${resolve(this.opts.project)}|${this.opts.agent ?? "auto"}|${commands ? "commands" : "-"}`).digest("hex").slice(0, 12);
|
|
1510
1744
|
return `${this.spaceKey()}#${who}@${process.pid}`;
|
|
1511
1745
|
}
|
|
1512
|
-
tellPerson(now) {
|
|
1746
|
+
tellPerson(now, o = {}) {
|
|
1513
1747
|
const r = this.readiness;
|
|
1514
1748
|
if (!r)
|
|
1515
1749
|
return;
|
|
1516
1750
|
const alerts = () => {
|
|
1517
|
-
if (!this.opts.notify || !decisionAlertsOn())
|
|
1751
|
+
if (o.quiet || !this.opts.notify || !decisionAlertsOn())
|
|
1518
1752
|
return false;
|
|
1519
1753
|
try {
|
|
1520
1754
|
return !this.opts.canAlert || this.opts.canAlert();
|
|
@@ -1536,7 +1770,7 @@ export class Worker {
|
|
|
1536
1770
|
const key = this.readinessKey();
|
|
1537
1771
|
const replaces = this.lastReadinessKey && this.lastReadinessKey !== key ? this.lastReadinessKey : undefined;
|
|
1538
1772
|
this.lastReadinessKey = key;
|
|
1539
|
-
const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}) });
|
|
1773
|
+
const t = readinessAlerts(policyHome(this.opts.home)).observe(key, { ok: r.ok, ...(r.code ? { code: r.code } : {}), ...(agent ? { agent } : {}), ...(r.fix ? { fix: r.fix } : {}), ...(r.unknown ? { unknown: true } : {}) }, { now, alerts, log: this.opts.log, ...(replaces ? { replaces } : {}), ...(r.code === "no_agent" && gitMissing() ? { gitMissing: true } : {}), ...(o.quiet ? { certain: true } : {}) });
|
|
1540
1774
|
this.alertPending = t.pending;
|
|
1541
1775
|
if (!t.notice)
|
|
1542
1776
|
return;
|
|
@@ -1580,30 +1814,56 @@ export class Worker {
|
|
|
1580
1814
|
}
|
|
1581
1815
|
}
|
|
1582
1816
|
outageDiagnosis(label, out, others, meanwhile) {
|
|
1817
|
+
const id = AGENTS.find((a) => a.label === label)?.id;
|
|
1818
|
+
if (isSignOut(out) || (id && this.signedOutHere.has(id))) {
|
|
1819
|
+
let why = (id && this.signedOutHere.has(id) && this.outages.get(id)?.why) || undefined;
|
|
1820
|
+
if (why === SIGNIN_REFUSED)
|
|
1821
|
+
why = undefined;
|
|
1822
|
+
if (!why && id) {
|
|
1823
|
+
const m = signedOutNow(id, { pantheonHome: this.stateDir() }) ?? signedOutChecking(id, { pantheonHome: this.stateDir() });
|
|
1824
|
+
if (m)
|
|
1825
|
+
why = this.signedOutWhy(id, m);
|
|
1826
|
+
}
|
|
1827
|
+
why ??= `is signed out on this machine — ${signInFixFor(id)}`;
|
|
1828
|
+
const rest = meanwhile ? `; the agent standing in for it cannot take this one (${meanwhile.replace(/\.$/, "")})` : others.length ? `; ${others.join("; ")}` : "";
|
|
1829
|
+
return {
|
|
1830
|
+
cause: "agent_unavailable",
|
|
1831
|
+
says: `${label} ${why}${rest}.`,
|
|
1832
|
+
fix: "It runs as soon as you sign in again — or answer it in a live session.",
|
|
1833
|
+
peer: {
|
|
1834
|
+
summary: `${this.cfg.member}'s agent is signed out — this waits until ${this.cfg.member} signs in`,
|
|
1835
|
+
body: `${this.cfg.member}'s worker could not start a run: ${label} is signed out on that machine. It is still owed; it runs once ${this.cfg.member} signs in again.`,
|
|
1836
|
+
},
|
|
1837
|
+
};
|
|
1838
|
+
}
|
|
1583
1839
|
const at = whenAgain(out.until);
|
|
1584
1840
|
const rest = meanwhile ? `; the agent standing in for it cannot take this one (${meanwhile.replace(/\.$/, "")})`
|
|
1585
1841
|
: others.length ? `; ${others.join("; ")}` : "; no other agent is installed for runs while you are away";
|
|
1586
1842
|
return {
|
|
1587
1843
|
cause: "agent_unavailable",
|
|
1588
|
-
says: `${label} is unavailable here until about ${at}
|
|
1844
|
+
says: `${label} ${outagePredicate(out.why)}, so it is unavailable here until about ${at}${rest}.`,
|
|
1589
1845
|
fix: `It runs by itself after ${at}. Sooner: sign in to another agent (cursor-agent, codex or copilot) — and for an agent without a sandbox, pantheon policy grant unsandboxed-runs — or answer it in a live session.`,
|
|
1590
1846
|
peer: {
|
|
1591
|
-
summary: `${this.cfg.member}'s agent is unavailable
|
|
1592
|
-
body: `${this.cfg.member}'s worker could not start a run: ${label} ${out.why}, and no other agent on that machine could take it. It is still owed; the worker tries again
|
|
1847
|
+
summary: `${this.cfg.member}'s agent is unavailable for ${forAbout(out.until)} — this waits`,
|
|
1848
|
+
body: `${this.cfg.member}'s worker could not start a run: ${label} ${outagePredicate(out.why)}, and no other agent on that machine could take it. It is still owed; the worker tries again in ${forAbout(out.until)}.`,
|
|
1593
1849
|
},
|
|
1594
1850
|
};
|
|
1595
1851
|
}
|
|
1596
1852
|
signedOutHere = new Set();
|
|
1853
|
+
allOutSignedOut() {
|
|
1854
|
+
const out = Object.keys(activeOutages(this.outages));
|
|
1855
|
+
return out.length > 0 && out.every((id) => this.signedOutHere.has(id));
|
|
1856
|
+
}
|
|
1597
1857
|
syncSignedOut() {
|
|
1598
1858
|
if (this.opts.agentCmd)
|
|
1599
1859
|
return;
|
|
1600
1860
|
for (const id of SIGNED_OUT_AGENTS) {
|
|
1601
|
-
const mark = signedOutNow(id, { pantheonHome: this.stateDir(), lift: true });
|
|
1861
|
+
const mark = signedOutNow(id, { pantheonHome: this.stateDir(), lift: true, env: process.env });
|
|
1602
1862
|
if (mark) {
|
|
1603
1863
|
const until = mark.at + retryAfter(mark);
|
|
1604
1864
|
const cur = this.outages.get(id);
|
|
1605
1865
|
if (!cur || cur.until < until)
|
|
1606
|
-
this.outages.set(id, { why: this.signedOutWhy(id, mark
|
|
1866
|
+
this.outages.set(id, { why: this.signedOutWhy(id, mark), until, signIn: true });
|
|
1607
1867
|
this.signedOutHere.add(id);
|
|
1608
1868
|
}
|
|
1609
1869
|
else if (this.signedOutHere.has(id)) {
|
|
@@ -1623,10 +1883,14 @@ export class Worker {
|
|
|
1623
1883
|
}
|
|
1624
1884
|
}
|
|
1625
1885
|
}
|
|
1626
|
-
signedOutWhy(id,
|
|
1886
|
+
signedOutWhy(id, mark) {
|
|
1887
|
+
const says = mark.says;
|
|
1888
|
+
const w = refusedWords(id, { line: says, ...(mark.via ? { env: mark.via } : {}), ...(mark.basis === "settings" ? { settings: true } : {}), ...(mark.helper ? { helper: true } : {}), ...(mark.basis === "token" ? { savedToken: true } : {}) });
|
|
1889
|
+
if (w)
|
|
1890
|
+
return `refused ${w.what} ("${says}") — ${w.fix}`;
|
|
1627
1891
|
if (id === "claude-code" && claudeTokenForRun(this.stateDir()))
|
|
1628
1892
|
return `refused the token saved for this worker ("${says}") — make a new one (claude setup-token, then pantheon worker token), or remove it to use your own sign-in: pantheon worker token --remove`;
|
|
1629
|
-
const how =
|
|
1893
|
+
const how = signInFixFor(id);
|
|
1630
1894
|
return `is signed out on this machine ("${says}") — ${how}${id === "claude-code" ? ", or keep it signed in for a year: claude setup-token, then pantheon worker token" : ""}`;
|
|
1631
1895
|
}
|
|
1632
1896
|
outagesPassed(c) {
|
|
@@ -1648,7 +1912,7 @@ export class Worker {
|
|
|
1648
1912
|
try {
|
|
1649
1913
|
this.opts.notify({
|
|
1650
1914
|
kind: "worker", who: "Your worker", heading: `${c.backend.label} is answering for Claude Code`, context: this.cfg.space,
|
|
1651
|
-
body: `Claude Code is
|
|
1915
|
+
body: `${isSignOut(first.out) ? `Claude Code ${outagePredicate(first.out.why === SIGNIN_REFUSED ? "is signed out on this machine" : first.out.why)}. Until it is signed in again` : `Claude Code ${outagePredicate(first.out.why)}, so it is unavailable until about ${whenAgain(first.out.until)}. Until then`} ${c.backend.label} takes teammates' requests on this machine, without Claude Code's sandbox, as your settings allow. To have them wait instead: pantheon policy revoke unsandboxed-runs`,
|
|
1652
1916
|
ttl: 0, sound: false,
|
|
1653
1917
|
});
|
|
1654
1918
|
}
|
|
@@ -1762,7 +2026,7 @@ export class Worker {
|
|
|
1762
2026
|
}
|
|
1763
2027
|
releaseHeld(id, failures, why) {
|
|
1764
2028
|
this.parkedNoSandbox.delete(id);
|
|
1765
|
-
this.failedRuns.set(id, failures);
|
|
2029
|
+
this.failedRuns.set(this.oKey(id), failures);
|
|
1766
2030
|
this.skipLogged.delete(`parked:${id}`);
|
|
1767
2031
|
this.skipLogged.delete(`refused:${id}`);
|
|
1768
2032
|
const act = this.parkedNoSandboxConts.get(id);
|
|
@@ -1789,7 +2053,7 @@ export class Worker {
|
|
|
1789
2053
|
}
|
|
1790
2054
|
if (!act && owedIds && !owedIds.has(id)) {
|
|
1791
2055
|
this.parkedNoSandbox.delete(id);
|
|
1792
|
-
this.failedRuns.delete(id);
|
|
2056
|
+
this.failedRuns.delete(this.oKey(id));
|
|
1793
2057
|
this.skipLogged.delete(`parked:${id}`);
|
|
1794
2058
|
continue;
|
|
1795
2059
|
}
|
|
@@ -1812,11 +2076,11 @@ export class Worker {
|
|
|
1812
2076
|
if (!ok)
|
|
1813
2077
|
this.ranNoSandboxCheckedAt = now;
|
|
1814
2078
|
this.loadContinued();
|
|
1815
|
-
const ran = (owed ?? []).filter((o) => this.
|
|
2079
|
+
const ran = (owed ?? []).filter((o) => this.marked("parked-ns", o.id) && !this.marked("unparked-ns", o.id));
|
|
1816
2080
|
if (!ran.length || (!ok && !this.claudeSandboxableNow()))
|
|
1817
2081
|
return;
|
|
1818
2082
|
for (const o of ran) {
|
|
1819
|
-
this.
|
|
2083
|
+
this.markFor("unparked-ns", o.id);
|
|
1820
2084
|
this.releaseHeld(o.id, 1, ok ? "unsandboxed runs are allowed here now" : "this Claude Code can run commands in its sandbox now");
|
|
1821
2085
|
}
|
|
1822
2086
|
}
|
|
@@ -1828,7 +2092,7 @@ export class Worker {
|
|
|
1828
2092
|
if (!changed)
|
|
1829
2093
|
return;
|
|
1830
2094
|
for (const id of this.parkedNotHere) {
|
|
1831
|
-
this.failedRuns.delete(id);
|
|
2095
|
+
this.failedRuns.delete(this.oKey(id));
|
|
1832
2096
|
this.skipLogged.delete(`parked:${id}`);
|
|
1833
2097
|
this.skipLogged.delete(`route:${id}`);
|
|
1834
2098
|
this.opts.log(`* this machine's folders changed — trying ${id} again (it was waiting for a repo that was not here)`);
|
|
@@ -1897,17 +2161,27 @@ export class Worker {
|
|
|
1897
2161
|
return;
|
|
1898
2162
|
}
|
|
1899
2163
|
this.loadContinued();
|
|
2164
|
+
const told = new Set();
|
|
1900
2165
|
for (const team of new Set([this.cfg.spaceId, this.cfg.space].filter((t) => !!t))) {
|
|
1901
2166
|
for (const [k, at] of Object.entries(no)) {
|
|
1902
2167
|
if (!k.startsWith(`${team}|`))
|
|
1903
2168
|
continue;
|
|
1904
2169
|
const member = k.slice(team.length + 1);
|
|
2170
|
+
if (told.has(member)) {
|
|
2171
|
+
this.markDone(`wake-no-said:${k}:${at}`);
|
|
2172
|
+
continue;
|
|
2173
|
+
}
|
|
1905
2174
|
if (!this.askedMembers.has(member))
|
|
1906
2175
|
continue;
|
|
1907
2176
|
const mark = `wake-no-said:${k}:${at}`;
|
|
1908
|
-
if (this.contDone.has(mark)
|
|
2177
|
+
if (this.contDone.has(mark)) {
|
|
2178
|
+
told.add(member);
|
|
2179
|
+
continue;
|
|
2180
|
+
}
|
|
2181
|
+
if (Date.now() - at > 24 * 3600_000)
|
|
1909
2182
|
continue;
|
|
1910
2183
|
this.markDone(mark);
|
|
2184
|
+
told.add(member);
|
|
1911
2185
|
await client.send({
|
|
1912
2186
|
act: "NOTE", goal: null, to: [member], after_seq: Math.max(client.readSeq, client.seq),
|
|
1913
2187
|
summary: `${this.cfg.member}'s machine will not start work for you while they are away`,
|
|
@@ -1916,9 +2190,151 @@ export class Worker {
|
|
|
1916
2190
|
}
|
|
1917
2191
|
}
|
|
1918
2192
|
}
|
|
2193
|
+
answeredHere(ref) {
|
|
2194
|
+
const k = `handled:${ref}`;
|
|
2195
|
+
return this.contDone.has(k) || this.continuedHas(k);
|
|
2196
|
+
}
|
|
2197
|
+
escalationAnswer(client, o, acts = client.mineSince(0)) {
|
|
2198
|
+
const signs = client.memberSigns(this.cfg.member);
|
|
2199
|
+
const as = (a, want) => {
|
|
2200
|
+
const t = client.trustOf(a).trust;
|
|
2201
|
+
if (t === want)
|
|
2202
|
+
return true;
|
|
2203
|
+
return !a.sig && !signs && t === "unsigned" && (want === "verified-run") === (a.role === "worker");
|
|
2204
|
+
};
|
|
2205
|
+
const byId = new Map(acts.map((a) => [a.id, a]));
|
|
2206
|
+
for (let i = acts.length - 1; i >= 0; i--) {
|
|
2207
|
+
const r = acts[i];
|
|
2208
|
+
if (r.act !== "ANSWER" || r.from !== this.cfg.member || !r.ref)
|
|
2209
|
+
continue;
|
|
2210
|
+
const e = byId.get(r.ref) ?? client.seenAct(r.ref);
|
|
2211
|
+
if (!e || e.act !== "ESCALATE" || e.from !== this.cfg.member)
|
|
2212
|
+
continue;
|
|
2213
|
+
if (e.ref ? e.ref !== o.ref : !this.hasMark(`${ESCALATED_FOR}${e.id}:${o.ref}`))
|
|
2214
|
+
continue;
|
|
2215
|
+
if (!as(e, "verified-run") || !as(r, "verified"))
|
|
2216
|
+
continue;
|
|
2217
|
+
return { escalation: e, answer: r };
|
|
2218
|
+
}
|
|
2219
|
+
return null;
|
|
2220
|
+
}
|
|
2221
|
+
decidedLookedUp = new Set();
|
|
2222
|
+
async reopenDecided(client, o) {
|
|
2223
|
+
let d = this.escalationAnswer(client, o);
|
|
2224
|
+
if (!d && !this.decidedLookedUp.has(o.ref)) {
|
|
2225
|
+
this.decidedLookedUp.add(o.ref);
|
|
2226
|
+
if (this.decidedLookedUp.size > 500)
|
|
2227
|
+
this.decidedLookedUp.delete(this.decidedLookedUp.values().next().value);
|
|
2228
|
+
const acts = await this.recentActs();
|
|
2229
|
+
d = this.escalationAnswer(client, o, acts.filter((a) => a.from === this.cfg.member).sort((x, y) => x.seq - y.seq));
|
|
2230
|
+
}
|
|
2231
|
+
if (!d)
|
|
2232
|
+
return;
|
|
2233
|
+
const mark = `decided:${d.answer.id}`;
|
|
2234
|
+
if (this.contDone.has(mark) || this.continuedHas(mark))
|
|
2235
|
+
return;
|
|
2236
|
+
this.markDone(mark);
|
|
2237
|
+
this.markDone(`${DECIDED_FOR}${o.ref}:${d.answer.id}`);
|
|
2238
|
+
this.decided.set(o.ref, d.answer.body);
|
|
2239
|
+
if (this.decided.size > 200)
|
|
2240
|
+
this.decided.delete(this.decided.keys().next().value);
|
|
2241
|
+
const handled = `handled:${o.ref}`;
|
|
2242
|
+
this.contDone.delete(handled);
|
|
2243
|
+
const homes = this.opts.home !== undefined ? [this.opts.home, undefined] : [undefined];
|
|
2244
|
+
for (const h of homes) {
|
|
2245
|
+
for (const k of this.contKeys())
|
|
2246
|
+
unmarkContinued(k, handled, h);
|
|
2247
|
+
try {
|
|
2248
|
+
unmarkAnswered(this.pinTeam, o.ref, h);
|
|
2249
|
+
}
|
|
2250
|
+
catch { }
|
|
2251
|
+
}
|
|
2252
|
+
this.skipLogged.delete(`handled:${o.id}`);
|
|
2253
|
+
this.opts.log(`> ${owedLabel(o.kind)} ${o.id}: its person answered the escalation (${d.answer.id}) — a run takes it again to pass that on`);
|
|
2254
|
+
}
|
|
2255
|
+
decided = new Map();
|
|
2256
|
+
hasMark(id) { return this.contDone.has(id) || this.continuedHas(id); }
|
|
2257
|
+
async decisionFor(client, o) {
|
|
2258
|
+
const kept = this.decided.get(o.ref);
|
|
2259
|
+
if (kept !== undefined)
|
|
2260
|
+
return kept;
|
|
2261
|
+
const seen = this.escalationAnswer(client, o)?.answer.body;
|
|
2262
|
+
if (seen !== undefined)
|
|
2263
|
+
return seen;
|
|
2264
|
+
this.loadContinued();
|
|
2265
|
+
const prefix = `${DECIDED_FOR}${o.ref}:`;
|
|
2266
|
+
const ids = [...this.contDone].filter((k) => k.startsWith(prefix)).map((k) => k.slice(prefix.length));
|
|
2267
|
+
const id = ids.at(-1);
|
|
2268
|
+
if (!id)
|
|
2269
|
+
return undefined;
|
|
2270
|
+
const answer = (await this.fetchActById(id));
|
|
2271
|
+
const escalation = answer?.ref ? (await this.fetchActById(answer.ref)) : null;
|
|
2272
|
+
if (!answer || !escalation)
|
|
2273
|
+
return undefined;
|
|
2274
|
+
return this.escalationAnswer(client, o, [escalation, answer])?.answer.body;
|
|
2275
|
+
}
|
|
2276
|
+
async recentActs() {
|
|
2277
|
+
try {
|
|
2278
|
+
const res = await signedRead(this.cfg, "/history", {}, { timeoutMs: 5000 });
|
|
2279
|
+
if (!res.ok)
|
|
2280
|
+
return [];
|
|
2281
|
+
return (await res.json()).acts ?? [];
|
|
2282
|
+
}
|
|
2283
|
+
catch {
|
|
2284
|
+
return [];
|
|
2285
|
+
}
|
|
2286
|
+
}
|
|
1919
2287
|
markDone(id) {
|
|
1920
2288
|
this.contDone.add(id);
|
|
1921
2289
|
markContinued(this.pinTeam, id, this.opts.home);
|
|
2290
|
+
if (WORKER_MARK.test(id))
|
|
2291
|
+
this.keepOwn();
|
|
2292
|
+
}
|
|
2293
|
+
async verifyOwn(o) {
|
|
2294
|
+
return verifyOwnWrites(o, this.cfg.member, (t) => configForTeam(t, this.cfg), refusedSaid(this.opts.log));
|
|
2295
|
+
}
|
|
2296
|
+
keepOwn() { try {
|
|
2297
|
+
persistOwn(this.stateDir());
|
|
2298
|
+
}
|
|
2299
|
+
catch { } }
|
|
2300
|
+
pendingRestore = null;
|
|
2301
|
+
pendingSaid = "";
|
|
2302
|
+
settlePendingRestore() {
|
|
2303
|
+
const p = this.pendingRestore;
|
|
2304
|
+
if (!p)
|
|
2305
|
+
return false;
|
|
2306
|
+
let changes = [];
|
|
2307
|
+
const only = { ...p.snap, files: p.snap.files.filter((f) => p.rels.includes(f.rel)), queueDone: true };
|
|
2308
|
+
try {
|
|
2309
|
+
changes = (this.opts.restore ?? restoreHome)(only);
|
|
2310
|
+
}
|
|
2311
|
+
catch {
|
|
2312
|
+
changes = p.rels.map((rel) => ({ rel, what: "reported", keys: [], failed: true }));
|
|
2313
|
+
}
|
|
2314
|
+
const left = changes.filter((c) => c.failed).map((c) => c.rel);
|
|
2315
|
+
if (!left.length) {
|
|
2316
|
+
this.pendingRestore = null;
|
|
2317
|
+
this.pendingSaid = "";
|
|
2318
|
+
clearPersistedGuard(this.stateDir());
|
|
2319
|
+
this.opts.log(`* put back what a teammate's request had locked: ${p.rels.join(", ")} — unattended runs start again`);
|
|
2320
|
+
try {
|
|
2321
|
+
this.opts.notify?.({ kind: "worker", who: "Your worker", heading: "Settings put back — runs start again", context: this.cfg.space, body: `What a teammate's request changed in ${p.rels.join(", ")} is put back now. Unattended runs start again.`, ttl: 0, sound: false });
|
|
2322
|
+
}
|
|
2323
|
+
catch { }
|
|
2324
|
+
return false;
|
|
2325
|
+
}
|
|
2326
|
+
p.rels = left;
|
|
2327
|
+
const key = left.join(",");
|
|
2328
|
+
if (this.pendingSaid !== key) {
|
|
2329
|
+
this.pendingSaid = key;
|
|
2330
|
+
const what = `A teammate's request locked ${left.join(", ")} in ${this.stateDir()} while it ran, so Pantheon could not put back what it changed there. No unattended run starts until it can; your worker tries again every few seconds.`;
|
|
2331
|
+
this.opts.log(`! holding every unattended run: ${what}`);
|
|
2332
|
+
try {
|
|
2333
|
+
this.opts.notify?.({ kind: "worker", who: "Your worker", heading: "Unattended runs are on hold", context: this.cfg.space, body: `${what} To fix: ${brokenStateFix(left)}`, ttl: 0, sound: true });
|
|
2334
|
+
}
|
|
2335
|
+
catch { }
|
|
2336
|
+
}
|
|
2337
|
+
return true;
|
|
1922
2338
|
}
|
|
1923
2339
|
async bringMachineAlong() {
|
|
1924
2340
|
if (this.opts.selfUpgrade?.run)
|
|
@@ -2126,6 +2542,7 @@ export class Worker {
|
|
|
2126
2542
|
if (open) {
|
|
2127
2543
|
if (open[1].refs && !open[1].refs.includes(o.ref)) {
|
|
2128
2544
|
open[1].refs.push(o.ref);
|
|
2545
|
+
askedHere.wake.set(open[0], canon(open[1]));
|
|
2129
2546
|
savePolicy(pol, this.opts.home);
|
|
2130
2547
|
}
|
|
2131
2548
|
this.alertWakeAsk(member, what, open[0]);
|
|
@@ -2165,9 +2582,10 @@ export class Worker {
|
|
|
2165
2582
|
const me2 = client.members.find((m) => m.member === member);
|
|
2166
2583
|
const teamBlocked = me2?.admitted === true && !me2.revoked_by && me2.may_wake === false;
|
|
2167
2584
|
(p.wakeAsks ??= {})[sent.id] = { team: this.cfg.spaceId ?? this.cfg.space, member, at: Date.now(), refs: [o.ref], ...(teamBlocked ? { teamBlocked: true } : {}) };
|
|
2168
|
-
askedHere.
|
|
2585
|
+
askedHere.wake.set(sent.id, canon(p.wakeAsks[sent.id]));
|
|
2169
2586
|
this.askedMembers.add(member);
|
|
2170
2587
|
savePolicy(p, this.opts.home);
|
|
2588
|
+
this.keepOwn();
|
|
2171
2589
|
this.markDone(`wake-asked:${sent.id}`);
|
|
2172
2590
|
this.opts.log(`* asked you whether ${member} may start work on this machine (their ${owedLabel(o.kind)} ${o.id} waits for the answer)`);
|
|
2173
2591
|
this.alertWakeAsk(member, what, sent.id);
|
|
@@ -2312,6 +2730,9 @@ export class Worker {
|
|
|
2312
2730
|
this.contDone.add(id);
|
|
2313
2731
|
markContinued(this.pinTeam, id, this.opts.home);
|
|
2314
2732
|
}
|
|
2733
|
+
for (const x of storedIn(dir))
|
|
2734
|
+
if (x.team === this.pinTeam)
|
|
2735
|
+
markContinued(storedKey(this.pinTeam), x.ref, this.opts.home);
|
|
2315
2736
|
}
|
|
2316
2737
|
catch { }
|
|
2317
2738
|
const { token: _t, privateKey: _k, pendingAuth: _p, delegate: _d, ...stub } = this.cfg;
|
|
@@ -2377,7 +2798,7 @@ export class Worker {
|
|
|
2377
2798
|
report.observe("down");
|
|
2378
2799
|
client.onChange = () => { nudger.observe(obligationSignature(client.obligations, client.connected)); report.observe(clientState(client)); };
|
|
2379
2800
|
try {
|
|
2380
|
-
this.recoverGuard();
|
|
2801
|
+
await this.recoverGuard();
|
|
2381
2802
|
await client.connect();
|
|
2382
2803
|
this.client = client;
|
|
2383
2804
|
await (this.opts.alertsOnly ? this.sayKeyChanges(client) : this.opts.housekeeping ? this.keepHouse(client) : this.serve(client));
|
|
@@ -2453,7 +2874,7 @@ export class Worker {
|
|
|
2453
2874
|
for (const w of p.warnings)
|
|
2454
2875
|
this.opts.log(` ! ${p.name}: ${w}`);
|
|
2455
2876
|
}
|
|
2456
|
-
this.recoverInterruptedRun();
|
|
2877
|
+
await this.recoverInterruptedRun(client);
|
|
2457
2878
|
await this.settleLeftHeld(client);
|
|
2458
2879
|
this.reportStanding(true);
|
|
2459
2880
|
if (this.opts.continuations !== false)
|
|
@@ -2461,11 +2882,26 @@ export class Worker {
|
|
|
2461
2882
|
this.writeState("watching", "waiting for work");
|
|
2462
2883
|
while (!this.stopped) {
|
|
2463
2884
|
const arrived = await client.next(5000);
|
|
2885
|
+
if (this.deadClaims.size)
|
|
2886
|
+
await this.releaseDeadClaims(client);
|
|
2464
2887
|
{
|
|
2465
2888
|
const held = this.tickHold;
|
|
2466
2889
|
this.tickHold = null;
|
|
2467
2890
|
this.reportStanding(false, held);
|
|
2468
2891
|
}
|
|
2892
|
+
if (this.guardWaiting) {
|
|
2893
|
+
if (Date.now() >= this.guardRetryAt)
|
|
2894
|
+
await this.recoverGuard();
|
|
2895
|
+
if (this.guardWaiting) {
|
|
2896
|
+
this.writeState("held", "putting back after an interrupted run waits for the relay");
|
|
2897
|
+
continue;
|
|
2898
|
+
}
|
|
2899
|
+
}
|
|
2900
|
+
if (policyUnreadable(loadPolicy(this.opts.home))) {
|
|
2901
|
+
this.sayBrokenState(["worker-policy.json"]);
|
|
2902
|
+
this.writeState("held", "worker-policy.json cannot be read");
|
|
2903
|
+
continue;
|
|
2904
|
+
}
|
|
2469
2905
|
if (client.refused && !this.refusedSaid) {
|
|
2470
2906
|
this.refusedSaid = true;
|
|
2471
2907
|
if (client.removedFrom)
|
|
@@ -2490,6 +2926,10 @@ export class Worker {
|
|
|
2490
2926
|
this.tickHold = { reason: "busy_elsewhere", detail: "another team's run is in progress on this machine" };
|
|
2491
2927
|
continue;
|
|
2492
2928
|
}
|
|
2929
|
+
if (this.settlePendingRestore()) {
|
|
2930
|
+
this.writeState("held", `${this.pendingRestore.rels.join(", ")} could not be put back after a run`);
|
|
2931
|
+
continue;
|
|
2932
|
+
}
|
|
2493
2933
|
const broken = this.brokenStateFiles();
|
|
2494
2934
|
if (broken.length) {
|
|
2495
2935
|
this.sayBrokenState(broken);
|
|
@@ -2506,7 +2946,9 @@ export class Worker {
|
|
|
2506
2946
|
this.busyObservedSince ??= Date.now();
|
|
2507
2947
|
else
|
|
2508
2948
|
this.busyObservedSince = null;
|
|
2949
|
+
this.learnRequests(client.myWork());
|
|
2509
2950
|
this.retryParkedForCommands(client.myWork());
|
|
2951
|
+
await this.askAgainForParked(client);
|
|
2510
2952
|
this.retryParkedForWorkspaces();
|
|
2511
2953
|
if (this.heldRestorePending && Date.now() - this.heldRestoreAt >= (this.opts.sandboxRecheckMs ?? 5 * 60_000))
|
|
2512
2954
|
await this.restoreHeldContinuations(client);
|
|
@@ -2520,12 +2962,20 @@ export class Worker {
|
|
|
2520
2962
|
this.sayKeyChanges(client);
|
|
2521
2963
|
this.syncSignedOut();
|
|
2522
2964
|
await this.settleMoot(client);
|
|
2965
|
+
await this.settleClosedQuestions(client);
|
|
2966
|
+
await this.settleSafety(client);
|
|
2523
2967
|
const aged = client
|
|
2524
2968
|
.myWork()
|
|
2525
2969
|
.filter((o) => o.kind !== "HUMAN" && o.kind !== "DELIVER" && !o.claimed_by)
|
|
2526
2970
|
.filter((o) => Date.now() - o.created_at >= grace)
|
|
2527
2971
|
.filter((o) => { const t = this.outageHold.get(o.id); if (t === undefined)
|
|
2528
2972
|
return true; if (t > Date.now()) {
|
|
2973
|
+
if (this.allOutSignedOut()) {
|
|
2974
|
+
if (!this.tickHold)
|
|
2975
|
+
this.tickHold = { reason: "not_signed_in", detail: "its agent is signed out" };
|
|
2976
|
+
this.decline(o.ref, "your agent is signed out");
|
|
2977
|
+
return false;
|
|
2978
|
+
}
|
|
2529
2979
|
if (!this.tickHold || (this.tickHold.reason === "outage" && (this.tickHold.until ?? Infinity) > t))
|
|
2530
2980
|
this.tickHold = { reason: "outage", until: t, detail: "its agents are out for now" };
|
|
2531
2981
|
this.decline(o.ref, "your agents are out for now");
|
|
@@ -2551,7 +3001,9 @@ export class Worker {
|
|
|
2551
3001
|
for (let i = start; i < aged.length; i++) {
|
|
2552
3002
|
const o = aged[i];
|
|
2553
3003
|
this.loadContinued();
|
|
2554
|
-
if (this.
|
|
3004
|
+
if (this.answeredHere(o.ref))
|
|
3005
|
+
await this.reopenDecided(client, o);
|
|
3006
|
+
if (this.answeredHere(o.ref)) {
|
|
2555
3007
|
if (!this.skipLogged.has(`handled:${o.id}`)) {
|
|
2556
3008
|
this.skipLogged.add(`handled:${o.id}`);
|
|
2557
3009
|
this.opts.log(`- not taking ${owedLabel(o.kind)} ${o.id}: this machine already answered the request it points at (${o.ref})`);
|
|
@@ -2734,7 +3186,9 @@ export class Worker {
|
|
|
2734
3186
|
const slot = this.takeSlot();
|
|
2735
3187
|
if ("held" in slot) {
|
|
2736
3188
|
this.writeState("watching", `waiting: ${slot.held}`);
|
|
2737
|
-
this.tickHold =
|
|
3189
|
+
this.tickHold = slot.held === PERSON_ANSWERING
|
|
3190
|
+
? { reason: "recently_active", detail: PERSON_ANSWERING, until: Date.now() + PERSON_HOLD_TTL_MS }
|
|
3191
|
+
: { reason: "busy_elsewhere", detail: "another run is in progress on this machine" };
|
|
2738
3192
|
if (continuation)
|
|
2739
3193
|
this.requeueContinuation(continuation, 5_000);
|
|
2740
3194
|
continue;
|
|
@@ -2820,6 +3274,9 @@ export class Worker {
|
|
|
2820
3274
|
this.report("idle");
|
|
2821
3275
|
if (candidate && job.ok)
|
|
2822
3276
|
this.markDone(`handled:${candidate.ref}`);
|
|
3277
|
+
if (candidate)
|
|
3278
|
+
for (const id of job.escalated ?? [])
|
|
3279
|
+
this.markDone(`${ESCALATED_FOR}${id}:${candidate.ref}`);
|
|
2823
3280
|
const ran = candidate ? candidateAct : continuation;
|
|
2824
3281
|
if (job.ok && ran?.sig) {
|
|
2825
3282
|
try {
|
|
@@ -2870,7 +3327,7 @@ export class Worker {
|
|
|
2870
3327
|
if (candidate)
|
|
2871
3328
|
this.decline(candidate.ref, job.diagnosis.cause === "repo_not_here" ? "the repo it is about is not on this computer" : "nothing on this computer can run it while you're away");
|
|
2872
3329
|
if (candidate) {
|
|
2873
|
-
this.failedRuns.set(candidate.id, 2);
|
|
3330
|
+
this.failedRuns.set(this.oKey(candidate.id), 2);
|
|
2874
3331
|
this.skipLogged.add(`parked:${candidate.id}`);
|
|
2875
3332
|
if (job.diagnosis.cause === "repo_not_here")
|
|
2876
3333
|
this.parkedNotHere.add(candidate.id);
|
|
@@ -2885,15 +3342,18 @@ export class Worker {
|
|
|
2885
3342
|
const attempts = this.failuresOf(candidate.id) + 1;
|
|
2886
3343
|
const failures = job.diagnosis?.cause === "no_commands" || job.diagnosis?.cause === "no_sandbox_commands" ? 2 : attempts;
|
|
2887
3344
|
if (job.diagnosis?.cause === "no_sandbox_commands")
|
|
2888
|
-
this.
|
|
2889
|
-
if (this.
|
|
2890
|
-
this.
|
|
2891
|
-
this.failedRuns.set(candidate.id, failures);
|
|
2892
|
-
this.
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
3345
|
+
this.markFor("parked-ns", candidate.id);
|
|
3346
|
+
if (this.marked("unparked-ns", candidate.id))
|
|
3347
|
+
this.markFor("parked2-ns", candidate.id);
|
|
3348
|
+
this.failedRuns.set(this.oKey(candidate.id), failures);
|
|
3349
|
+
this.markFor(failures >= 2 ? "parked" : "failed1", candidate.id);
|
|
3350
|
+
const under = this.unparkedUnder.get(this.oKey(candidate.id));
|
|
3351
|
+
if (under)
|
|
3352
|
+
this.markDone(`parked2@${under}:${this.oKey(candidate.id)}`);
|
|
3353
|
+
else if (this.marked("unparked", candidate.id))
|
|
3354
|
+
this.markFor("parked2", candidate.id);
|
|
3355
|
+
if (job.diagnosis?.cause === "no_commands")
|
|
3356
|
+
this.markFor("parked-nc", candidate.id);
|
|
2897
3357
|
if (this.failedRuns.size > 200) {
|
|
2898
3358
|
this.failedRuns.delete(this.failedRuns.keys().next().value);
|
|
2899
3359
|
}
|
|
@@ -2905,6 +3365,22 @@ export class Worker {
|
|
|
2905
3365
|
}
|
|
2906
3366
|
finally {
|
|
2907
3367
|
this.running = false;
|
|
3368
|
+
if (this.stopped) {
|
|
3369
|
+
if (continuation) {
|
|
3370
|
+
try {
|
|
3371
|
+
await client.releaseClaim(`cont:${continuation.id}`);
|
|
3372
|
+
}
|
|
3373
|
+
catch { }
|
|
3374
|
+
}
|
|
3375
|
+
else if (candidate) {
|
|
3376
|
+
this.deadClaims.set(candidate.id, { session: null, until: Date.now() + LIMITS.claimTtlMs + 60_000 });
|
|
3377
|
+
for (let i = 0; i < 12 && this.deadClaims.has(candidate.id); i++) {
|
|
3378
|
+
await this.releaseDeadClaims(client);
|
|
3379
|
+
if (this.deadClaims.has(candidate.id))
|
|
3380
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
3381
|
+
}
|
|
3382
|
+
}
|
|
3383
|
+
}
|
|
2908
3384
|
this.endAttempt();
|
|
2909
3385
|
this.giveSlot(slot.token);
|
|
2910
3386
|
if (continuation)
|
|
@@ -2936,7 +3412,7 @@ export class Worker {
|
|
|
2936
3412
|
return { held: why };
|
|
2937
3413
|
}
|
|
2938
3414
|
if ("held" in lease)
|
|
2939
|
-
return { held: `a run for "${lease.held}" is in progress in another pantheon worker on this machine` };
|
|
3415
|
+
return { held: lease.held === PERSON_ANSWERING ? PERSON_ANSWERING : `a run for "${lease.held}" is in progress in another pantheon worker on this machine` };
|
|
2940
3416
|
fleetLock = this;
|
|
2941
3417
|
const dir = this.stateDir(), token = lease.token;
|
|
2942
3418
|
this.heartbeatAt = Date.now();
|
|
@@ -2966,13 +3442,13 @@ export class Worker {
|
|
|
2966
3442
|
liveRuns.add(pid);
|
|
2967
3443
|
installRunReaper();
|
|
2968
3444
|
this.attemptPid = pid;
|
|
2969
|
-
this.runRecord = { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, ...(workerIdentity() ? { worker_id: workerIdentity() } : {}), obligation: attempt.obligation, ...(attempt.ref ? { ref: attempt.ref } : {}), ...(attempt.spend ? { spend: attempt.spend } : {}), ...(attempt.limit ? { limit: attempt.limit } : {}), ...(attempt.sandbox ? { sandbox: attempt.sandbox } : {}) };
|
|
3445
|
+
this.runRecord = { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, ...(workerIdentity() ? { worker_id: workerIdentity() } : {}), obligation: attempt.obligation, ...(this.oKey(attempt.obligation) !== attempt.obligation ? { okey: this.oKey(attempt.obligation) } : {}), ...(attempt.ref ? { ref: attempt.ref } : {}), ...(attempt.spend ? { spend: attempt.spend } : {}), ...(attempt.limit ? { limit: attempt.limit } : {}), ...(attempt.sandbox ? { sandbox: attempt.sandbox } : {}) };
|
|
2970
3446
|
writeRunRecord(this.stateDir(), this.runRecord);
|
|
2971
3447
|
}
|
|
2972
3448
|
runRecord = null;
|
|
2973
3449
|
recordChanged = false;
|
|
2974
3450
|
recordOnDisk() { try {
|
|
2975
|
-
return
|
|
3451
|
+
return readStateText(join(this.stateDir(), RUN_RECORD));
|
|
2976
3452
|
}
|
|
2977
3453
|
catch {
|
|
2978
3454
|
return null;
|
|
@@ -3005,8 +3481,20 @@ export class Worker {
|
|
|
3005
3481
|
this.attemptPid = null;
|
|
3006
3482
|
this.runRecord = null;
|
|
3007
3483
|
}
|
|
3008
|
-
|
|
3009
|
-
|
|
3484
|
+
guardWaiting = false;
|
|
3485
|
+
guardRetryAt = 0;
|
|
3486
|
+
async recoverGuard() {
|
|
3487
|
+
const g = loadPersistedGuard(this.stateDir());
|
|
3488
|
+
const v = g?.own && g.snapshot && !sameProcess(g.worker, g.workerId) ? await this.verifyOwn(g.own) : undefined;
|
|
3489
|
+
if (v === "unreachable") {
|
|
3490
|
+
if (!this.guardWaiting)
|
|
3491
|
+
this.opts.log(" guard a run was interrupted with its worker; putting Pantheon's files back waits until the relay can be asked — unattended runs wait meanwhile");
|
|
3492
|
+
this.guardWaiting = true;
|
|
3493
|
+
this.guardRetryAt = Date.now() + 30_000;
|
|
3494
|
+
return;
|
|
3495
|
+
}
|
|
3496
|
+
this.guardWaiting = false;
|
|
3497
|
+
const r = recoverInterruptedGuard(this.stateDir(), Date.now(), restoreHome, v);
|
|
3010
3498
|
if (r.outcome === "none" || r.outcome === "live")
|
|
3011
3499
|
return;
|
|
3012
3500
|
const tell = (heading, body) => {
|
|
@@ -3021,6 +3509,12 @@ export class Worker {
|
|
|
3021
3509
|
this.opts.log(` guard a snapshot from a run on ${new Date(r.at).toLocaleString()} was not restored — ${r.outcome === "stale" ? "too old to be that run's" : "you have changed your setup since"}; dropped`);
|
|
3022
3510
|
return;
|
|
3023
3511
|
}
|
|
3512
|
+
if (r.outcome === "pending") {
|
|
3513
|
+
for (const l of r.said)
|
|
3514
|
+
this.opts.log(`! the run interrupted with its worker: put back ${l}`);
|
|
3515
|
+
this.pendingRestore = { snap: r.snapshot, rels: r.rels };
|
|
3516
|
+
return;
|
|
3517
|
+
}
|
|
3024
3518
|
if (r.outcome === "failed") {
|
|
3025
3519
|
this.opts.log(`! could not put Pantheon's own files back after the interrupted run (try ${r.attempts} of ${GUARD_RESTORE_TRIES}): ${r.why}`);
|
|
3026
3520
|
if (r.dropped)
|
|
@@ -3035,11 +3529,11 @@ export class Worker {
|
|
|
3035
3529
|
this.opts.log(`! the run interrupted with its worker: put back ${l}`);
|
|
3036
3530
|
tell("Your worker stopped mid-run — settings put back", `${putBackSaid(r.said)} If you made that change yourself, make it again.`);
|
|
3037
3531
|
}
|
|
3038
|
-
recoverInterruptedRun() {
|
|
3532
|
+
async recoverInterruptedRun(client) {
|
|
3039
3533
|
const dir = this.stateDir(), f = join(dir, RUN_RECORD);
|
|
3040
3534
|
let r;
|
|
3041
3535
|
try {
|
|
3042
|
-
r = JSON.parse(
|
|
3536
|
+
r = JSON.parse(readStateText(f));
|
|
3043
3537
|
}
|
|
3044
3538
|
catch {
|
|
3045
3539
|
return;
|
|
@@ -3059,21 +3553,81 @@ export class Worker {
|
|
|
3059
3553
|
rmSync(f, { force: true });
|
|
3060
3554
|
}
|
|
3061
3555
|
catch { }
|
|
3556
|
+
const replied = r.obligation.startsWith("cont:") ? null : await this.runReplied(client, r);
|
|
3062
3557
|
if (typeof r.spend === "string") {
|
|
3558
|
+
const summary = replied ? "answered (its worker went before the run ended)" : "the worker went before the run ended";
|
|
3559
|
+
try {
|
|
3560
|
+
closeStartedRun(r.spend, summary, { at: r.at, goal: null, obligation: r.obligation, ok: !!replied, triggeredBy: "unknown", summary, space: this.spaceKey(), member: r.member }, this.opts.home, !!replied);
|
|
3561
|
+
}
|
|
3562
|
+
catch { }
|
|
3563
|
+
}
|
|
3564
|
+
if (r.obligation.startsWith("cont:")) {
|
|
3063
3565
|
try {
|
|
3064
|
-
|
|
3566
|
+
await client.releaseClaim(r.obligation);
|
|
3065
3567
|
}
|
|
3066
3568
|
catch { }
|
|
3067
3569
|
}
|
|
3570
|
+
else {
|
|
3571
|
+
const held = client.obligations.find((o) => o.id === r.obligation || (typeof r.okey === "string" && `r:${o.kind}:${o.ref}` === r.okey));
|
|
3572
|
+
if (held) {
|
|
3573
|
+
this.deadClaims.set(held.id, { session: null, until: Date.now() + LIMITS.claimTtlMs + 60_000 });
|
|
3574
|
+
await this.releaseDeadClaims(client);
|
|
3575
|
+
}
|
|
3576
|
+
}
|
|
3068
3577
|
if (r.obligation.startsWith("cont:")) {
|
|
3069
3578
|
this.opts.log(`! the last run (${r.obligation}) ended with its worker — counted, not run again`);
|
|
3070
3579
|
return;
|
|
3071
3580
|
}
|
|
3581
|
+
if (replied) {
|
|
3582
|
+
if (r.ref)
|
|
3583
|
+
this.markDone(`handled:${r.ref}`);
|
|
3584
|
+
this.opts.log(` the last run for ${r.obligation} sent its ${replied} before its worker went — counted as handled`);
|
|
3585
|
+
return;
|
|
3586
|
+
}
|
|
3587
|
+
if (typeof r.okey === "string" && r.okey.startsWith("r:"))
|
|
3588
|
+
this.obKeys.set(r.obligation, r.okey);
|
|
3072
3589
|
const failures = this.failuresOf(r.obligation) + 1;
|
|
3073
|
-
this.failedRuns.set(r.obligation, failures);
|
|
3074
|
-
this.
|
|
3590
|
+
this.failedRuns.set(this.oKey(r.obligation), failures);
|
|
3591
|
+
this.markFor(failures >= 2 ? "parked" : "failed1", r.obligation);
|
|
3075
3592
|
this.opts.log(`! the last run for ${r.obligation} ended with its worker (stopped, restarted or killed mid-run) — counted as a failed attempt${failures >= 2 ? "; parked: it is not run again" : ""}`);
|
|
3076
3593
|
}
|
|
3594
|
+
async runReplied(client, r) {
|
|
3595
|
+
const ref = r.ref, at = r.at;
|
|
3596
|
+
if (typeof ref !== "string" || typeof at !== "number")
|
|
3597
|
+
return null;
|
|
3598
|
+
if (client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" && (o.id === r.obligation || o.ref === ref)))
|
|
3599
|
+
return null;
|
|
3600
|
+
try {
|
|
3601
|
+
const res = await signedRead(this.cfg, "/history", {}, { timeoutMs: 5000 });
|
|
3602
|
+
if (!res.ok)
|
|
3603
|
+
return null;
|
|
3604
|
+
const j = await res.json();
|
|
3605
|
+
const a = (j.acts ?? []).find((x) => x.ref === ref && x.from === this.cfg.member && x.ts >= at && x.sig?.role === "run" && x.act !== "NOTE" && x.act !== "RETRACT");
|
|
3606
|
+
return a ? a.act : null;
|
|
3607
|
+
}
|
|
3608
|
+
catch {
|
|
3609
|
+
return null;
|
|
3610
|
+
}
|
|
3611
|
+
}
|
|
3612
|
+
deadClaims = new Map();
|
|
3613
|
+
async releaseDeadClaims(client) {
|
|
3614
|
+
for (const [id, d] of [...this.deadClaims]) {
|
|
3615
|
+
const by = client.obligations.find((o) => o.id === id)?.claimed_by;
|
|
3616
|
+
if (Date.now() > d.until || !by || by.role !== "worker" || by.session === client.session || (d.session !== null && by.session !== d.session)) {
|
|
3617
|
+
this.deadClaims.delete(id);
|
|
3618
|
+
continue;
|
|
3619
|
+
}
|
|
3620
|
+
d.session = by.session;
|
|
3621
|
+
if (client.roster.some((m) => m.sessions.some((s) => s.id === by.session)))
|
|
3622
|
+
continue;
|
|
3623
|
+
this.deadClaims.delete(id);
|
|
3624
|
+
try {
|
|
3625
|
+
if (await client.releaseClaim(id))
|
|
3626
|
+
this.opts.log(` let go of ${id}: the run that held it went with its worker — it is picked up again now, not in ${Math.round(LIMITS.claimTtlMs / 60_000)} minutes`);
|
|
3627
|
+
}
|
|
3628
|
+
catch { }
|
|
3629
|
+
}
|
|
3630
|
+
}
|
|
3077
3631
|
stateDir() {
|
|
3078
3632
|
return this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
|
|
3079
3633
|
}
|
|
@@ -3107,7 +3661,7 @@ export class Worker {
|
|
|
3107
3661
|
}
|
|
3108
3662
|
brokenStateFiles() {
|
|
3109
3663
|
repairRunLease(this.stateDir());
|
|
3110
|
-
return brokenStateFiles(this.stateDir());
|
|
3664
|
+
return brokenStateFiles(this.stateDir(), GUARDED_FILES);
|
|
3111
3665
|
}
|
|
3112
3666
|
hooksInstalledHere() {
|
|
3113
3667
|
try {
|
|
@@ -3128,7 +3682,8 @@ export class Worker {
|
|
|
3128
3682
|
: `(the triggering act ${printable(o.ref)} is not in this session's buffer; the relay's one-line summary of it follows)\n${o.body}`;
|
|
3129
3683
|
const act = (base ?? undefined);
|
|
3130
3684
|
const needs = workNeedsCommands(o.kind, trigger, o.kind === "REVISE" && act?.ref ? client.seenAct(act.ref)?.act : act?.act);
|
|
3131
|
-
|
|
3685
|
+
const decided = await this.decisionFor(client, o);
|
|
3686
|
+
return this.runJob(client, o, persona, policy, (branch, canPush) => buildBrief(o, trigger, branch, persona, canPush, decided), true, needs, trigger, act);
|
|
3132
3687
|
}
|
|
3133
3688
|
async handleContinuation(client, act, persona, policy) {
|
|
3134
3689
|
this.markDone(act.id);
|
|
@@ -3146,7 +3701,7 @@ export class Worker {
|
|
|
3146
3701
|
}
|
|
3147
3702
|
spentOnce = {};
|
|
3148
3703
|
policyBytes() { try {
|
|
3149
|
-
return
|
|
3704
|
+
return readStateText(policyPath(this.opts.home));
|
|
3150
3705
|
}
|
|
3151
3706
|
catch {
|
|
3152
3707
|
return null;
|
|
@@ -3543,6 +4098,7 @@ export class Worker {
|
|
|
3543
4098
|
}
|
|
3544
4099
|
let detail;
|
|
3545
4100
|
let transcript = "";
|
|
4101
|
+
resetOwn();
|
|
3546
4102
|
const homeBefore = snapshotHome(this.stateDir());
|
|
3547
4103
|
const guardHome = this.stateDir();
|
|
3548
4104
|
try {
|
|
@@ -3658,7 +4214,7 @@ export class Worker {
|
|
|
3658
4214
|
this.cleanupStep("checking Pantheon's own files", () => {
|
|
3659
4215
|
const ok = this.restoreHomeAfterRun(homeBefore, o);
|
|
3660
4216
|
pendingGuards.delete(guardHome);
|
|
3661
|
-
if (ok)
|
|
4217
|
+
if (ok && !this.pendingRestore)
|
|
3662
4218
|
clearPersistedGuard(guardHome);
|
|
3663
4219
|
this.markRunRestored();
|
|
3664
4220
|
});
|
|
@@ -3701,7 +4257,10 @@ export class Worker {
|
|
|
3701
4257
|
const held = await this.settleHeld(client, heldRun ? { run: heldRun, canPush: canPush && !massDelete, massDelete } : null);
|
|
3702
4258
|
const settled = [...held.sent, ...held.waiting];
|
|
3703
4259
|
const settledIds = new Set(settled.map((a) => a.id));
|
|
3704
|
-
const
|
|
4260
|
+
const runId = this.runKey?.runId;
|
|
4261
|
+
const elsewhere = o.goal == null ? [] : client.mineSince(startedAt).filter((a) => a.goal !== o.goal && !settledIds.has(a.id)
|
|
4262
|
+
&& ((!!o.ref && a.ref === o.ref) || (!!runId && a.goal == null && a.sig?.role === "run" && a.sig.cert?.run === runId)));
|
|
4263
|
+
const sent = [...client.mineSince(startedAt, o.goal).filter((a) => !settledIds.has(a.id)), ...elsewhere, ...settled.filter((a) => o.goal == null || a.goal === o.goal)];
|
|
3705
4264
|
const repliedWith = sent.find((a) => ["DELIVERED", "ANSWER", "ESCALATE", "ACCEPT", "OBJECT", "PROPOSE", "CONTRACT", "CLOSE", "WITHDRAW"].includes(a.act) || (a.act === "NOTE" && !!a.ref));
|
|
3706
4265
|
const replied = requireReply ? client.repliedTo(o) || !!repliedWith : true;
|
|
3707
4266
|
if (requireReply && replied && !client.repliedTo(o) && repliedWith) {
|
|
@@ -3748,9 +4307,10 @@ export class Worker {
|
|
|
3748
4307
|
const sentAnywhere = client.mineSince(startedAt).some((a) => !runKeys || !a.sig || a.sig.role === "run");
|
|
3749
4308
|
const outage = !ok && !exited && !sentAnywhere && choice && !/\(stopped after \d+s\)/.test(detail.split("\n")[0] ?? "") ? detectOutage(choice.backend.id, detail) : null;
|
|
3750
4309
|
const refusedLog = snap ? join(scratch, `${basename(worktree)}.${choice?.backend.id}-refused.run.log`) : join(worktree, `.pantheon-run.${choice?.backend.id}-refused.log`);
|
|
4310
|
+
const hintLed = outage?.why === SIGNIN_REFUSED && !this.opts.agentCmd && signedOutChecking(choice.backend.id, { pantheonHome: this.stateDir() })?.lifted === "hint";
|
|
3751
4311
|
if (outage) {
|
|
3752
4312
|
this.outages.set(choice.backend.id, outage);
|
|
3753
|
-
if (outage.why === SIGNIN_REFUSED && !this.opts.agentCmd && markSignedOut(choice.backend.id, detail, { pantheonHome: this.stateDir() })) {
|
|
4313
|
+
if (outage.why === SIGNIN_REFUSED && !this.opts.agentCmd && markSignedOut(choice.backend.id, detail, { pantheonHome: this.stateDir(), env: process.env })) {
|
|
3754
4314
|
this.syncSignedOut();
|
|
3755
4315
|
this.readinessAt = 0;
|
|
3756
4316
|
}
|
|
@@ -3758,22 +4318,26 @@ export class Worker {
|
|
|
3758
4318
|
if (Date.now() - startedAt < ONCE_GIVE_BACK_MS)
|
|
3759
4319
|
this.restoreOnces();
|
|
3760
4320
|
}
|
|
3761
|
-
if (outage && (this.outageRetries.get(o.id) ?? 0) < 3) {
|
|
3762
|
-
|
|
4321
|
+
if (outage && (hintLed || (this.outageRetries.get(o.id) ?? 0) < 3)) {
|
|
4322
|
+
if (!hintLed)
|
|
4323
|
+
this.outageRetries.set(o.id, (this.outageRetries.get(o.id) ?? 0) + 1);
|
|
3763
4324
|
if (this.outageRetries.size > 200)
|
|
3764
4325
|
this.outageRetries.delete(this.outageRetries.keys().next().value);
|
|
3765
|
-
this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over until about ${whenAgain(outage.until)}; the request goes to the next agent here (transcript: ${refusedLog})`);
|
|
4326
|
+
this.opts.log(`! ${choice.backend.label} refused the run for ${o.id}: ${outage.why} — passed over ${outage.why === SIGNIN_REFUSED ? "until it is signed in again" : `until about ${whenAgain(outage.until)}`}; the request goes to the next agent here (transcript: ${refusedLog})`);
|
|
3766
4327
|
return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
|
|
3767
4328
|
}
|
|
3768
4329
|
const quiet = ok && !requireReply && sent.length === 0;
|
|
3769
4330
|
const diagnosis = ok ? undefined : outage ? this.outageDiagnosis(choice.backend.label, outage, [], "it was handed to another agent three times already") : diagnoseRun({ exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands,
|
|
4331
|
+
...(agentId === "claude-code" && !this.opts.agentCmd && claudeTokenForRun(this.stateDir())
|
|
4332
|
+
? { env: { ...process.env, CLAUDE_CODE_OAUTH_TOKEN: undefined }, savedToken: true } : { env: process.env }),
|
|
4333
|
+
...(agentId === "claude-code" ? { claudeSettings: claudeSettingsAuth({ env: process.env }) } : {}),
|
|
3770
4334
|
...(/\(stopped after \d+s\)/.test(detail.split("\n")[0] ?? "") ? { timedOut: { minutes: runLimit.minutes, from: runLimit.from } } : {}),
|
|
3771
4335
|
...(bashDroppedForSandbox ? { commandsNeedSandbox: byCli ? "agent-cmd" : "claude" } : {}), ...(agentId ? { agent: agentId } : {}), ...(place.kind === "folder" ? { folder: true } : {}),
|
|
3772
4336
|
...(agentId === "claude-code" && allowBash ? { allowedRules: bashRules } : {}), ...(baseRef ? { baseRef } : {}) }, this.opts.probe ?? gitProbe);
|
|
3773
4337
|
if (ok && agentId && !this.opts.agentCmd)
|
|
3774
4338
|
clearSignedOut(agentId, { pantheonHome: this.stateDir() });
|
|
3775
4339
|
if (!ok && diagnosis?.cause === "not_signed_in" && agentId && !this.opts.agentCmd
|
|
3776
|
-
&& markSignedOut(agentId, detail, { pantheonHome: this.stateDir() })) {
|
|
4340
|
+
&& markSignedOut(agentId, detail, { pantheonHome: this.stateDir(), env: process.env })) {
|
|
3777
4341
|
this.opts.log(`! ${choice?.backend.label ?? agentId} is signed out on this machine: requests wait until it is signed in again${agentId === "claude-code" ? " (pantheon worker token keeps Claude Code signed in for a year)" : ""}`);
|
|
3778
4342
|
this.syncSignedOut();
|
|
3779
4343
|
this.readinessAt = 0;
|
|
@@ -3788,7 +4352,8 @@ export class Worker {
|
|
|
3788
4352
|
this.opts.log(quiet ? `< read ${o.id} on ${branch} — the thread was already settled, nothing sent`
|
|
3789
4353
|
: ok ? `< handled ${o.id} on ${branch}`
|
|
3790
4354
|
: `! ${o.id} stopped [${diagnosis.cause}]: ${diagnosis.says} (transcript: ${runLog})`);
|
|
3791
|
-
|
|
4355
|
+
const escalated = sent.filter((a) => a.act === "ESCALATE" && !a.ref && a.from === this.cfg.member && (a.sig?.role === "run" || a.role === "worker")).map((a) => a.id);
|
|
4356
|
+
return { obligation: o, branch, worktree, ok, detail, ...(diagnosis ? { diagnosis } : {}), ...(escalated.length ? { escalated } : {}) };
|
|
3792
4357
|
}
|
|
3793
4358
|
catch (e) {
|
|
3794
4359
|
if (!recorded) {
|
|
@@ -3817,7 +4382,7 @@ export class Worker {
|
|
|
3817
4382
|
restoreHomeAfterRun(before, o) {
|
|
3818
4383
|
let changes = [];
|
|
3819
4384
|
try {
|
|
3820
|
-
changes = restoreHome(before);
|
|
4385
|
+
changes = (this.opts.restore ?? restoreHome)(before);
|
|
3821
4386
|
}
|
|
3822
4387
|
catch (e) {
|
|
3823
4388
|
this.opts.log(`! could not check Pantheon's own files after the run: ${e.message}`);
|
|
@@ -3825,6 +4390,14 @@ export class Worker {
|
|
|
3825
4390
|
}
|
|
3826
4391
|
if (!changes.length)
|
|
3827
4392
|
return true;
|
|
4393
|
+
const failed = changes.filter((c) => c.failed).map((c) => c.rel);
|
|
4394
|
+
if (failed.length) {
|
|
4395
|
+
this.pendingRestore = { snap: { ...before, files: before.files.filter((f) => failed.includes(f.rel)) }, rels: failed };
|
|
4396
|
+
try {
|
|
4397
|
+
persistPending(this.stateDir(), failed);
|
|
4398
|
+
}
|
|
4399
|
+
catch { }
|
|
4400
|
+
}
|
|
3828
4401
|
for (const c of changes)
|
|
3829
4402
|
this.opts.log(`${c.what === "reported" || c.quiet ? " " : "!"} ${o.id}: ${describeChange(c)}`);
|
|
3830
4403
|
const taken = changes.filter((c) => c.what !== "reported" && !c.quiet);
|
|
@@ -4120,7 +4693,7 @@ export class Worker {
|
|
|
4120
4693
|
if (this.running || !client.connected)
|
|
4121
4694
|
return;
|
|
4122
4695
|
try {
|
|
4123
|
-
const r = JSON.parse(
|
|
4696
|
+
const r = JSON.parse(readStateText(join(this.stateDir(), RUN_RECORD)));
|
|
4124
4697
|
if (r && r.space === this.cfg.space && Number.isInteger(r.worker) && r.worker !== process.pid && pidAlive(r.worker))
|
|
4125
4698
|
return;
|
|
4126
4699
|
}
|
|
@@ -4197,9 +4770,11 @@ export class Worker {
|
|
|
4197
4770
|
}
|
|
4198
4771
|
if (!owed)
|
|
4199
4772
|
return;
|
|
4200
|
-
|
|
4773
|
+
this.learnRequests([o]);
|
|
4774
|
+
const key = `refused:${this.oKey(o.id)}`;
|
|
4201
4775
|
const space = this.spaceKey();
|
|
4202
|
-
|
|
4776
|
+
const told = loadContinued(space, this.opts.home);
|
|
4777
|
+
if (told.has(key) || told.has(`refused:${o.id}`))
|
|
4203
4778
|
return;
|
|
4204
4779
|
const peer = client.recentFrom(o.ref);
|
|
4205
4780
|
try {
|
|
@@ -4226,7 +4801,7 @@ export class Worker {
|
|
|
4226
4801
|
if (!client.connected)
|
|
4227
4802
|
return;
|
|
4228
4803
|
const policy = loadPolicy(this.opts.home);
|
|
4229
|
-
const teams = [this.cfg.spaceId
|
|
4804
|
+
const teams = this.cfg.spaceId ? [this.cfg.spaceId] : [this.cfg.space];
|
|
4230
4805
|
const reports = [
|
|
4231
4806
|
...Object.entries(policy.stuckReports ?? {}).filter(([, r]) => teams.includes(r.team))
|
|
4232
4807
|
.map(([act, r]) => [act, [r.ref], "stuck"]),
|
|
@@ -4240,31 +4815,43 @@ export class Worker {
|
|
|
4240
4815
|
return;
|
|
4241
4816
|
const now = Date.now();
|
|
4242
4817
|
const done = [];
|
|
4818
|
+
const waitingHere = (ref) => client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" && o.ref === ref);
|
|
4819
|
+
const stillNeeds = (cap, except) => Object.entries(policy.stuckReports ?? {}).some(([a, r]) => a !== except && r.cap === cap && (teams.includes(r.team) ? waitingHere(r.ref) : true)) ||
|
|
4820
|
+
(cap === "run_commands" && client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" &&
|
|
4821
|
+
!(policy.stuckReports?.[except]?.ref === o.ref) && this.marked("parked-nc", o.id) && this.failuresOf(o.id) >= 2));
|
|
4243
4822
|
for (const [act, refs, kind] of reports) {
|
|
4244
4823
|
const member = kind === "wake" ? policy.wakeAsks?.[act]?.member : undefined;
|
|
4245
4824
|
const owed = client.obligations.some((o) => o.owner === this.cfg.member && o.kind !== "HUMAN" &&
|
|
4246
4825
|
(refs.includes(o.ref) || (!!member && (client.seenAct(o.ref)?.from ?? /^([a-z0-9._-]+)\s/i.exec(o.body)?.[1]) === member)));
|
|
4247
|
-
|
|
4248
|
-
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
|
|
4252
|
-
|
|
4253
|
-
|
|
4254
|
-
|
|
4826
|
+
const asksFor = kind === "stuck" ? Object.entries(policy.asks).find(([, a]) => a?.act === act)?.[0] : undefined;
|
|
4827
|
+
const settled = member ? wakeSettled(policy, [this.cfg.spaceId, this.cfg.space], member) : null;
|
|
4828
|
+
if (!settled) {
|
|
4829
|
+
if (owed || (asksFor && stillNeeds(asksFor, act))) {
|
|
4830
|
+
this.mootSince.delete(act);
|
|
4831
|
+
continue;
|
|
4832
|
+
}
|
|
4833
|
+
const since = this.mootSince.get(act);
|
|
4834
|
+
if (since === undefined) {
|
|
4835
|
+
this.mootSince.set(act, now);
|
|
4836
|
+
continue;
|
|
4837
|
+
}
|
|
4838
|
+
if (now - since < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
|
|
4839
|
+
continue;
|
|
4255
4840
|
}
|
|
4256
|
-
if (now - since < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
|
|
4257
|
-
continue;
|
|
4258
4841
|
if ((this.mootRetryAt.get(act) ?? 0) > now)
|
|
4259
4842
|
continue;
|
|
4260
4843
|
const open = client.obligations.some((o) => o.kind === "HUMAN" && o.ref === act);
|
|
4261
4844
|
if (open) {
|
|
4262
4845
|
try {
|
|
4263
4846
|
await client.send({ act: "RETRACT", goal: null, ref: act, after_seq: Math.max(client.readSeq, client.seq),
|
|
4264
|
-
body:
|
|
4265
|
-
?
|
|
4266
|
-
:
|
|
4267
|
-
|
|
4847
|
+
body: settled
|
|
4848
|
+
? `No longer needed: settled on this computer — ${settled === "yes" ? `${member} may start work here (pantheon wake allow ${member})` : `no to unattended runs from ${member} here (pantheon wake deny ${member})`}.`
|
|
4849
|
+
: kind === "wake"
|
|
4850
|
+
? "No longer needed: the request this was about isn't waiting here any more (it was answered, withdrawn or closed). Nothing was changed on this computer — you'll be asked again if they need it."
|
|
4851
|
+
: "No longer needed: the request this was about isn't waiting here any more (it was answered, withdrawn or closed). Nothing was changed on this computer." });
|
|
4852
|
+
this.opts.log(settled
|
|
4853
|
+
? `* withdrew ${act}: you settled it here (pantheon wake ${settled === "yes" ? "allow" : "deny"} ${member})`
|
|
4854
|
+
: `* withdrew ${act}: the request${refs.length > 1 ? "s" : ""} it was about (${refs.join(", ")}) ${refs.length > 1 ? "are" : "is"} finished`);
|
|
4268
4855
|
}
|
|
4269
4856
|
catch (e) {
|
|
4270
4857
|
this.mootRetryAt.set(act, now + 5 * 60_000);
|
|
@@ -4280,6 +4867,7 @@ export class Worker {
|
|
|
4280
4867
|
return;
|
|
4281
4868
|
const fresh = loadPolicy(this.opts.home);
|
|
4282
4869
|
for (const [act, kind] of done) {
|
|
4870
|
+
withdrawnHere.add(act);
|
|
4283
4871
|
if (kind === "wake")
|
|
4284
4872
|
delete fresh.wakeAsks?.[act];
|
|
4285
4873
|
else {
|
|
@@ -4291,11 +4879,28 @@ export class Worker {
|
|
|
4291
4879
|
}
|
|
4292
4880
|
savePolicy(fresh, this.opts.home);
|
|
4293
4881
|
}
|
|
4294
|
-
|
|
4295
|
-
|
|
4882
|
+
safetyDue = new Map();
|
|
4883
|
+
stuckDue = new Map();
|
|
4884
|
+
async ownOnRecord(client, id, waitMs) {
|
|
4885
|
+
const r = (await client.seenOwn(id, waitMs).catch(() => null)) ?? (await client.storedAct(id).catch(() => null));
|
|
4886
|
+
return r ? { id: r.id, seq: r.seq } : null;
|
|
4887
|
+
}
|
|
4888
|
+
safetyRaised = new Map();
|
|
4889
|
+
async reportStuck(client, o, d, attempts, again = 0, known) {
|
|
4890
|
+
this.learnRequests([o]);
|
|
4891
|
+
const safety = SAFETY_CAUSES.has(d.cause);
|
|
4892
|
+
const key = safety ? `stuck-safety:${this.oKey(o.id)}:${d.cause}` : `stuck:${this.oKey(o.id)}`;
|
|
4296
4893
|
const space = this.spaceKey();
|
|
4297
|
-
|
|
4894
|
+
const said = loadContinued(space, this.opts.home);
|
|
4895
|
+
if (!again && (said.has(key) || (!safety && said.has(`stuck:${o.id}`))))
|
|
4298
4896
|
return;
|
|
4897
|
+
if (safety && !again) {
|
|
4898
|
+
this.safetyDue.set(key, { o, d });
|
|
4899
|
+
try {
|
|
4900
|
+
markContinued(space, `stuck-safety-due:${this.oKey(o.id)}:${d.cause}`, this.opts.home);
|
|
4901
|
+
}
|
|
4902
|
+
catch { }
|
|
4903
|
+
}
|
|
4299
4904
|
const policy = loadPolicy(this.opts.home);
|
|
4300
4905
|
const asking = !!d.capability && mayAsk(policy, d.capability);
|
|
4301
4906
|
const what = clip(plainRequest(o.body, this.cfg.member), 160);
|
|
@@ -4309,27 +4914,217 @@ export class Worker {
|
|
|
4309
4914
|
...(d.dir && d.lines?.length ? [``, `On this computer, in ${d.dir}:`, ...d.lines.map((l) => ` ${l}`)] : []),
|
|
4310
4915
|
``,
|
|
4311
4916
|
`The request is still open, and the teammate who asked hasn't been told. It won't be retried on its own.`,
|
|
4917
|
+
...(again ? [``, safety ? `Raised again: the earlier warning about this was cleared without your answer.`
|
|
4918
|
+
: `Asked again: the "just this once" went to another request, and this one still needs commands.`] : []),
|
|
4312
4919
|
];
|
|
4313
4920
|
if (asking) {
|
|
4314
4921
|
lines.push(``, `Also: may ${this.cfg.member}'s agent run commands on this computer (tests, commit, push)? Answer "allow always", "allow once", or "deny" — in the menu bar, or with pantheon approvals.`, `Today it may only edit files, which is why it couldn't commit or push. Allowing it means: ${COMMAND_GRANT_SCOPE}`, `Answer on this computer: an answer from anywhere else settles this but grants nothing.`);
|
|
4315
4922
|
}
|
|
4316
4923
|
try {
|
|
4317
|
-
const
|
|
4318
|
-
|
|
4924
|
+
const fit = (t) => t.length <= LIMITS.bodyChars - 100 ? t : `${t.slice(0, LIMITS.bodyChars - 300)}\n\n(cut here: too long for one message — the rest is on this computer)`;
|
|
4925
|
+
const frame = (goal, short = false, ref = o.ref || null) => ({
|
|
4926
|
+
act: "ESCALATE", goal, ref, after_seq: Math.max(client.readSeq, client.seq),
|
|
4319
4927
|
summary: clip(this.ledgerPaths(`${this.cfg.member}'s agent is stuck: ${d.says}`), 190),
|
|
4320
|
-
body: this.ledgerPaths(lines.join("\n")),
|
|
4928
|
+
body: fit(this.ledgerPaths(short ? [lines[0], ``, `Why: ${clip(d.says, 600)}`, `What to do: ${clip(d.fix, 600)}`].join("\n") : lines.join("\n"))),
|
|
4929
|
+
...(safety ? { meta: { safety_stop: d.cause } } : {}),
|
|
4930
|
+
});
|
|
4931
|
+
const passing = (m) => !/^[A-Z_]+:/.test(m) || /^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(m);
|
|
4932
|
+
const before = known ? await this.ownOnRecord(client, known, 500) : null;
|
|
4933
|
+
const sent = before ?? await client.send(frame(o.goal)).catch((e) => {
|
|
4934
|
+
if (!safety || passing(e.message))
|
|
4935
|
+
return Promise.reject(e);
|
|
4936
|
+
return client.send(frame(null, false, null)).catch((e2) => passing(e2.message) ? Promise.reject(e2) : client.send(frame(null, true, null)));
|
|
4321
4937
|
});
|
|
4322
4938
|
markContinued(space, key, this.opts.home);
|
|
4939
|
+
this.stuckDue.delete(key);
|
|
4940
|
+
if (safety) {
|
|
4941
|
+
this.safetyDue.delete(key);
|
|
4942
|
+
this.safetyRaised.set(sent.id, { o, d, seq: sent.seq, again });
|
|
4943
|
+
}
|
|
4323
4944
|
const fresh = loadPolicy(this.opts.home);
|
|
4324
|
-
if (asking && d.capability)
|
|
4945
|
+
if (asking && d.capability) {
|
|
4325
4946
|
recordAsk(fresh, d.capability, sent.id);
|
|
4326
|
-
|
|
4327
|
-
|
|
4947
|
+
askedHere.cap.set(d.capability, canon(fresh.asks[d.capability]));
|
|
4948
|
+
}
|
|
4949
|
+
if (o.kind !== "CONTINUE" && !safety) {
|
|
4950
|
+
const rec = { ref: o.ref, team: this.cfg.spaceId ?? this.cfg.space, at: Date.now(), ...(d.capability ? { cap: d.capability } : {}) };
|
|
4951
|
+
(fresh.stuckReports ??= {})[sent.id] = rec;
|
|
4952
|
+
askedHere.stuck.set(sent.id, canon(rec));
|
|
4953
|
+
}
|
|
4954
|
+
this.keepOwn();
|
|
4328
4955
|
savePolicy(fresh, this.opts.home);
|
|
4329
4956
|
this.opts.log(`* escalated ${o.id}: ${d.cause}${asking ? ` (and asked for ${d.capability})` : ""}`);
|
|
4330
4957
|
}
|
|
4331
4958
|
catch (e) {
|
|
4332
|
-
|
|
4959
|
+
const msg = e.message;
|
|
4960
|
+
const lasting = safety && /^[A-Z_]+:/.test(msg) && !/^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(msg);
|
|
4961
|
+
if (lasting) {
|
|
4962
|
+
markContinued(space, key, this.opts.home);
|
|
4963
|
+
this.safetyDue.delete(key);
|
|
4964
|
+
try {
|
|
4965
|
+
const note = { says: clip(d.says, 400), fix: clip(d.fix, 500), at: Date.now(), team: this.cfg.spaceId ?? this.cfg.space };
|
|
4966
|
+
const fresh = loadPolicy(this.opts.home);
|
|
4967
|
+
(fresh.safetyNotes ??= {})[key] = note;
|
|
4968
|
+
askedHere.safety.set(key, canon(note));
|
|
4969
|
+
savePolicy(fresh, this.opts.home);
|
|
4970
|
+
this.keepOwn();
|
|
4971
|
+
}
|
|
4972
|
+
catch { }
|
|
4973
|
+
try {
|
|
4974
|
+
this.opts.notify?.({ kind: "worker", who: "Your worker", heading: "A safety warning could not reach your board", context: this.cfg.space,
|
|
4975
|
+
body: `${clip(d.says, 300)}\nWhat to do: ${clip(d.fix, 400)}`, ttl: 0, sound: true });
|
|
4976
|
+
}
|
|
4977
|
+
catch { }
|
|
4978
|
+
}
|
|
4979
|
+
const unanswered = !safety && /^NO_REPLY\b/.test(msg) ? failedActId(e) : null;
|
|
4980
|
+
if (unanswered) {
|
|
4981
|
+
const on = await this.ownOnRecord(client, unanswered, 2_500);
|
|
4982
|
+
if (on) {
|
|
4983
|
+
await this.reportStuck(client, o, d, attempts, again, on.id);
|
|
4984
|
+
return;
|
|
4985
|
+
}
|
|
4986
|
+
}
|
|
4987
|
+
const retry = !safety && (!!unanswered || !/^[A-Z_]+:/.test(msg) || /^(RATE_LIMIT|STORAGE|STALE_VIEW|TOO_MANY_REFUSALS):/.test(msg));
|
|
4988
|
+
if (retry)
|
|
4989
|
+
this.stuckDue.set(key, { o, d, attempts, again, next: Date.now() + 60_000, ...(unanswered ? { known: unanswered } : known ? { known } : {}) });
|
|
4990
|
+
else if (!safety)
|
|
4991
|
+
this.stuckDue.delete(key);
|
|
4992
|
+
this.opts.log(`! could not report the stuck obligation on the ledger: ${msg}${safety ? lasting ? " — a safety warning: shown on this computer instead" : " — a safety warning: tried again until it is said" : retry ? " — tried again in a minute" : ""}`);
|
|
4993
|
+
}
|
|
4994
|
+
}
|
|
4995
|
+
async settleSafety(client) {
|
|
4996
|
+
if (!client.connected)
|
|
4997
|
+
return;
|
|
4998
|
+
for (const [key, s] of [...this.stuckDue]) {
|
|
4999
|
+
if (loadContinued(this.spaceKey(), this.opts.home).has(key)) {
|
|
5000
|
+
this.stuckDue.delete(key);
|
|
5001
|
+
continue;
|
|
5002
|
+
}
|
|
5003
|
+
const now = client.myWork().find((x) => this.oKey(x.id) === this.oKey(s.o.id));
|
|
5004
|
+
if (!now) {
|
|
5005
|
+
this.stuckDue.delete(key);
|
|
5006
|
+
continue;
|
|
5007
|
+
}
|
|
5008
|
+
if (s.next > Date.now())
|
|
5009
|
+
continue;
|
|
5010
|
+
s.next = Date.now() + 60_000;
|
|
5011
|
+
await this.reportStuck(client, now, s.d, s.attempts, s.again, s.known);
|
|
5012
|
+
}
|
|
5013
|
+
for (const [key, { o, d }] of [...this.safetyDue]) {
|
|
5014
|
+
if (loadContinued(this.spaceKey(), this.opts.home).has(key)) {
|
|
5015
|
+
this.safetyDue.delete(key);
|
|
5016
|
+
continue;
|
|
5017
|
+
}
|
|
5018
|
+
const now = client.myWork().find((x) => this.oKey(x.id) === this.oKey(o.id)) ?? o;
|
|
5019
|
+
await this.reportStuck(client, now, d, 0);
|
|
5020
|
+
}
|
|
5021
|
+
if (!this.safetyDiskRead) {
|
|
5022
|
+
this.safetyDiskRead = true;
|
|
5023
|
+
const marks = loadContinued(this.spaceKey(), this.opts.home);
|
|
5024
|
+
for (const m of marks) {
|
|
5025
|
+
if (!m.startsWith("stuck-safety-due:"))
|
|
5026
|
+
continue;
|
|
5027
|
+
const rest = m.slice("stuck-safety-due:".length);
|
|
5028
|
+
const cause = rest.slice(rest.lastIndexOf(":") + 1), okey = rest.slice(0, rest.lastIndexOf(":"));
|
|
5029
|
+
if (!SAFETY_CAUSES.has(cause) || marks.has(`stuck-safety:${okey}:${cause}`))
|
|
5030
|
+
continue;
|
|
5031
|
+
const o = client.myWork().find((x) => this.oKey(x.id) === okey)
|
|
5032
|
+
?? { id: `gone:${okey}`, kind: "ANSWER", owner: this.cfg.member, ref: "", goal: null, body: "a request that is no longer waiting", created_at: 0 };
|
|
5033
|
+
if (o.id.startsWith("gone:"))
|
|
5034
|
+
this.obKeys.set(o.id, okey);
|
|
5035
|
+
this.safetyDue.set(`stuck-safety:${okey}:${cause}`, { o, d: safetyAgainDiagnosis(cause, this.cfg.member) });
|
|
5036
|
+
}
|
|
5037
|
+
}
|
|
5038
|
+
for (const [act, r] of [...this.safetyRaised]) {
|
|
5039
|
+
if (client.obligations.some((x) => x.kind === "HUMAN" && x.ref === act)) {
|
|
5040
|
+
r.goneAt = undefined;
|
|
5041
|
+
continue;
|
|
5042
|
+
}
|
|
5043
|
+
const now = Date.now();
|
|
5044
|
+
if (r.goneAt === undefined) {
|
|
5045
|
+
r.goneAt = now;
|
|
5046
|
+
continue;
|
|
5047
|
+
}
|
|
5048
|
+
if (now - r.goneAt < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
|
|
5049
|
+
continue;
|
|
5050
|
+
if ((r.nextAt ?? 0) > now)
|
|
5051
|
+
continue;
|
|
5052
|
+
const settled = await this.personSettled(act, r.seq);
|
|
5053
|
+
if (settled === null) {
|
|
5054
|
+
r.wait = Math.min((r.wait ?? 30_000) * 2, 30 * 60_000);
|
|
5055
|
+
r.nextAt = now + r.wait;
|
|
5056
|
+
continue;
|
|
5057
|
+
}
|
|
5058
|
+
this.safetyRaised.delete(act);
|
|
5059
|
+
if (settled || r.again >= SAFETY_RERAISE_MAX)
|
|
5060
|
+
continue;
|
|
5061
|
+
const o = client.myWork().find((x) => this.oKey(x.id) === this.oKey(r.o.id)) ?? r.o;
|
|
5062
|
+
this.opts.log(`! the safety warning ${act} left the board without your answer — raising it again`);
|
|
5063
|
+
await this.reportStuck(client, o, r.d, 0, r.again + 1);
|
|
5064
|
+
}
|
|
5065
|
+
}
|
|
5066
|
+
safetyDiskRead = false;
|
|
5067
|
+
closedSince = new Map();
|
|
5068
|
+
async settleClosedQuestions(client) {
|
|
5069
|
+
if (!client.connected)
|
|
5070
|
+
return;
|
|
5071
|
+
const p = loadPolicy(this.opts.home);
|
|
5072
|
+
const teams = this.cfg.spaceId ? [this.cfg.spaceId] : [this.cfg.space];
|
|
5073
|
+
const mine = [
|
|
5074
|
+
...Object.entries(p.asks).filter(([, a]) => !!a?.act && teams.includes(p.stuckReports?.[a.act]?.team ?? ""))
|
|
5075
|
+
.map(([cap, a]) => [a.act, "cap", cap]),
|
|
5076
|
+
...Object.entries(p.wakeAsks ?? {}).filter(([, a]) => teams.includes(a.team)).map(([id]) => [id, "wake", id]),
|
|
5077
|
+
];
|
|
5078
|
+
const now = Date.now();
|
|
5079
|
+
const gone = [];
|
|
5080
|
+
for (const [act, kind, key] of mine) {
|
|
5081
|
+
if (client.obligations.some((o) => o.kind === "HUMAN" && o.ref === act)) {
|
|
5082
|
+
this.closedSince.delete(act);
|
|
5083
|
+
continue;
|
|
5084
|
+
}
|
|
5085
|
+
const since = this.closedSince.get(act);
|
|
5086
|
+
if (since === undefined) {
|
|
5087
|
+
this.closedSince.set(act, now);
|
|
5088
|
+
continue;
|
|
5089
|
+
}
|
|
5090
|
+
if (now - since < (this.opts.mootGraceMs ?? MOOT_GRACE_MS))
|
|
5091
|
+
continue;
|
|
5092
|
+
gone.push([act, kind, key]);
|
|
5093
|
+
}
|
|
5094
|
+
if (!gone.length)
|
|
5095
|
+
return;
|
|
5096
|
+
const fresh = loadPolicy(this.opts.home);
|
|
5097
|
+
for (const [act, kind, key] of gone) {
|
|
5098
|
+
this.closedSince.delete(act);
|
|
5099
|
+
withdrawnHere.add(act);
|
|
5100
|
+
if (kind === "wake")
|
|
5101
|
+
delete fresh.wakeAsks?.[act];
|
|
5102
|
+
else if (fresh.asks[key]?.act === act)
|
|
5103
|
+
delete fresh.asks[key];
|
|
5104
|
+
this.opts.log(`* ${act}: settled on the record — no longer waiting for an answer here`);
|
|
5105
|
+
}
|
|
5106
|
+
savePolicy(fresh, this.opts.home);
|
|
5107
|
+
}
|
|
5108
|
+
async personSettled(act, seq) {
|
|
5109
|
+
let since = Math.max(0, seq - 1);
|
|
5110
|
+
try {
|
|
5111
|
+
for (let page = 0; page < 40; page++) {
|
|
5112
|
+
const res = await signedRead(this.cfg, "/history", { since: String(since) }, { timeoutMs: 5000 });
|
|
5113
|
+
if (!res.ok)
|
|
5114
|
+
return null;
|
|
5115
|
+
const j = await res.json();
|
|
5116
|
+
const acts = j.acts ?? [];
|
|
5117
|
+
if (acts.some((a) => a.ref === act && (a.act === "ANSWER" || a.act === "RETRACT") && a.sig?.role !== "run" && a.from === this.cfg.member))
|
|
5118
|
+
return true;
|
|
5119
|
+
const last = acts.length ? acts[acts.length - 1].seq : since;
|
|
5120
|
+
if (!acts.length || last <= since || (typeof j.head === "number" && last >= j.head))
|
|
5121
|
+
return false;
|
|
5122
|
+
since = last;
|
|
5123
|
+
}
|
|
5124
|
+
return null;
|
|
5125
|
+
}
|
|
5126
|
+
catch {
|
|
5127
|
+
return null;
|
|
4333
5128
|
}
|
|
4334
5129
|
}
|
|
4335
5130
|
ledgerPaths(text) {
|
|
@@ -4499,8 +5294,12 @@ export class Worker {
|
|
|
4499
5294
|
}
|
|
4500
5295
|
if (denies.length) {
|
|
4501
5296
|
const fresh = loadPolicy(home);
|
|
4502
|
-
for (const [act, body] of denies)
|
|
4503
|
-
applyAnswer(fresh, act, body);
|
|
5297
|
+
for (const [act, body] of denies) {
|
|
5298
|
+
const r = applyAnswer(fresh, act, body);
|
|
5299
|
+
withdrawnHere.add(act);
|
|
5300
|
+
if (r)
|
|
5301
|
+
noesHere.add(`denied:${r.cap}`);
|
|
5302
|
+
}
|
|
4504
5303
|
savePolicy(fresh, home);
|
|
4505
5304
|
}
|
|
4506
5305
|
}
|
|
@@ -4889,7 +5688,7 @@ export class Worker {
|
|
|
4889
5688
|
cursorSaid = 0;
|
|
4890
5689
|
readCursor() {
|
|
4891
5690
|
try {
|
|
4892
|
-
const j = JSON.parse(
|
|
5691
|
+
const j = JSON.parse(readStateText(this.cursorFile()));
|
|
4893
5692
|
const v = j[this.spaceKey()] ?? j[this.cfg.space];
|
|
4894
5693
|
return typeof v === "number" && v > 0 ? v : 0;
|
|
4895
5694
|
}
|
|
@@ -4904,7 +5703,7 @@ export class Worker {
|
|
|
4904
5703
|
try {
|
|
4905
5704
|
let j = {};
|
|
4906
5705
|
try {
|
|
4907
|
-
j = JSON.parse(
|
|
5706
|
+
j = JSON.parse(readStateText(this.cursorFile()));
|
|
4908
5707
|
}
|
|
4909
5708
|
catch { }
|
|
4910
5709
|
j[this.spaceKey()] = seq;
|
|
@@ -4929,7 +5728,7 @@ export class Worker {
|
|
|
4929
5728
|
return false;
|
|
4930
5729
|
}
|
|
4931
5730
|
catch { }
|
|
4932
|
-
if (this.
|
|
5731
|
+
if (this.answeredHere(ref))
|
|
4933
5732
|
return false;
|
|
4934
5733
|
const c = this.client;
|
|
4935
5734
|
if (c && !ref.startsWith("cont:")) {
|
|
@@ -5066,4 +5865,14 @@ export class Worker {
|
|
|
5066
5865
|
catch { }
|
|
5067
5866
|
}
|
|
5068
5867
|
}
|
|
5868
|
+
export function forAbout(until, now = Date.now()) {
|
|
5869
|
+
const min = Math.max(1, Math.round((until - now) / 60_000));
|
|
5870
|
+
if (min < 60)
|
|
5871
|
+
return `about ${min} minute${min === 1 ? "" : "s"}`;
|
|
5872
|
+
const h = Math.round(min / 60);
|
|
5873
|
+
if (h < 48)
|
|
5874
|
+
return `about ${h} hour${h === 1 ? "" : "s"}`;
|
|
5875
|
+
const d = Math.round(h / 24);
|
|
5876
|
+
return `about ${d} days`;
|
|
5877
|
+
}
|
|
5069
5878
|
//# sourceMappingURL=worker.js.map
|