@join-pantheon/cli 0.33.5 → 0.33.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +34 -2
- package/dist/agents.js +114 -13
- package/dist/board.js +4 -0
- package/dist/claude-confine.js +46 -2
- package/dist/cli.js +194 -23
- package/dist/client.js +143 -6
- package/dist/doctor.js +49 -7
- package/dist/file-lock.js +2 -2
- package/dist/git-safe.js +5 -1
- package/dist/home-guard.js +57 -2
- package/dist/home-safety.js +2 -0
- package/dist/linux-sandbox.js +71 -0
- package/dist/machine.js +18 -0
- package/dist/mcp.js +13 -5
- package/dist/notifier.js +57 -7
- package/dist/persona.js +34 -10
- package/dist/pickup.js +82 -0
- package/dist/profile.js +24 -1
- package/dist/protocol.js +19 -2
- package/dist/relay.js +128 -20
- package/dist/removed.js +132 -0
- package/dist/run-failure.js +13 -0
- package/dist/setup-guide.js +27 -4
- package/dist/store.js +59 -0
- package/dist/tray.js +45 -7
- package/dist/usage.js +97 -0
- package/dist/which.js +3 -2
- package/dist/worker-fleet.js +33 -3
- package/dist/worker-policy.js +58 -0
- package/dist/worker.js +215 -29
- package/helper/macos/prebuilt/manifest.json +1 -1
- package/package.json +1 -1
package/dist/worker.js
CHANGED
|
@@ -3,7 +3,7 @@ import { execFileSync } from "node:child_process";
|
|
|
3
3
|
import { chmodSync, lstatSync, mkdirSync, writeFileSync, existsSync, readdirSync, readFileSync, readlinkSync, realpathSync, rmSync, statSync } from "node:fs";
|
|
4
4
|
import { homedir } from "node:os";
|
|
5
5
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
6
|
-
import { detectClaudeFeatures, describeFeatures, ensureRunTmp, NO_CLAUDE_FEATURES, runTmpDir } from "./claude-confine.js";
|
|
6
|
+
import { absentDeniedPaths, detectClaudeFeatures, describeFeatures, ensureRunTmp, NO_CLAUDE_FEATURES, removeSandboxLeftovers, runTmpDir } from "./claude-confine.js";
|
|
7
7
|
import { snapshotHome, restoreHome, describeChange, trustWrite, askedHere } from "./home-guard.js";
|
|
8
8
|
import { PantheonClient } from "./client.js";
|
|
9
9
|
import { createHash, randomBytes } from "node:crypto";
|
|
@@ -14,12 +14,13 @@ import { brokenStateFiles, brokenStateFix, pidAlive, readStateFile, withFileLock
|
|
|
14
14
|
import { mkdirInside, writeInside } from "./safe-write.js";
|
|
15
15
|
import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
|
|
16
16
|
import { trusted } from "./verify-act.js";
|
|
17
|
-
import { headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
17
|
+
import { headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
|
|
18
18
|
import { activeOutages, detectOutage, earliest, whenAgain } from "./agent-outage.js";
|
|
19
19
|
import { agentSpawn, CmdLineUnsafe } from "./which.js";
|
|
20
20
|
import { VERSION } from "./version.js";
|
|
21
|
-
import { applyAnswer, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, } from "./worker-policy.js";
|
|
21
|
+
import { applyAnswer, CAPABILITY_ARGS, CARRY_REASON, carryOverWakers, dropLedgerGrants, hasWakeOnce, isApprovedWaker, mayAskWake, takeWakeOnce, OLD_RELAY_WAKE, relayKnowsAdmitters, loadPolicy, mayAsk, needsSelfUpgrade, parseGrantAnswer, recordAsk, savePolicy, useGrant, WOKEN_RUN_ENV, takeTeamFlagOnYes, unsandboxedAllowed, policyPath, isNewer, RUN_MINUTES, effectiveRunMinutes, effectiveIdleSeconds, readinessRecheckPath, describeRunMinutes, parseRunMinutes, } from "./worker-policy.js";
|
|
22
22
|
import { runtimeNodeVersion } from "./managed.js";
|
|
23
|
+
import { removalWords, removedLine, removedNotice, takeTold } from "./removed.js";
|
|
23
24
|
import { agentHookStates, repairHook, repairNote } from "./notify.js";
|
|
24
25
|
import { fence, assess } from "./untrusted.js";
|
|
25
26
|
import { allowed, closeStartedRun, finishRun, recordRun, startRun } from "./spend.js";
|
|
@@ -40,6 +41,7 @@ import { isContinuationFor, LIMITS, shouldDeferToHuman } from "./protocol.js";
|
|
|
40
41
|
import { proxyEnvFor } from "./network.js";
|
|
41
42
|
import { signedRead } from "./reads.js";
|
|
42
43
|
import { commandState } from "./command-state.js";
|
|
44
|
+
import { hostEnv } from "./host-env.js";
|
|
43
45
|
export function heldNote() {
|
|
44
46
|
return `
|
|
45
47
|
|
|
@@ -839,9 +841,18 @@ Act, reply if a reply is owed, then finish.`;
|
|
|
839
841
|
}
|
|
840
842
|
export const TASK_POINTER = "Your task for this run is in the file .pantheon-task.md in the current directory. Read that file first and follow it: it is the whole brief.";
|
|
841
843
|
let fleetLock = null;
|
|
842
|
-
export const RUN_TIMEOUT_MS =
|
|
844
|
+
export const RUN_TIMEOUT_MS = RUN_MINUTES.default * 60_000;
|
|
845
|
+
export const RUN_TIMEOUT_MAX_MS = RUN_MINUTES.max * 60_000;
|
|
846
|
+
function recordLimitMs(r) {
|
|
847
|
+
const l = typeof r.limit === "number" && Number.isFinite(r.limit) && r.limit > 0 ? Math.min(r.limit, RUN_TIMEOUT_MAX_MS) : RUN_TIMEOUT_MAX_MS;
|
|
848
|
+
return l + 5 * 60_000;
|
|
849
|
+
}
|
|
843
850
|
export const RUN_SLOT = "run-slot.json";
|
|
844
851
|
export const RUN_LEASE_RENEW_MS = 60_000;
|
|
852
|
+
export const WORKER_STATE_EVERY_MS = 5 * 60_000;
|
|
853
|
+
export const WORKER_READINESS_MS = 10 * 60_000;
|
|
854
|
+
export const WORKER_READINESS_RETRY_MS = 60_000;
|
|
855
|
+
export const RUN_HEARTBEAT_MS = 4 * 60_000;
|
|
845
856
|
const RUN_LEASE_FRESH_MS = 10 * 60_000;
|
|
846
857
|
function readRunLease(f) {
|
|
847
858
|
let text;
|
|
@@ -962,7 +973,7 @@ export function killRecordedRun(dir, now = Date.now()) {
|
|
|
962
973
|
rmSync(f, { force: true });
|
|
963
974
|
if (!r || !Number.isInteger(r.pid) || typeof r.at !== "number" || !Number.isFinite(r.at) || r.at > now + 60_000)
|
|
964
975
|
return null;
|
|
965
|
-
if (now - r.at >
|
|
976
|
+
if (now - r.at > recordLimitMs(r))
|
|
966
977
|
return null;
|
|
967
978
|
if (!recordedRunAlive(r.pid))
|
|
968
979
|
return null;
|
|
@@ -988,7 +999,7 @@ export function runInFlight(dir, now = Date.now()) {
|
|
|
988
999
|
}
|
|
989
1000
|
if (!r || !Number.isInteger(r.pid) || typeof r.at !== "number" || !Number.isFinite(r.at))
|
|
990
1001
|
return null;
|
|
991
|
-
if (now - r.at >
|
|
1002
|
+
if (now - r.at > recordLimitMs(r))
|
|
992
1003
|
return null;
|
|
993
1004
|
if (!recordedRunAlive(r.pid))
|
|
994
1005
|
return null;
|
|
@@ -1147,6 +1158,82 @@ export class Worker {
|
|
|
1147
1158
|
fix: "Install and sign in to another agent CLI (claude, codex, cursor-agent or copilot) for woken runs; until then requests wait for a live session.",
|
|
1148
1159
|
};
|
|
1149
1160
|
}
|
|
1161
|
+
lastReport = null;
|
|
1162
|
+
report(state, detail, until, more = {}) {
|
|
1163
|
+
if (state === "idle" && this.readiness && !this.readiness.ok) {
|
|
1164
|
+
state = "unavailable";
|
|
1165
|
+
detail = "can't take requests yet";
|
|
1166
|
+
more = { ...(this.readiness.code ? { reason: this.readiness.code } : {}) };
|
|
1167
|
+
}
|
|
1168
|
+
this.lastReport = { state, ...(detail ? { detail } : {}), ...(until ? { until } : {}), ...more, at: Date.now() };
|
|
1169
|
+
this.client?.reportWorkerState(state, detail, until, this.idleMs(), more);
|
|
1170
|
+
}
|
|
1171
|
+
tickHold = null;
|
|
1172
|
+
readiness = null;
|
|
1173
|
+
readinessAt = 0;
|
|
1174
|
+
checkReadiness(force) {
|
|
1175
|
+
const now = Date.now();
|
|
1176
|
+
const every = this.readiness && !this.readiness.ok ? WORKER_READINESS_RETRY_MS : WORKER_READINESS_MS;
|
|
1177
|
+
let marked = false;
|
|
1178
|
+
try {
|
|
1179
|
+
const t = statSync(readinessRecheckPath(this.opts.home)).mtimeMs;
|
|
1180
|
+
marked = t > this.readinessAt && t <= now + 1_000;
|
|
1181
|
+
}
|
|
1182
|
+
catch { }
|
|
1183
|
+
if (!force && !marked && now - this.readinessAt < every)
|
|
1184
|
+
return;
|
|
1185
|
+
if ((force || marked) && now - this.readinessAt < 15_000)
|
|
1186
|
+
return;
|
|
1187
|
+
this.readinessAt = now;
|
|
1188
|
+
if (force || marked)
|
|
1189
|
+
forgetMissingBinaries();
|
|
1190
|
+
if (this.opts.agentCmd || this.opts.housekeeping || this.opts.alertsOnly) {
|
|
1191
|
+
this.readiness = { ok: true, short: "" };
|
|
1192
|
+
return;
|
|
1193
|
+
}
|
|
1194
|
+
try {
|
|
1195
|
+
const pathOnly = this.readiness && !this.readiness.ok && (this.readiness.code === "no_agent" || this.readiness.code === "kiro_only") && !marked && !force;
|
|
1196
|
+
if (pathOnly && !anyUnattendedAgentOnPath())
|
|
1197
|
+
return;
|
|
1198
|
+
let commandsOn = false;
|
|
1199
|
+
try {
|
|
1200
|
+
commandsOn = commandState({ home: this.opts.home, project: this.opts.project, allowBash: this.opts.allowBash }).level !== "off";
|
|
1201
|
+
}
|
|
1202
|
+
catch { }
|
|
1203
|
+
const r = unattendedReadiness({ prefer: loadProfile(this.cfg).agent.primary_surface, unsandboxedOk: this.unsandboxedOk(), pinned: this.opts.agent ?? "auto", ...(this.opts.choose ? { choose: this.opts.choose } : {}),
|
|
1204
|
+
commandsOn, inContainer: hostEnv().kind === "container", ...(this.opts.sandboxProbe ? { sandboxProbe: this.opts.sandboxProbe } : {}) });
|
|
1205
|
+
if (!r.ok && (this.readiness?.ok ?? true))
|
|
1206
|
+
this.opts.log(`! teammates are told this machine can't take requests yet: ${r.short} — ${r.fix}`);
|
|
1207
|
+
if (r.ok && this.readiness && !this.readiness.ok)
|
|
1208
|
+
this.opts.log(` agent ready again — teammates are told this machine takes requests`);
|
|
1209
|
+
this.readiness = { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}) };
|
|
1210
|
+
}
|
|
1211
|
+
catch {
|
|
1212
|
+
this.readiness = null;
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
reportStanding(first, held) {
|
|
1216
|
+
const now = Date.now();
|
|
1217
|
+
this.checkReadiness(first);
|
|
1218
|
+
if (this.running)
|
|
1219
|
+
return;
|
|
1220
|
+
const last = this.lastReport;
|
|
1221
|
+
const stale = !last || now - last.at >= WORKER_STATE_EVERY_MS;
|
|
1222
|
+
if (held) {
|
|
1223
|
+
const same = last?.state === "holding" && last.reason === held.reason && last.until === held.until;
|
|
1224
|
+
if (!same || stale)
|
|
1225
|
+
this.report("holding", held.detail, held.until, { reason: held.reason });
|
|
1226
|
+
return;
|
|
1227
|
+
}
|
|
1228
|
+
const holdOver = last?.state === "holding" && (held === null || (!!last.until && last.until <= now));
|
|
1229
|
+
const flipped = !!last && ((last.state === "unavailable") !== (this.readiness ? !this.readiness.ok : false));
|
|
1230
|
+
if (first || stale || holdOver || flipped) {
|
|
1231
|
+
if (!last || holdOver || last.state !== "holding")
|
|
1232
|
+
this.report("idle");
|
|
1233
|
+
else
|
|
1234
|
+
this.report(last.state, last.detail, last.until, { ...(last.reason ? { reason: last.reason } : {}) });
|
|
1235
|
+
}
|
|
1236
|
+
}
|
|
1150
1237
|
unsandboxedOk() {
|
|
1151
1238
|
try {
|
|
1152
1239
|
return unsandboxedAllowed(loadPolicy(this.opts.home));
|
|
@@ -1794,7 +1881,7 @@ export class Worker {
|
|
|
1794
1881
|
const base = this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
|
|
1795
1882
|
return join(base, "run-homes", encodeURIComponent(this.cfg.spaceId ?? this.cfg.space));
|
|
1796
1883
|
}
|
|
1797
|
-
openRunHome(cfg = this.cfg, features = []) {
|
|
1884
|
+
openRunHome(cfg = this.cfg, features = [], limitMs = this.runLimit(undefined).ms) {
|
|
1798
1885
|
const dir = this.runHomePath();
|
|
1799
1886
|
try {
|
|
1800
1887
|
if (!lstatSync(dir).isDirectory())
|
|
@@ -1807,7 +1894,7 @@ export class Worker {
|
|
|
1807
1894
|
if (features.includes("delegate") && cfg.privateKey) {
|
|
1808
1895
|
const run = newKeypair();
|
|
1809
1896
|
const team = cfg.spaceId ?? this.client?.teamKey() ?? cfg.space;
|
|
1810
|
-
const expires = Date.now() +
|
|
1897
|
+
const expires = Date.now() + limitMs + 10 * 60_000;
|
|
1811
1898
|
const cert = signDelegate(cfg.privateKey, team, cfg.member, run.publicKey, randomBytes(8).toString("hex"), expires);
|
|
1812
1899
|
written = { ...cfg, privateKey: "", delegate: { privateKey: run.privateKey, cert }, ...(cfg.spaceId ? {} : { spaceId: team }) };
|
|
1813
1900
|
}
|
|
@@ -1958,6 +2045,10 @@ export class Worker {
|
|
|
1958
2045
|
this.opts.log(` agent ${agentLabel}`);
|
|
1959
2046
|
const cmds = commandState({ home: this.opts.home, project: this.opts.project, allowBash: this.opts.allowBash });
|
|
1960
2047
|
this.opts.log(` policy ${this.opts.maxRunsPerHour} runs/hour, one at a time, commands ${cmds.level === "off" ? "off: edits only (bash gated)" : cmds.summary}`);
|
|
2048
|
+
{
|
|
2049
|
+
const lim = this.runLimit(undefined);
|
|
2050
|
+
this.opts.log(` limit ${describeRunMinutes(lim)}${lim.from === "flag" ? "" : " — change it: pantheon policy set run-minutes <5-180>"}`);
|
|
2051
|
+
}
|
|
1961
2052
|
this.carryWakers(client);
|
|
1962
2053
|
const personas = this.personas();
|
|
1963
2054
|
if (this.opts.persona && !personaByName(personas, this.opts.persona)) {
|
|
@@ -1976,14 +2067,23 @@ export class Worker {
|
|
|
1976
2067
|
this.opts.log(` ! ${p.name}: ${w}`);
|
|
1977
2068
|
}
|
|
1978
2069
|
this.recoverInterruptedRun();
|
|
2070
|
+
this.reportStanding(true);
|
|
1979
2071
|
if (this.opts.continuations !== false)
|
|
1980
2072
|
await this.restoreHeldContinuations(client);
|
|
1981
2073
|
this.writeState("watching", "waiting for work");
|
|
1982
2074
|
while (!this.stopped) {
|
|
1983
2075
|
const arrived = await client.next(5000);
|
|
2076
|
+
{
|
|
2077
|
+
const held = this.tickHold;
|
|
2078
|
+
this.tickHold = null;
|
|
2079
|
+
this.reportStanding(false, held);
|
|
2080
|
+
}
|
|
1984
2081
|
if (client.refused && !this.refusedSaid) {
|
|
1985
2082
|
this.refusedSaid = true;
|
|
1986
|
-
|
|
2083
|
+
if (client.removedFrom)
|
|
2084
|
+
this.sayRemoved(client.removedFrom.by);
|
|
2085
|
+
else
|
|
2086
|
+
this.opts.log(`! the relay refused this machine (${client.refused}): ${String(client.lastError ?? "").replace(/^[A-Z_]+: /, "")} — it keeps trying, every ${Math.round(PantheonClient.REFUSED_RETRY_MS / 60_000)} minutes after the first ${Math.round(PantheonClient.REFUSED_GRACE_MS / 60_000)}`);
|
|
1987
2087
|
}
|
|
1988
2088
|
else if (!client.refused)
|
|
1989
2089
|
this.refusedSaid = false;
|
|
@@ -1996,6 +2096,7 @@ export class Worker {
|
|
|
1996
2096
|
continue;
|
|
1997
2097
|
if (fleetLock && fleetLock !== this) {
|
|
1998
2098
|
this.writeState("watching", `waiting: a run for "${fleetLock.cfg.space}" is in progress`);
|
|
2099
|
+
this.tickHold = { reason: "busy_elsewhere", detail: "another team's run is in progress on this machine" };
|
|
1999
2100
|
continue;
|
|
2000
2101
|
}
|
|
2001
2102
|
const broken = this.brokenStateFiles();
|
|
@@ -2029,8 +2130,11 @@ export class Worker {
|
|
|
2029
2130
|
.filter((o) => o.kind !== "HUMAN" && o.kind !== "DELIVER" && !o.claimed_by)
|
|
2030
2131
|
.filter((o) => Date.now() - o.created_at >= grace)
|
|
2031
2132
|
.filter((o) => { const t = this.outageHold.get(o.id); if (t === undefined)
|
|
2032
|
-
return true; if (t > Date.now())
|
|
2033
|
-
|
|
2133
|
+
return true; if (t > Date.now()) {
|
|
2134
|
+
if (!this.tickHold || (this.tickHold.reason === "outage" && (this.tickHold.until ?? Infinity) > t))
|
|
2135
|
+
this.tickHold = { reason: "outage", until: t, detail: "its agents are out for now" };
|
|
2136
|
+
return false;
|
|
2137
|
+
} this.outageHold.delete(o.id); return true; })
|
|
2034
2138
|
.filter((o) => {
|
|
2035
2139
|
if (this.failuresOf(o.id) < 2)
|
|
2036
2140
|
return true;
|
|
@@ -2073,8 +2177,10 @@ export class Worker {
|
|
|
2073
2177
|
await this.reportRefused(client, o, held, true);
|
|
2074
2178
|
}
|
|
2075
2179
|
}
|
|
2076
|
-
else
|
|
2180
|
+
else {
|
|
2077
2181
|
await this.askWake(client, o, refusedWake.askable);
|
|
2182
|
+
this.tickHold = { reason: "awaiting_wake_approval", detail: "asking its person whether this teammate may start work here" };
|
|
2183
|
+
}
|
|
2078
2184
|
}
|
|
2079
2185
|
continue;
|
|
2080
2186
|
}
|
|
@@ -2142,7 +2248,7 @@ export class Worker {
|
|
|
2142
2248
|
}
|
|
2143
2249
|
const busyTtlMs = this.opts.busyTtlMs ?? LIMITS.activityBusyTtlMs;
|
|
2144
2250
|
if (me && shouldDeferToHuman(me.sessions, me.work, {
|
|
2145
|
-
idleMs: this.
|
|
2251
|
+
idleMs: this.idleMs(), busyTtlMs, busySince: this.busyObservedSince, hooksInstalled,
|
|
2146
2252
|
})) {
|
|
2147
2253
|
const id = candidate?.id ?? `cont:${continuation.id}`;
|
|
2148
2254
|
const busySince = me.work?.busy_since ?? this.busyObservedSince;
|
|
@@ -2151,13 +2257,15 @@ export class Worker {
|
|
|
2151
2257
|
? ` (for ${Math.round((Date.now() - busySince) / 60_000)}m — read as ended after ${Math.round(busyTtlMs / 60_000)}m with no Stop)`
|
|
2152
2258
|
: ""}`
|
|
2153
2259
|
: me.work?.idle_since
|
|
2154
|
-
? `your session went idle ${Math.round((Date.now() - me.work.idle_since) / 1000)}s ago (< ${Math.round(this.
|
|
2260
|
+
? `your session went idle ${Math.round((Date.now() - me.work.idle_since) / 1000)}s ago (< ${Math.round(this.idleMs() / 1000)}s)`
|
|
2155
2261
|
: "you have a live session and no activity signal (no wake-up hook on this machine — pantheon hook install)";
|
|
2156
|
-
const until = !me.work?.busy && me.work?.idle_since ? me.work.idle_since + this.
|
|
2262
|
+
const until = !me.work?.busy && me.work?.idle_since ? me.work.idle_since + this.idleMs() : undefined;
|
|
2157
2263
|
const holdKey = `${me.work?.busy ? "busy" : "idle"}:${until ?? 0}`;
|
|
2264
|
+
this.tickHold = { reason: me.work?.busy ? "busy_turn" : me.work?.idle_since ? "recently_active" : "no_signal", ...(until ? { until } : {}),
|
|
2265
|
+
detail: me.work?.busy ? "a turn is in flight in their live session" : me.work?.idle_since ? `their session went idle recently; waits ${Math.round(this.idleMs() / 1000)}s of quiet` : "their live session gave no activity signal" };
|
|
2158
2266
|
if (this.lastHold !== holdKey) {
|
|
2159
2267
|
this.lastHold = holdKey;
|
|
2160
|
-
this.
|
|
2268
|
+
this.reportStanding(false, this.tickHold);
|
|
2161
2269
|
}
|
|
2162
2270
|
if (!this.skipLogged.has(`defer:${id}`)) {
|
|
2163
2271
|
this.skipLogged.add(`defer:${id}`);
|
|
@@ -2185,6 +2293,7 @@ export class Worker {
|
|
|
2185
2293
|
const slot = this.takeSlot();
|
|
2186
2294
|
if ("held" in slot) {
|
|
2187
2295
|
this.writeState("watching", `waiting: ${slot.held}`);
|
|
2296
|
+
this.tickHold = { reason: "busy_elsewhere", detail: "another run is in progress on this machine" };
|
|
2188
2297
|
if (continuation)
|
|
2189
2298
|
this.requeueContinuation(continuation, 5_000);
|
|
2190
2299
|
continue;
|
|
@@ -2198,6 +2307,7 @@ export class Worker {
|
|
|
2198
2307
|
: `! not spawning: ${gate.why}. Raise it with --max-runs / --max-runs-day${persona ? ` (persona "${persona.name}" may cap it lower)` : ""}, or leave it.`);
|
|
2199
2308
|
}
|
|
2200
2309
|
this.writeState("capped", gate.why);
|
|
2310
|
+
this.tickHold = { reason: "capped", detail: "its run cap for now" };
|
|
2201
2311
|
if (continuation)
|
|
2202
2312
|
this.requeueContinuation(continuation, 60_000);
|
|
2203
2313
|
continue;
|
|
@@ -2227,6 +2337,8 @@ export class Worker {
|
|
|
2227
2337
|
break;
|
|
2228
2338
|
}
|
|
2229
2339
|
}
|
|
2340
|
+
if (this.readiness && !this.readiness.ok)
|
|
2341
|
+
this.checkReadiness(true);
|
|
2230
2342
|
this.running = true;
|
|
2231
2343
|
this.writeState("running", candidate ? `${candidate.kind} ${candidate.id}` : `CONTINUE ${continuation.id}`);
|
|
2232
2344
|
let job;
|
|
@@ -2252,7 +2364,7 @@ export class Worker {
|
|
|
2252
2364
|
}
|
|
2253
2365
|
this.runProject = null;
|
|
2254
2366
|
this.done.push(job);
|
|
2255
|
-
this.
|
|
2367
|
+
this.report("idle");
|
|
2256
2368
|
if (candidate && job.ok)
|
|
2257
2369
|
this.markDone(`handled:${candidate.ref}`);
|
|
2258
2370
|
const ran = candidate ? candidateAct : continuation;
|
|
@@ -2331,6 +2443,8 @@ export class Worker {
|
|
|
2331
2443
|
this.running = false;
|
|
2332
2444
|
this.endAttempt();
|
|
2333
2445
|
this.giveSlot(slot.token);
|
|
2446
|
+
if (continuation)
|
|
2447
|
+
client.forgetClaim(`cont:${continuation.id}`);
|
|
2334
2448
|
}
|
|
2335
2449
|
this.writeState("watching", "waiting for work");
|
|
2336
2450
|
if (this.opts.once)
|
|
@@ -2361,7 +2475,14 @@ export class Worker {
|
|
|
2361
2475
|
return { held: `a run for "${lease.held}" is in progress in another pantheon worker on this machine` };
|
|
2362
2476
|
fleetLock = this;
|
|
2363
2477
|
const dir = this.stateDir(), token = lease.token;
|
|
2364
|
-
this.
|
|
2478
|
+
this.heartbeatAt = Date.now();
|
|
2479
|
+
this.leaseTimer = setInterval(() => {
|
|
2480
|
+
renewRunLease(dir, token);
|
|
2481
|
+
if (Date.now() - this.heartbeatAt >= RUN_HEARTBEAT_MS) {
|
|
2482
|
+
this.heartbeatAt = Date.now();
|
|
2483
|
+
this.runHeartbeat();
|
|
2484
|
+
}
|
|
2485
|
+
}, this.opts.runLeaseRenewMs ?? RUN_LEASE_RENEW_MS);
|
|
2365
2486
|
this.leaseTimer.unref?.();
|
|
2366
2487
|
return lease;
|
|
2367
2488
|
}
|
|
@@ -2381,7 +2502,7 @@ export class Worker {
|
|
|
2381
2502
|
liveRuns.add(pid);
|
|
2382
2503
|
installRunReaper();
|
|
2383
2504
|
this.attemptPid = pid;
|
|
2384
|
-
writeRunRecord(this.stateDir(), { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, obligation: attempt.obligation, ...(attempt.spend ? { spend: attempt.spend } : {}) });
|
|
2505
|
+
writeRunRecord(this.stateDir(), { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, obligation: attempt.obligation, ...(attempt.spend ? { spend: attempt.spend } : {}), ...(attempt.limit ? { limit: attempt.limit } : {}) });
|
|
2385
2506
|
}
|
|
2386
2507
|
untrackRun(pid) {
|
|
2387
2508
|
this.runPids.delete(pid);
|
|
@@ -2407,7 +2528,7 @@ export class Worker {
|
|
|
2407
2528
|
return;
|
|
2408
2529
|
if (!Number.isInteger(r.worker) || r.worker === process.pid || pidAlive(r.worker))
|
|
2409
2530
|
return;
|
|
2410
|
-
if (Number.isInteger(r.pid) && typeof r.at === "number" && Date.now() - r.at <=
|
|
2531
|
+
if (Number.isInteger(r.pid) && typeof r.at === "number" && Date.now() - r.at <= recordLimitMs(r) && recordedRunAlive(r.pid))
|
|
2411
2532
|
killRunTree(r.pid);
|
|
2412
2533
|
try {
|
|
2413
2534
|
rmSync(f, { force: true });
|
|
@@ -2459,7 +2580,7 @@ export class Worker {
|
|
|
2459
2580
|
async handle(client, o, persona, policy, checked = null) {
|
|
2460
2581
|
this.opts.log(`> woken by ${owedLabel(o.kind)} ${o.id} (${o.body})${persona ? ` — persona "${persona.name}"` : ""} — checking whether it can run here`);
|
|
2461
2582
|
this.lastHold = "";
|
|
2462
|
-
this.
|
|
2583
|
+
this.report("running", `${o.kind} ${o.id}`, undefined, { ref: o.ref, ...(o.goal ? { goal: o.goal } : {}) });
|
|
2463
2584
|
const fetched = checked ?? (client.seenAct(o.ref) ? null : (await this.fetchActById(o.ref)));
|
|
2464
2585
|
const base = checked ?? client.seenAct(o.ref) ?? fetched;
|
|
2465
2586
|
const trigger = base && typeof base.body === "string" && typeof base.act === "string"
|
|
@@ -2478,7 +2599,7 @@ export class Worker {
|
|
|
2478
2599
|
};
|
|
2479
2600
|
this.opts.log(`> continuing from ${act.act} ${act.id} by ${act.from}${persona ? ` — persona "${persona.name}"` : ""}`);
|
|
2480
2601
|
this.lastHold = "";
|
|
2481
|
-
this.
|
|
2602
|
+
this.report("running", `${act.act} ${act.id}`, undefined, { ref: act.id, ...(act.goal ? { goal: act.goal } : {}) });
|
|
2482
2603
|
const trigger = actLine(act);
|
|
2483
2604
|
const needs = workNeedsCommands("CONTINUE", trigger, act.act);
|
|
2484
2605
|
return this.runJob(client, o, persona, policy, (branch, canPush) => buildContinuationBrief(act, trigger, branch, persona, canPush), false, needs, trigger, act);
|
|
@@ -2727,12 +2848,14 @@ export class Worker {
|
|
|
2727
2848
|
let plannedFiles;
|
|
2728
2849
|
let bashRules = [];
|
|
2729
2850
|
let fenced = null;
|
|
2851
|
+
let sandboxAbsent = [];
|
|
2730
2852
|
if (!client.features.includes("delegate") && client.signaturesExpected()) {
|
|
2731
2853
|
const says = "this team's relay signs messages but did not offer a key for the run, so the run would have needed your own key — it was not started";
|
|
2732
2854
|
this.opts.log(`! ${says}`);
|
|
2733
2855
|
return { obligation: o, branch, worktree, ok: false, refused: true, detail: says, diagnosis: { cause: "no_unattended_agent", says, fix: "Check the relay's version with its operator (pantheon doctor); runs start again when it offers run keys." } };
|
|
2734
2856
|
}
|
|
2735
|
-
const
|
|
2857
|
+
const runLimit = this.runLimit(persona);
|
|
2858
|
+
const runHome = this.openRunHome(runCfg, client.features, runLimit.ms);
|
|
2736
2859
|
try {
|
|
2737
2860
|
if (holdRepoConfig) {
|
|
2738
2861
|
fenced = quarantineRepoAgentConfig(worktree, runHome);
|
|
@@ -2765,6 +2888,17 @@ export class Worker {
|
|
|
2765
2888
|
...(features ? { claude: { features, pantheonHome: this.stateDir(), home: personHome, userSettings: join(personHome, ".claude", "settings.json"), ...(() => { const c = adoptedCommon(worktree); return c ? { gitCommon: c } : {}; })() } } : {}),
|
|
2766
2889
|
});
|
|
2767
2890
|
plannedFiles = plan.writeFiles;
|
|
2891
|
+
{
|
|
2892
|
+
const gc = adoptedCommon(worktree);
|
|
2893
|
+
if (gc) {
|
|
2894
|
+
try {
|
|
2895
|
+
sandboxAbsent = absentDeniedPaths(gc);
|
|
2896
|
+
}
|
|
2897
|
+
catch {
|
|
2898
|
+
sandboxAbsent = [];
|
|
2899
|
+
}
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2768
2902
|
for (const n of plan.notes ?? [])
|
|
2769
2903
|
if (!this.notesSaid.has(n)) {
|
|
2770
2904
|
this.notesSaid.add(n);
|
|
@@ -2827,7 +2961,7 @@ export class Worker {
|
|
|
2827
2961
|
}
|
|
2828
2962
|
const pid = child.pid;
|
|
2829
2963
|
if (pid)
|
|
2830
|
-
this.trackRun(pid, { obligation: o.id, spend: spendId });
|
|
2964
|
+
this.trackRun(pid, { obligation: o.id, spend: spendId, limit: runLimit.ms });
|
|
2831
2965
|
let head = "", tail = "";
|
|
2832
2966
|
const take = (b) => {
|
|
2833
2967
|
const s = b.toString();
|
|
@@ -2845,7 +2979,7 @@ export class Worker {
|
|
|
2845
2979
|
let settled = false;
|
|
2846
2980
|
let exitGrace = null;
|
|
2847
2981
|
let exitCode = null;
|
|
2848
|
-
const limit =
|
|
2982
|
+
const limit = runLimit.ms;
|
|
2849
2983
|
const finish = (code, signal) => {
|
|
2850
2984
|
if (settled)
|
|
2851
2985
|
return;
|
|
@@ -2880,6 +3014,11 @@ export class Worker {
|
|
|
2880
3014
|
});
|
|
2881
3015
|
}
|
|
2882
3016
|
finally {
|
|
3017
|
+
this.cleanupStep("removing the sandbox's empty mount points", () => {
|
|
3018
|
+
const gone = removeSandboxLeftovers(sandboxAbsent);
|
|
3019
|
+
if (gone.length)
|
|
3020
|
+
this.opts.log(` confine removed ${gone.length} empty mount point${gone.length === 1 ? "" : "s"} the sandbox left: ${gone.map((g) => basename(g)).join(", ")}`);
|
|
3021
|
+
});
|
|
2883
3022
|
this.cleanupStep("putting the repo's agent config back", () => { if (fenced)
|
|
2884
3023
|
for (const l of fenced.restore().lines)
|
|
2885
3024
|
this.opts.log(` config ${l}`); });
|
|
@@ -2966,7 +3105,9 @@ export class Worker {
|
|
|
2966
3105
|
return { obligation: o, branch, worktree, ok: false, detail, agentOut: true };
|
|
2967
3106
|
}
|
|
2968
3107
|
const quiet = ok && !requireReply && sent.length === 0;
|
|
2969
|
-
const diagnosis = ok ? undefined : outage ? this.outageDiagnosis(choice.backend.label, outage, [], "it was handed to another agent three times already") : diagnoseRun({ exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands,
|
|
3108
|
+
const diagnosis = ok ? undefined : outage ? this.outageDiagnosis(choice.backend.label, outage, [], "it was handed to another agent three times already") : diagnoseRun({ exited, replied, allowBash, branch, project, worktree, detail, transcript, runLog, needsCommands,
|
|
3109
|
+
...(/\(stopped after \d+s\)/.test(detail.split("\n")[0] ?? "") ? { timedOut: { minutes: runLimit.minutes, from: runLimit.from } } : {}),
|
|
3110
|
+
...(bashDroppedForSandbox ? { commandsNeedSandbox: byCli ? "agent-cmd" : "claude" } : {}), ...(agentId ? { agent: agentId } : {}), ...(place.kind === "folder" ? { folder: true } : {}),
|
|
2970
3111
|
...(agentId === "claude-code" && allowBash ? { allowedRules: bashRules } : {}), ...(baseRef ? { baseRef } : {}) }, this.opts.probe ?? gitProbe);
|
|
2971
3112
|
if (this.opts.notify && !quiet && (!ok || loadProfile(this.cfg).notifications?.worker_done)) {
|
|
2972
3113
|
try {
|
|
@@ -3016,8 +3157,8 @@ export class Worker {
|
|
|
3016
3157
|
if (!changes.length)
|
|
3017
3158
|
return;
|
|
3018
3159
|
for (const c of changes)
|
|
3019
|
-
this.opts.log(`${c.what === "reported" ? " " : "!"} ${o.id}: ${describeChange(c)}`);
|
|
3020
|
-
const taken = changes.filter((c) => c.what !== "reported");
|
|
3160
|
+
this.opts.log(`${c.what === "reported" || c.quiet ? " " : "!"} ${o.id}: ${describeChange(c)}`);
|
|
3161
|
+
const taken = changes.filter((c) => c.what !== "reported" && !c.quiet);
|
|
3021
3162
|
if (taken.length && this.opts.notify) {
|
|
3022
3163
|
try {
|
|
3023
3164
|
this.opts.notify({
|
|
@@ -3523,6 +3664,16 @@ export class Worker {
|
|
|
3523
3664
|
}
|
|
3524
3665
|
}
|
|
3525
3666
|
refusedSaid = false;
|
|
3667
|
+
sayRemoved(by) {
|
|
3668
|
+
const team = { space: this.cfg.space, spaceId: this.cfg.spaceId ?? null, relay: this.cfg.relay, member: this.cfg.member };
|
|
3669
|
+
const r = removalWords(team, by);
|
|
3670
|
+
this.opts.log(`! ${removedLine(r)}`);
|
|
3671
|
+
try {
|
|
3672
|
+
if (this.opts.notify && takeTold(team))
|
|
3673
|
+
this.opts.notify(removedNotice(r));
|
|
3674
|
+
}
|
|
3675
|
+
catch { }
|
|
3676
|
+
}
|
|
3526
3677
|
oldRelaySaid = false;
|
|
3527
3678
|
dropLedgerGrants() {
|
|
3528
3679
|
try {
|
|
@@ -3677,6 +3828,40 @@ export class Worker {
|
|
|
3677
3828
|
const r = upgrade(undefined, version ? { version } : {});
|
|
3678
3829
|
return { ok: r.ok, alreadyCurrent: r.alreadyCurrent, from: r.from, to: r.to, ...(r.needsPerson ? { needsPerson: r.needsPerson } : {}) };
|
|
3679
3830
|
}
|
|
3831
|
+
idleMs() {
|
|
3832
|
+
if (this.opts.idleMs !== undefined)
|
|
3833
|
+
return this.opts.idleMs;
|
|
3834
|
+
let policy;
|
|
3835
|
+
try {
|
|
3836
|
+
policy = loadPolicy(this.opts.home);
|
|
3837
|
+
}
|
|
3838
|
+
catch {
|
|
3839
|
+
policy = undefined;
|
|
3840
|
+
}
|
|
3841
|
+
return effectiveIdleSeconds({ ...(policy ? { policy } : {}) }).seconds * 1000;
|
|
3842
|
+
}
|
|
3843
|
+
runLimit(persona) {
|
|
3844
|
+
let policy;
|
|
3845
|
+
try {
|
|
3846
|
+
policy = loadPolicy(this.opts.home);
|
|
3847
|
+
}
|
|
3848
|
+
catch {
|
|
3849
|
+
policy = undefined;
|
|
3850
|
+
}
|
|
3851
|
+
const flag = this.opts.runMinutes !== undefined ? parseRunMinutes(this.opts.runMinutes) ?? undefined : undefined;
|
|
3852
|
+
const r = effectiveRunMinutes({ ...(flag !== undefined ? { flag } : {}), ...(policy ? { policy } : {}), ...(persona?.maxRunMinutes !== undefined ? { persona: persona.maxRunMinutes } : {}) });
|
|
3853
|
+
return { ...r, ms: this.opts.runTimeoutMs ?? r.minutes * 60_000 };
|
|
3854
|
+
}
|
|
3855
|
+
runHeartbeat() {
|
|
3856
|
+
if (!this.running || !this.lastState.startsWith("running|"))
|
|
3857
|
+
return;
|
|
3858
|
+
const detail = this.lastState.slice("running|".length);
|
|
3859
|
+
this.lastState = "";
|
|
3860
|
+
this.writeState("running", detail);
|
|
3861
|
+
const l = this.lastReport;
|
|
3862
|
+
this.report("running", detail, undefined, { ...(l?.ref ? { ref: l.ref } : {}), ...(l?.goal ? { goal: l.goal } : {}) });
|
|
3863
|
+
}
|
|
3864
|
+
heartbeatAt = 0;
|
|
3680
3865
|
writeState(state, detail) {
|
|
3681
3866
|
const line = `${state}|${detail}`;
|
|
3682
3867
|
if (line === this.lastState)
|
|
@@ -3685,7 +3870,8 @@ export class Worker {
|
|
|
3685
3870
|
try {
|
|
3686
3871
|
const base = this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
|
|
3687
3872
|
mkdirSync(base, { recursive: true });
|
|
3688
|
-
writeFileSync(join(base, "worker-state.json"), JSON.stringify({ at: Date.now(), state, detail, member: this.cfg.member
|
|
3873
|
+
writeFileSync(join(base, "worker-state.json"), JSON.stringify({ at: Date.now(), state, detail, member: this.cfg.member,
|
|
3874
|
+
...(this.opts.runMinutes !== undefined ? { run_minutes: this.opts.runMinutes } : {}) }) + "\n");
|
|
3689
3875
|
}
|
|
3690
3876
|
catch { }
|
|
3691
3877
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@join-pantheon/cli",
|
|
3
|
-
"version": "0.33.
|
|
3
|
+
"version": "0.33.6",
|
|
4
4
|
"description": "Let your coding agent collaborate directly with your teammates' agents — ask, agree an interface, and deliver, without a human relaying messages.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|