pacework 1.2.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/pacework-runner-ctl.mjs +11 -12
- package/bin/pacework-runner.mjs +631 -95
- package/lib/automation-limits.mjs +23 -0
- package/lib/git-lock.mjs +83 -0
- package/lib/pr-base-branch.mjs +84 -0
- package/lib/push-retry.mjs +108 -0
- package/lib/run-limits.mjs +302 -0
- package/lib/runner-control.mjs +55 -0
- package/lib/skills.mjs +142 -0
- package/lib/triage-worker.mjs +29 -4
- package/lib/usage-limit.mjs +23 -0
- package/lib/version.mjs +27 -0
- package/package.json +1 -1
package/bin/pacework-runner.mjs
CHANGED
|
@@ -9,9 +9,28 @@ import process from "node:process"
|
|
|
9
9
|
import { spawn, execFileSync } from "node:child_process"
|
|
10
10
|
import { AsyncLocalStorage } from "node:async_hooks"
|
|
11
11
|
import { createLogger } from "../lib/logger.mjs"
|
|
12
|
-
import {
|
|
12
|
+
import { detectRunUsageLimit, USAGE_LIMIT_MARKER } from "../lib/usage-limit.mjs"
|
|
13
|
+
import {
|
|
14
|
+
buildImpactReport,
|
|
15
|
+
checkRunLimits,
|
|
16
|
+
commitStoppedWork,
|
|
17
|
+
costLimitStop,
|
|
18
|
+
describeRunLimits,
|
|
19
|
+
hasCommitsSince,
|
|
20
|
+
holdsUnpushedStoppedWork,
|
|
21
|
+
listRunChangedFiles,
|
|
22
|
+
resolveRunLimits,
|
|
23
|
+
tripwireErrorMessage,
|
|
24
|
+
tripwireHeadline,
|
|
25
|
+
} from "../lib/run-limits.mjs"
|
|
26
|
+
import { fetchGithubDefaultBranch, readOriginHeadBranch, resolvePrBaseBranch } from "../lib/pr-base-branch.mjs"
|
|
27
|
+
import { clearPendingPush, describeGitFailure, isTransientGitHubFailure, readPendingPush, retryTransient, writePendingPush } from "../lib/push-retry.mjs"
|
|
13
28
|
import { prepareRunEnvironment } from "../lib/environment.mjs"
|
|
29
|
+
import { installSkills, parseSkillsUsed, skillsPromptSection } from "../lib/skills.mjs"
|
|
30
|
+
import { isGitLockError, repoLockKey, retryGitLock, withRepoLock } from "../lib/git-lock.mjs"
|
|
31
|
+
import { describeRegistration, isRevokedError, registrationChanges, REVOKED_MESSAGE, runEndedOnServer } from "../lib/runner-control.mjs"
|
|
14
32
|
import { runTriageLoop, TRIAGE_EXECUTORS } from "../lib/triage-worker.mjs"
|
|
33
|
+
import { automationDenyRules, automationLimits } from "../lib/automation-limits.mjs"
|
|
15
34
|
|
|
16
35
|
function parseArgs(argv) {
|
|
17
36
|
const args = {}
|
|
@@ -343,10 +362,66 @@ function sleep(ms) {
|
|
|
343
362
|
}
|
|
344
363
|
|
|
345
364
|
/**
|
|
346
|
-
*
|
|
347
|
-
*
|
|
365
|
+
* This runner's pause for an executor's plan limit: { until (ms), executor }, or
|
|
366
|
+
* null. `executor` null means every executor. Runs of the limited executor on
|
|
367
|
+
* the machine's own plan would fail the same way until the limit resets; the
|
|
368
|
+
* server holds those back and says so on each claim, so an admin can clear it.
|
|
348
369
|
*/
|
|
349
|
-
let
|
|
370
|
+
let usageLimitPause = null
|
|
371
|
+
|
|
372
|
+
/** Whether `pause` leaves this runner nothing to claim but work paid with a key. */
|
|
373
|
+
function pauseCoversEveryExecutor(pause) {
|
|
374
|
+
if (!pause) return false
|
|
375
|
+
return pause.executor === null || installedExecutors.every(executor => executor === pause.executor)
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Mark a run failure as already reported to the server and logged, so the
|
|
380
|
+
* claim loop's catch-all does not report it again as a setup failure.
|
|
381
|
+
*/
|
|
382
|
+
function reportedRunFailure(error) {
|
|
383
|
+
const reported = error instanceof Error ? error : new Error(String(error))
|
|
384
|
+
reported.runFailureReported = true
|
|
385
|
+
return reported
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
/**
|
|
389
|
+
* Follow the server's usage-limit pause from a claim response. The server holds
|
|
390
|
+
* the pause (it outlives a restart, and an admin can clear it in the app).
|
|
391
|
+
* Logs only when the pause starts, changes or ends.
|
|
392
|
+
*/
|
|
393
|
+
function syncUsageLimitPause(claim, logger) {
|
|
394
|
+
const serverUntil = claim.backpressure === "backoff" && claim.available_after
|
|
395
|
+
? new Date(claim.available_after).getTime()
|
|
396
|
+
: 0
|
|
397
|
+
const previous = usageLimitPause
|
|
398
|
+
if (serverUntil > Date.now()) {
|
|
399
|
+
const executor = claim.backoff_executor ?? null
|
|
400
|
+
if (previous?.until === serverUntil && previous.executor === executor) return
|
|
401
|
+
usageLimitPause = { until: serverUntil, executor }
|
|
402
|
+
logger.info("runner.paused", executor
|
|
403
|
+
? `${executorDisplayName(executor)} reported its usage limit; its runs on this machine's plan wait until the limit resets`
|
|
404
|
+
: "An AI tool reported its usage limit; runs on this machine's plan wait until the limit resets", {
|
|
405
|
+
meta: { until: new Date(serverUntil).toISOString(), executor, reason: claim.backoff_reason ?? "usage_limit" },
|
|
406
|
+
})
|
|
407
|
+
return
|
|
408
|
+
}
|
|
409
|
+
// Other back-pressure (capacity, draining) is answered before the pause
|
|
410
|
+
// counts, so it says nothing about it. A plain "nothing to claim", or a
|
|
411
|
+
// claimed run the pause would have held back, says the server has none.
|
|
412
|
+
if (!previous || claim.backpressure) return
|
|
413
|
+
if (claim.run) {
|
|
414
|
+
const heldBack = !runCredentials(claim.run)
|
|
415
|
+
&& (previous.executor === null || previous.executor === executorForRun(claim.run))
|
|
416
|
+
if (!heldBack) return
|
|
417
|
+
}
|
|
418
|
+
usageLimitPause = null
|
|
419
|
+
logger.info("runner.resumed", previous.until > Date.now()
|
|
420
|
+
? "The usage-limit pause was cleared; taking all work again"
|
|
421
|
+
: "The usage limit has reset; taking all work again", {
|
|
422
|
+
meta: { executor: previous.executor },
|
|
423
|
+
})
|
|
424
|
+
}
|
|
350
425
|
|
|
351
426
|
function formatLocalTime(date) {
|
|
352
427
|
return date.toLocaleString(undefined, { weekday: "short", hour: "2-digit", minute: "2-digit", timeZoneName: "short" })
|
|
@@ -381,6 +456,8 @@ function parseClaudeJsonResult(stdout) {
|
|
|
381
456
|
return {
|
|
382
457
|
text: typeof parsed.result === "string" ? parsed.result : "",
|
|
383
458
|
isError: parsed.is_error === true,
|
|
459
|
+
// "error_max_budget_usd" when --max-budget-usd ended the run.
|
|
460
|
+
subtype: typeof parsed.subtype === "string" ? parsed.subtype : null,
|
|
384
461
|
usage: {
|
|
385
462
|
cost_usd: typeof parsed.total_cost_usd === "number" ? parsed.total_cost_usd : null,
|
|
386
463
|
input_tokens: inputTokens.length ? inputTokens.reduce((a, b) => a + b, 0) : null,
|
|
@@ -757,6 +834,13 @@ function isAutomationSmokeTestIssue(run) {
|
|
|
757
834
|
return title.includes(automationSmokeTestMarker) || description.includes(automationSmokeTestMarker)
|
|
758
835
|
}
|
|
759
836
|
|
|
837
|
+
/** Per running run: stop its agent because of an admin's runner-wide decision. */
|
|
838
|
+
const adminStoppers = new Map()
|
|
839
|
+
|
|
840
|
+
function stopAllRuns(reason) {
|
|
841
|
+
for (const stopRun of adminStoppers.values()) stopRun(reason)
|
|
842
|
+
}
|
|
843
|
+
|
|
760
844
|
async function requestJson(url, init = {}) {
|
|
761
845
|
const response = await fetch(url, init)
|
|
762
846
|
let payload = null
|
|
@@ -774,7 +858,9 @@ async function requestJson(url, init = {}) {
|
|
|
774
858
|
: ""
|
|
775
859
|
const errorLabel = typeof payload?.error === "string" ? payload.error : ""
|
|
776
860
|
const errorMessage = errorDetail || errorLabel || `Request failed: ${response.status}`
|
|
777
|
-
|
|
861
|
+
const error = new Error(errorMessage)
|
|
862
|
+
error.status = response.status
|
|
863
|
+
throw error
|
|
778
864
|
}
|
|
779
865
|
|
|
780
866
|
return payload
|
|
@@ -888,7 +974,24 @@ async function registerRunner() {
|
|
|
888
974
|
|
|
889
975
|
async function ensureRegisteredRunner() {
|
|
890
976
|
const state = loadState()
|
|
891
|
-
if (state?.runner_id && state?.runner_secret && state?.runner_mcp_key) return
|
|
977
|
+
if (!(state?.runner_id && state?.runner_secret && state?.runner_mcp_key)) return registerRunner()
|
|
978
|
+
const changes = registrationChanges(state, {
|
|
979
|
+
name: runnerName,
|
|
980
|
+
machine_name: machineName,
|
|
981
|
+
repo_root: repoRoot,
|
|
982
|
+
local_executor: defaultExecutor,
|
|
983
|
+
max_parallel_runs: maxParallelRuns,
|
|
984
|
+
labels,
|
|
985
|
+
})
|
|
986
|
+
if (changes.length === 0) return state
|
|
987
|
+
// Registering again updates this machine's entry in place (the server
|
|
988
|
+
// matches it by machine and repo root) and saves the new settings. Without
|
|
989
|
+
// that, a new --name or RUNNER_LABELS changed nothing on the server.
|
|
990
|
+
if (!workspaceApiKey) {
|
|
991
|
+
console.warn(`Runner settings changed (${changes.join(", ")}) but there is no workspace key to register them with; the server keeps the old ones.`)
|
|
992
|
+
return state
|
|
993
|
+
}
|
|
994
|
+
console.log(`Runner settings changed (${changes.join(", ")}); updating them in pacework.`)
|
|
892
995
|
return registerRunner()
|
|
893
996
|
}
|
|
894
997
|
|
|
@@ -1266,33 +1369,53 @@ async function fetchGithubToken(state, workspaceId, projectId, teamId = null, ta
|
|
|
1266
1369
|
*/
|
|
1267
1370
|
const GIT_IGNORE_STORED_AUTH = ["-c", "http.https://github.com/.extraheader="]
|
|
1268
1371
|
|
|
1269
|
-
async function pushBranchToGitHub(worktreePath, branch, githubToken, repo) {
|
|
1372
|
+
async function pushBranchToGitHub(worktreePath, branch, githubToken, repo, onRetry = () => {}) {
|
|
1373
|
+
// Worktrees of one repository share its config, remote URL included: a run
|
|
1374
|
+
// pushing while another swaps the URL would push without the token, or leave
|
|
1375
|
+
// it behind. Pushes and checkout setup take turns (lib/git-lock.mjs).
|
|
1376
|
+
return withRepoLock(repoLockKey(worktreePath), () => pushBranchWithToken(worktreePath, branch, githubToken, repo, onRetry))
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
async function pushBranchWithToken(worktreePath, branch, githubToken, repo, onRetry) {
|
|
1270
1380
|
// Set remote URL with embedded token for auth, then push
|
|
1271
1381
|
const remoteUrl = `https://x-access-token:${githubToken}@github.com/${repo.owner}/${repo.repo}.git`
|
|
1272
|
-
await execCommand(
|
|
1382
|
+
await retryGitLock(() => execCommand(
|
|
1273
1383
|
"git",
|
|
1274
1384
|
["-C", worktreePath, "remote", "set-url", "origin", remoteUrl],
|
|
1275
1385
|
worktreePath,
|
|
1276
|
-
)
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1283
|
-
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1386
|
+
))
|
|
1387
|
+
try {
|
|
1388
|
+
// A GitHub 500 or a dropped connection is retried a few times before the
|
|
1389
|
+
// run gives up; one hiccup used to throw a finished run away.
|
|
1390
|
+
await retryTransient(async () => {
|
|
1391
|
+
// start_work creates the branch on GitHub before the runner has a local clone
|
|
1392
|
+
// of it, so --force-with-lease sees a remote it has no ref for and rejects the
|
|
1393
|
+
// push. Fetch first so the lease has something to compare against.
|
|
1394
|
+
await execCommandSafe(
|
|
1395
|
+
"git",
|
|
1396
|
+
["-C", worktreePath, ...GIT_IGNORE_STORED_AUTH, "fetch", "origin", branch],
|
|
1397
|
+
worktreePath,
|
|
1398
|
+
)
|
|
1399
|
+
await execCommand(
|
|
1400
|
+
"git",
|
|
1401
|
+
["-C", worktreePath, ...GIT_IGNORE_STORED_AUTH, "push", "--set-upstream", "origin", branch, "--force-with-lease"],
|
|
1402
|
+
worktreePath,
|
|
1403
|
+
)
|
|
1404
|
+
}, {
|
|
1405
|
+
onRetry,
|
|
1406
|
+
// `--set-upstream` writes the shared config: a lock held by another git
|
|
1407
|
+
// process is worth waiting out too.
|
|
1408
|
+
isTransient: (error) => isGitLockError(error) || isTransientGitHubFailure(error instanceof Error ? error.message : String(error)),
|
|
1409
|
+
})
|
|
1410
|
+
} finally {
|
|
1411
|
+
// Restore remote URL without token for safety, also when the push failed.
|
|
1412
|
+
// Retried on a held lock: giving up would leave the token in the config.
|
|
1413
|
+
await retryGitLock(() => execCommand(
|
|
1414
|
+
"git",
|
|
1415
|
+
["-C", worktreePath, "remote", "set-url", "origin", `https://github.com/${repo.owner}/${repo.repo}.git`],
|
|
1416
|
+
worktreePath,
|
|
1417
|
+
)).catch(() => {})
|
|
1418
|
+
}
|
|
1296
1419
|
}
|
|
1297
1420
|
|
|
1298
1421
|
/** A section the agent left empty or answered with "none" says nothing to a reader. */
|
|
@@ -1699,6 +1822,15 @@ async function prepareWorktree(run) {
|
|
|
1699
1822
|
|
|
1700
1823
|
ensureDir(path.dirname(worktreePath))
|
|
1701
1824
|
|
|
1825
|
+
// Runs that start together take turns per repository, and a lock held by
|
|
1826
|
+
// another git process is waited out (see lib/git-lock.mjs). Each step below
|
|
1827
|
+
// checks what already exists, so a retry picks up where a failure left off.
|
|
1828
|
+
await withRepoLock(repoLockKey(runRepoRoot), () => retryGitLock(() => prepareWorktreeCheckout(run, runRepoRoot, worktreePath, branch)))
|
|
1829
|
+
|
|
1830
|
+
return { runRoot, worktreePath, branch, localCheckoutMode, gitAvailable: true }
|
|
1831
|
+
}
|
|
1832
|
+
|
|
1833
|
+
async function prepareWorktreeCheckout(run, runRepoRoot, worktreePath, branch) {
|
|
1702
1834
|
// Pull requests merge on GitHub, so origin/* in this checkout falls behind.
|
|
1703
1835
|
// A stale base gave builders an old main to work on, and gave reviewers a
|
|
1704
1836
|
// `git diff origin/HEAD..HEAD` full of work that had already merged.
|
|
@@ -1725,7 +1857,9 @@ async function prepareWorktree(run) {
|
|
|
1725
1857
|
// Branch exists from a prior run — reuse it instead of -b
|
|
1726
1858
|
await execCommand("git", ["-C", runRepoRoot, "worktree", "add", worktreePath, branch], runRepoRoot)
|
|
1727
1859
|
} else {
|
|
1728
|
-
|
|
1860
|
+
// --no-track: the run's branch must not follow the base branch, and
|
|
1861
|
+
// recording an upstream writes .git/config, the lock runs collided on.
|
|
1862
|
+
await execCommand("git", ["-C", runRepoRoot, "worktree", "add", "--no-track", "-b", branch, worktreePath, upToDateBase], runRepoRoot)
|
|
1729
1863
|
}
|
|
1730
1864
|
} else if (worktreeKnown) {
|
|
1731
1865
|
const checkedOutBranch = await currentBranch(worktreePath)
|
|
@@ -1743,8 +1877,6 @@ async function prepareWorktree(run) {
|
|
|
1743
1877
|
)
|
|
1744
1878
|
}
|
|
1745
1879
|
}
|
|
1746
|
-
|
|
1747
|
-
return { runRoot, worktreePath, branch, localCheckoutMode, gitAvailable: true }
|
|
1748
1880
|
}
|
|
1749
1881
|
|
|
1750
1882
|
function buildPrompt(run, branch, worktreePath, options = {}) {
|
|
@@ -1755,7 +1887,9 @@ function buildPrompt(run, branch, worktreePath, options = {}) {
|
|
|
1755
1887
|
const gitAvailable = options.gitAvailable !== false
|
|
1756
1888
|
const workspaceContext = options.workspaceContext || null
|
|
1757
1889
|
const greenfieldWorkspace = workspaceContext?.is_empty_workspace === true
|
|
1758
|
-
|
|
1890
|
+
// The agent's standing instructions; servers before skill profiles were
|
|
1891
|
+
// retired sent them as skill_profile.system_prompt.
|
|
1892
|
+
const skillPrompt = (agent?.instructions ?? agent?.skill_profile?.system_prompt)?.trim()
|
|
1759
1893
|
const base = skillPrompt ? `${skillPrompt}\n\n---\n\n` : ""
|
|
1760
1894
|
const codingRules = Array.isArray(run?.coding_rules) ? run.coding_rules : []
|
|
1761
1895
|
const reviewerContext = run?.reviewer_context || null
|
|
@@ -1860,6 +1994,13 @@ work that is already on the base branch is the worst thing a pull request can do
|
|
|
1860
1994
|
.join("\n")
|
|
1861
1995
|
: ""
|
|
1862
1996
|
const answersGuidance = clarificationAnswers(run)
|
|
1997
|
+
// Told up front so an agent facing an oversized issue stops and says so,
|
|
1998
|
+
// instead of finding out from the tripwire.
|
|
1999
|
+
const runLimitsGuidance = !isReviewer && options.runLimitsText
|
|
2000
|
+
? `- Run limits: ${options.runLimitsText}. The runner stops the run as soon as one is crossed.\n`
|
|
2001
|
+
+ ` If this issue clearly needs more than that, do not start building: say in your summary that it\n`
|
|
2002
|
+
+ ` should be split, and propose the subtasks.`
|
|
2003
|
+
: ""
|
|
1863
2004
|
const environmentGuidance = run.__environmentNote
|
|
1864
2005
|
? `- Environment setup did not fully work. Say so in your summary rather than treating it as a bug in the code:\n ${run.__environmentNote.replace(/\n/g, "\n ")}`
|
|
1865
2006
|
: ""
|
|
@@ -1888,7 +2029,7 @@ Execution rules:
|
|
|
1888
2029
|
- Local checkout mode: ${localCheckoutMode}
|
|
1889
2030
|
${checkoutModeGuidance}
|
|
1890
2031
|
${existingCheckoutGuidance}
|
|
1891
|
-
${optionalGuidance ? `${optionalGuidance}\n` : ""}${environmentServices.length > 0 ? `- Services are running for this checkout; their addresses are in ${environmentServices.join(", ")}.\n` : ""}${environmentGuidance ? `${environmentGuidance}\n` : ""}${feedbackGuidance ? `${feedbackGuidance}\n` : ""}${codingRulesGuidance ? `${codingRulesGuidance}\n` : ""}${reviewerGuidance ? `${reviewerGuidance}\n` : ""}- Do not open a PR yet.
|
|
2032
|
+
${optionalGuidance ? `${optionalGuidance}\n` : ""}${environmentServices.length > 0 ? `- Services are running for this checkout; their addresses are in ${environmentServices.join(", ")}.\n` : ""}${environmentGuidance ? `${environmentGuidance}\n` : ""}${feedbackGuidance ? `${feedbackGuidance}\n` : ""}${codingRulesGuidance ? `${codingRulesGuidance}\n` : ""}${runLimitsGuidance ? `${runLimitsGuidance}\n` : ""}${reviewerGuidance ? `${reviewerGuidance}\n` : ""}- Do not open a PR yet.
|
|
1892
2033
|
- Treat explicit issue-description constraints as higher priority than generic instructions.
|
|
1893
2034
|
- ${executionModeGuidance}
|
|
1894
2035
|
${isReviewer || answersGuidance ? "" : `- ${ASK_WHEN_UNCLEAR.replace(/\n/g, "\n ")}\n`}- End with a concise summary of what changed, what was verified, and any remaining gaps.
|
|
@@ -1941,6 +2082,8 @@ function claudeRunSettings(run) {
|
|
|
1941
2082
|
],
|
|
1942
2083
|
}
|
|
1943
2084
|
}
|
|
2085
|
+
const deny = automationDenyRules(run)
|
|
2086
|
+
if (deny.length) settings.permissions = { ...(settings.permissions || {}), deny }
|
|
1944
2087
|
if (runCredentials(run)?.provider === "anthropic") {
|
|
1945
2088
|
// ANTHROPIC_API_KEY alone loses to a signed-in Claude subscription, so the
|
|
1946
2089
|
// run would be billed to the machine's plan. A key helper always wins. It
|
|
@@ -1950,8 +2093,30 @@ function claudeRunSettings(run) {
|
|
|
1950
2093
|
return Object.keys(settings).length ? JSON.stringify(settings) : null
|
|
1951
2094
|
}
|
|
1952
2095
|
|
|
1953
|
-
|
|
2096
|
+
/** Whether this machine's Claude Code takes --max-budget-usd; an older one would refuse to start. */
|
|
2097
|
+
let claudeMaxBudgetSupport = null
|
|
2098
|
+
function claudeSupportsMaxBudget() {
|
|
2099
|
+
if (claudeMaxBudgetSupport === null) {
|
|
2100
|
+
try {
|
|
2101
|
+
const help = execFileSync(claudeCodeBin, ["--help"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 15_000 })
|
|
2102
|
+
claudeMaxBudgetSupport = help.includes("--max-budget-usd")
|
|
2103
|
+
} catch {
|
|
2104
|
+
claudeMaxBudgetSupport = false
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
return claudeMaxBudgetSupport
|
|
2108
|
+
}
|
|
2109
|
+
|
|
2110
|
+
/**
|
|
2111
|
+
* @param {{ maxCostUsd?: number | null }} [options] The run's cost limit, for
|
|
2112
|
+
* Claude Code: it ends the run itself once the limit is spent.
|
|
2113
|
+
*/
|
|
2114
|
+
function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey, options = {}) {
|
|
1954
2115
|
const requestedModel = resolveRequestedModel(run)
|
|
2116
|
+
const limits = automationLimits(run)
|
|
2117
|
+
if (limits && (!limits.web || limits.code !== "write") && activeExecutor() !== "claude_code" && activeExecutor() !== "codex") {
|
|
2118
|
+
throw new Error("This automation limits web or code access, which only Claude Code and Codex runners can enforce. Run it on one of those, or allow web and code access.")
|
|
2119
|
+
}
|
|
1955
2120
|
|
|
1956
2121
|
if (activeExecutor() === "codex") {
|
|
1957
2122
|
const argsList = [
|
|
@@ -1960,8 +2125,18 @@ function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey) {
|
|
|
1960
2125
|
// use its own current default rather than a name hardcoded here.
|
|
1961
2126
|
...(requestedModel ? ["-m", requestedModel] : []),
|
|
1962
2127
|
"--ephemeral",
|
|
2128
|
+
// An automation without code write runs read-only, which also cuts the network.
|
|
2129
|
+
...(limits && limits.code !== "write"
|
|
2130
|
+
? ["-s", "read-only", "-c", "approval_policy=\"never\""]
|
|
2131
|
+
: limits && !limits.web
|
|
2132
|
+
? [
|
|
2133
|
+
"-s", "workspace-write",
|
|
2134
|
+
"-c", "approval_policy=\"never\"",
|
|
2135
|
+
"-c", "sandbox_workspace_write.network_access=false",
|
|
2136
|
+
"-c", `sandbox_workspace_write.writable_roots=${JSON.stringify([path.join(run.__repoRoot || repoRoot, ".git")])}`,
|
|
2137
|
+
]
|
|
1963
2138
|
// Codex's sandbox cannot allow single domains, so network stays on.
|
|
1964
|
-
|
|
2139
|
+
: runSandboxed(run)
|
|
1965
2140
|
? [
|
|
1966
2141
|
"-s", "workspace-write",
|
|
1967
2142
|
"-c", "approval_policy=\"never\"",
|
|
@@ -2012,6 +2187,9 @@ function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey) {
|
|
|
2012
2187
|
if (requestedModel) {
|
|
2013
2188
|
argsList.push("--model", requestedModel)
|
|
2014
2189
|
}
|
|
2190
|
+
if (typeof options.maxCostUsd === "number") {
|
|
2191
|
+
argsList.push("--max-budget-usd", String(options.maxCostUsd))
|
|
2192
|
+
}
|
|
2015
2193
|
return {
|
|
2016
2194
|
command: claudeCodeBin,
|
|
2017
2195
|
args: argsList,
|
|
@@ -2536,6 +2714,15 @@ async function runPlannerExecutor(run, state) {
|
|
|
2536
2714
|
let plannerMcpBackups = null
|
|
2537
2715
|
let plannerUsage = null
|
|
2538
2716
|
|
|
2717
|
+
// An admin stopping or disconnecting this runner ends the run from outside.
|
|
2718
|
+
adminStoppers.set(run.id, (reason) => {
|
|
2719
|
+
cancelRequested = true
|
|
2720
|
+
if (childProcess && !childProcess.killed) {
|
|
2721
|
+
appendLogTail(logState, `> ${reason}; terminating executor process...`)
|
|
2722
|
+
requestChildProcessShutdown(childProcess)
|
|
2723
|
+
}
|
|
2724
|
+
})
|
|
2725
|
+
|
|
2539
2726
|
function appendHeartbeat(force = false) {
|
|
2540
2727
|
if (!childProcess || childProcess.killed) return
|
|
2541
2728
|
const now = Date.now()
|
|
@@ -2550,7 +2737,7 @@ async function runPlannerExecutor(run, state) {
|
|
|
2550
2737
|
async function flushPlannerLog() {
|
|
2551
2738
|
appendHeartbeat()
|
|
2552
2739
|
const response = await postPlannerLog(run, logState.tail)
|
|
2553
|
-
if (response
|
|
2740
|
+
if (runEndedOnServer(response)) {
|
|
2554
2741
|
cancelRequested = true
|
|
2555
2742
|
if (childProcess && !childProcess.killed) {
|
|
2556
2743
|
appendLogTail(logState, "> cancel requested; terminating executor process...")
|
|
@@ -2694,6 +2881,9 @@ async function runPlannerExecutor(run, state) {
|
|
|
2694
2881
|
}
|
|
2695
2882
|
|
|
2696
2883
|
await flushPlannerLog().catch(() => {})
|
|
2884
|
+
// Canceled while the agent was finishing (or it exited cleanly on the
|
|
2885
|
+
// signal): its plan is not applied.
|
|
2886
|
+
if (cancelRequested) throw new Error("Canceled by admin.")
|
|
2697
2887
|
|
|
2698
2888
|
const fileContent = fs.existsSync(outputFile) ? fs.readFileSync(outputFile, "utf8").trim() : ""
|
|
2699
2889
|
const plannerResponse = staticResponse || fileContent || stdioSummary.trim()
|
|
@@ -2736,7 +2926,7 @@ async function runPlannerExecutor(run, state) {
|
|
|
2736
2926
|
canceled: cancelRequested,
|
|
2737
2927
|
},
|
|
2738
2928
|
}, error)
|
|
2739
|
-
throw error
|
|
2929
|
+
throw reportedRunFailure(error)
|
|
2740
2930
|
} finally {
|
|
2741
2931
|
clearInterval(logInterval)
|
|
2742
2932
|
if (runTimeoutHandle) clearTimeout(runTimeoutHandle)
|
|
@@ -2835,7 +3025,7 @@ async function runExecutor(run, state) {
|
|
|
2835
3025
|
// Resolve the repo this ticket belongs to before preparing a checkout, so the
|
|
2836
3026
|
// code is written in the same repo the PR will be opened against.
|
|
2837
3027
|
try {
|
|
2838
|
-
const { token, repo } = await fetchGithubToken(
|
|
3028
|
+
const { token, repo, baseBranch } = await fetchGithubToken(
|
|
2839
3029
|
state,
|
|
2840
3030
|
run.workspace_id,
|
|
2841
3031
|
run.task?.project_id ?? null,
|
|
@@ -2844,6 +3034,7 @@ async function runExecutor(run, state) {
|
|
|
2844
3034
|
)
|
|
2845
3035
|
run.__githubToken = token ?? null
|
|
2846
3036
|
run.__repo = repo ?? null
|
|
3037
|
+
run.__baseBranch = baseBranch ?? null
|
|
2847
3038
|
run.__repoRoot = await resolveRunRepoRoot(run, state)
|
|
2848
3039
|
if (run.__repoRoot !== repoRoot) {
|
|
2849
3040
|
runLogger.info("runner.repo.resolved", "Working in the ticket's repository", {
|
|
@@ -2893,6 +3084,9 @@ async function runExecutor(run, state) {
|
|
|
2893
3084
|
|
|
2894
3085
|
runLogger.info("runner.run.prepare", "Preparing worktree")
|
|
2895
3086
|
const { runRoot, worktreePath, branch, localCheckoutMode, gitAvailable } = await prepareWorktree(run)
|
|
3087
|
+
// Where a commit that could not reach GitHub is noted; only this issue's own
|
|
3088
|
+
// checkout has a folder of its own to hold the note.
|
|
3089
|
+
const pendingPushDir = gitAvailable && branch && localCheckoutMode !== "repo_root" ? path.dirname(worktreePath) : null
|
|
2896
3090
|
const prepared = await prepareRunEnvironment({
|
|
2897
3091
|
environment: run.environment,
|
|
2898
3092
|
checkoutPath: worktreePath,
|
|
@@ -2921,8 +3115,37 @@ async function runExecutor(run, state) {
|
|
|
2921
3115
|
run.reviewer_context = { ...(run.reviewer_context || {}), pr_diff: prDiff.diff, pr_diff_truncated: prDiff.truncated === true }
|
|
2922
3116
|
}
|
|
2923
3117
|
}
|
|
2924
|
-
const
|
|
2925
|
-
|
|
3118
|
+
const runLimits = resolveRunLimits({ args, env: process.env, profile: runnerProfile })
|
|
3119
|
+
// Scouts survey the codebase and may touch many files on purpose.
|
|
3120
|
+
const fileLimitApplies = gitAvailable && getAgentSkillProfileSlug(run.agent) !== "scout"
|
|
3121
|
+
// The commit the run starts from: only the files this run changes count.
|
|
3122
|
+
const runStartSha = gitAvailable
|
|
3123
|
+
? ((await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null)
|
|
3124
|
+
: null
|
|
3125
|
+
// Only Claude Code can stop itself at a cost, and another provider's cost is
|
|
3126
|
+
// priced at Anthropic's rates, so it cannot be held to a dollar limit.
|
|
3127
|
+
const costLimitApplies = runLimits.maxCostUsd !== null
|
|
3128
|
+
&& activeExecutor() === "claude_code"
|
|
3129
|
+
&& !usesThirdPartyModels(run)
|
|
3130
|
+
&& claudeSupportsMaxBudget()
|
|
3131
|
+
const enforcedLimits = { files: fileLimitApplies, cost: costLimitApplies, time: true }
|
|
3132
|
+
const runLimitsText = describeRunLimits(runLimits, enforcedLimits)
|
|
3133
|
+
// The agent's skills, written where Claude Code and Codex load skills on demand
|
|
3134
|
+
// and kept out of git (PWK-145). Only a git checkout can keep them out of the change.
|
|
3135
|
+
const skillInstall = gitAvailable
|
|
3136
|
+
? installSkills(worktreePath, run.skills, { log: (line) => runLogger.info("runner.run.skills", line) })
|
|
3137
|
+
: { installed: [], skipped: [], cleanup() {} }
|
|
3138
|
+
const skillsSection = skillsPromptSection(
|
|
3139
|
+
(run.skills ?? []).filter((skill) => skillInstall.installed.includes(skill.slug)),
|
|
3140
|
+
{ purpose: isReviewRun(run) ? "review" : "build" },
|
|
3141
|
+
)
|
|
3142
|
+
const prompt = [
|
|
3143
|
+
buildPrompt(run, branch, worktreePath, { localCheckoutMode, gitAvailable, workspaceContext, runLimitsText }),
|
|
3144
|
+
skillsSection,
|
|
3145
|
+
].filter(Boolean).join("\n\n")
|
|
3146
|
+
const execSpec = buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey, {
|
|
3147
|
+
maxCostUsd: costLimitApplies ? runLimits.maxCostUsd : null,
|
|
3148
|
+
})
|
|
2926
3149
|
// These steps run before the executor spawns; when one stalls the run sits in
|
|
2927
3150
|
// "running" with no output, so each is traced.
|
|
2928
3151
|
runLogger.info("runner.run.prepare", "Fetching MCP servers")
|
|
@@ -2939,6 +3162,8 @@ async function runExecutor(run, state) {
|
|
|
2939
3162
|
|
|
2940
3163
|
const logState = { tail: "" }
|
|
2941
3164
|
const startedAt = Date.now()
|
|
3165
|
+
/** Where the run is relative to the AI tool: only its own failure can be a usage limit. */
|
|
3166
|
+
let executorPhase = "before"
|
|
2942
3167
|
let stdioSummary = ""
|
|
2943
3168
|
// Claude Code's JSON result arrives on stdout only; stderr would break parsing.
|
|
2944
3169
|
let stdoutRaw = ""
|
|
@@ -2946,7 +3171,18 @@ async function runExecutor(run, state) {
|
|
|
2946
3171
|
let cancelRequested = false
|
|
2947
3172
|
let childProcess = null
|
|
2948
3173
|
let lastHeartbeatLogAt = 0
|
|
3174
|
+
|
|
3175
|
+
// An admin stopping or disconnecting this runner ends the run from outside.
|
|
3176
|
+
adminStoppers.set(run.id, (reason) => {
|
|
3177
|
+
cancelRequested = true
|
|
3178
|
+
if (childProcess && !childProcess.killed) {
|
|
3179
|
+
appendLogTail(logState, `> ${reason}; terminating executor process...`)
|
|
3180
|
+
requestChildProcessShutdown(childProcess)
|
|
3181
|
+
}
|
|
3182
|
+
})
|
|
2949
3183
|
let activityThreadCommentId = run.activity_thread_comment_id || null
|
|
3184
|
+
/** The limit that stopped this run, once one has. */
|
|
3185
|
+
let limitStop = null
|
|
2950
3186
|
|
|
2951
3187
|
function appendHeartbeat(force = false) {
|
|
2952
3188
|
if (!childProcess || childProcess.killed) return
|
|
@@ -2962,7 +3198,7 @@ async function runExecutor(run, state) {
|
|
|
2962
3198
|
async function flushRunnerLog() {
|
|
2963
3199
|
appendHeartbeat()
|
|
2964
3200
|
const response = await postRunLog(state, run.id, logState.tail)
|
|
2965
|
-
if (response
|
|
3201
|
+
if (runEndedOnServer(response)) {
|
|
2966
3202
|
cancelRequested = true
|
|
2967
3203
|
if (childProcess && !childProcess.killed) {
|
|
2968
3204
|
appendLogTail(logState, "> cancel requested; terminating executor process...")
|
|
@@ -2975,6 +3211,36 @@ async function runExecutor(run, state) {
|
|
|
2975
3211
|
flushRunnerLog().catch(() => {})
|
|
2976
3212
|
}, 5000)
|
|
2977
3213
|
|
|
3214
|
+
// Files and time are checked while the executor works, so a run that is too
|
|
3215
|
+
// big stops then, not after it has spent all its time and money. Cost is
|
|
3216
|
+
// Claude Code's own check (--max-budget-usd).
|
|
3217
|
+
let limitCheckInFlight = false
|
|
3218
|
+
async function checkLimitsWhileRunning() {
|
|
3219
|
+
if (limitStop || limitCheckInFlight || !childProcess || childProcess.killed) return
|
|
3220
|
+
limitCheckInFlight = true
|
|
3221
|
+
try {
|
|
3222
|
+
const changed = fileLimitApplies && runLimits.maxChangedFiles !== null
|
|
3223
|
+
? await listRunChangedFiles(worktreePath, runStartSha)
|
|
3224
|
+
: []
|
|
3225
|
+
const stop = checkRunLimits(
|
|
3226
|
+
{ ...runLimits, maxChangedFiles: fileLimitApplies ? runLimits.maxChangedFiles : null, maxCostUsd: null },
|
|
3227
|
+
{ changedFileCount: changed.length, elapsedMs: Date.now() - startedAt },
|
|
3228
|
+
)
|
|
3229
|
+
if (!stop || limitStop || !childProcess || childProcess.killed) return
|
|
3230
|
+
limitStop = stop
|
|
3231
|
+
appendLogTail(logState, `> tripwire: stopping the run because ${stop.reason}`)
|
|
3232
|
+
runLogger.warn("runner.tripwire.tripped", `Run stopped at its ${stop.kind} limit`, {
|
|
3233
|
+
meta: { kind: stop.kind, limit: stop.limit, actual: stop.actual },
|
|
3234
|
+
})
|
|
3235
|
+
requestChildProcessShutdown(childProcess)
|
|
3236
|
+
} finally {
|
|
3237
|
+
limitCheckInFlight = false
|
|
3238
|
+
}
|
|
3239
|
+
}
|
|
3240
|
+
const limitInterval = setInterval(() => {
|
|
3241
|
+
checkLimitsWhileRunning().catch(() => {})
|
|
3242
|
+
}, RUN_LIMIT_CHECK_INTERVAL_MS)
|
|
3243
|
+
|
|
2978
3244
|
// Hard timeout — kill the run if it hangs
|
|
2979
3245
|
let runTimeoutHandle = null
|
|
2980
3246
|
const runTimeoutPromise = new Promise((_, reject) => {
|
|
@@ -2997,13 +3263,15 @@ async function runExecutor(run, state) {
|
|
|
2997
3263
|
appendLogTail(logState, `> worktree: ${worktreePath}`)
|
|
2998
3264
|
appendLogTail(logState, `> branch: ${branch || "(none)"}`)
|
|
2999
3265
|
appendLogTail(logState, `> executor: ${activeExecutor()}`)
|
|
3266
|
+
appendLogTail(logState, `> run limits: ${runLimitsText}`)
|
|
3000
3267
|
await flushRunnerLog().catch(() => {})
|
|
3001
3268
|
await postIssueMilestone(
|
|
3002
3269
|
mcpKey,
|
|
3003
3270
|
run.task_id,
|
|
3004
|
-
isReviewRun(run)
|
|
3271
|
+
(isReviewRun(run)
|
|
3005
3272
|
? `AI review started in ${gitAvailable ? (branch || worktreePath) : worktreePath}.`
|
|
3006
|
-
: `Implementation started in ${gitAvailable ? (branch || worktreePath) : worktreePath}
|
|
3273
|
+
: `Implementation started in ${gitAvailable ? (branch || worktreePath) : worktreePath}.`)
|
|
3274
|
+
+ ` Run limits: ${runLimitsText}.`,
|
|
3007
3275
|
"info",
|
|
3008
3276
|
)
|
|
3009
3277
|
if (!isReviewRun(run)) {
|
|
@@ -3021,7 +3289,27 @@ async function runExecutor(run, state) {
|
|
|
3021
3289
|
}
|
|
3022
3290
|
await flushRunnerLog().catch(() => {})
|
|
3023
3291
|
|
|
3024
|
-
|
|
3292
|
+
// A previous run finished this work but GitHub would not take it. Publish
|
|
3293
|
+
// that commit instead of running the agent again.
|
|
3294
|
+
const pendingPush = pendingPushDir && !isReviewRun(run)
|
|
3295
|
+
? readPendingPush(pendingPushDir, {
|
|
3296
|
+
taskId: run.task_id,
|
|
3297
|
+
branch,
|
|
3298
|
+
head: (await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null,
|
|
3299
|
+
})
|
|
3300
|
+
: null
|
|
3301
|
+
|
|
3302
|
+
if (pendingPush) {
|
|
3303
|
+
appendLogTail(logState, `> commit ${pendingPush.head.slice(0, 7)} from run ${pendingPush.run_id} was never published; publishing it without running the agent`)
|
|
3304
|
+
await postIssueMilestone(
|
|
3305
|
+
mcpKey,
|
|
3306
|
+
run.task_id,
|
|
3307
|
+
`Publishing commit ${pendingPush.head.slice(0, 7)}, which an earlier run finished but could not push. The agent is not run again.`,
|
|
3308
|
+
"info",
|
|
3309
|
+
).catch(() => {})
|
|
3310
|
+
stdioSummary = pendingPush.summary || "Published the commit an earlier run finished but could not push."
|
|
3311
|
+
if (execSpec.summaryMode === "file") fs.writeFileSync(outputFile, stdioSummary)
|
|
3312
|
+
} else if (automationSmokeTestMode && isAutomationSmokeTestIssue(run)) {
|
|
3025
3313
|
const smokeResult = await runSmokeTestExecution(run, state, { activityThreadCommentId })
|
|
3026
3314
|
stdioSummary = smokeResult.summary
|
|
3027
3315
|
appendLogTail(logState, smokeResult.summary)
|
|
@@ -3034,6 +3322,7 @@ async function runExecutor(run, state) {
|
|
|
3034
3322
|
appendLogTail(logState, `> cwd: ${worktreePath}`)
|
|
3035
3323
|
appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
|
|
3036
3324
|
await flushRunnerLog().catch(() => {})
|
|
3325
|
+
executorPhase = "running"
|
|
3037
3326
|
await Promise.race([runTimeoutPromise, new Promise((resolve, reject) => {
|
|
3038
3327
|
const child = spawn(execSpec.command, execSpec.args, {
|
|
3039
3328
|
cwd: worktreePath,
|
|
@@ -3081,12 +3370,24 @@ async function runExecutor(run, state) {
|
|
|
3081
3370
|
if (result.usage.cost_usd !== null) {
|
|
3082
3371
|
appendLogTail(logState, `> cost: $${result.usage.cost_usd.toFixed(4)}, ${result.usage.num_turns ?? "?"} turns`)
|
|
3083
3372
|
}
|
|
3373
|
+
if (result.subtype === "error_max_budget_usd" && costLimitApplies && !limitStop) {
|
|
3374
|
+
limitStop = costLimitStop(runLimits.maxCostUsd, result.usage.cost_usd)
|
|
3375
|
+
appendLogTail(logState, `> tripwire: the run stopped because ${limitStop.reason}`)
|
|
3376
|
+
}
|
|
3377
|
+
if (limitStop) {
|
|
3378
|
+
reject(new Error(tripwireHeadline(limitStop)))
|
|
3379
|
+
return
|
|
3380
|
+
}
|
|
3084
3381
|
if (code === 0 && result.isError) {
|
|
3085
3382
|
reject(new Error(formatChildExitError(activeExecutor(), "0 (reported an error)", result.text)))
|
|
3086
3383
|
return
|
|
3087
3384
|
}
|
|
3088
3385
|
}
|
|
3089
3386
|
}
|
|
3387
|
+
if (limitStop) {
|
|
3388
|
+
reject(new Error(tripwireHeadline(limitStop)))
|
|
3389
|
+
return
|
|
3390
|
+
}
|
|
3090
3391
|
if (code === 0) {
|
|
3091
3392
|
resolve()
|
|
3092
3393
|
return
|
|
@@ -3100,7 +3401,11 @@ async function runExecutor(run, state) {
|
|
|
3100
3401
|
})])
|
|
3101
3402
|
}
|
|
3102
3403
|
|
|
3404
|
+
executorPhase = "finished"
|
|
3103
3405
|
await flushRunnerLog().catch(() => {})
|
|
3406
|
+
// An agent that exits cleanly on the cancel signal, or finishes as the
|
|
3407
|
+
// cancel arrives, must not have its work committed, pushed or opened.
|
|
3408
|
+
if (cancelRequested) throw new Error("Canceled by admin.")
|
|
3104
3409
|
|
|
3105
3410
|
const summary = execSpec.summaryMode === "file"
|
|
3106
3411
|
? (fs.existsSync(outputFile) ? fs.readFileSync(outputFile, "utf8").trim() : "Run finished without a final summary.")
|
|
@@ -3192,17 +3497,21 @@ async function runExecutor(run, state) {
|
|
|
3192
3497
|
.filter(Boolean))
|
|
3193
3498
|
: []
|
|
3194
3499
|
|
|
3195
|
-
// TRIPWIRE PROTOCOL
|
|
3196
|
-
|
|
3197
|
-
if (
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3205
|
-
|
|
3500
|
+
// TRIPWIRE PROTOCOL: the last look, for files changed after the live
|
|
3501
|
+
// check's last one. The work is saved below, never reset away.
|
|
3502
|
+
if (fileLimitApplies && runLimits.maxChangedFiles !== null) {
|
|
3503
|
+
const runChangedFiles = await listRunChangedFiles(worktreePath, runStartSha)
|
|
3504
|
+
const stop = checkRunLimits(
|
|
3505
|
+
{ maxChangedFiles: runLimits.maxChangedFiles, maxCostUsd: null, maxMinutes: null },
|
|
3506
|
+
{ changedFileCount: runChangedFiles.length },
|
|
3507
|
+
)
|
|
3508
|
+
if (stop) {
|
|
3509
|
+
limitStop = stop
|
|
3510
|
+
runLogger.warn("runner.tripwire.tripped", `Run stopped at its ${stop.kind} limit`, {
|
|
3511
|
+
meta: { kind: stop.kind, limit: stop.limit, actual: stop.actual },
|
|
3512
|
+
})
|
|
3513
|
+
throw new Error(tripwireHeadline(stop))
|
|
3514
|
+
}
|
|
3206
3515
|
}
|
|
3207
3516
|
|
|
3208
3517
|
|
|
@@ -3288,22 +3597,46 @@ async function runExecutor(run, state) {
|
|
|
3288
3597
|
).catch(() => {})
|
|
3289
3598
|
}
|
|
3290
3599
|
|
|
3600
|
+
// Last look before anything leaves the machine: a cancel that arrived
|
|
3601
|
+
// while the work was being committed still stops the push.
|
|
3602
|
+
if (autoPushOnSuccessFlag && somethingToPush) {
|
|
3603
|
+
await flushRunnerLog().catch(() => {})
|
|
3604
|
+
if (cancelRequested) throw new Error("Canceled by admin.")
|
|
3605
|
+
}
|
|
3606
|
+
|
|
3291
3607
|
if (autoPushOnSuccessFlag && somethingToPush && branch && gitAvailable && !isReviewerRun2) {
|
|
3608
|
+
/** How far publishing got, so a failure names the step that broke. */
|
|
3609
|
+
let publishStage = null
|
|
3292
3610
|
try {
|
|
3293
3611
|
const githubToken = run.__githubToken
|
|
3294
3612
|
const repo = run.__repo
|
|
3295
3613
|
|
|
3296
3614
|
if (githubToken && repo?.owner && repo?.repo) {
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3615
|
+
publishStage = "push"
|
|
3616
|
+
await pushBranchToGitHub(worktreePath, branch, githubToken, repo, ({ attempt, delayMs, error }) => {
|
|
3617
|
+
runLogger.warn("runner.run.push_retry", `GitHub push failed (attempt ${attempt}), retrying in ${Math.round(delayMs / 1000)}s`, {
|
|
3618
|
+
meta: { branch, reason: describeGitFailure(error instanceof Error ? error.message : String(error), 300) },
|
|
3619
|
+
})
|
|
3620
|
+
})
|
|
3621
|
+
publishStage = "pull_request"
|
|
3622
|
+
|
|
3623
|
+
// Base the PR on the target repo's base branch, never on the branch
|
|
3624
|
+
// the runner's own checkout happens to be on.
|
|
3625
|
+
const base = await resolvePrBaseBranch({
|
|
3626
|
+
configured: run.__baseBranch,
|
|
3627
|
+
githubDefault: () => fetchGithubDefaultBranch(githubToken, repo),
|
|
3628
|
+
originHead: () => readOriginHeadBranch(run.__repoRoot || repoRoot),
|
|
3629
|
+
})
|
|
3630
|
+
runLogger.info("runner.pr.base_branch", `Opening the pull request against ${base.branch} (${base.source})`, {
|
|
3631
|
+
meta: { base_branch: base.branch, source: base.source, errors: base.errors.length ? base.errors : undefined },
|
|
3632
|
+
})
|
|
3633
|
+
const pr = await retryTransient(() => createGithubPr(githubToken, repo, branch, base.branch, run, {
|
|
3302
3634
|
summary,
|
|
3303
3635
|
changedFiles,
|
|
3304
3636
|
diffStat: diffStat.stdout,
|
|
3305
3637
|
verificationSummary: runnerVerificationSummary,
|
|
3306
|
-
})
|
|
3638
|
+
}))
|
|
3639
|
+
if (pendingPushDir) clearPendingPush(pendingPushDir)
|
|
3307
3640
|
|
|
3308
3641
|
pushResult = { pushed: true, repo: `${repo.owner}/${repo.repo}`, pr_url: pr.html_url, pr_number: pr.number }
|
|
3309
3642
|
|
|
@@ -3350,14 +3683,37 @@ async function runExecutor(run, state) {
|
|
|
3350
3683
|
})
|
|
3351
3684
|
}
|
|
3352
3685
|
} catch (pushError) {
|
|
3353
|
-
//
|
|
3354
|
-
const
|
|
3355
|
-
|
|
3356
|
-
|
|
3686
|
+
// describeGitFailure masks the token a remote URL carries.
|
|
3687
|
+
const reason = describeGitFailure(pushError instanceof Error ? pushError.message : String(pushError))
|
|
3688
|
+
const attempts = Number(pushError?.attempts) || 1
|
|
3689
|
+
const tries = attempts > 1 ? ` after ${attempts} attempts` : ""
|
|
3690
|
+
const head = (await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null
|
|
3691
|
+
// Keep what the run produced next to the checkout, so the next run of
|
|
3692
|
+
// this issue publishes the commit instead of paying for the work again.
|
|
3693
|
+
const kept = Boolean(pendingPushDir && head)
|
|
3694
|
+
if (kept) {
|
|
3695
|
+
writePendingPush(pendingPushDir, {
|
|
3696
|
+
task_id: run.task_id,
|
|
3697
|
+
run_id: run.id,
|
|
3698
|
+
branch,
|
|
3699
|
+
head,
|
|
3700
|
+
summary,
|
|
3701
|
+
changed_files: changedFiles,
|
|
3702
|
+
verification_summary: runnerVerificationSummary,
|
|
3703
|
+
})
|
|
3704
|
+
}
|
|
3705
|
+
const keptNote = kept
|
|
3706
|
+
? ` The commit ${head.slice(0, 7)} is kept on ${runnerName}; re-run the agent and it is ${publishStage === "pull_request" ? "opened as a pull request" : "pushed"} without redoing the work.`
|
|
3707
|
+
: ""
|
|
3357
3708
|
// Work that never reached GitHub is not finished: completing the run
|
|
3358
3709
|
// sent an empty branch to review, which asked for changes until the
|
|
3359
3710
|
// revision limit ran out. Fail it with the reason instead.
|
|
3360
|
-
|
|
3711
|
+
if (publishStage === "pull_request") {
|
|
3712
|
+
runLogger.error("runner.run.pr_failed", "Branch pushed but the pull request could not be opened", { meta: { branch, attempts } })
|
|
3713
|
+
throw new Error(`The branch was pushed to GitHub but the pull request could not be opened${tries}: ${reason}.${keptNote}`)
|
|
3714
|
+
}
|
|
3715
|
+
runLogger.error("runner.run.push_failed", "Failed to push branch to GitHub", { meta: { branch, attempts } })
|
|
3716
|
+
throw new Error(`The work was committed but could not be pushed to GitHub${tries}: ${reason}.${keptNote}`)
|
|
3361
3717
|
}
|
|
3362
3718
|
}
|
|
3363
3719
|
|
|
@@ -3415,6 +3771,8 @@ async function runExecutor(run, state) {
|
|
|
3415
3771
|
await completeRun(state, run.id, {
|
|
3416
3772
|
status: "completed",
|
|
3417
3773
|
result_summary: truncate(summary, 4000),
|
|
3774
|
+
// Which of its skills the agent says it used, from its report's last line.
|
|
3775
|
+
skills_used: parseSkillsUsed(summary, skillInstall.installed),
|
|
3418
3776
|
usage: runUsage,
|
|
3419
3777
|
branch_name: branch,
|
|
3420
3778
|
changed_files: changedFiles,
|
|
@@ -3468,20 +3826,56 @@ async function runExecutor(run, state) {
|
|
|
3468
3826
|
return { summary, branch, worktreePath, artifactsFile, reportFile, verificationSummary, commitResult, pushResult }
|
|
3469
3827
|
} catch (error) {
|
|
3470
3828
|
await flushRunnerLog().catch(() => {})
|
|
3471
|
-
|
|
3472
|
-
const
|
|
3829
|
+
// A stop at a limit keeps the work and explains itself on the issue.
|
|
3830
|
+
const stopped = limitStop
|
|
3831
|
+
? await reportStoppedRun({
|
|
3832
|
+
run,
|
|
3833
|
+
stop: limitStop,
|
|
3834
|
+
runLimits,
|
|
3835
|
+
enforcedLimits,
|
|
3836
|
+
worktreePath,
|
|
3837
|
+
branch,
|
|
3838
|
+
gitAvailable,
|
|
3839
|
+
localCheckoutMode,
|
|
3840
|
+
runStartSha,
|
|
3841
|
+
elapsedMs: Date.now() - startedAt,
|
|
3842
|
+
costUsd: runUsage?.cost_usd ?? null,
|
|
3843
|
+
runRoot,
|
|
3844
|
+
mcpKey,
|
|
3845
|
+
runLogger,
|
|
3846
|
+
}).catch((reportError) => {
|
|
3847
|
+
runLogger.error("runner.tripwire.report_failed", "Could not save or report the stopped run", undefined, reportError)
|
|
3848
|
+
return null
|
|
3849
|
+
})
|
|
3850
|
+
: null
|
|
3851
|
+
const rawMessage = stopped?.errorMessage
|
|
3852
|
+
?? (limitStop ? tripwireHeadline(limitStop) : null)
|
|
3853
|
+
?? (cancelRequested ? "Canceled by admin." : (error instanceof Error ? error.message : String(error)))
|
|
3854
|
+
const usageLimit = detectRunUsageLimit({
|
|
3855
|
+
phase: executorPhase,
|
|
3856
|
+
// The runner stopped the tool itself; its last words are not a usage limit.
|
|
3857
|
+
canceled: cancelRequested || Boolean(limitStop),
|
|
3858
|
+
text: `${rawMessage}\n${logState.tail}`,
|
|
3859
|
+
})
|
|
3473
3860
|
// A key's own limit is that key's business: pausing the runner would also
|
|
3474
3861
|
// stop agents on the machine's subscription and on other keys. The server
|
|
3475
3862
|
// switches the agent to its fallback key, or retries at the reset.
|
|
3476
3863
|
const paysWithKey = Boolean(runCredentials(run))
|
|
3477
|
-
// The
|
|
3864
|
+
// The marker is what tells the server this is a usage limit; it reads
|
|
3865
|
+
// "Resets at <ISO>" to schedule the retry for that moment.
|
|
3478
3866
|
const message = usageLimit
|
|
3479
|
-
?
|
|
3867
|
+
? `${USAGE_LIMIT_MARKER}${paysWithKey ? `this agent's ${run.api_credentials.provider} key` : executorDisplayName()}. Resets at ${usageLimit.resetAt.toISOString()}.\n\n${rawMessage}`
|
|
3480
3868
|
: rawMessage
|
|
3481
3869
|
if (usageLimit && !paysWithKey) {
|
|
3482
|
-
|
|
3483
|
-
|
|
3484
|
-
|
|
3870
|
+
const executor = activeExecutor()
|
|
3871
|
+
// A pause still running for another executor widens to all of them.
|
|
3872
|
+
const current = usageLimitPause && usageLimitPause.until > Date.now() ? usageLimitPause : null
|
|
3873
|
+
usageLimitPause = {
|
|
3874
|
+
until: Math.max(current?.until ?? 0, usageLimit.resetAt.getTime()),
|
|
3875
|
+
executor: !current || current.executor === executor ? executor : null,
|
|
3876
|
+
}
|
|
3877
|
+
runLogger.warn("runner.usage_limit", `${executorDisplayName()} reported its usage limit; holding back its runs on this machine's plan`, {
|
|
3878
|
+
meta: { executor, reset_at: usageLimit.resetAt.toISOString(), stated: usageLimit.stated },
|
|
3485
3879
|
})
|
|
3486
3880
|
}
|
|
3487
3881
|
await postIssueMilestone(
|
|
@@ -3490,7 +3884,10 @@ async function runExecutor(run, state) {
|
|
|
3490
3884
|
usageLimit
|
|
3491
3885
|
? `${paysWithKey ? "The agent's key" : executorDisplayName()} hit its usage limit. This picks up again automatically `
|
|
3492
3886
|
+ `${usageLimit.stated ? "when it resets" : "after a short wait"}, at ${formatLocalTime(usageLimit.resetAt)}, or sooner on a fallback key.`
|
|
3493
|
-
:
|
|
3887
|
+
: limitStop
|
|
3888
|
+
? `Run stopped by tripwire because ${limitStop.reason}. `
|
|
3889
|
+
+ (stopped?.reportPosted ? "The Impact Report is in the comments." : "The Impact Report is on the run in Execution details.")
|
|
3890
|
+
: `Execution ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
|
|
3494
3891
|
"warning",
|
|
3495
3892
|
)
|
|
3496
3893
|
// No add_comment here: the failure is already on the run record (shown in
|
|
@@ -3501,6 +3898,9 @@ async function runExecutor(run, state) {
|
|
|
3501
3898
|
status: "error",
|
|
3502
3899
|
error_message: truncate(message, 4000),
|
|
3503
3900
|
usage: runUsage,
|
|
3901
|
+
// Where a stopped run's work was saved, so the issue links to it.
|
|
3902
|
+
branch_name: stopped?.branchName ?? undefined,
|
|
3903
|
+
changed_files: stopped?.changedFiles ?? undefined,
|
|
3504
3904
|
}).catch(() => {})
|
|
3505
3905
|
|
|
3506
3906
|
runLogger.error("runner.run.failed", "Local runner execution failed", {
|
|
@@ -3508,16 +3908,118 @@ async function runExecutor(run, state) {
|
|
|
3508
3908
|
branch,
|
|
3509
3909
|
worktree_path: worktreePath,
|
|
3510
3910
|
canceled: cancelRequested,
|
|
3911
|
+
executor_phase: executorPhase,
|
|
3912
|
+
usage_limit: Boolean(usageLimit),
|
|
3511
3913
|
},
|
|
3512
3914
|
}, error)
|
|
3513
|
-
throw error
|
|
3915
|
+
throw reportedRunFailure(error)
|
|
3514
3916
|
} finally {
|
|
3515
3917
|
clearInterval(logInterval)
|
|
3918
|
+
clearInterval(limitInterval)
|
|
3516
3919
|
if (runTimeoutHandle) clearTimeout(runTimeoutHandle)
|
|
3517
3920
|
cleanupExecutorMcp(mcpSideEffectBackups)
|
|
3921
|
+
try { skillInstall.cleanup() } catch (error) { runLogger.warn("runner.run.skills", "Could not remove the run's skill files", undefined, error) }
|
|
3922
|
+
}
|
|
3923
|
+
}
|
|
3924
|
+
|
|
3925
|
+
/**
|
|
3926
|
+
* Keep a stopped run's work: commit it to the ticket branch and push that
|
|
3927
|
+
* branch. No pull request is opened, so it does not go to review. A reset used
|
|
3928
|
+
* to throw the work away, and uncommitted work in a ticket checkout is easy to
|
|
3929
|
+
* lose.
|
|
3930
|
+
*/
|
|
3931
|
+
async function saveStoppedWork({ run, stop, worktreePath, branch, gitAvailable, localCheckoutMode, runStartSha }) {
|
|
3932
|
+
if (!gitAvailable) {
|
|
3933
|
+
return { kind: "left_in_place", worktreePath, error: "The folder is not a git repository." }
|
|
3934
|
+
}
|
|
3935
|
+
const dirty = (await execCommandSafe("git", ["-C", worktreePath, "status", "--porcelain"], worktreePath))?.stdout?.trim()
|
|
3936
|
+
// The repository root is the user's own checkout and branch, and a review
|
|
3937
|
+
// shares the build's branch: a commit there is not the runner's to make.
|
|
3938
|
+
if (localCheckoutMode === "repo_root" || isReviewRun(run)) {
|
|
3939
|
+
return dirty
|
|
3940
|
+
? {
|
|
3941
|
+
kind: "left_in_place",
|
|
3942
|
+
worktreePath,
|
|
3943
|
+
error: localCheckoutMode === "repo_root"
|
|
3944
|
+
? "The run works in the repository root, so the runner does not commit for it."
|
|
3945
|
+
: "Review runs are not committed.",
|
|
3946
|
+
}
|
|
3947
|
+
: { kind: "none" }
|
|
3948
|
+
}
|
|
3949
|
+
const commit = await commitStoppedWork(worktreePath, {
|
|
3950
|
+
subject: `${run.task?.identifier || run.task_id} WIP: stopped by tripwire (${stop.kind} limit)`,
|
|
3951
|
+
body: `${tripwireHeadline(stop)}\n\nRun: ${run.id}`,
|
|
3952
|
+
})
|
|
3953
|
+
if (commit.error) return { kind: "uncommitted", worktreePath, error: commit.error }
|
|
3954
|
+
if (!commit.committed && !(await hasCommitsSince(worktreePath, runStartSha))) return { kind: "none" }
|
|
3955
|
+
|
|
3956
|
+
const sha = commit.sha
|
|
3957
|
+
?? ((await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null)
|
|
3958
|
+
const committed = (error) => ({ kind: "committed", sha, branch, worktreePath, error })
|
|
3959
|
+
const repo = run.__repo
|
|
3960
|
+
if (!branch) return committed("the run has no branch to push")
|
|
3961
|
+
if (!autoPushOnSuccessFlag) return committed("pushing is turned off on this runner (auto-push-on-success)")
|
|
3962
|
+
if (!run.__githubToken || !repo?.owner || !repo?.repo) return committed("no GitHub repository is connected for this issue")
|
|
3963
|
+
try {
|
|
3964
|
+
await pushBranchToGitHub(worktreePath, branch, run.__githubToken, repo)
|
|
3965
|
+
return { kind: "pushed", sha, branch, repo: `${repo.owner}/${repo.repo}` }
|
|
3966
|
+
} catch (error) {
|
|
3967
|
+
// Never let a token in a remote URL reach the report.
|
|
3968
|
+
const reason = String(error instanceof Error ? error.message : error)
|
|
3969
|
+
.replace(/x-access-token:[^@\s]+@/g, "x-access-token:***@")
|
|
3970
|
+
.replace(/\s+/g, " ")
|
|
3971
|
+
return committed(`the push failed: ${truncate(reason, 200)}`)
|
|
3972
|
+
}
|
|
3973
|
+
}
|
|
3974
|
+
|
|
3975
|
+
/**
|
|
3976
|
+
* After a run is stopped at a limit: save its work, write the Impact Report
|
|
3977
|
+
* next to the run and post it on the issue. Posted before the run completes,
|
|
3978
|
+
* because completing revokes the run's key.
|
|
3979
|
+
*/
|
|
3980
|
+
async function reportStoppedRun(input) {
|
|
3981
|
+
const { run, stop, gitAvailable, worktreePath, runStartSha, runRoot, mcpKey, runLogger } = input
|
|
3982
|
+
const changedFiles = gitAvailable ? await listRunChangedFiles(worktreePath, runStartSha) : []
|
|
3983
|
+
const saved = await saveStoppedWork(input)
|
|
3984
|
+
const report = buildImpactReport({
|
|
3985
|
+
stop,
|
|
3986
|
+
limits: input.runLimits,
|
|
3987
|
+
enforced: input.enforcedLimits,
|
|
3988
|
+
elapsedMs: input.elapsedMs,
|
|
3989
|
+
costUsd: input.costUsd,
|
|
3990
|
+
changedFiles,
|
|
3991
|
+
saved,
|
|
3992
|
+
executor: executorDisplayName(),
|
|
3993
|
+
agentName: run.agent?.name || null,
|
|
3994
|
+
})
|
|
3995
|
+
try {
|
|
3996
|
+
fs.mkdirSync(runRoot, { recursive: true })
|
|
3997
|
+
fs.writeFileSync(path.join(runRoot, "impact-report.md"), report)
|
|
3998
|
+
} catch {
|
|
3999
|
+
// The copy on the issue and the run is the one that matters.
|
|
4000
|
+
}
|
|
4001
|
+
let reportPosted = false
|
|
4002
|
+
if (mcpKey && run.task_id) {
|
|
4003
|
+
reportPosted = await callMcpTool(mcpKey, "add_comment", { issue_id: run.task_id, content: report }, run.agent_id)
|
|
4004
|
+
.then(() => true)
|
|
4005
|
+
.catch((error) => {
|
|
4006
|
+
runLogger.warn("runner.tripwire.comment_failed", "Could not post the Impact Report on the issue", undefined, error)
|
|
4007
|
+
return false
|
|
4008
|
+
})
|
|
4009
|
+
}
|
|
4010
|
+
runLogger.info("runner.tripwire.saved", `Stopped run's work: ${saved.kind}`, {
|
|
4011
|
+
meta: { kind: saved.kind, branch: saved.branch ?? null, sha: saved.sha ?? null, changed_file_count: changedFiles.length },
|
|
4012
|
+
})
|
|
4013
|
+
return {
|
|
4014
|
+
errorMessage: tripwireErrorMessage(stop, report),
|
|
4015
|
+
changedFiles,
|
|
4016
|
+
branchName: saved.kind === "pushed" || saved.kind === "committed" ? saved.branch ?? null : null,
|
|
4017
|
+
reportPosted,
|
|
3518
4018
|
}
|
|
3519
4019
|
}
|
|
3520
4020
|
|
|
4021
|
+
// How often a running run is checked against its file and time limits.
|
|
4022
|
+
const RUN_LIMIT_CHECK_INTERVAL_MS = 15_000
|
|
3521
4023
|
// How long (ms) before we consider a run stuck and mark it error
|
|
3522
4024
|
const RUN_TIMEOUT_MS = Number(args["run-timeout-ms"] || process.env.RUNNER_RUN_TIMEOUT_MS || 4 * 60 * 60 * 1000) // 4 hour default
|
|
3523
4025
|
const CLAIM_ERROR_BACKOFF_MS = Number(args["claim-error-backoff-ms"] || process.env.RUNNER_CLAIM_ERROR_BACKOFF_MS || 5000)
|
|
@@ -3558,6 +4060,7 @@ async function cleanupFinishedCheckouts(state, logger) {
|
|
|
3558
4060
|
if (!issue || !["done", "canceled"].includes(issue.status) || issue.branch_name !== branch) continue
|
|
3559
4061
|
const dirty = (await execCommandSafe("git", ["-C", worktree, "status", "--porcelain"], worktree))?.stdout?.trim()
|
|
3560
4062
|
if (dirty) continue
|
|
4063
|
+
if (await holdsUnpushedStoppedWork(worktree)) continue
|
|
3561
4064
|
const gone = await execCommandSafe("git", ["-C", root, "worktree", "remove", worktree], root)
|
|
3562
4065
|
if (!gone) continue
|
|
3563
4066
|
// Squash merges leave the branch "unmerged" to git, hence -D.
|
|
@@ -3608,6 +4111,7 @@ async function main() {
|
|
|
3608
4111
|
local_executor: state.local_executor,
|
|
3609
4112
|
},
|
|
3610
4113
|
})
|
|
4114
|
+
if (!cloudState) getRunnerLogger(state).info("runner.registration", describeRegistration(state), { meta: { name: state.name, labels: state.labels ?? [] } })
|
|
3611
4115
|
|
|
3612
4116
|
if (registerOnly) return
|
|
3613
4117
|
|
|
@@ -3632,17 +4136,47 @@ async function main() {
|
|
|
3632
4136
|
let lastCheckoutCleanup = 0
|
|
3633
4137
|
let currentControlState = "active"
|
|
3634
4138
|
|
|
4139
|
+
// An admin disconnected this runner: it stops its agents and waits without
|
|
4140
|
+
// calling the server. Exiting would only have the service manager restart it.
|
|
4141
|
+
let revoked = false
|
|
4142
|
+
function markRevoked(error) {
|
|
4143
|
+
if (revoked) return
|
|
4144
|
+
revoked = true
|
|
4145
|
+
getRunnerLogger(state).error("runner.revoked", REVOKED_MESSAGE, undefined, error)
|
|
4146
|
+
stopAllRuns("this runner was disconnected from the workspace")
|
|
4147
|
+
}
|
|
4148
|
+
|
|
4149
|
+
// An admin stopped this runner: the server pauses it and cancel-requests its
|
|
4150
|
+
// runs; stopping the agents here too means they end now, not at the next log flush.
|
|
4151
|
+
let stopRequested = false
|
|
4152
|
+
function syncStopRequest(payload) {
|
|
4153
|
+
if (!payload || typeof payload.stop_requested !== "boolean" || payload.stop_requested === stopRequested) return
|
|
4154
|
+
stopRequested = payload.stop_requested
|
|
4155
|
+
if (stopRequested) {
|
|
4156
|
+
getRunnerLogger(state).warn("runner.stopped", "An admin stopped this runner: ending its runs and taking no new work until it is resumed")
|
|
4157
|
+
stopAllRuns("an admin stopped this runner")
|
|
4158
|
+
} else {
|
|
4159
|
+
getRunnerLogger(state).info("runner.resumed", "This runner was resumed and takes work again")
|
|
4160
|
+
}
|
|
4161
|
+
}
|
|
4162
|
+
|
|
3635
4163
|
async function withRunnerAuth(operation) {
|
|
3636
4164
|
try {
|
|
3637
4165
|
return await operation(state)
|
|
3638
4166
|
} catch (error) {
|
|
3639
|
-
if (
|
|
4167
|
+
if (isRevokedError(error)) markRevoked(error)
|
|
4168
|
+
if (revoked || !isInvalidRunnerCredentialsError(error) || !workspaceApiKey) {
|
|
3640
4169
|
throw error
|
|
3641
4170
|
}
|
|
3642
4171
|
|
|
3643
4172
|
getRunnerLogger(state).warn("runner.credentials.invalid", "Runner credentials were rejected; re-registering", undefined, error)
|
|
3644
4173
|
clearState()
|
|
3645
|
-
|
|
4174
|
+
// Revoking clears the secret, so its first sign is "invalid
|
|
4175
|
+
// credentials"; registering again is then refused as revoked.
|
|
4176
|
+
state = await registerRunner().catch((registerError) => {
|
|
4177
|
+
if (isRevokedError(registerError)) markRevoked(registerError)
|
|
4178
|
+
throw registerError
|
|
4179
|
+
})
|
|
3646
4180
|
getRunnerLogger(state).info("runner.reregistered", "Runner re-registered", {
|
|
3647
4181
|
meta: {
|
|
3648
4182
|
local_executor: state.local_executor,
|
|
@@ -3656,6 +4190,7 @@ async function main() {
|
|
|
3656
4190
|
try {
|
|
3657
4191
|
const payload = await withRunnerAuth((runnerState) => heartbeat(runnerState, status))
|
|
3658
4192
|
currentControlState = payload?.control_state || currentControlState
|
|
4193
|
+
syncStopRequest(payload)
|
|
3659
4194
|
return payload
|
|
3660
4195
|
} catch {
|
|
3661
4196
|
return null
|
|
@@ -3715,7 +4250,8 @@ async function main() {
|
|
|
3715
4250
|
else if (event === "completed") triageLogger().info("runner.job.complete", "Triage job completed", jobMeta(details.job, { duration_ms: details.durationMs, confidence: details.confidence }))
|
|
3716
4251
|
else if (event === "rejected") triageLogger().error("runner.job.error", `Triage result rejected (${details.status})`, jobMeta(details.job, { duration_ms: details.durationMs }))
|
|
3717
4252
|
else if (event === "failed") triageLogger().error("runner.job.error", details.message, jobMeta(details.job, { duration_ms: details.durationMs }))
|
|
3718
|
-
else if (event === "auth_failed") triageLogger().
|
|
4253
|
+
else if (event === "auth_failed") triageLogger().error("runner.triage.auth_failed", `Triage is paused: the server rejected this runner's key (${details.status}). Retrying every ${Math.round(details.retryMs / 60000)} minutes; run \`npx pacework login\` if it keeps failing.`)
|
|
4254
|
+
else if (event === "auth_recovered") triageLogger().info("runner.triage.auth_recovered", "Triage resumed: the server accepts this runner's key again")
|
|
3719
4255
|
},
|
|
3720
4256
|
}).catch((error) => triageLogger().warn("runner.triage.stopped", "Triage polling stopped", undefined, error))
|
|
3721
4257
|
} else {
|
|
@@ -3724,6 +4260,10 @@ async function main() {
|
|
|
3724
4260
|
}
|
|
3725
4261
|
|
|
3726
4262
|
while (!shuttingDown) {
|
|
4263
|
+
if (revoked) {
|
|
4264
|
+
await sleep(60_000)
|
|
4265
|
+
continue
|
|
4266
|
+
}
|
|
3727
4267
|
const runnerStatus = activeRuns.size > 0 ? "busy" : "online"
|
|
3728
4268
|
await heartbeatAndSync(runnerStatus)
|
|
3729
4269
|
let claimFailed = false
|
|
@@ -3742,16 +4282,18 @@ async function main() {
|
|
|
3742
4282
|
})
|
|
3743
4283
|
}
|
|
3744
4284
|
|
|
3745
|
-
//
|
|
3746
|
-
// until the reset. Work paid with a key or token is not
|
|
3747
|
-
// ask once a minute
|
|
3748
|
-
|
|
3749
|
-
|
|
4285
|
+
// Every executor here is at its plan's limit: the machine's subscription
|
|
4286
|
+
// takes nothing until the reset. Work paid with a key or token is not
|
|
4287
|
+
// limited by it, so ask once a minute; the server offers only that kind
|
|
4288
|
+
// while paused, and says when the pause ends or an admin clears it.
|
|
4289
|
+
// With another executor still free, keep claiming as usual.
|
|
4290
|
+
const pausedFor = (usageLimitPause?.until ?? 0) - Date.now()
|
|
4291
|
+
if (pausedFor > 0 && pauseCoversEveryExecutor(usageLimitPause)) {
|
|
3750
4292
|
await sleep(Math.min(pausedFor, 60_000))
|
|
3751
4293
|
if (shuttingDown) continue
|
|
3752
4294
|
}
|
|
3753
4295
|
|
|
3754
|
-
while (!shuttingDown && activeRuns.size < maxParallelRuns && launchedRuns < maxRuns) {
|
|
4296
|
+
while (!shuttingDown && !revoked && !stopRequested && activeRuns.size < maxParallelRuns && launchedRuns < maxRuns) {
|
|
3755
4297
|
let claim = null
|
|
3756
4298
|
try {
|
|
3757
4299
|
claim = await withRunnerAuth((runnerState) => claimNext({ ...runnerState }, activeRuns.size))
|
|
@@ -3766,17 +4308,8 @@ async function main() {
|
|
|
3766
4308
|
break
|
|
3767
4309
|
}
|
|
3768
4310
|
currentControlState = claim.control_state || currentControlState
|
|
3769
|
-
|
|
3770
|
-
|
|
3771
|
-
// possibly before a restart). Wait it out instead of asking again.
|
|
3772
|
-
const until = new Date(claim.available_after).getTime()
|
|
3773
|
-
if (until > Date.now() && until > usageLimitPausedUntil) {
|
|
3774
|
-
usageLimitPausedUntil = until
|
|
3775
|
-
getRunnerLogger(state).info("runner.paused", "Paused until the usage limit resets", {
|
|
3776
|
-
meta: { until: new Date(until).toISOString() },
|
|
3777
|
-
})
|
|
3778
|
-
}
|
|
3779
|
-
}
|
|
4311
|
+
syncStopRequest(claim)
|
|
4312
|
+
syncUsageLimitPause(claim, getRunnerLogger(state))
|
|
3780
4313
|
if (!claim.run) break
|
|
3781
4314
|
|
|
3782
4315
|
const run = claim.run
|
|
@@ -3813,6 +4346,8 @@ async function main() {
|
|
|
3813
4346
|
() => run.run_kind === "planner" ? runPlannerExecutor(run, state) : runExecutor(run, state),
|
|
3814
4347
|
)
|
|
3815
4348
|
.catch(async (error) => {
|
|
4349
|
+
// The run's own handler already reported and logged it.
|
|
4350
|
+
if (error?.runFailureReported) return
|
|
3816
4351
|
// A failure before the executor spawns (worktree setup, repo
|
|
3817
4352
|
// resolution) never reaches runExecutor's own handler, so the run
|
|
3818
4353
|
// would sit in "running" until the 4-hour timeout. Mark it failed.
|
|
@@ -3830,6 +4365,7 @@ async function main() {
|
|
|
3830
4365
|
})
|
|
3831
4366
|
.finally(() => {
|
|
3832
4367
|
activeRuns.delete(run.id)
|
|
4368
|
+
adminStoppers.delete(run.id)
|
|
3833
4369
|
})
|
|
3834
4370
|
|
|
3835
4371
|
activeRuns.set(run.id, runPromise)
|