pacework 1.2.0 → 1.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,9 +9,28 @@ import process from "node:process"
9
9
  import { spawn, execFileSync } from "node:child_process"
10
10
  import { AsyncLocalStorage } from "node:async_hooks"
11
11
  import { createLogger } from "../lib/logger.mjs"
12
- import { detectUsageLimit } from "../lib/usage-limit.mjs"
12
+ import { detectRunUsageLimit, USAGE_LIMIT_MARKER } from "../lib/usage-limit.mjs"
13
+ import {
14
+ buildImpactReport,
15
+ checkRunLimits,
16
+ commitStoppedWork,
17
+ costLimitStop,
18
+ describeRunLimits,
19
+ hasCommitsSince,
20
+ holdsUnpushedStoppedWork,
21
+ listRunChangedFiles,
22
+ resolveRunLimits,
23
+ tripwireErrorMessage,
24
+ tripwireHeadline,
25
+ } from "../lib/run-limits.mjs"
26
+ import { fetchGithubDefaultBranch, readOriginHeadBranch, resolvePrBaseBranch } from "../lib/pr-base-branch.mjs"
27
+ import { clearPendingPush, describeGitFailure, isTransientGitHubFailure, readPendingPush, retryTransient, writePendingPush } from "../lib/push-retry.mjs"
13
28
  import { prepareRunEnvironment } from "../lib/environment.mjs"
29
+ import { installSkills, parseSkillsUsed, skillsPromptSection } from "../lib/skills.mjs"
30
+ import { isGitLockError, repoLockKey, retryGitLock, withRepoLock } from "../lib/git-lock.mjs"
31
+ import { describeRegistration, isRevokedError, registrationChanges, REVOKED_MESSAGE, runEndedOnServer } from "../lib/runner-control.mjs"
14
32
  import { runTriageLoop, TRIAGE_EXECUTORS } from "../lib/triage-worker.mjs"
33
+ import { automationDenyRules, automationLimits } from "../lib/automation-limits.mjs"
15
34
 
16
35
  function parseArgs(argv) {
17
36
  const args = {}
@@ -343,10 +362,66 @@ function sleep(ms) {
343
362
  }
344
363
 
345
364
  /**
346
- * Until when this runner claims nothing because the executor's plan limit was
347
- * hit. Every run would fail the same way until the limit resets.
365
+ * This runner's pause for an executor's plan limit: { until (ms), executor }, or
366
+ * null. `executor` null means every executor. Runs of the limited executor on
367
+ * the machine's own plan would fail the same way until the limit resets; the
368
+ * server holds those back and says so on each claim, so an admin can clear it.
348
369
  */
349
- let usageLimitPausedUntil = 0
370
+ let usageLimitPause = null
371
+
372
+ /** Whether `pause` leaves this runner nothing to claim but work paid with a key. */
373
+ function pauseCoversEveryExecutor(pause) {
374
+ if (!pause) return false
375
+ return pause.executor === null || installedExecutors.every(executor => executor === pause.executor)
376
+ }
377
+
378
+ /**
379
+ * Mark a run failure as already reported to the server and logged, so the
380
+ * claim loop's catch-all does not report it again as a setup failure.
381
+ */
382
+ function reportedRunFailure(error) {
383
+ const reported = error instanceof Error ? error : new Error(String(error))
384
+ reported.runFailureReported = true
385
+ return reported
386
+ }
387
+
388
+ /**
389
+ * Follow the server's usage-limit pause from a claim response. The server holds
390
+ * the pause (it outlives a restart, and an admin can clear it in the app).
391
+ * Logs only when the pause starts, changes or ends.
392
+ */
393
+ function syncUsageLimitPause(claim, logger) {
394
+ const serverUntil = claim.backpressure === "backoff" && claim.available_after
395
+ ? new Date(claim.available_after).getTime()
396
+ : 0
397
+ const previous = usageLimitPause
398
+ if (serverUntil > Date.now()) {
399
+ const executor = claim.backoff_executor ?? null
400
+ if (previous?.until === serverUntil && previous.executor === executor) return
401
+ usageLimitPause = { until: serverUntil, executor }
402
+ logger.info("runner.paused", executor
403
+ ? `${executorDisplayName(executor)} reported its usage limit; its runs on this machine's plan wait until the limit resets`
404
+ : "An AI tool reported its usage limit; runs on this machine's plan wait until the limit resets", {
405
+ meta: { until: new Date(serverUntil).toISOString(), executor, reason: claim.backoff_reason ?? "usage_limit" },
406
+ })
407
+ return
408
+ }
409
+ // Other back-pressure (capacity, draining) is answered before the pause
410
+ // counts, so it says nothing about it. A plain "nothing to claim", or a
411
+ // claimed run the pause would have held back, says the server has none.
412
+ if (!previous || claim.backpressure) return
413
+ if (claim.run) {
414
+ const heldBack = !runCredentials(claim.run)
415
+ && (previous.executor === null || previous.executor === executorForRun(claim.run))
416
+ if (!heldBack) return
417
+ }
418
+ usageLimitPause = null
419
+ logger.info("runner.resumed", previous.until > Date.now()
420
+ ? "The usage-limit pause was cleared; taking all work again"
421
+ : "The usage limit has reset; taking all work again", {
422
+ meta: { executor: previous.executor },
423
+ })
424
+ }
350
425
 
351
426
  function formatLocalTime(date) {
352
427
  return date.toLocaleString(undefined, { weekday: "short", hour: "2-digit", minute: "2-digit", timeZoneName: "short" })
@@ -381,6 +456,8 @@ function parseClaudeJsonResult(stdout) {
381
456
  return {
382
457
  text: typeof parsed.result === "string" ? parsed.result : "",
383
458
  isError: parsed.is_error === true,
459
+ // "error_max_budget_usd" when --max-budget-usd ended the run.
460
+ subtype: typeof parsed.subtype === "string" ? parsed.subtype : null,
384
461
  usage: {
385
462
  cost_usd: typeof parsed.total_cost_usd === "number" ? parsed.total_cost_usd : null,
386
463
  input_tokens: inputTokens.length ? inputTokens.reduce((a, b) => a + b, 0) : null,
@@ -757,6 +834,13 @@ function isAutomationSmokeTestIssue(run) {
757
834
  return title.includes(automationSmokeTestMarker) || description.includes(automationSmokeTestMarker)
758
835
  }
759
836
 
837
+ /** Per running run: stop its agent because of an admin's runner-wide decision. */
838
+ const adminStoppers = new Map()
839
+
840
+ function stopAllRuns(reason) {
841
+ for (const stopRun of adminStoppers.values()) stopRun(reason)
842
+ }
843
+
760
844
  async function requestJson(url, init = {}) {
761
845
  const response = await fetch(url, init)
762
846
  let payload = null
@@ -774,7 +858,9 @@ async function requestJson(url, init = {}) {
774
858
  : ""
775
859
  const errorLabel = typeof payload?.error === "string" ? payload.error : ""
776
860
  const errorMessage = errorDetail || errorLabel || `Request failed: ${response.status}`
777
- throw new Error(errorMessage)
861
+ const error = new Error(errorMessage)
862
+ error.status = response.status
863
+ throw error
778
864
  }
779
865
 
780
866
  return payload
@@ -888,7 +974,24 @@ async function registerRunner() {
888
974
 
889
975
  async function ensureRegisteredRunner() {
890
976
  const state = loadState()
891
- if (state?.runner_id && state?.runner_secret && state?.runner_mcp_key) return state
977
+ if (!(state?.runner_id && state?.runner_secret && state?.runner_mcp_key)) return registerRunner()
978
+ const changes = registrationChanges(state, {
979
+ name: runnerName,
980
+ machine_name: machineName,
981
+ repo_root: repoRoot,
982
+ local_executor: defaultExecutor,
983
+ max_parallel_runs: maxParallelRuns,
984
+ labels,
985
+ })
986
+ if (changes.length === 0) return state
987
+ // Registering again updates this machine's entry in place (the server
988
+ // matches it by machine and repo root) and saves the new settings. Without
989
+ // that, a new --name or RUNNER_LABELS changed nothing on the server.
990
+ if (!workspaceApiKey) {
991
+ console.warn(`Runner settings changed (${changes.join(", ")}) but there is no workspace key to register them with; the server keeps the old ones.`)
992
+ return state
993
+ }
994
+ console.log(`Runner settings changed (${changes.join(", ")}); updating them in pacework.`)
892
995
  return registerRunner()
893
996
  }
894
997
 
@@ -1266,33 +1369,53 @@ async function fetchGithubToken(state, workspaceId, projectId, teamId = null, ta
1266
1369
  */
1267
1370
  const GIT_IGNORE_STORED_AUTH = ["-c", "http.https://github.com/.extraheader="]
1268
1371
 
1269
- async function pushBranchToGitHub(worktreePath, branch, githubToken, repo) {
1372
+ async function pushBranchToGitHub(worktreePath, branch, githubToken, repo, onRetry = () => {}) {
1373
+ // Worktrees of one repository share its config, remote URL included: a run
1374
+ // pushing while another swaps the URL would push without the token, or leave
1375
+ // it behind. Pushes and checkout setup take turns (lib/git-lock.mjs).
1376
+ return withRepoLock(repoLockKey(worktreePath), () => pushBranchWithToken(worktreePath, branch, githubToken, repo, onRetry))
1377
+ }
1378
+
1379
+ async function pushBranchWithToken(worktreePath, branch, githubToken, repo, onRetry) {
1270
1380
  // Set remote URL with embedded token for auth, then push
1271
1381
  const remoteUrl = `https://x-access-token:${githubToken}@github.com/${repo.owner}/${repo.repo}.git`
1272
- await execCommand(
1382
+ await retryGitLock(() => execCommand(
1273
1383
  "git",
1274
1384
  ["-C", worktreePath, "remote", "set-url", "origin", remoteUrl],
1275
1385
  worktreePath,
1276
- )
1277
- // start_work creates the branch on GitHub before the runner has a local clone
1278
- // of it, so --force-with-lease sees a remote it has no ref for and rejects the
1279
- // push. Fetch first so the lease has something to compare against.
1280
- await execCommandSafe(
1281
- "git",
1282
- ["-C", worktreePath, ...GIT_IGNORE_STORED_AUTH, "fetch", "origin", branch],
1283
- worktreePath,
1284
- )
1285
- await execCommand(
1286
- "git",
1287
- ["-C", worktreePath, ...GIT_IGNORE_STORED_AUTH, "push", "--set-upstream", "origin", branch, "--force-with-lease"],
1288
- worktreePath,
1289
- )
1290
- // Restore remote URL without token for safety
1291
- await execCommandSafe(
1292
- "git",
1293
- ["-C", worktreePath, "remote", "set-url", "origin", `https://github.com/${repo.owner}/${repo.repo}.git`],
1294
- worktreePath,
1295
- )
1386
+ ))
1387
+ try {
1388
+ // A GitHub 500 or a dropped connection is retried a few times before the
1389
+ // run gives up; one hiccup used to throw a finished run away.
1390
+ await retryTransient(async () => {
1391
+ // start_work creates the branch on GitHub before the runner has a local clone
1392
+ // of it, so --force-with-lease sees a remote it has no ref for and rejects the
1393
+ // push. Fetch first so the lease has something to compare against.
1394
+ await execCommandSafe(
1395
+ "git",
1396
+ ["-C", worktreePath, ...GIT_IGNORE_STORED_AUTH, "fetch", "origin", branch],
1397
+ worktreePath,
1398
+ )
1399
+ await execCommand(
1400
+ "git",
1401
+ ["-C", worktreePath, ...GIT_IGNORE_STORED_AUTH, "push", "--set-upstream", "origin", branch, "--force-with-lease"],
1402
+ worktreePath,
1403
+ )
1404
+ }, {
1405
+ onRetry,
1406
+ // `--set-upstream` writes the shared config: a lock held by another git
1407
+ // process is worth waiting out too.
1408
+ isTransient: (error) => isGitLockError(error) || isTransientGitHubFailure(error instanceof Error ? error.message : String(error)),
1409
+ })
1410
+ } finally {
1411
+ // Restore remote URL without token for safety, also when the push failed.
1412
+ // Retried on a held lock: giving up would leave the token in the config.
1413
+ await retryGitLock(() => execCommand(
1414
+ "git",
1415
+ ["-C", worktreePath, "remote", "set-url", "origin", `https://github.com/${repo.owner}/${repo.repo}.git`],
1416
+ worktreePath,
1417
+ )).catch(() => {})
1418
+ }
1296
1419
  }
1297
1420
 
1298
1421
  /** A section the agent left empty or answered with "none" says nothing to a reader. */
@@ -1699,6 +1822,15 @@ async function prepareWorktree(run) {
1699
1822
 
1700
1823
  ensureDir(path.dirname(worktreePath))
1701
1824
 
1825
+ // Runs that start together take turns per repository, and a lock held by
1826
+ // another git process is waited out (see lib/git-lock.mjs). Each step below
1827
+ // checks what already exists, so a retry picks up where a failure left off.
1828
+ await withRepoLock(repoLockKey(runRepoRoot), () => retryGitLock(() => prepareWorktreeCheckout(run, runRepoRoot, worktreePath, branch)))
1829
+
1830
+ return { runRoot, worktreePath, branch, localCheckoutMode, gitAvailable: true }
1831
+ }
1832
+
1833
+ async function prepareWorktreeCheckout(run, runRepoRoot, worktreePath, branch) {
1702
1834
  // Pull requests merge on GitHub, so origin/* in this checkout falls behind.
1703
1835
  // A stale base gave builders an old main to work on, and gave reviewers a
1704
1836
  // `git diff origin/HEAD..HEAD` full of work that had already merged.
@@ -1725,7 +1857,9 @@ async function prepareWorktree(run) {
1725
1857
  // Branch exists from a prior run — reuse it instead of -b
1726
1858
  await execCommand("git", ["-C", runRepoRoot, "worktree", "add", worktreePath, branch], runRepoRoot)
1727
1859
  } else {
1728
- await execCommand("git", ["-C", runRepoRoot, "worktree", "add", "-b", branch, worktreePath, upToDateBase], runRepoRoot)
1860
+ // --no-track: the run's branch must not follow the base branch, and
1861
+ // recording an upstream writes .git/config, the lock runs collided on.
1862
+ await execCommand("git", ["-C", runRepoRoot, "worktree", "add", "--no-track", "-b", branch, worktreePath, upToDateBase], runRepoRoot)
1729
1863
  }
1730
1864
  } else if (worktreeKnown) {
1731
1865
  const checkedOutBranch = await currentBranch(worktreePath)
@@ -1743,8 +1877,6 @@ async function prepareWorktree(run) {
1743
1877
  )
1744
1878
  }
1745
1879
  }
1746
-
1747
- return { runRoot, worktreePath, branch, localCheckoutMode, gitAvailable: true }
1748
1880
  }
1749
1881
 
1750
1882
  function buildPrompt(run, branch, worktreePath, options = {}) {
@@ -1755,7 +1887,9 @@ function buildPrompt(run, branch, worktreePath, options = {}) {
1755
1887
  const gitAvailable = options.gitAvailable !== false
1756
1888
  const workspaceContext = options.workspaceContext || null
1757
1889
  const greenfieldWorkspace = workspaceContext?.is_empty_workspace === true
1758
- const skillPrompt = agent?.skill_profile?.system_prompt?.trim()
1890
+ // The agent's standing instructions; servers before skill profiles were
1891
+ // retired sent them as skill_profile.system_prompt.
1892
+ const skillPrompt = (agent?.instructions ?? agent?.skill_profile?.system_prompt)?.trim()
1759
1893
  const base = skillPrompt ? `${skillPrompt}\n\n---\n\n` : ""
1760
1894
  const codingRules = Array.isArray(run?.coding_rules) ? run.coding_rules : []
1761
1895
  const reviewerContext = run?.reviewer_context || null
@@ -1860,6 +1994,13 @@ work that is already on the base branch is the worst thing a pull request can do
1860
1994
  .join("\n")
1861
1995
  : ""
1862
1996
  const answersGuidance = clarificationAnswers(run)
1997
+ // Told up front so an agent facing an oversized issue stops and says so,
1998
+ // instead of finding out from the tripwire.
1999
+ const runLimitsGuidance = !isReviewer && options.runLimitsText
2000
+ ? `- Run limits: ${options.runLimitsText}. The runner stops the run as soon as one is crossed.\n`
2001
+ + ` If this issue clearly needs more than that, do not start building: say in your summary that it\n`
2002
+ + ` should be split, and propose the subtasks.`
2003
+ : ""
1863
2004
  const environmentGuidance = run.__environmentNote
1864
2005
  ? `- Environment setup did not fully work. Say so in your summary rather than treating it as a bug in the code:\n ${run.__environmentNote.replace(/\n/g, "\n ")}`
1865
2006
  : ""
@@ -1888,7 +2029,7 @@ Execution rules:
1888
2029
  - Local checkout mode: ${localCheckoutMode}
1889
2030
  ${checkoutModeGuidance}
1890
2031
  ${existingCheckoutGuidance}
1891
- ${optionalGuidance ? `${optionalGuidance}\n` : ""}${environmentServices.length > 0 ? `- Services are running for this checkout; their addresses are in ${environmentServices.join(", ")}.\n` : ""}${environmentGuidance ? `${environmentGuidance}\n` : ""}${feedbackGuidance ? `${feedbackGuidance}\n` : ""}${codingRulesGuidance ? `${codingRulesGuidance}\n` : ""}${reviewerGuidance ? `${reviewerGuidance}\n` : ""}- Do not open a PR yet.
2032
+ ${optionalGuidance ? `${optionalGuidance}\n` : ""}${environmentServices.length > 0 ? `- Services are running for this checkout; their addresses are in ${environmentServices.join(", ")}.\n` : ""}${environmentGuidance ? `${environmentGuidance}\n` : ""}${feedbackGuidance ? `${feedbackGuidance}\n` : ""}${codingRulesGuidance ? `${codingRulesGuidance}\n` : ""}${runLimitsGuidance ? `${runLimitsGuidance}\n` : ""}${reviewerGuidance ? `${reviewerGuidance}\n` : ""}- Do not open a PR yet.
1892
2033
  - Treat explicit issue-description constraints as higher priority than generic instructions.
1893
2034
  - ${executionModeGuidance}
1894
2035
  ${isReviewer || answersGuidance ? "" : `- ${ASK_WHEN_UNCLEAR.replace(/\n/g, "\n ")}\n`}- End with a concise summary of what changed, what was verified, and any remaining gaps.
@@ -1941,6 +2082,8 @@ function claudeRunSettings(run) {
1941
2082
  ],
1942
2083
  }
1943
2084
  }
2085
+ const deny = automationDenyRules(run)
2086
+ if (deny.length) settings.permissions = { ...(settings.permissions || {}), deny }
1944
2087
  if (runCredentials(run)?.provider === "anthropic") {
1945
2088
  // ANTHROPIC_API_KEY alone loses to a signed-in Claude subscription, so the
1946
2089
  // run would be billed to the machine's plan. A key helper always wins. It
@@ -1950,8 +2093,30 @@ function claudeRunSettings(run) {
1950
2093
  return Object.keys(settings).length ? JSON.stringify(settings) : null
1951
2094
  }
1952
2095
 
1953
- function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey) {
2096
+ /** Whether this machine's Claude Code takes --max-budget-usd; an older one would refuse to start. */
2097
+ let claudeMaxBudgetSupport = null
2098
+ function claudeSupportsMaxBudget() {
2099
+ if (claudeMaxBudgetSupport === null) {
2100
+ try {
2101
+ const help = execFileSync(claudeCodeBin, ["--help"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 15_000 })
2102
+ claudeMaxBudgetSupport = help.includes("--max-budget-usd")
2103
+ } catch {
2104
+ claudeMaxBudgetSupport = false
2105
+ }
2106
+ }
2107
+ return claudeMaxBudgetSupport
2108
+ }
2109
+
2110
+ /**
2111
+ * @param {{ maxCostUsd?: number | null }} [options] The run's cost limit, for
2112
+ * Claude Code: it ends the run itself once the limit is spent.
2113
+ */
2114
+ function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey, options = {}) {
1954
2115
  const requestedModel = resolveRequestedModel(run)
2116
+ const limits = automationLimits(run)
2117
+ if (limits && (!limits.web || limits.code !== "write") && activeExecutor() !== "claude_code" && activeExecutor() !== "codex") {
2118
+ throw new Error("This automation limits web or code access, which only Claude Code and Codex runners can enforce. Run it on one of those, or allow web and code access.")
2119
+ }
1955
2120
 
1956
2121
  if (activeExecutor() === "codex") {
1957
2122
  const argsList = [
@@ -1960,8 +2125,18 @@ function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey) {
1960
2125
  // use its own current default rather than a name hardcoded here.
1961
2126
  ...(requestedModel ? ["-m", requestedModel] : []),
1962
2127
  "--ephemeral",
2128
+ // An automation without code write runs read-only, which also cuts the network.
2129
+ ...(limits && limits.code !== "write"
2130
+ ? ["-s", "read-only", "-c", "approval_policy=\"never\""]
2131
+ : limits && !limits.web
2132
+ ? [
2133
+ "-s", "workspace-write",
2134
+ "-c", "approval_policy=\"never\"",
2135
+ "-c", "sandbox_workspace_write.network_access=false",
2136
+ "-c", `sandbox_workspace_write.writable_roots=${JSON.stringify([path.join(run.__repoRoot || repoRoot, ".git")])}`,
2137
+ ]
1963
2138
  // Codex's sandbox cannot allow single domains, so network stays on.
1964
- ...(runSandboxed(run)
2139
+ : runSandboxed(run)
1965
2140
  ? [
1966
2141
  "-s", "workspace-write",
1967
2142
  "-c", "approval_policy=\"never\"",
@@ -2012,6 +2187,9 @@ function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey) {
2012
2187
  if (requestedModel) {
2013
2188
  argsList.push("--model", requestedModel)
2014
2189
  }
2190
+ if (typeof options.maxCostUsd === "number") {
2191
+ argsList.push("--max-budget-usd", String(options.maxCostUsd))
2192
+ }
2015
2193
  return {
2016
2194
  command: claudeCodeBin,
2017
2195
  args: argsList,
@@ -2536,6 +2714,15 @@ async function runPlannerExecutor(run, state) {
2536
2714
  let plannerMcpBackups = null
2537
2715
  let plannerUsage = null
2538
2716
 
2717
+ // An admin stopping or disconnecting this runner ends the run from outside.
2718
+ adminStoppers.set(run.id, (reason) => {
2719
+ cancelRequested = true
2720
+ if (childProcess && !childProcess.killed) {
2721
+ appendLogTail(logState, `> ${reason}; terminating executor process...`)
2722
+ requestChildProcessShutdown(childProcess)
2723
+ }
2724
+ })
2725
+
2539
2726
  function appendHeartbeat(force = false) {
2540
2727
  if (!childProcess || childProcess.killed) return
2541
2728
  const now = Date.now()
@@ -2550,7 +2737,7 @@ async function runPlannerExecutor(run, state) {
2550
2737
  async function flushPlannerLog() {
2551
2738
  appendHeartbeat()
2552
2739
  const response = await postPlannerLog(run, logState.tail)
2553
- if (response?.cancel_requested) {
2740
+ if (runEndedOnServer(response)) {
2554
2741
  cancelRequested = true
2555
2742
  if (childProcess && !childProcess.killed) {
2556
2743
  appendLogTail(logState, "> cancel requested; terminating executor process...")
@@ -2694,6 +2881,9 @@ async function runPlannerExecutor(run, state) {
2694
2881
  }
2695
2882
 
2696
2883
  await flushPlannerLog().catch(() => {})
2884
+ // Canceled while the agent was finishing (or it exited cleanly on the
2885
+ // signal): its plan is not applied.
2886
+ if (cancelRequested) throw new Error("Canceled by admin.")
2697
2887
 
2698
2888
  const fileContent = fs.existsSync(outputFile) ? fs.readFileSync(outputFile, "utf8").trim() : ""
2699
2889
  const plannerResponse = staticResponse || fileContent || stdioSummary.trim()
@@ -2736,7 +2926,7 @@ async function runPlannerExecutor(run, state) {
2736
2926
  canceled: cancelRequested,
2737
2927
  },
2738
2928
  }, error)
2739
- throw error
2929
+ throw reportedRunFailure(error)
2740
2930
  } finally {
2741
2931
  clearInterval(logInterval)
2742
2932
  if (runTimeoutHandle) clearTimeout(runTimeoutHandle)
@@ -2835,7 +3025,7 @@ async function runExecutor(run, state) {
2835
3025
  // Resolve the repo this ticket belongs to before preparing a checkout, so the
2836
3026
  // code is written in the same repo the PR will be opened against.
2837
3027
  try {
2838
- const { token, repo } = await fetchGithubToken(
3028
+ const { token, repo, baseBranch } = await fetchGithubToken(
2839
3029
  state,
2840
3030
  run.workspace_id,
2841
3031
  run.task?.project_id ?? null,
@@ -2844,6 +3034,7 @@ async function runExecutor(run, state) {
2844
3034
  )
2845
3035
  run.__githubToken = token ?? null
2846
3036
  run.__repo = repo ?? null
3037
+ run.__baseBranch = baseBranch ?? null
2847
3038
  run.__repoRoot = await resolveRunRepoRoot(run, state)
2848
3039
  if (run.__repoRoot !== repoRoot) {
2849
3040
  runLogger.info("runner.repo.resolved", "Working in the ticket's repository", {
@@ -2893,6 +3084,9 @@ async function runExecutor(run, state) {
2893
3084
 
2894
3085
  runLogger.info("runner.run.prepare", "Preparing worktree")
2895
3086
  const { runRoot, worktreePath, branch, localCheckoutMode, gitAvailable } = await prepareWorktree(run)
3087
+ // Where a commit that could not reach GitHub is noted; only this issue's own
3088
+ // checkout has a folder of its own to hold the note.
3089
+ const pendingPushDir = gitAvailable && branch && localCheckoutMode !== "repo_root" ? path.dirname(worktreePath) : null
2896
3090
  const prepared = await prepareRunEnvironment({
2897
3091
  environment: run.environment,
2898
3092
  checkoutPath: worktreePath,
@@ -2921,8 +3115,37 @@ async function runExecutor(run, state) {
2921
3115
  run.reviewer_context = { ...(run.reviewer_context || {}), pr_diff: prDiff.diff, pr_diff_truncated: prDiff.truncated === true }
2922
3116
  }
2923
3117
  }
2924
- const prompt = buildPrompt(run, branch, worktreePath, { localCheckoutMode, gitAvailable, workspaceContext })
2925
- const execSpec = buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey)
3118
+ const runLimits = resolveRunLimits({ args, env: process.env, profile: runnerProfile })
3119
+ // Scouts survey the codebase and may touch many files on purpose.
3120
+ const fileLimitApplies = gitAvailable && getAgentSkillProfileSlug(run.agent) !== "scout"
3121
+ // The commit the run starts from: only the files this run changes count.
3122
+ const runStartSha = gitAvailable
3123
+ ? ((await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null)
3124
+ : null
3125
+ // Only Claude Code can stop itself at a cost, and another provider's cost is
3126
+ // priced at Anthropic's rates, so it cannot be held to a dollar limit.
3127
+ const costLimitApplies = runLimits.maxCostUsd !== null
3128
+ && activeExecutor() === "claude_code"
3129
+ && !usesThirdPartyModels(run)
3130
+ && claudeSupportsMaxBudget()
3131
+ const enforcedLimits = { files: fileLimitApplies, cost: costLimitApplies, time: true }
3132
+ const runLimitsText = describeRunLimits(runLimits, enforcedLimits)
3133
+ // The agent's skills, written where Claude Code and Codex load skills on demand
3134
+ // and kept out of git (PWK-145). Only a git checkout can keep them out of the change.
3135
+ const skillInstall = gitAvailable
3136
+ ? installSkills(worktreePath, run.skills, { log: (line) => runLogger.info("runner.run.skills", line) })
3137
+ : { installed: [], skipped: [], cleanup() {} }
3138
+ const skillsSection = skillsPromptSection(
3139
+ (run.skills ?? []).filter((skill) => skillInstall.installed.includes(skill.slug)),
3140
+ { purpose: isReviewRun(run) ? "review" : "build" },
3141
+ )
3142
+ const prompt = [
3143
+ buildPrompt(run, branch, worktreePath, { localCheckoutMode, gitAvailable, workspaceContext, runLimitsText }),
3144
+ skillsSection,
3145
+ ].filter(Boolean).join("\n\n")
3146
+ const execSpec = buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey, {
3147
+ maxCostUsd: costLimitApplies ? runLimits.maxCostUsd : null,
3148
+ })
2926
3149
  // These steps run before the executor spawns; when one stalls the run sits in
2927
3150
  // "running" with no output, so each is traced.
2928
3151
  runLogger.info("runner.run.prepare", "Fetching MCP servers")
@@ -2939,6 +3162,8 @@ async function runExecutor(run, state) {
2939
3162
 
2940
3163
  const logState = { tail: "" }
2941
3164
  const startedAt = Date.now()
3165
+ /** Where the run is relative to the AI tool: only its own failure can be a usage limit. */
3166
+ let executorPhase = "before"
2942
3167
  let stdioSummary = ""
2943
3168
  // Claude Code's JSON result arrives on stdout only; stderr would break parsing.
2944
3169
  let stdoutRaw = ""
@@ -2946,7 +3171,18 @@ async function runExecutor(run, state) {
2946
3171
  let cancelRequested = false
2947
3172
  let childProcess = null
2948
3173
  let lastHeartbeatLogAt = 0
3174
+
3175
+ // An admin stopping or disconnecting this runner ends the run from outside.
3176
+ adminStoppers.set(run.id, (reason) => {
3177
+ cancelRequested = true
3178
+ if (childProcess && !childProcess.killed) {
3179
+ appendLogTail(logState, `> ${reason}; terminating executor process...`)
3180
+ requestChildProcessShutdown(childProcess)
3181
+ }
3182
+ })
2949
3183
  let activityThreadCommentId = run.activity_thread_comment_id || null
3184
+ /** The limit that stopped this run, once one has. */
3185
+ let limitStop = null
2950
3186
 
2951
3187
  function appendHeartbeat(force = false) {
2952
3188
  if (!childProcess || childProcess.killed) return
@@ -2962,7 +3198,7 @@ async function runExecutor(run, state) {
2962
3198
  async function flushRunnerLog() {
2963
3199
  appendHeartbeat()
2964
3200
  const response = await postRunLog(state, run.id, logState.tail)
2965
- if (response?.cancel_requested) {
3201
+ if (runEndedOnServer(response)) {
2966
3202
  cancelRequested = true
2967
3203
  if (childProcess && !childProcess.killed) {
2968
3204
  appendLogTail(logState, "> cancel requested; terminating executor process...")
@@ -2975,6 +3211,36 @@ async function runExecutor(run, state) {
2975
3211
  flushRunnerLog().catch(() => {})
2976
3212
  }, 5000)
2977
3213
 
3214
+ // Files and time are checked while the executor works, so a run that is too
3215
+ // big stops then, not after it has spent all its time and money. Cost is
3216
+ // Claude Code's own check (--max-budget-usd).
3217
+ let limitCheckInFlight = false
3218
+ async function checkLimitsWhileRunning() {
3219
+ if (limitStop || limitCheckInFlight || !childProcess || childProcess.killed) return
3220
+ limitCheckInFlight = true
3221
+ try {
3222
+ const changed = fileLimitApplies && runLimits.maxChangedFiles !== null
3223
+ ? await listRunChangedFiles(worktreePath, runStartSha)
3224
+ : []
3225
+ const stop = checkRunLimits(
3226
+ { ...runLimits, maxChangedFiles: fileLimitApplies ? runLimits.maxChangedFiles : null, maxCostUsd: null },
3227
+ { changedFileCount: changed.length, elapsedMs: Date.now() - startedAt },
3228
+ )
3229
+ if (!stop || limitStop || !childProcess || childProcess.killed) return
3230
+ limitStop = stop
3231
+ appendLogTail(logState, `> tripwire: stopping the run because ${stop.reason}`)
3232
+ runLogger.warn("runner.tripwire.tripped", `Run stopped at its ${stop.kind} limit`, {
3233
+ meta: { kind: stop.kind, limit: stop.limit, actual: stop.actual },
3234
+ })
3235
+ requestChildProcessShutdown(childProcess)
3236
+ } finally {
3237
+ limitCheckInFlight = false
3238
+ }
3239
+ }
3240
+ const limitInterval = setInterval(() => {
3241
+ checkLimitsWhileRunning().catch(() => {})
3242
+ }, RUN_LIMIT_CHECK_INTERVAL_MS)
3243
+
2978
3244
  // Hard timeout — kill the run if it hangs
2979
3245
  let runTimeoutHandle = null
2980
3246
  const runTimeoutPromise = new Promise((_, reject) => {
@@ -2997,13 +3263,15 @@ async function runExecutor(run, state) {
2997
3263
  appendLogTail(logState, `> worktree: ${worktreePath}`)
2998
3264
  appendLogTail(logState, `> branch: ${branch || "(none)"}`)
2999
3265
  appendLogTail(logState, `> executor: ${activeExecutor()}`)
3266
+ appendLogTail(logState, `> run limits: ${runLimitsText}`)
3000
3267
  await flushRunnerLog().catch(() => {})
3001
3268
  await postIssueMilestone(
3002
3269
  mcpKey,
3003
3270
  run.task_id,
3004
- isReviewRun(run)
3271
+ (isReviewRun(run)
3005
3272
  ? `AI review started in ${gitAvailable ? (branch || worktreePath) : worktreePath}.`
3006
- : `Implementation started in ${gitAvailable ? (branch || worktreePath) : worktreePath}.`,
3273
+ : `Implementation started in ${gitAvailable ? (branch || worktreePath) : worktreePath}.`)
3274
+ + ` Run limits: ${runLimitsText}.`,
3007
3275
  "info",
3008
3276
  )
3009
3277
  if (!isReviewRun(run)) {
@@ -3021,7 +3289,27 @@ async function runExecutor(run, state) {
3021
3289
  }
3022
3290
  await flushRunnerLog().catch(() => {})
3023
3291
 
3024
- if (automationSmokeTestMode && isAutomationSmokeTestIssue(run)) {
3292
+ // A previous run finished this work but GitHub would not take it. Publish
3293
+ // that commit instead of running the agent again.
3294
+ const pendingPush = pendingPushDir && !isReviewRun(run)
3295
+ ? readPendingPush(pendingPushDir, {
3296
+ taskId: run.task_id,
3297
+ branch,
3298
+ head: (await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null,
3299
+ })
3300
+ : null
3301
+
3302
+ if (pendingPush) {
3303
+ appendLogTail(logState, `> commit ${pendingPush.head.slice(0, 7)} from run ${pendingPush.run_id} was never published; publishing it without running the agent`)
3304
+ await postIssueMilestone(
3305
+ mcpKey,
3306
+ run.task_id,
3307
+ `Publishing commit ${pendingPush.head.slice(0, 7)}, which an earlier run finished but could not push. The agent is not run again.`,
3308
+ "info",
3309
+ ).catch(() => {})
3310
+ stdioSummary = pendingPush.summary || "Published the commit an earlier run finished but could not push."
3311
+ if (execSpec.summaryMode === "file") fs.writeFileSync(outputFile, stdioSummary)
3312
+ } else if (automationSmokeTestMode && isAutomationSmokeTestIssue(run)) {
3025
3313
  const smokeResult = await runSmokeTestExecution(run, state, { activityThreadCommentId })
3026
3314
  stdioSummary = smokeResult.summary
3027
3315
  appendLogTail(logState, smokeResult.summary)
@@ -3034,6 +3322,7 @@ async function runExecutor(run, state) {
3034
3322
  appendLogTail(logState, `> cwd: ${worktreePath}`)
3035
3323
  appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
3036
3324
  await flushRunnerLog().catch(() => {})
3325
+ executorPhase = "running"
3037
3326
  await Promise.race([runTimeoutPromise, new Promise((resolve, reject) => {
3038
3327
  const child = spawn(execSpec.command, execSpec.args, {
3039
3328
  cwd: worktreePath,
@@ -3081,12 +3370,24 @@ async function runExecutor(run, state) {
3081
3370
  if (result.usage.cost_usd !== null) {
3082
3371
  appendLogTail(logState, `> cost: $${result.usage.cost_usd.toFixed(4)}, ${result.usage.num_turns ?? "?"} turns`)
3083
3372
  }
3373
+ if (result.subtype === "error_max_budget_usd" && costLimitApplies && !limitStop) {
3374
+ limitStop = costLimitStop(runLimits.maxCostUsd, result.usage.cost_usd)
3375
+ appendLogTail(logState, `> tripwire: the run stopped because ${limitStop.reason}`)
3376
+ }
3377
+ if (limitStop) {
3378
+ reject(new Error(tripwireHeadline(limitStop)))
3379
+ return
3380
+ }
3084
3381
  if (code === 0 && result.isError) {
3085
3382
  reject(new Error(formatChildExitError(activeExecutor(), "0 (reported an error)", result.text)))
3086
3383
  return
3087
3384
  }
3088
3385
  }
3089
3386
  }
3387
+ if (limitStop) {
3388
+ reject(new Error(tripwireHeadline(limitStop)))
3389
+ return
3390
+ }
3090
3391
  if (code === 0) {
3091
3392
  resolve()
3092
3393
  return
@@ -3100,7 +3401,11 @@ async function runExecutor(run, state) {
3100
3401
  })])
3101
3402
  }
3102
3403
 
3404
+ executorPhase = "finished"
3103
3405
  await flushRunnerLog().catch(() => {})
3406
+ // An agent that exits cleanly on the cancel signal, or finishes as the
3407
+ // cancel arrives, must not have its work committed, pushed or opened.
3408
+ if (cancelRequested) throw new Error("Canceled by admin.")
3104
3409
 
3105
3410
  const summary = execSpec.summaryMode === "file"
3106
3411
  ? (fs.existsSync(outputFile) ? fs.readFileSync(outputFile, "utf8").trim() : "Run finished without a final summary.")
@@ -3192,17 +3497,21 @@ async function runExecutor(run, state) {
3192
3497
  .filter(Boolean))
3193
3498
  : []
3194
3499
 
3195
- // TRIPWIRE PROTOCOL
3196
- const isScout = getAgentSkillProfileSlug(run.agent) === "scout"
3197
- if (gitAvailable && changedFiles.length > 20 && !isScout) {
3198
- runLogger.error("runner.tripwire.tripped", "Agent tripwire activated: Too many file changes.", {
3199
- meta: { changed_files_count: changedFiles.length, files: changedFiles }
3200
- })
3201
- // Reset to prevent dirty state from corrupting the workspace
3202
- await execCommand("git", ["-C", worktreePath, "reset", "--hard"], worktreePath).catch(() => {})
3203
- await execCommand("git", ["-C", worktreePath, "clean", "-fd"], worktreePath).catch(() => {})
3204
-
3205
- throw new Error(`TRIPWIRE ACTIVATED: Agent attempted to modify ${changedFiles.length} files, exceeding the limit. An Impact Report has been generated.`)
3500
+ // TRIPWIRE PROTOCOL: the last look, for files changed after the live
3501
+ // check's last one. The work is saved below, never reset away.
3502
+ if (fileLimitApplies && runLimits.maxChangedFiles !== null) {
3503
+ const runChangedFiles = await listRunChangedFiles(worktreePath, runStartSha)
3504
+ const stop = checkRunLimits(
3505
+ { maxChangedFiles: runLimits.maxChangedFiles, maxCostUsd: null, maxMinutes: null },
3506
+ { changedFileCount: runChangedFiles.length },
3507
+ )
3508
+ if (stop) {
3509
+ limitStop = stop
3510
+ runLogger.warn("runner.tripwire.tripped", `Run stopped at its ${stop.kind} limit`, {
3511
+ meta: { kind: stop.kind, limit: stop.limit, actual: stop.actual },
3512
+ })
3513
+ throw new Error(tripwireHeadline(stop))
3514
+ }
3206
3515
  }
3207
3516
 
3208
3517
 
@@ -3288,22 +3597,46 @@ async function runExecutor(run, state) {
3288
3597
  ).catch(() => {})
3289
3598
  }
3290
3599
 
3600
+ // Last look before anything leaves the machine: a cancel that arrived
3601
+ // while the work was being committed still stops the push.
3602
+ if (autoPushOnSuccessFlag && somethingToPush) {
3603
+ await flushRunnerLog().catch(() => {})
3604
+ if (cancelRequested) throw new Error("Canceled by admin.")
3605
+ }
3606
+
3291
3607
  if (autoPushOnSuccessFlag && somethingToPush && branch && gitAvailable && !isReviewerRun2) {
3608
+ /** How far publishing got, so a failure names the step that broke. */
3609
+ let publishStage = null
3292
3610
  try {
3293
3611
  const githubToken = run.__githubToken
3294
3612
  const repo = run.__repo
3295
3613
 
3296
3614
  if (githubToken && repo?.owner && repo?.repo) {
3297
- await pushBranchToGitHub(worktreePath, branch, githubToken, repo)
3298
-
3299
- // Base the PR on the target repo's default branch, not the runner's.
3300
- const defaultBranch = await detectDefaultBranch(run.__repoRoot || repoRoot)
3301
- const pr = await createGithubPr(githubToken, repo, branch, defaultBranch, run, {
3615
+ publishStage = "push"
3616
+ await pushBranchToGitHub(worktreePath, branch, githubToken, repo, ({ attempt, delayMs, error }) => {
3617
+ runLogger.warn("runner.run.push_retry", `GitHub push failed (attempt ${attempt}), retrying in ${Math.round(delayMs / 1000)}s`, {
3618
+ meta: { branch, reason: describeGitFailure(error instanceof Error ? error.message : String(error), 300) },
3619
+ })
3620
+ })
3621
+ publishStage = "pull_request"
3622
+
3623
+ // Base the PR on the target repo's base branch, never on the branch
3624
+ // the runner's own checkout happens to be on.
3625
+ const base = await resolvePrBaseBranch({
3626
+ configured: run.__baseBranch,
3627
+ githubDefault: () => fetchGithubDefaultBranch(githubToken, repo),
3628
+ originHead: () => readOriginHeadBranch(run.__repoRoot || repoRoot),
3629
+ })
3630
+ runLogger.info("runner.pr.base_branch", `Opening the pull request against ${base.branch} (${base.source})`, {
3631
+ meta: { base_branch: base.branch, source: base.source, errors: base.errors.length ? base.errors : undefined },
3632
+ })
3633
+ const pr = await retryTransient(() => createGithubPr(githubToken, repo, branch, base.branch, run, {
3302
3634
  summary,
3303
3635
  changedFiles,
3304
3636
  diffStat: diffStat.stdout,
3305
3637
  verificationSummary: runnerVerificationSummary,
3306
- })
3638
+ }))
3639
+ if (pendingPushDir) clearPendingPush(pendingPushDir)
3307
3640
 
3308
3641
  pushResult = { pushed: true, repo: `${repo.owner}/${repo.repo}`, pr_url: pr.html_url, pr_number: pr.number }
3309
3642
 
@@ -3350,14 +3683,37 @@ async function runExecutor(run, state) {
3350
3683
  })
3351
3684
  }
3352
3685
  } catch (pushError) {
3353
- // Never let a token in a remote URL reach a log or the issue.
3354
- const pushMessage = (pushError instanceof Error ? pushError.message : String(pushError))
3355
- .replace(/x-access-token:[^@\s]+@/g, "x-access-token:***@")
3356
- runLogger.error("runner.run.push_failed", "Failed to push branch to GitHub", { meta: { branch } })
3686
+ // describeGitFailure masks the token a remote URL carries.
3687
+ const reason = describeGitFailure(pushError instanceof Error ? pushError.message : String(pushError))
3688
+ const attempts = Number(pushError?.attempts) || 1
3689
+ const tries = attempts > 1 ? ` after ${attempts} attempts` : ""
3690
+ const head = (await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null
3691
+ // Keep what the run produced next to the checkout, so the next run of
3692
+ // this issue publishes the commit instead of paying for the work again.
3693
+ const kept = Boolean(pendingPushDir && head)
3694
+ if (kept) {
3695
+ writePendingPush(pendingPushDir, {
3696
+ task_id: run.task_id,
3697
+ run_id: run.id,
3698
+ branch,
3699
+ head,
3700
+ summary,
3701
+ changed_files: changedFiles,
3702
+ verification_summary: runnerVerificationSummary,
3703
+ })
3704
+ }
3705
+ const keptNote = kept
3706
+ ? ` The commit ${head.slice(0, 7)} is kept on ${runnerName}; re-run the agent and it is ${publishStage === "pull_request" ? "opened as a pull request" : "pushed"} without redoing the work.`
3707
+ : ""
3357
3708
  // Work that never reached GitHub is not finished: completing the run
3358
3709
  // sent an empty branch to review, which asked for changes until the
3359
3710
  // revision limit ran out. Fail it with the reason instead.
3360
- throw new Error(`The work was committed but could not be pushed to GitHub: ${truncate(pushMessage.replace(/\s+/g, " "), 400)}`)
3711
+ if (publishStage === "pull_request") {
3712
+ runLogger.error("runner.run.pr_failed", "Branch pushed but the pull request could not be opened", { meta: { branch, attempts } })
3713
+ throw new Error(`The branch was pushed to GitHub but the pull request could not be opened${tries}: ${reason}.${keptNote}`)
3714
+ }
3715
+ runLogger.error("runner.run.push_failed", "Failed to push branch to GitHub", { meta: { branch, attempts } })
3716
+ throw new Error(`The work was committed but could not be pushed to GitHub${tries}: ${reason}.${keptNote}`)
3361
3717
  }
3362
3718
  }
3363
3719
 
@@ -3415,6 +3771,8 @@ async function runExecutor(run, state) {
3415
3771
  await completeRun(state, run.id, {
3416
3772
  status: "completed",
3417
3773
  result_summary: truncate(summary, 4000),
3774
+ // Which of its skills the agent says it used, from its report's last line.
3775
+ skills_used: parseSkillsUsed(summary, skillInstall.installed),
3418
3776
  usage: runUsage,
3419
3777
  branch_name: branch,
3420
3778
  changed_files: changedFiles,
@@ -3468,20 +3826,56 @@ async function runExecutor(run, state) {
3468
3826
  return { summary, branch, worktreePath, artifactsFile, reportFile, verificationSummary, commitResult, pushResult }
3469
3827
  } catch (error) {
3470
3828
  await flushRunnerLog().catch(() => {})
3471
- const rawMessage = cancelRequested ? "Canceled by admin." : (error instanceof Error ? error.message : String(error))
3472
- const usageLimit = cancelRequested ? null : detectUsageLimit(`${rawMessage}\n${logState.tail}`)
3829
+ // A stop at a limit keeps the work and explains itself on the issue.
3830
+ const stopped = limitStop
3831
+ ? await reportStoppedRun({
3832
+ run,
3833
+ stop: limitStop,
3834
+ runLimits,
3835
+ enforcedLimits,
3836
+ worktreePath,
3837
+ branch,
3838
+ gitAvailable,
3839
+ localCheckoutMode,
3840
+ runStartSha,
3841
+ elapsedMs: Date.now() - startedAt,
3842
+ costUsd: runUsage?.cost_usd ?? null,
3843
+ runRoot,
3844
+ mcpKey,
3845
+ runLogger,
3846
+ }).catch((reportError) => {
3847
+ runLogger.error("runner.tripwire.report_failed", "Could not save or report the stopped run", undefined, reportError)
3848
+ return null
3849
+ })
3850
+ : null
3851
+ const rawMessage = stopped?.errorMessage
3852
+ ?? (limitStop ? tripwireHeadline(limitStop) : null)
3853
+ ?? (cancelRequested ? "Canceled by admin." : (error instanceof Error ? error.message : String(error)))
3854
+ const usageLimit = detectRunUsageLimit({
3855
+ phase: executorPhase,
3856
+ // The runner stopped the tool itself; its last words are not a usage limit.
3857
+ canceled: cancelRequested || Boolean(limitStop),
3858
+ text: `${rawMessage}\n${logState.tail}`,
3859
+ })
3473
3860
  // A key's own limit is that key's business: pausing the runner would also
3474
3861
  // stop agents on the machine's subscription and on other keys. The server
3475
3862
  // switches the agent to its fallback key, or retries at the reset.
3476
3863
  const paysWithKey = Boolean(runCredentials(run))
3477
- // The server reads "Resets at <ISO>" to schedule the retry for that moment.
3864
+ // The marker is what tells the server this is a usage limit; it reads
3865
+ // "Resets at <ISO>" to schedule the retry for that moment.
3478
3866
  const message = usageLimit
3479
- ? `Usage limit reached on ${paysWithKey ? `this agent's ${run.api_credentials.provider} key` : executorDisplayName()}. Resets at ${usageLimit.resetAt.toISOString()}.\n\n${rawMessage}`
3867
+ ? `${USAGE_LIMIT_MARKER}${paysWithKey ? `this agent's ${run.api_credentials.provider} key` : executorDisplayName()}. Resets at ${usageLimit.resetAt.toISOString()}.\n\n${rawMessage}`
3480
3868
  : rawMessage
3481
3869
  if (usageLimit && !paysWithKey) {
3482
- usageLimitPausedUntil = Math.max(usageLimitPausedUntil, usageLimit.resetAt.getTime())
3483
- runLogger.warn("runner.usage_limit", "Executor hit its usage limit; pausing claims", {
3484
- meta: { reset_at: usageLimit.resetAt.toISOString(), stated: usageLimit.stated },
3870
+ const executor = activeExecutor()
3871
+ // A pause still running for another executor widens to all of them.
3872
+ const current = usageLimitPause && usageLimitPause.until > Date.now() ? usageLimitPause : null
3873
+ usageLimitPause = {
3874
+ until: Math.max(current?.until ?? 0, usageLimit.resetAt.getTime()),
3875
+ executor: !current || current.executor === executor ? executor : null,
3876
+ }
3877
+ runLogger.warn("runner.usage_limit", `${executorDisplayName()} reported its usage limit; holding back its runs on this machine's plan`, {
3878
+ meta: { executor, reset_at: usageLimit.resetAt.toISOString(), stated: usageLimit.stated },
3485
3879
  })
3486
3880
  }
3487
3881
  await postIssueMilestone(
@@ -3490,7 +3884,10 @@ async function runExecutor(run, state) {
3490
3884
  usageLimit
3491
3885
  ? `${paysWithKey ? "The agent's key" : executorDisplayName()} hit its usage limit. This picks up again automatically `
3492
3886
  + `${usageLimit.stated ? "when it resets" : "after a short wait"}, at ${formatLocalTime(usageLimit.resetAt)}, or sooner on a fallback key.`
3493
- : `Execution ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
3887
+ : limitStop
3888
+ ? `Run stopped by tripwire because ${limitStop.reason}. `
3889
+ + (stopped?.reportPosted ? "The Impact Report is in the comments." : "The Impact Report is on the run in Execution details.")
3890
+ : `Execution ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
3494
3891
  "warning",
3495
3892
  )
3496
3893
  // No add_comment here: the failure is already on the run record (shown in
@@ -3501,6 +3898,9 @@ async function runExecutor(run, state) {
3501
3898
  status: "error",
3502
3899
  error_message: truncate(message, 4000),
3503
3900
  usage: runUsage,
3901
+ // Where a stopped run's work was saved, so the issue links to it.
3902
+ branch_name: stopped?.branchName ?? undefined,
3903
+ changed_files: stopped?.changedFiles ?? undefined,
3504
3904
  }).catch(() => {})
3505
3905
 
3506
3906
  runLogger.error("runner.run.failed", "Local runner execution failed", {
@@ -3508,16 +3908,118 @@ async function runExecutor(run, state) {
3508
3908
  branch,
3509
3909
  worktree_path: worktreePath,
3510
3910
  canceled: cancelRequested,
3911
+ executor_phase: executorPhase,
3912
+ usage_limit: Boolean(usageLimit),
3511
3913
  },
3512
3914
  }, error)
3513
- throw error
3915
+ throw reportedRunFailure(error)
3514
3916
  } finally {
3515
3917
  clearInterval(logInterval)
3918
+ clearInterval(limitInterval)
3516
3919
  if (runTimeoutHandle) clearTimeout(runTimeoutHandle)
3517
3920
  cleanupExecutorMcp(mcpSideEffectBackups)
3921
+ try { skillInstall.cleanup() } catch (error) { runLogger.warn("runner.run.skills", "Could not remove the run's skill files", undefined, error) }
3922
+ }
3923
+ }
3924
+
3925
+ /**
3926
+ * Keep a stopped run's work: commit it to the ticket branch and push that
3927
+ * branch. No pull request is opened, so it does not go to review. A reset used
3928
+ * to throw the work away, and uncommitted work in a ticket checkout is easy to
3929
+ * lose.
3930
+ */
3931
+ async function saveStoppedWork({ run, stop, worktreePath, branch, gitAvailable, localCheckoutMode, runStartSha }) {
3932
+ if (!gitAvailable) {
3933
+ return { kind: "left_in_place", worktreePath, error: "The folder is not a git repository." }
3934
+ }
3935
+ const dirty = (await execCommandSafe("git", ["-C", worktreePath, "status", "--porcelain"], worktreePath))?.stdout?.trim()
3936
+ // The repository root is the user's own checkout and branch, and a review
3937
+ // shares the build's branch: a commit there is not the runner's to make.
3938
+ if (localCheckoutMode === "repo_root" || isReviewRun(run)) {
3939
+ return dirty
3940
+ ? {
3941
+ kind: "left_in_place",
3942
+ worktreePath,
3943
+ error: localCheckoutMode === "repo_root"
3944
+ ? "The run works in the repository root, so the runner does not commit for it."
3945
+ : "Review runs are not committed.",
3946
+ }
3947
+ : { kind: "none" }
3948
+ }
3949
+ const commit = await commitStoppedWork(worktreePath, {
3950
+ subject: `${run.task?.identifier || run.task_id} WIP: stopped by tripwire (${stop.kind} limit)`,
3951
+ body: `${tripwireHeadline(stop)}\n\nRun: ${run.id}`,
3952
+ })
3953
+ if (commit.error) return { kind: "uncommitted", worktreePath, error: commit.error }
3954
+ if (!commit.committed && !(await hasCommitsSince(worktreePath, runStartSha))) return { kind: "none" }
3955
+
3956
+ const sha = commit.sha
3957
+ ?? ((await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null)
3958
+ const committed = (error) => ({ kind: "committed", sha, branch, worktreePath, error })
3959
+ const repo = run.__repo
3960
+ if (!branch) return committed("the run has no branch to push")
3961
+ if (!autoPushOnSuccessFlag) return committed("pushing is turned off on this runner (auto-push-on-success)")
3962
+ if (!run.__githubToken || !repo?.owner || !repo?.repo) return committed("no GitHub repository is connected for this issue")
3963
+ try {
3964
+ await pushBranchToGitHub(worktreePath, branch, run.__githubToken, repo)
3965
+ return { kind: "pushed", sha, branch, repo: `${repo.owner}/${repo.repo}` }
3966
+ } catch (error) {
3967
+ // Never let a token in a remote URL reach the report.
3968
+ const reason = String(error instanceof Error ? error.message : error)
3969
+ .replace(/x-access-token:[^@\s]+@/g, "x-access-token:***@")
3970
+ .replace(/\s+/g, " ")
3971
+ return committed(`the push failed: ${truncate(reason, 200)}`)
3972
+ }
3973
+ }
3974
+
3975
+ /**
3976
+ * After a run is stopped at a limit: save its work, write the Impact Report
3977
+ * next to the run and post it on the issue. Posted before the run completes,
3978
+ * because completing revokes the run's key.
3979
+ */
3980
+ async function reportStoppedRun(input) {
3981
+ const { run, stop, gitAvailable, worktreePath, runStartSha, runRoot, mcpKey, runLogger } = input
3982
+ const changedFiles = gitAvailable ? await listRunChangedFiles(worktreePath, runStartSha) : []
3983
+ const saved = await saveStoppedWork(input)
3984
+ const report = buildImpactReport({
3985
+ stop,
3986
+ limits: input.runLimits,
3987
+ enforced: input.enforcedLimits,
3988
+ elapsedMs: input.elapsedMs,
3989
+ costUsd: input.costUsd,
3990
+ changedFiles,
3991
+ saved,
3992
+ executor: executorDisplayName(),
3993
+ agentName: run.agent?.name || null,
3994
+ })
3995
+ try {
3996
+ fs.mkdirSync(runRoot, { recursive: true })
3997
+ fs.writeFileSync(path.join(runRoot, "impact-report.md"), report)
3998
+ } catch {
3999
+ // The copy on the issue and the run is the one that matters.
4000
+ }
4001
+ let reportPosted = false
4002
+ if (mcpKey && run.task_id) {
4003
+ reportPosted = await callMcpTool(mcpKey, "add_comment", { issue_id: run.task_id, content: report }, run.agent_id)
4004
+ .then(() => true)
4005
+ .catch((error) => {
4006
+ runLogger.warn("runner.tripwire.comment_failed", "Could not post the Impact Report on the issue", undefined, error)
4007
+ return false
4008
+ })
4009
+ }
4010
+ runLogger.info("runner.tripwire.saved", `Stopped run's work: ${saved.kind}`, {
4011
+ meta: { kind: saved.kind, branch: saved.branch ?? null, sha: saved.sha ?? null, changed_file_count: changedFiles.length },
4012
+ })
4013
+ return {
4014
+ errorMessage: tripwireErrorMessage(stop, report),
4015
+ changedFiles,
4016
+ branchName: saved.kind === "pushed" || saved.kind === "committed" ? saved.branch ?? null : null,
4017
+ reportPosted,
3518
4018
  }
3519
4019
  }
3520
4020
 
4021
+ // How often a running run is checked against its file and time limits.
4022
+ const RUN_LIMIT_CHECK_INTERVAL_MS = 15_000
3521
4023
  // How long (ms) before we consider a run stuck and mark it error
3522
4024
  const RUN_TIMEOUT_MS = Number(args["run-timeout-ms"] || process.env.RUNNER_RUN_TIMEOUT_MS || 4 * 60 * 60 * 1000) // 4 hour default
3523
4025
  const CLAIM_ERROR_BACKOFF_MS = Number(args["claim-error-backoff-ms"] || process.env.RUNNER_CLAIM_ERROR_BACKOFF_MS || 5000)
@@ -3558,6 +4060,7 @@ async function cleanupFinishedCheckouts(state, logger) {
3558
4060
  if (!issue || !["done", "canceled"].includes(issue.status) || issue.branch_name !== branch) continue
3559
4061
  const dirty = (await execCommandSafe("git", ["-C", worktree, "status", "--porcelain"], worktree))?.stdout?.trim()
3560
4062
  if (dirty) continue
4063
+ if (await holdsUnpushedStoppedWork(worktree)) continue
3561
4064
  const gone = await execCommandSafe("git", ["-C", root, "worktree", "remove", worktree], root)
3562
4065
  if (!gone) continue
3563
4066
  // Squash merges leave the branch "unmerged" to git, hence -D.
@@ -3608,6 +4111,7 @@ async function main() {
3608
4111
  local_executor: state.local_executor,
3609
4112
  },
3610
4113
  })
4114
+ if (!cloudState) getRunnerLogger(state).info("runner.registration", describeRegistration(state), { meta: { name: state.name, labels: state.labels ?? [] } })
3611
4115
 
3612
4116
  if (registerOnly) return
3613
4117
 
@@ -3632,17 +4136,47 @@ async function main() {
3632
4136
  let lastCheckoutCleanup = 0
3633
4137
  let currentControlState = "active"
3634
4138
 
4139
+ // An admin disconnected this runner: it stops its agents and waits without
4140
+ // calling the server. Exiting would only have the service manager restart it.
4141
+ let revoked = false
4142
+ function markRevoked(error) {
4143
+ if (revoked) return
4144
+ revoked = true
4145
+ getRunnerLogger(state).error("runner.revoked", REVOKED_MESSAGE, undefined, error)
4146
+ stopAllRuns("this runner was disconnected from the workspace")
4147
+ }
4148
+
4149
+ // An admin stopped this runner: the server pauses it and cancel-requests its
4150
+ // runs; stopping the agents here too means they end now, not at the next log flush.
4151
+ let stopRequested = false
4152
+ function syncStopRequest(payload) {
4153
+ if (!payload || typeof payload.stop_requested !== "boolean" || payload.stop_requested === stopRequested) return
4154
+ stopRequested = payload.stop_requested
4155
+ if (stopRequested) {
4156
+ getRunnerLogger(state).warn("runner.stopped", "An admin stopped this runner: ending its runs and taking no new work until it is resumed")
4157
+ stopAllRuns("an admin stopped this runner")
4158
+ } else {
4159
+ getRunnerLogger(state).info("runner.resumed", "This runner was resumed and takes work again")
4160
+ }
4161
+ }
4162
+
3635
4163
  async function withRunnerAuth(operation) {
3636
4164
  try {
3637
4165
  return await operation(state)
3638
4166
  } catch (error) {
3639
- if (!isInvalidRunnerCredentialsError(error) || !workspaceApiKey) {
4167
+ if (isRevokedError(error)) markRevoked(error)
4168
+ if (revoked || !isInvalidRunnerCredentialsError(error) || !workspaceApiKey) {
3640
4169
  throw error
3641
4170
  }
3642
4171
 
3643
4172
  getRunnerLogger(state).warn("runner.credentials.invalid", "Runner credentials were rejected; re-registering", undefined, error)
3644
4173
  clearState()
3645
- state = await registerRunner()
4174
+ // Revoking clears the secret, so its first sign is "invalid
4175
+ // credentials"; registering again is then refused as revoked.
4176
+ state = await registerRunner().catch((registerError) => {
4177
+ if (isRevokedError(registerError)) markRevoked(registerError)
4178
+ throw registerError
4179
+ })
3646
4180
  getRunnerLogger(state).info("runner.reregistered", "Runner re-registered", {
3647
4181
  meta: {
3648
4182
  local_executor: state.local_executor,
@@ -3656,6 +4190,7 @@ async function main() {
3656
4190
  try {
3657
4191
  const payload = await withRunnerAuth((runnerState) => heartbeat(runnerState, status))
3658
4192
  currentControlState = payload?.control_state || currentControlState
4193
+ syncStopRequest(payload)
3659
4194
  return payload
3660
4195
  } catch {
3661
4196
  return null
@@ -3715,7 +4250,8 @@ async function main() {
3715
4250
  else if (event === "completed") triageLogger().info("runner.job.complete", "Triage job completed", jobMeta(details.job, { duration_ms: details.durationMs, confidence: details.confidence }))
3716
4251
  else if (event === "rejected") triageLogger().error("runner.job.error", `Triage result rejected (${details.status})`, jobMeta(details.job, { duration_ms: details.durationMs }))
3717
4252
  else if (event === "failed") triageLogger().error("runner.job.error", details.message, jobMeta(details.job, { duration_ms: details.durationMs }))
3718
- else if (event === "auth_failed") triageLogger().warn("runner.triage.auth_failed", `Triage polling stopped: the server rejected this runner's key (${details.status})`)
4253
+ else if (event === "auth_failed") triageLogger().error("runner.triage.auth_failed", `Triage is paused: the server rejected this runner's key (${details.status}). Retrying every ${Math.round(details.retryMs / 60000)} minutes; run \`npx pacework login\` if it keeps failing.`)
4254
+ else if (event === "auth_recovered") triageLogger().info("runner.triage.auth_recovered", "Triage resumed: the server accepts this runner's key again")
3719
4255
  },
3720
4256
  }).catch((error) => triageLogger().warn("runner.triage.stopped", "Triage polling stopped", undefined, error))
3721
4257
  } else {
@@ -3724,6 +4260,10 @@ async function main() {
3724
4260
  }
3725
4261
 
3726
4262
  while (!shuttingDown) {
4263
+ if (revoked) {
4264
+ await sleep(60_000)
4265
+ continue
4266
+ }
3727
4267
  const runnerStatus = activeRuns.size > 0 ? "busy" : "online"
3728
4268
  await heartbeatAndSync(runnerStatus)
3729
4269
  let claimFailed = false
@@ -3742,16 +4282,18 @@ async function main() {
3742
4282
  })
3743
4283
  }
3744
4284
 
3745
- // Paused for this machine's usage limit: its subscription takes nothing
3746
- // until the reset. Work paid with a key or token is not limited by it, so
3747
- // ask once a minute — the server offers only that kind while paused.
3748
- const pausedFor = usageLimitPausedUntil - Date.now()
3749
- if (pausedFor > 0) {
4285
+ // Every executor here is at its plan's limit: the machine's subscription
4286
+ // takes nothing until the reset. Work paid with a key or token is not
4287
+ // limited by it, so ask once a minute; the server offers only that kind
4288
+ // while paused, and says when the pause ends or an admin clears it.
4289
+ // With another executor still free, keep claiming as usual.
4290
+ const pausedFor = (usageLimitPause?.until ?? 0) - Date.now()
4291
+ if (pausedFor > 0 && pauseCoversEveryExecutor(usageLimitPause)) {
3750
4292
  await sleep(Math.min(pausedFor, 60_000))
3751
4293
  if (shuttingDown) continue
3752
4294
  }
3753
4295
 
3754
- while (!shuttingDown && activeRuns.size < maxParallelRuns && launchedRuns < maxRuns) {
4296
+ while (!shuttingDown && !revoked && !stopRequested && activeRuns.size < maxParallelRuns && launchedRuns < maxRuns) {
3755
4297
  let claim = null
3756
4298
  try {
3757
4299
  claim = await withRunnerAuth((runnerState) => claimNext({ ...runnerState }, activeRuns.size))
@@ -3766,17 +4308,8 @@ async function main() {
3766
4308
  break
3767
4309
  }
3768
4310
  currentControlState = claim.control_state || currentControlState
3769
- if (claim.backpressure === "backoff" && claim.available_after) {
3770
- // The server holds this runner back (a usage limit reported earlier,
3771
- // possibly before a restart). Wait it out instead of asking again.
3772
- const until = new Date(claim.available_after).getTime()
3773
- if (until > Date.now() && until > usageLimitPausedUntil) {
3774
- usageLimitPausedUntil = until
3775
- getRunnerLogger(state).info("runner.paused", "Paused until the usage limit resets", {
3776
- meta: { until: new Date(until).toISOString() },
3777
- })
3778
- }
3779
- }
4311
+ syncStopRequest(claim)
4312
+ syncUsageLimitPause(claim, getRunnerLogger(state))
3780
4313
  if (!claim.run) break
3781
4314
 
3782
4315
  const run = claim.run
@@ -3813,6 +4346,8 @@ async function main() {
3813
4346
  () => run.run_kind === "planner" ? runPlannerExecutor(run, state) : runExecutor(run, state),
3814
4347
  )
3815
4348
  .catch(async (error) => {
4349
+ // The run's own handler already reported and logged it.
4350
+ if (error?.runFailureReported) return
3816
4351
  // A failure before the executor spawns (worktree setup, repo
3817
4352
  // resolution) never reaches runExecutor's own handler, so the run
3818
4353
  // would sit in "running" until the 4-hour timeout. Mark it failed.
@@ -3830,6 +4365,7 @@ async function main() {
3830
4365
  })
3831
4366
  .finally(() => {
3832
4367
  activeRuns.delete(run.id)
4368
+ adminStoppers.delete(run.id)
3833
4369
  })
3834
4370
 
3835
4371
  activeRuns.set(run.id, runPromise)