pacework 1.1.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,9 +9,28 @@ import process from "node:process"
9
9
  import { spawn, execFileSync } from "node:child_process"
10
10
  import { AsyncLocalStorage } from "node:async_hooks"
11
11
  import { createLogger } from "../lib/logger.mjs"
12
- import { detectUsageLimit } from "../lib/usage-limit.mjs"
12
+ import { detectRunUsageLimit, USAGE_LIMIT_MARKER } from "../lib/usage-limit.mjs"
13
+ import {
14
+ buildImpactReport,
15
+ checkRunLimits,
16
+ commitStoppedWork,
17
+ costLimitStop,
18
+ describeRunLimits,
19
+ hasCommitsSince,
20
+ holdsUnpushedStoppedWork,
21
+ listRunChangedFiles,
22
+ resolveRunLimits,
23
+ tripwireErrorMessage,
24
+ tripwireHeadline,
25
+ } from "../lib/run-limits.mjs"
26
+ import { fetchGithubDefaultBranch, readOriginHeadBranch, resolvePrBaseBranch } from "../lib/pr-base-branch.mjs"
27
+ import { clearPendingPush, describeGitFailure, isTransientGitHubFailure, readPendingPush, retryTransient, writePendingPush } from "../lib/push-retry.mjs"
13
28
  import { prepareRunEnvironment } from "../lib/environment.mjs"
29
+ import { installSkills, parseSkillsUsed, skillsPromptSection } from "../lib/skills.mjs"
30
+ import { isGitLockError, repoLockKey, retryGitLock, withRepoLock } from "../lib/git-lock.mjs"
31
+ import { describeRegistration, isRevokedError, registrationChanges, REVOKED_MESSAGE, runEndedOnServer } from "../lib/runner-control.mjs"
14
32
  import { runTriageLoop, TRIAGE_EXECUTORS } from "../lib/triage-worker.mjs"
33
+ import { automationDenyRules, automationLimits } from "../lib/automation-limits.mjs"
15
34
 
16
35
  function parseArgs(argv) {
17
36
  const args = {}
@@ -343,10 +362,66 @@ function sleep(ms) {
343
362
  }
344
363
 
345
364
  /**
346
- * Until when this runner claims nothing because the executor's plan limit was
347
- * hit. Every run would fail the same way until the limit resets.
365
+ * This runner's pause for an executor's plan limit: { until (ms), executor }, or
366
+ * null. `executor` null means every executor. Runs of the limited executor on
367
+ * the machine's own plan would fail the same way until the limit resets; the
368
+ * server holds those back and says so on each claim, so an admin can clear it.
348
369
  */
349
- let usageLimitPausedUntil = 0
370
+ let usageLimitPause = null
371
+
372
+ /** Whether `pause` leaves this runner nothing to claim but work paid with a key. */
373
+ function pauseCoversEveryExecutor(pause) {
374
+ if (!pause) return false
375
+ return pause.executor === null || installedExecutors.every(executor => executor === pause.executor)
376
+ }
377
+
378
+ /**
379
+ * Mark a run failure as already reported to the server and logged, so the
380
+ * claim loop's catch-all does not report it again as a setup failure.
381
+ */
382
+ function reportedRunFailure(error) {
383
+ const reported = error instanceof Error ? error : new Error(String(error))
384
+ reported.runFailureReported = true
385
+ return reported
386
+ }
387
+
388
+ /**
389
+ * Follow the server's usage-limit pause from a claim response. The server holds
390
+ * the pause (it outlives a restart, and an admin can clear it in the app).
391
+ * Logs only when the pause starts, changes or ends.
392
+ */
393
+ function syncUsageLimitPause(claim, logger) {
394
+ const serverUntil = claim.backpressure === "backoff" && claim.available_after
395
+ ? new Date(claim.available_after).getTime()
396
+ : 0
397
+ const previous = usageLimitPause
398
+ if (serverUntil > Date.now()) {
399
+ const executor = claim.backoff_executor ?? null
400
+ if (previous?.until === serverUntil && previous.executor === executor) return
401
+ usageLimitPause = { until: serverUntil, executor }
402
+ logger.info("runner.paused", executor
403
+ ? `${executorDisplayName(executor)} reported its usage limit; its runs on this machine's plan wait until the limit resets`
404
+ : "An AI tool reported its usage limit; runs on this machine's plan wait until the limit resets", {
405
+ meta: { until: new Date(serverUntil).toISOString(), executor, reason: claim.backoff_reason ?? "usage_limit" },
406
+ })
407
+ return
408
+ }
409
+ // Other back-pressure (capacity, draining) is answered before the pause
410
+ // counts, so it says nothing about it. A plain "nothing to claim", or a
411
+ // claimed run the pause would have held back, says the server has none.
412
+ if (!previous || claim.backpressure) return
413
+ if (claim.run) {
414
+ const heldBack = !runCredentials(claim.run)
415
+ && (previous.executor === null || previous.executor === executorForRun(claim.run))
416
+ if (!heldBack) return
417
+ }
418
+ usageLimitPause = null
419
+ logger.info("runner.resumed", previous.until > Date.now()
420
+ ? "The usage-limit pause was cleared; taking all work again"
421
+ : "The usage limit has reset; taking all work again", {
422
+ meta: { executor: previous.executor },
423
+ })
424
+ }
350
425
 
351
426
  function formatLocalTime(date) {
352
427
  return date.toLocaleString(undefined, { weekday: "short", hour: "2-digit", minute: "2-digit", timeZoneName: "short" })
@@ -381,6 +456,8 @@ function parseClaudeJsonResult(stdout) {
381
456
  return {
382
457
  text: typeof parsed.result === "string" ? parsed.result : "",
383
458
  isError: parsed.is_error === true,
459
+ // "error_max_budget_usd" when --max-budget-usd ended the run.
460
+ subtype: typeof parsed.subtype === "string" ? parsed.subtype : null,
384
461
  usage: {
385
462
  cost_usd: typeof parsed.total_cost_usd === "number" ? parsed.total_cost_usd : null,
386
463
  input_tokens: inputTokens.length ? inputTokens.reduce((a, b) => a + b, 0) : null,
@@ -757,6 +834,13 @@ function isAutomationSmokeTestIssue(run) {
757
834
  return title.includes(automationSmokeTestMarker) || description.includes(automationSmokeTestMarker)
758
835
  }
759
836
 
837
+ /** Per running run: stop its agent because of an admin's runner-wide decision. */
838
+ const adminStoppers = new Map()
839
+
840
+ function stopAllRuns(reason) {
841
+ for (const stopRun of adminStoppers.values()) stopRun(reason)
842
+ }
843
+
760
844
  async function requestJson(url, init = {}) {
761
845
  const response = await fetch(url, init)
762
846
  let payload = null
@@ -774,7 +858,9 @@ async function requestJson(url, init = {}) {
774
858
  : ""
775
859
  const errorLabel = typeof payload?.error === "string" ? payload.error : ""
776
860
  const errorMessage = errorDetail || errorLabel || `Request failed: ${response.status}`
777
- throw new Error(errorMessage)
861
+ const error = new Error(errorMessage)
862
+ error.status = response.status
863
+ throw error
778
864
  }
779
865
 
780
866
  return payload
@@ -888,7 +974,24 @@ async function registerRunner() {
888
974
 
889
975
  async function ensureRegisteredRunner() {
890
976
  const state = loadState()
891
- if (state?.runner_id && state?.runner_secret && state?.runner_mcp_key) return state
977
+ if (!(state?.runner_id && state?.runner_secret && state?.runner_mcp_key)) return registerRunner()
978
+ const changes = registrationChanges(state, {
979
+ name: runnerName,
980
+ machine_name: machineName,
981
+ repo_root: repoRoot,
982
+ local_executor: defaultExecutor,
983
+ max_parallel_runs: maxParallelRuns,
984
+ labels,
985
+ })
986
+ if (changes.length === 0) return state
987
+ // Registering again updates this machine's entry in place (the server
988
+ // matches it by machine and repo root) and saves the new settings. Without
989
+ // that, a new --name or RUNNER_LABELS changed nothing on the server.
990
+ if (!workspaceApiKey) {
991
+ console.warn(`Runner settings changed (${changes.join(", ")}) but there is no workspace key to register them with; the server keeps the old ones.`)
992
+ return state
993
+ }
994
+ console.log(`Runner settings changed (${changes.join(", ")}); updating them in pacework.`)
892
995
  return registerRunner()
893
996
  }
894
997
 
@@ -1266,33 +1369,53 @@ async function fetchGithubToken(state, workspaceId, projectId, teamId = null, ta
1266
1369
  */
1267
1370
  const GIT_IGNORE_STORED_AUTH = ["-c", "http.https://github.com/.extraheader="]
1268
1371
 
1269
- async function pushBranchToGitHub(worktreePath, branch, githubToken, repo) {
1372
+ async function pushBranchToGitHub(worktreePath, branch, githubToken, repo, onRetry = () => {}) {
1373
+ // Worktrees of one repository share its config, remote URL included: a run
1374
+ // pushing while another swaps the URL would push without the token, or leave
1375
+ // it behind. Pushes and checkout setup take turns (lib/git-lock.mjs).
1376
+ return withRepoLock(repoLockKey(worktreePath), () => pushBranchWithToken(worktreePath, branch, githubToken, repo, onRetry))
1377
+ }
1378
+
1379
+ async function pushBranchWithToken(worktreePath, branch, githubToken, repo, onRetry) {
1270
1380
  // Set remote URL with embedded token for auth, then push
1271
1381
  const remoteUrl = `https://x-access-token:${githubToken}@github.com/${repo.owner}/${repo.repo}.git`
1272
- await execCommand(
1382
+ await retryGitLock(() => execCommand(
1273
1383
  "git",
1274
1384
  ["-C", worktreePath, "remote", "set-url", "origin", remoteUrl],
1275
1385
  worktreePath,
1276
- )
1277
- // start_work creates the branch on GitHub before the runner has a local clone
1278
- // of it, so --force-with-lease sees a remote it has no ref for and rejects the
1279
- // push. Fetch first so the lease has something to compare against.
1280
- await execCommandSafe(
1281
- "git",
1282
- ["-C", worktreePath, ...GIT_IGNORE_STORED_AUTH, "fetch", "origin", branch],
1283
- worktreePath,
1284
- )
1285
- await execCommand(
1286
- "git",
1287
- ["-C", worktreePath, ...GIT_IGNORE_STORED_AUTH, "push", "--set-upstream", "origin", branch, "--force-with-lease"],
1288
- worktreePath,
1289
- )
1290
- // Restore remote URL without token for safety
1291
- await execCommandSafe(
1292
- "git",
1293
- ["-C", worktreePath, "remote", "set-url", "origin", `https://github.com/${repo.owner}/${repo.repo}.git`],
1294
- worktreePath,
1295
- )
1386
+ ))
1387
+ try {
1388
+ // A GitHub 500 or a dropped connection is retried a few times before the
1389
+ // run gives up; one hiccup used to throw a finished run away.
1390
+ await retryTransient(async () => {
1391
+ // start_work creates the branch on GitHub before the runner has a local clone
1392
+ // of it, so --force-with-lease sees a remote it has no ref for and rejects the
1393
+ // push. Fetch first so the lease has something to compare against.
1394
+ await execCommandSafe(
1395
+ "git",
1396
+ ["-C", worktreePath, ...GIT_IGNORE_STORED_AUTH, "fetch", "origin", branch],
1397
+ worktreePath,
1398
+ )
1399
+ await execCommand(
1400
+ "git",
1401
+ ["-C", worktreePath, ...GIT_IGNORE_STORED_AUTH, "push", "--set-upstream", "origin", branch, "--force-with-lease"],
1402
+ worktreePath,
1403
+ )
1404
+ }, {
1405
+ onRetry,
1406
+ // `--set-upstream` writes the shared config: a lock held by another git
1407
+ // process is worth waiting out too.
1408
+ isTransient: (error) => isGitLockError(error) || isTransientGitHubFailure(error instanceof Error ? error.message : String(error)),
1409
+ })
1410
+ } finally {
1411
+ // Restore remote URL without token for safety, also when the push failed.
1412
+ // Retried on a held lock: giving up would leave the token in the config.
1413
+ await retryGitLock(() => execCommand(
1414
+ "git",
1415
+ ["-C", worktreePath, "remote", "set-url", "origin", `https://github.com/${repo.owner}/${repo.repo}.git`],
1416
+ worktreePath,
1417
+ )).catch(() => {})
1418
+ }
1296
1419
  }
1297
1420
 
1298
1421
  /** A section the agent left empty or answered with "none" says nothing to a reader. */
@@ -1699,6 +1822,15 @@ async function prepareWorktree(run) {
1699
1822
 
1700
1823
  ensureDir(path.dirname(worktreePath))
1701
1824
 
1825
+ // Runs that start together take turns per repository, and a lock held by
1826
+ // another git process is waited out (see lib/git-lock.mjs). Each step below
1827
+ // checks what already exists, so a retry picks up where a failure left off.
1828
+ await withRepoLock(repoLockKey(runRepoRoot), () => retryGitLock(() => prepareWorktreeCheckout(run, runRepoRoot, worktreePath, branch)))
1829
+
1830
+ return { runRoot, worktreePath, branch, localCheckoutMode, gitAvailable: true }
1831
+ }
1832
+
1833
+ async function prepareWorktreeCheckout(run, runRepoRoot, worktreePath, branch) {
1702
1834
  // Pull requests merge on GitHub, so origin/* in this checkout falls behind.
1703
1835
  // A stale base gave builders an old main to work on, and gave reviewers a
1704
1836
  // `git diff origin/HEAD..HEAD` full of work that had already merged.
@@ -1725,7 +1857,9 @@ async function prepareWorktree(run) {
1725
1857
  // Branch exists from a prior run — reuse it instead of -b
1726
1858
  await execCommand("git", ["-C", runRepoRoot, "worktree", "add", worktreePath, branch], runRepoRoot)
1727
1859
  } else {
1728
- await execCommand("git", ["-C", runRepoRoot, "worktree", "add", "-b", branch, worktreePath, upToDateBase], runRepoRoot)
1860
+ // --no-track: the run's branch must not follow the base branch, and
1861
+ // recording an upstream writes .git/config, the lock runs collided on.
1862
+ await execCommand("git", ["-C", runRepoRoot, "worktree", "add", "--no-track", "-b", branch, worktreePath, upToDateBase], runRepoRoot)
1729
1863
  }
1730
1864
  } else if (worktreeKnown) {
1731
1865
  const checkedOutBranch = await currentBranch(worktreePath)
@@ -1743,8 +1877,6 @@ async function prepareWorktree(run) {
1743
1877
  )
1744
1878
  }
1745
1879
  }
1746
-
1747
- return { runRoot, worktreePath, branch, localCheckoutMode, gitAvailable: true }
1748
1880
  }
1749
1881
 
1750
1882
  function buildPrompt(run, branch, worktreePath, options = {}) {
@@ -1755,7 +1887,9 @@ function buildPrompt(run, branch, worktreePath, options = {}) {
1755
1887
  const gitAvailable = options.gitAvailable !== false
1756
1888
  const workspaceContext = options.workspaceContext || null
1757
1889
  const greenfieldWorkspace = workspaceContext?.is_empty_workspace === true
1758
- const skillPrompt = agent?.skill_profile?.system_prompt?.trim()
1890
+ // The agent's standing instructions; servers before skill profiles were
1891
+ // retired sent them as skill_profile.system_prompt.
1892
+ const skillPrompt = (agent?.instructions ?? agent?.skill_profile?.system_prompt)?.trim()
1759
1893
  const base = skillPrompt ? `${skillPrompt}\n\n---\n\n` : ""
1760
1894
  const codingRules = Array.isArray(run?.coding_rules) ? run.coding_rules : []
1761
1895
  const reviewerContext = run?.reviewer_context || null
@@ -1860,6 +1994,13 @@ work that is already on the base branch is the worst thing a pull request can do
1860
1994
  .join("\n")
1861
1995
  : ""
1862
1996
  const answersGuidance = clarificationAnswers(run)
1997
+ // Told up front so an agent facing an oversized issue stops and says so,
1998
+ // instead of finding out from the tripwire.
1999
+ const runLimitsGuidance = !isReviewer && options.runLimitsText
2000
+ ? `- Run limits: ${options.runLimitsText}. The runner stops the run as soon as one is crossed.\n`
2001
+ + ` If this issue clearly needs more than that, do not start building: say in your summary that it\n`
2002
+ + ` should be split, and propose the subtasks.`
2003
+ : ""
1863
2004
  const environmentGuidance = run.__environmentNote
1864
2005
  ? `- Environment setup did not fully work. Say so in your summary rather than treating it as a bug in the code:\n ${run.__environmentNote.replace(/\n/g, "\n ")}`
1865
2006
  : ""
@@ -1888,7 +2029,7 @@ Execution rules:
1888
2029
  - Local checkout mode: ${localCheckoutMode}
1889
2030
  ${checkoutModeGuidance}
1890
2031
  ${existingCheckoutGuidance}
1891
- ${optionalGuidance ? `${optionalGuidance}\n` : ""}${environmentServices.length > 0 ? `- Services are running for this checkout; their addresses are in ${environmentServices.join(", ")}.\n` : ""}${environmentGuidance ? `${environmentGuidance}\n` : ""}${feedbackGuidance ? `${feedbackGuidance}\n` : ""}${codingRulesGuidance ? `${codingRulesGuidance}\n` : ""}${reviewerGuidance ? `${reviewerGuidance}\n` : ""}- Do not open a PR yet.
2032
+ ${optionalGuidance ? `${optionalGuidance}\n` : ""}${environmentServices.length > 0 ? `- Services are running for this checkout; their addresses are in ${environmentServices.join(", ")}.\n` : ""}${environmentGuidance ? `${environmentGuidance}\n` : ""}${feedbackGuidance ? `${feedbackGuidance}\n` : ""}${codingRulesGuidance ? `${codingRulesGuidance}\n` : ""}${runLimitsGuidance ? `${runLimitsGuidance}\n` : ""}${reviewerGuidance ? `${reviewerGuidance}\n` : ""}- Do not open a PR yet.
1892
2033
  - Treat explicit issue-description constraints as higher priority than generic instructions.
1893
2034
  - ${executionModeGuidance}
1894
2035
  ${isReviewer || answersGuidance ? "" : `- ${ASK_WHEN_UNCLEAR.replace(/\n/g, "\n ")}\n`}- End with a concise summary of what changed, what was verified, and any remaining gaps.
@@ -1941,6 +2082,8 @@ function claudeRunSettings(run) {
1941
2082
  ],
1942
2083
  }
1943
2084
  }
2085
+ const deny = automationDenyRules(run)
2086
+ if (deny.length) settings.permissions = { ...(settings.permissions || {}), deny }
1944
2087
  if (runCredentials(run)?.provider === "anthropic") {
1945
2088
  // ANTHROPIC_API_KEY alone loses to a signed-in Claude subscription, so the
1946
2089
  // run would be billed to the machine's plan. A key helper always wins. It
@@ -1950,8 +2093,30 @@ function claudeRunSettings(run) {
1950
2093
  return Object.keys(settings).length ? JSON.stringify(settings) : null
1951
2094
  }
1952
2095
 
1953
- function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey) {
2096
+ /** Whether this machine's Claude Code takes --max-budget-usd; an older one would refuse to start. */
2097
+ let claudeMaxBudgetSupport = null
2098
+ function claudeSupportsMaxBudget() {
2099
+ if (claudeMaxBudgetSupport === null) {
2100
+ try {
2101
+ const help = execFileSync(claudeCodeBin, ["--help"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"], timeout: 15_000 })
2102
+ claudeMaxBudgetSupport = help.includes("--max-budget-usd")
2103
+ } catch {
2104
+ claudeMaxBudgetSupport = false
2105
+ }
2106
+ }
2107
+ return claudeMaxBudgetSupport
2108
+ }
2109
+
2110
+ /**
2111
+ * @param {{ maxCostUsd?: number | null }} [options] The run's cost limit, for
2112
+ * Claude Code: it ends the run itself once the limit is spent.
2113
+ */
2114
+ function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey, options = {}) {
1954
2115
  const requestedModel = resolveRequestedModel(run)
2116
+ const limits = automationLimits(run)
2117
+ if (limits && (!limits.web || limits.code !== "write") && activeExecutor() !== "claude_code" && activeExecutor() !== "codex") {
2118
+ throw new Error("This automation limits web or code access, which only Claude Code and Codex runners can enforce. Run it on one of those, or allow web and code access.")
2119
+ }
1955
2120
 
1956
2121
  if (activeExecutor() === "codex") {
1957
2122
  const argsList = [
@@ -1960,8 +2125,18 @@ function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey) {
1960
2125
  // use its own current default rather than a name hardcoded here.
1961
2126
  ...(requestedModel ? ["-m", requestedModel] : []),
1962
2127
  "--ephemeral",
2128
+ // An automation without code write runs read-only, which also cuts the network.
2129
+ ...(limits && limits.code !== "write"
2130
+ ? ["-s", "read-only", "-c", "approval_policy=\"never\""]
2131
+ : limits && !limits.web
2132
+ ? [
2133
+ "-s", "workspace-write",
2134
+ "-c", "approval_policy=\"never\"",
2135
+ "-c", "sandbox_workspace_write.network_access=false",
2136
+ "-c", `sandbox_workspace_write.writable_roots=${JSON.stringify([path.join(run.__repoRoot || repoRoot, ".git")])}`,
2137
+ ]
1963
2138
  // Codex's sandbox cannot allow single domains, so network stays on.
1964
- ...(runSandboxed(run)
2139
+ : runSandboxed(run)
1965
2140
  ? [
1966
2141
  "-s", "workspace-write",
1967
2142
  "-c", "approval_policy=\"never\"",
@@ -2012,6 +2187,9 @@ function buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey) {
2012
2187
  if (requestedModel) {
2013
2188
  argsList.push("--model", requestedModel)
2014
2189
  }
2190
+ if (typeof options.maxCostUsd === "number") {
2191
+ argsList.push("--max-budget-usd", String(options.maxCostUsd))
2192
+ }
2015
2193
  return {
2016
2194
  command: claudeCodeBin,
2017
2195
  args: argsList,
@@ -2536,6 +2714,15 @@ async function runPlannerExecutor(run, state) {
2536
2714
  let plannerMcpBackups = null
2537
2715
  let plannerUsage = null
2538
2716
 
2717
+ // An admin stopping or disconnecting this runner ends the run from outside.
2718
+ adminStoppers.set(run.id, (reason) => {
2719
+ cancelRequested = true
2720
+ if (childProcess && !childProcess.killed) {
2721
+ appendLogTail(logState, `> ${reason}; terminating executor process...`)
2722
+ requestChildProcessShutdown(childProcess)
2723
+ }
2724
+ })
2725
+
2539
2726
  function appendHeartbeat(force = false) {
2540
2727
  if (!childProcess || childProcess.killed) return
2541
2728
  const now = Date.now()
@@ -2550,7 +2737,7 @@ async function runPlannerExecutor(run, state) {
2550
2737
  async function flushPlannerLog() {
2551
2738
  appendHeartbeat()
2552
2739
  const response = await postPlannerLog(run, logState.tail)
2553
- if (response?.cancel_requested) {
2740
+ if (runEndedOnServer(response)) {
2554
2741
  cancelRequested = true
2555
2742
  if (childProcess && !childProcess.killed) {
2556
2743
  appendLogTail(logState, "> cancel requested; terminating executor process...")
@@ -2694,6 +2881,9 @@ async function runPlannerExecutor(run, state) {
2694
2881
  }
2695
2882
 
2696
2883
  await flushPlannerLog().catch(() => {})
2884
+ // Canceled while the agent was finishing (or it exited cleanly on the
2885
+ // signal): its plan is not applied.
2886
+ if (cancelRequested) throw new Error("Canceled by admin.")
2697
2887
 
2698
2888
  const fileContent = fs.existsSync(outputFile) ? fs.readFileSync(outputFile, "utf8").trim() : ""
2699
2889
  const plannerResponse = staticResponse || fileContent || stdioSummary.trim()
@@ -2736,7 +2926,7 @@ async function runPlannerExecutor(run, state) {
2736
2926
  canceled: cancelRequested,
2737
2927
  },
2738
2928
  }, error)
2739
- throw error
2929
+ throw reportedRunFailure(error)
2740
2930
  } finally {
2741
2931
  clearInterval(logInterval)
2742
2932
  if (runTimeoutHandle) clearTimeout(runTimeoutHandle)
@@ -2835,7 +3025,7 @@ async function runExecutor(run, state) {
2835
3025
  // Resolve the repo this ticket belongs to before preparing a checkout, so the
2836
3026
  // code is written in the same repo the PR will be opened against.
2837
3027
  try {
2838
- const { token, repo } = await fetchGithubToken(
3028
+ const { token, repo, baseBranch } = await fetchGithubToken(
2839
3029
  state,
2840
3030
  run.workspace_id,
2841
3031
  run.task?.project_id ?? null,
@@ -2844,6 +3034,7 @@ async function runExecutor(run, state) {
2844
3034
  )
2845
3035
  run.__githubToken = token ?? null
2846
3036
  run.__repo = repo ?? null
3037
+ run.__baseBranch = baseBranch ?? null
2847
3038
  run.__repoRoot = await resolveRunRepoRoot(run, state)
2848
3039
  if (run.__repoRoot !== repoRoot) {
2849
3040
  runLogger.info("runner.repo.resolved", "Working in the ticket's repository", {
@@ -2893,6 +3084,9 @@ async function runExecutor(run, state) {
2893
3084
 
2894
3085
  runLogger.info("runner.run.prepare", "Preparing worktree")
2895
3086
  const { runRoot, worktreePath, branch, localCheckoutMode, gitAvailable } = await prepareWorktree(run)
3087
+ // Where a commit that could not reach GitHub is noted; only this issue's own
3088
+ // checkout has a folder of its own to hold the note.
3089
+ const pendingPushDir = gitAvailable && branch && localCheckoutMode !== "repo_root" ? path.dirname(worktreePath) : null
2896
3090
  const prepared = await prepareRunEnvironment({
2897
3091
  environment: run.environment,
2898
3092
  checkoutPath: worktreePath,
@@ -2921,8 +3115,34 @@ async function runExecutor(run, state) {
2921
3115
  run.reviewer_context = { ...(run.reviewer_context || {}), pr_diff: prDiff.diff, pr_diff_truncated: prDiff.truncated === true }
2922
3116
  }
2923
3117
  }
2924
- const prompt = buildPrompt(run, branch, worktreePath, { localCheckoutMode, gitAvailable, workspaceContext })
2925
- const execSpec = buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey)
3118
+ const runLimits = resolveRunLimits({ args, env: process.env, profile: runnerProfile })
3119
+ // Scouts survey the codebase and may touch many files on purpose.
3120
+ const fileLimitApplies = gitAvailable && getAgentSkillProfileSlug(run.agent) !== "scout"
3121
+ // The commit the run starts from: only the files this run changes count.
3122
+ const runStartSha = gitAvailable
3123
+ ? ((await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null)
3124
+ : null
3125
+ // Only Claude Code can stop itself at a cost, and another provider's cost is
3126
+ // priced at Anthropic's rates, so it cannot be held to a dollar limit.
3127
+ const costLimitApplies = runLimits.maxCostUsd !== null
3128
+ && activeExecutor() === "claude_code"
3129
+ && !usesThirdPartyModels(run)
3130
+ && claudeSupportsMaxBudget()
3131
+ const enforcedLimits = { files: fileLimitApplies, cost: costLimitApplies, time: true }
3132
+ const runLimitsText = describeRunLimits(runLimits, enforcedLimits)
3133
+ // The agent's skills, written where Claude Code and Codex load skills on demand
3134
+ // and kept out of git (PWK-145). Only a git checkout can keep them out of the change.
3135
+ const skillInstall = gitAvailable
3136
+ ? installSkills(worktreePath, run.skills, { log: (line) => runLogger.info("runner.run.skills", line) })
3137
+ : { installed: [], skipped: [], cleanup() {} }
3138
+ const skillsSection = skillsPromptSection((run.skills ?? []).filter((skill) => skillInstall.installed.includes(skill.slug)))
3139
+ const prompt = [
3140
+ buildPrompt(run, branch, worktreePath, { localCheckoutMode, gitAvailable, workspaceContext, runLimitsText }),
3141
+ skillsSection,
3142
+ ].filter(Boolean).join("\n\n")
3143
+ const execSpec = buildExecutorCommand(run, worktreePath, prompt, outputFile, mcpKey, {
3144
+ maxCostUsd: costLimitApplies ? runLimits.maxCostUsd : null,
3145
+ })
2926
3146
  // These steps run before the executor spawns; when one stalls the run sits in
2927
3147
  // "running" with no output, so each is traced.
2928
3148
  runLogger.info("runner.run.prepare", "Fetching MCP servers")
@@ -2939,6 +3159,8 @@ async function runExecutor(run, state) {
2939
3159
 
2940
3160
  const logState = { tail: "" }
2941
3161
  const startedAt = Date.now()
3162
+ /** Where the run is relative to the AI tool: only its own failure can be a usage limit. */
3163
+ let executorPhase = "before"
2942
3164
  let stdioSummary = ""
2943
3165
  // Claude Code's JSON result arrives on stdout only; stderr would break parsing.
2944
3166
  let stdoutRaw = ""
@@ -2946,7 +3168,18 @@ async function runExecutor(run, state) {
2946
3168
  let cancelRequested = false
2947
3169
  let childProcess = null
2948
3170
  let lastHeartbeatLogAt = 0
3171
+
3172
+ // An admin stopping or disconnecting this runner ends the run from outside.
3173
+ adminStoppers.set(run.id, (reason) => {
3174
+ cancelRequested = true
3175
+ if (childProcess && !childProcess.killed) {
3176
+ appendLogTail(logState, `> ${reason}; terminating executor process...`)
3177
+ requestChildProcessShutdown(childProcess)
3178
+ }
3179
+ })
2949
3180
  let activityThreadCommentId = run.activity_thread_comment_id || null
3181
+ /** The limit that stopped this run, once one has. */
3182
+ let limitStop = null
2950
3183
 
2951
3184
  function appendHeartbeat(force = false) {
2952
3185
  if (!childProcess || childProcess.killed) return
@@ -2962,7 +3195,7 @@ async function runExecutor(run, state) {
2962
3195
  async function flushRunnerLog() {
2963
3196
  appendHeartbeat()
2964
3197
  const response = await postRunLog(state, run.id, logState.tail)
2965
- if (response?.cancel_requested) {
3198
+ if (runEndedOnServer(response)) {
2966
3199
  cancelRequested = true
2967
3200
  if (childProcess && !childProcess.killed) {
2968
3201
  appendLogTail(logState, "> cancel requested; terminating executor process...")
@@ -2975,6 +3208,36 @@ async function runExecutor(run, state) {
2975
3208
  flushRunnerLog().catch(() => {})
2976
3209
  }, 5000)
2977
3210
 
3211
+ // Files and time are checked while the executor works, so a run that is too
3212
+ // big stops then, not after it has spent all its time and money. Cost is
3213
+ // Claude Code's own check (--max-budget-usd).
3214
+ let limitCheckInFlight = false
3215
+ async function checkLimitsWhileRunning() {
3216
+ if (limitStop || limitCheckInFlight || !childProcess || childProcess.killed) return
3217
+ limitCheckInFlight = true
3218
+ try {
3219
+ const changed = fileLimitApplies && runLimits.maxChangedFiles !== null
3220
+ ? await listRunChangedFiles(worktreePath, runStartSha)
3221
+ : []
3222
+ const stop = checkRunLimits(
3223
+ { ...runLimits, maxChangedFiles: fileLimitApplies ? runLimits.maxChangedFiles : null, maxCostUsd: null },
3224
+ { changedFileCount: changed.length, elapsedMs: Date.now() - startedAt },
3225
+ )
3226
+ if (!stop || limitStop || !childProcess || childProcess.killed) return
3227
+ limitStop = stop
3228
+ appendLogTail(logState, `> tripwire: stopping the run because ${stop.reason}`)
3229
+ runLogger.warn("runner.tripwire.tripped", `Run stopped at its ${stop.kind} limit`, {
3230
+ meta: { kind: stop.kind, limit: stop.limit, actual: stop.actual },
3231
+ })
3232
+ requestChildProcessShutdown(childProcess)
3233
+ } finally {
3234
+ limitCheckInFlight = false
3235
+ }
3236
+ }
3237
+ const limitInterval = setInterval(() => {
3238
+ checkLimitsWhileRunning().catch(() => {})
3239
+ }, RUN_LIMIT_CHECK_INTERVAL_MS)
3240
+
2978
3241
  // Hard timeout — kill the run if it hangs
2979
3242
  let runTimeoutHandle = null
2980
3243
  const runTimeoutPromise = new Promise((_, reject) => {
@@ -2997,13 +3260,15 @@ async function runExecutor(run, state) {
2997
3260
  appendLogTail(logState, `> worktree: ${worktreePath}`)
2998
3261
  appendLogTail(logState, `> branch: ${branch || "(none)"}`)
2999
3262
  appendLogTail(logState, `> executor: ${activeExecutor()}`)
3263
+ appendLogTail(logState, `> run limits: ${runLimitsText}`)
3000
3264
  await flushRunnerLog().catch(() => {})
3001
3265
  await postIssueMilestone(
3002
3266
  mcpKey,
3003
3267
  run.task_id,
3004
- isReviewRun(run)
3268
+ (isReviewRun(run)
3005
3269
  ? `AI review started in ${gitAvailable ? (branch || worktreePath) : worktreePath}.`
3006
- : `Implementation started in ${gitAvailable ? (branch || worktreePath) : worktreePath}.`,
3270
+ : `Implementation started in ${gitAvailable ? (branch || worktreePath) : worktreePath}.`)
3271
+ + ` Run limits: ${runLimitsText}.`,
3007
3272
  "info",
3008
3273
  )
3009
3274
  if (!isReviewRun(run)) {
@@ -3021,7 +3286,27 @@ async function runExecutor(run, state) {
3021
3286
  }
3022
3287
  await flushRunnerLog().catch(() => {})
3023
3288
 
3024
- if (automationSmokeTestMode && isAutomationSmokeTestIssue(run)) {
3289
+ // A previous run finished this work but GitHub would not take it. Publish
3290
+ // that commit instead of running the agent again.
3291
+ const pendingPush = pendingPushDir && !isReviewRun(run)
3292
+ ? readPendingPush(pendingPushDir, {
3293
+ taskId: run.task_id,
3294
+ branch,
3295
+ head: (await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null,
3296
+ })
3297
+ : null
3298
+
3299
+ if (pendingPush) {
3300
+ appendLogTail(logState, `> commit ${pendingPush.head.slice(0, 7)} from run ${pendingPush.run_id} was never published; publishing it without running the agent`)
3301
+ await postIssueMilestone(
3302
+ mcpKey,
3303
+ run.task_id,
3304
+ `Publishing commit ${pendingPush.head.slice(0, 7)}, which an earlier run finished but could not push. The agent is not run again.`,
3305
+ "info",
3306
+ ).catch(() => {})
3307
+ stdioSummary = pendingPush.summary || "Published the commit an earlier run finished but could not push."
3308
+ if (execSpec.summaryMode === "file") fs.writeFileSync(outputFile, stdioSummary)
3309
+ } else if (automationSmokeTestMode && isAutomationSmokeTestIssue(run)) {
3025
3310
  const smokeResult = await runSmokeTestExecution(run, state, { activityThreadCommentId })
3026
3311
  stdioSummary = smokeResult.summary
3027
3312
  appendLogTail(logState, smokeResult.summary)
@@ -3034,6 +3319,7 @@ async function runExecutor(run, state) {
3034
3319
  appendLogTail(logState, `> cwd: ${worktreePath}`)
3035
3320
  appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
3036
3321
  await flushRunnerLog().catch(() => {})
3322
+ executorPhase = "running"
3037
3323
  await Promise.race([runTimeoutPromise, new Promise((resolve, reject) => {
3038
3324
  const child = spawn(execSpec.command, execSpec.args, {
3039
3325
  cwd: worktreePath,
@@ -3081,12 +3367,24 @@ async function runExecutor(run, state) {
3081
3367
  if (result.usage.cost_usd !== null) {
3082
3368
  appendLogTail(logState, `> cost: $${result.usage.cost_usd.toFixed(4)}, ${result.usage.num_turns ?? "?"} turns`)
3083
3369
  }
3370
+ if (result.subtype === "error_max_budget_usd" && costLimitApplies && !limitStop) {
3371
+ limitStop = costLimitStop(runLimits.maxCostUsd, result.usage.cost_usd)
3372
+ appendLogTail(logState, `> tripwire: the run stopped because ${limitStop.reason}`)
3373
+ }
3374
+ if (limitStop) {
3375
+ reject(new Error(tripwireHeadline(limitStop)))
3376
+ return
3377
+ }
3084
3378
  if (code === 0 && result.isError) {
3085
3379
  reject(new Error(formatChildExitError(activeExecutor(), "0 (reported an error)", result.text)))
3086
3380
  return
3087
3381
  }
3088
3382
  }
3089
3383
  }
3384
+ if (limitStop) {
3385
+ reject(new Error(tripwireHeadline(limitStop)))
3386
+ return
3387
+ }
3090
3388
  if (code === 0) {
3091
3389
  resolve()
3092
3390
  return
@@ -3100,7 +3398,11 @@ async function runExecutor(run, state) {
3100
3398
  })])
3101
3399
  }
3102
3400
 
3401
+ executorPhase = "finished"
3103
3402
  await flushRunnerLog().catch(() => {})
3403
+ // An agent that exits cleanly on the cancel signal, or finishes as the
3404
+ // cancel arrives, must not have its work committed, pushed or opened.
3405
+ if (cancelRequested) throw new Error("Canceled by admin.")
3104
3406
 
3105
3407
  const summary = execSpec.summaryMode === "file"
3106
3408
  ? (fs.existsSync(outputFile) ? fs.readFileSync(outputFile, "utf8").trim() : "Run finished without a final summary.")
@@ -3192,17 +3494,21 @@ async function runExecutor(run, state) {
3192
3494
  .filter(Boolean))
3193
3495
  : []
3194
3496
 
3195
- // TRIPWIRE PROTOCOL
3196
- const isScout = getAgentSkillProfileSlug(run.agent) === "scout"
3197
- if (gitAvailable && changedFiles.length > 20 && !isScout) {
3198
- runLogger.error("runner.tripwire.tripped", "Agent tripwire activated: Too many file changes.", {
3199
- meta: { changed_files_count: changedFiles.length, files: changedFiles }
3200
- })
3201
- // Reset to prevent dirty state from corrupting the workspace
3202
- await execCommand("git", ["-C", worktreePath, "reset", "--hard"], worktreePath).catch(() => {})
3203
- await execCommand("git", ["-C", worktreePath, "clean", "-fd"], worktreePath).catch(() => {})
3204
-
3205
- throw new Error(`TRIPWIRE ACTIVATED: Agent attempted to modify ${changedFiles.length} files, exceeding the limit. An Impact Report has been generated.`)
3497
+ // TRIPWIRE PROTOCOL: the last look, for files changed after the live
3498
+ // check's last one. The work is saved below, never reset away.
3499
+ if (fileLimitApplies && runLimits.maxChangedFiles !== null) {
3500
+ const runChangedFiles = await listRunChangedFiles(worktreePath, runStartSha)
3501
+ const stop = checkRunLimits(
3502
+ { maxChangedFiles: runLimits.maxChangedFiles, maxCostUsd: null, maxMinutes: null },
3503
+ { changedFileCount: runChangedFiles.length },
3504
+ )
3505
+ if (stop) {
3506
+ limitStop = stop
3507
+ runLogger.warn("runner.tripwire.tripped", `Run stopped at its ${stop.kind} limit`, {
3508
+ meta: { kind: stop.kind, limit: stop.limit, actual: stop.actual },
3509
+ })
3510
+ throw new Error(tripwireHeadline(stop))
3511
+ }
3206
3512
  }
3207
3513
 
3208
3514
 
@@ -3288,22 +3594,46 @@ async function runExecutor(run, state) {
3288
3594
  ).catch(() => {})
3289
3595
  }
3290
3596
 
3597
+ // Last look before anything leaves the machine: a cancel that arrived
3598
+ // while the work was being committed still stops the push.
3599
+ if (autoPushOnSuccessFlag && somethingToPush) {
3600
+ await flushRunnerLog().catch(() => {})
3601
+ if (cancelRequested) throw new Error("Canceled by admin.")
3602
+ }
3603
+
3291
3604
  if (autoPushOnSuccessFlag && somethingToPush && branch && gitAvailable && !isReviewerRun2) {
3605
+ /** How far publishing got, so a failure names the step that broke. */
3606
+ let publishStage = null
3292
3607
  try {
3293
3608
  const githubToken = run.__githubToken
3294
3609
  const repo = run.__repo
3295
3610
 
3296
3611
  if (githubToken && repo?.owner && repo?.repo) {
3297
- await pushBranchToGitHub(worktreePath, branch, githubToken, repo)
3298
-
3299
- // Base the PR on the target repo's default branch, not the runner's.
3300
- const defaultBranch = await detectDefaultBranch(run.__repoRoot || repoRoot)
3301
- const pr = await createGithubPr(githubToken, repo, branch, defaultBranch, run, {
3612
+ publishStage = "push"
3613
+ await pushBranchToGitHub(worktreePath, branch, githubToken, repo, ({ attempt, delayMs, error }) => {
3614
+ runLogger.warn("runner.run.push_retry", `GitHub push failed (attempt ${attempt}), retrying in ${Math.round(delayMs / 1000)}s`, {
3615
+ meta: { branch, reason: describeGitFailure(error instanceof Error ? error.message : String(error), 300) },
3616
+ })
3617
+ })
3618
+ publishStage = "pull_request"
3619
+
3620
+ // Base the PR on the target repo's base branch, never on the branch
3621
+ // the runner's own checkout happens to be on.
3622
+ const base = await resolvePrBaseBranch({
3623
+ configured: run.__baseBranch,
3624
+ githubDefault: () => fetchGithubDefaultBranch(githubToken, repo),
3625
+ originHead: () => readOriginHeadBranch(run.__repoRoot || repoRoot),
3626
+ })
3627
+ runLogger.info("runner.pr.base_branch", `Opening the pull request against ${base.branch} (${base.source})`, {
3628
+ meta: { base_branch: base.branch, source: base.source, errors: base.errors.length ? base.errors : undefined },
3629
+ })
3630
+ const pr = await retryTransient(() => createGithubPr(githubToken, repo, branch, base.branch, run, {
3302
3631
  summary,
3303
3632
  changedFiles,
3304
3633
  diffStat: diffStat.stdout,
3305
3634
  verificationSummary: runnerVerificationSummary,
3306
- })
3635
+ }))
3636
+ if (pendingPushDir) clearPendingPush(pendingPushDir)
3307
3637
 
3308
3638
  pushResult = { pushed: true, repo: `${repo.owner}/${repo.repo}`, pr_url: pr.html_url, pr_number: pr.number }
3309
3639
 
@@ -3350,14 +3680,37 @@ async function runExecutor(run, state) {
3350
3680
  })
3351
3681
  }
3352
3682
  } catch (pushError) {
3353
- // Never let a token in a remote URL reach a log or the issue.
3354
- const pushMessage = (pushError instanceof Error ? pushError.message : String(pushError))
3355
- .replace(/x-access-token:[^@\s]+@/g, "x-access-token:***@")
3356
- runLogger.error("runner.run.push_failed", "Failed to push branch to GitHub", { meta: { branch } })
3683
+ // describeGitFailure masks the token a remote URL carries.
3684
+ const reason = describeGitFailure(pushError instanceof Error ? pushError.message : String(pushError))
3685
+ const attempts = Number(pushError?.attempts) || 1
3686
+ const tries = attempts > 1 ? ` after ${attempts} attempts` : ""
3687
+ const head = (await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null
3688
+ // Keep what the run produced next to the checkout, so the next run of
3689
+ // this issue publishes the commit instead of paying for the work again.
3690
+ const kept = Boolean(pendingPushDir && head)
3691
+ if (kept) {
3692
+ writePendingPush(pendingPushDir, {
3693
+ task_id: run.task_id,
3694
+ run_id: run.id,
3695
+ branch,
3696
+ head,
3697
+ summary,
3698
+ changed_files: changedFiles,
3699
+ verification_summary: runnerVerificationSummary,
3700
+ })
3701
+ }
3702
+ const keptNote = kept
3703
+ ? ` The commit ${head.slice(0, 7)} is kept on ${runnerName}; re-run the agent and it is ${publishStage === "pull_request" ? "opened as a pull request" : "pushed"} without redoing the work.`
3704
+ : ""
3357
3705
  // Work that never reached GitHub is not finished: completing the run
3358
3706
  // sent an empty branch to review, which asked for changes until the
3359
3707
  // revision limit ran out. Fail it with the reason instead.
3360
- throw new Error(`The work was committed but could not be pushed to GitHub: ${truncate(pushMessage.replace(/\s+/g, " "), 400)}`)
3708
+ if (publishStage === "pull_request") {
3709
+ runLogger.error("runner.run.pr_failed", "Branch pushed but the pull request could not be opened", { meta: { branch, attempts } })
3710
+ throw new Error(`The branch was pushed to GitHub but the pull request could not be opened${tries}: ${reason}.${keptNote}`)
3711
+ }
3712
+ runLogger.error("runner.run.push_failed", "Failed to push branch to GitHub", { meta: { branch, attempts } })
3713
+ throw new Error(`The work was committed but could not be pushed to GitHub${tries}: ${reason}.${keptNote}`)
3361
3714
  }
3362
3715
  }
3363
3716
 
@@ -3415,6 +3768,8 @@ async function runExecutor(run, state) {
3415
3768
  await completeRun(state, run.id, {
3416
3769
  status: "completed",
3417
3770
  result_summary: truncate(summary, 4000),
3771
+ // Which of its skills the agent says it used, from its report's last line.
3772
+ skills_used: parseSkillsUsed(summary, skillInstall.installed),
3418
3773
  usage: runUsage,
3419
3774
  branch_name: branch,
3420
3775
  changed_files: changedFiles,
@@ -3468,20 +3823,56 @@ async function runExecutor(run, state) {
3468
3823
  return { summary, branch, worktreePath, artifactsFile, reportFile, verificationSummary, commitResult, pushResult }
3469
3824
  } catch (error) {
3470
3825
  await flushRunnerLog().catch(() => {})
3471
- const rawMessage = cancelRequested ? "Canceled by admin." : (error instanceof Error ? error.message : String(error))
3472
- const usageLimit = cancelRequested ? null : detectUsageLimit(`${rawMessage}\n${logState.tail}`)
3826
+ // A stop at a limit keeps the work and explains itself on the issue.
3827
+ const stopped = limitStop
3828
+ ? await reportStoppedRun({
3829
+ run,
3830
+ stop: limitStop,
3831
+ runLimits,
3832
+ enforcedLimits,
3833
+ worktreePath,
3834
+ branch,
3835
+ gitAvailable,
3836
+ localCheckoutMode,
3837
+ runStartSha,
3838
+ elapsedMs: Date.now() - startedAt,
3839
+ costUsd: runUsage?.cost_usd ?? null,
3840
+ runRoot,
3841
+ mcpKey,
3842
+ runLogger,
3843
+ }).catch((reportError) => {
3844
+ runLogger.error("runner.tripwire.report_failed", "Could not save or report the stopped run", undefined, reportError)
3845
+ return null
3846
+ })
3847
+ : null
3848
+ const rawMessage = stopped?.errorMessage
3849
+ ?? (limitStop ? tripwireHeadline(limitStop) : null)
3850
+ ?? (cancelRequested ? "Canceled by admin." : (error instanceof Error ? error.message : String(error)))
3851
+ const usageLimit = detectRunUsageLimit({
3852
+ phase: executorPhase,
3853
+ // The runner stopped the tool itself; its last words are not a usage limit.
3854
+ canceled: cancelRequested || Boolean(limitStop),
3855
+ text: `${rawMessage}\n${logState.tail}`,
3856
+ })
3473
3857
  // A key's own limit is that key's business: pausing the runner would also
3474
3858
  // stop agents on the machine's subscription and on other keys. The server
3475
3859
  // switches the agent to its fallback key, or retries at the reset.
3476
3860
  const paysWithKey = Boolean(runCredentials(run))
3477
- // The server reads "Resets at <ISO>" to schedule the retry for that moment.
3861
+ // The marker is what tells the server this is a usage limit; it reads
3862
+ // "Resets at <ISO>" to schedule the retry for that moment.
3478
3863
  const message = usageLimit
3479
- ? `Usage limit reached on ${paysWithKey ? `this agent's ${run.api_credentials.provider} key` : executorDisplayName()}. Resets at ${usageLimit.resetAt.toISOString()}.\n\n${rawMessage}`
3864
+ ? `${USAGE_LIMIT_MARKER}${paysWithKey ? `this agent's ${run.api_credentials.provider} key` : executorDisplayName()}. Resets at ${usageLimit.resetAt.toISOString()}.\n\n${rawMessage}`
3480
3865
  : rawMessage
3481
3866
  if (usageLimit && !paysWithKey) {
3482
- usageLimitPausedUntil = Math.max(usageLimitPausedUntil, usageLimit.resetAt.getTime())
3483
- runLogger.warn("runner.usage_limit", "Executor hit its usage limit; pausing claims", {
3484
- meta: { reset_at: usageLimit.resetAt.toISOString(), stated: usageLimit.stated },
3867
+ const executor = activeExecutor()
3868
+ // A pause still running for another executor widens to all of them.
3869
+ const current = usageLimitPause && usageLimitPause.until > Date.now() ? usageLimitPause : null
3870
+ usageLimitPause = {
3871
+ until: Math.max(current?.until ?? 0, usageLimit.resetAt.getTime()),
3872
+ executor: !current || current.executor === executor ? executor : null,
3873
+ }
3874
+ runLogger.warn("runner.usage_limit", `${executorDisplayName()} reported its usage limit; holding back its runs on this machine's plan`, {
3875
+ meta: { executor, reset_at: usageLimit.resetAt.toISOString(), stated: usageLimit.stated },
3485
3876
  })
3486
3877
  }
3487
3878
  await postIssueMilestone(
@@ -3490,7 +3881,10 @@ async function runExecutor(run, state) {
3490
3881
  usageLimit
3491
3882
  ? `${paysWithKey ? "The agent's key" : executorDisplayName()} hit its usage limit. This picks up again automatically `
3492
3883
  + `${usageLimit.stated ? "when it resets" : "after a short wait"}, at ${formatLocalTime(usageLimit.resetAt)}, or sooner on a fallback key.`
3493
- : `Execution ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
3884
+ : limitStop
3885
+ ? `Run stopped by tripwire because ${limitStop.reason}. `
3886
+ + (stopped?.reportPosted ? "The Impact Report is in the comments." : "The Impact Report is on the run in Execution details.")
3887
+ : `Execution ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
3494
3888
  "warning",
3495
3889
  )
3496
3890
  // No add_comment here: the failure is already on the run record (shown in
@@ -3501,6 +3895,9 @@ async function runExecutor(run, state) {
3501
3895
  status: "error",
3502
3896
  error_message: truncate(message, 4000),
3503
3897
  usage: runUsage,
3898
+ // Where a stopped run's work was saved, so the issue links to it.
3899
+ branch_name: stopped?.branchName ?? undefined,
3900
+ changed_files: stopped?.changedFiles ?? undefined,
3504
3901
  }).catch(() => {})
3505
3902
 
3506
3903
  runLogger.error("runner.run.failed", "Local runner execution failed", {
@@ -3508,16 +3905,118 @@ async function runExecutor(run, state) {
3508
3905
  branch,
3509
3906
  worktree_path: worktreePath,
3510
3907
  canceled: cancelRequested,
3908
+ executor_phase: executorPhase,
3909
+ usage_limit: Boolean(usageLimit),
3511
3910
  },
3512
3911
  }, error)
3513
- throw error
3912
+ throw reportedRunFailure(error)
3514
3913
  } finally {
3515
3914
  clearInterval(logInterval)
3915
+ clearInterval(limitInterval)
3516
3916
  if (runTimeoutHandle) clearTimeout(runTimeoutHandle)
3517
3917
  cleanupExecutorMcp(mcpSideEffectBackups)
3918
+ try { skillInstall.cleanup() } catch (error) { runLogger.warn("runner.run.skills", "Could not remove the run's skill files", undefined, error) }
3919
+ }
3920
+ }
3921
+
3922
+ /**
3923
+ * Keep a stopped run's work: commit it to the ticket branch and push that
3924
+ * branch. No pull request is opened, so it does not go to review. A reset used
3925
+ * to throw the work away, and uncommitted work in a ticket checkout is easy to
3926
+ * lose.
3927
+ */
3928
+ async function saveStoppedWork({ run, stop, worktreePath, branch, gitAvailable, localCheckoutMode, runStartSha }) {
3929
+ if (!gitAvailable) {
3930
+ return { kind: "left_in_place", worktreePath, error: "The folder is not a git repository." }
3931
+ }
3932
+ const dirty = (await execCommandSafe("git", ["-C", worktreePath, "status", "--porcelain"], worktreePath))?.stdout?.trim()
3933
+ // The repository root is the user's own checkout and branch, and a review
3934
+ // shares the build's branch: a commit there is not the runner's to make.
3935
+ if (localCheckoutMode === "repo_root" || isReviewRun(run)) {
3936
+ return dirty
3937
+ ? {
3938
+ kind: "left_in_place",
3939
+ worktreePath,
3940
+ error: localCheckoutMode === "repo_root"
3941
+ ? "The run works in the repository root, so the runner does not commit for it."
3942
+ : "Review runs are not committed.",
3943
+ }
3944
+ : { kind: "none" }
3945
+ }
3946
+ const commit = await commitStoppedWork(worktreePath, {
3947
+ subject: `${run.task?.identifier || run.task_id} WIP: stopped by tripwire (${stop.kind} limit)`,
3948
+ body: `${tripwireHeadline(stop)}\n\nRun: ${run.id}`,
3949
+ })
3950
+ if (commit.error) return { kind: "uncommitted", worktreePath, error: commit.error }
3951
+ if (!commit.committed && !(await hasCommitsSince(worktreePath, runStartSha))) return { kind: "none" }
3952
+
3953
+ const sha = commit.sha
3954
+ ?? ((await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null)
3955
+ const committed = (error) => ({ kind: "committed", sha, branch, worktreePath, error })
3956
+ const repo = run.__repo
3957
+ if (!branch) return committed("the run has no branch to push")
3958
+ if (!autoPushOnSuccessFlag) return committed("pushing is turned off on this runner (auto-push-on-success)")
3959
+ if (!run.__githubToken || !repo?.owner || !repo?.repo) return committed("no GitHub repository is connected for this issue")
3960
+ try {
3961
+ await pushBranchToGitHub(worktreePath, branch, run.__githubToken, repo)
3962
+ return { kind: "pushed", sha, branch, repo: `${repo.owner}/${repo.repo}` }
3963
+ } catch (error) {
3964
+ // Never let a token in a remote URL reach the report.
3965
+ const reason = String(error instanceof Error ? error.message : error)
3966
+ .replace(/x-access-token:[^@\s]+@/g, "x-access-token:***@")
3967
+ .replace(/\s+/g, " ")
3968
+ return committed(`the push failed: ${truncate(reason, 200)}`)
3969
+ }
3970
+ }
3971
+
3972
+ /**
3973
+ * After a run is stopped at a limit: save its work, write the Impact Report
3974
+ * next to the run and post it on the issue. Posted before the run completes,
3975
+ * because completing revokes the run's key.
3976
+ */
3977
+ async function reportStoppedRun(input) {
3978
+ const { run, stop, gitAvailable, worktreePath, runStartSha, runRoot, mcpKey, runLogger } = input
3979
+ const changedFiles = gitAvailable ? await listRunChangedFiles(worktreePath, runStartSha) : []
3980
+ const saved = await saveStoppedWork(input)
3981
+ const report = buildImpactReport({
3982
+ stop,
3983
+ limits: input.runLimits,
3984
+ enforced: input.enforcedLimits,
3985
+ elapsedMs: input.elapsedMs,
3986
+ costUsd: input.costUsd,
3987
+ changedFiles,
3988
+ saved,
3989
+ executor: executorDisplayName(),
3990
+ agentName: run.agent?.name || null,
3991
+ })
3992
+ try {
3993
+ fs.mkdirSync(runRoot, { recursive: true })
3994
+ fs.writeFileSync(path.join(runRoot, "impact-report.md"), report)
3995
+ } catch {
3996
+ // The copy on the issue and the run is the one that matters.
3997
+ }
3998
+ let reportPosted = false
3999
+ if (mcpKey && run.task_id) {
4000
+ reportPosted = await callMcpTool(mcpKey, "add_comment", { issue_id: run.task_id, content: report }, run.agent_id)
4001
+ .then(() => true)
4002
+ .catch((error) => {
4003
+ runLogger.warn("runner.tripwire.comment_failed", "Could not post the Impact Report on the issue", undefined, error)
4004
+ return false
4005
+ })
4006
+ }
4007
+ runLogger.info("runner.tripwire.saved", `Stopped run's work: ${saved.kind}`, {
4008
+ meta: { kind: saved.kind, branch: saved.branch ?? null, sha: saved.sha ?? null, changed_file_count: changedFiles.length },
4009
+ })
4010
+ return {
4011
+ errorMessage: tripwireErrorMessage(stop, report),
4012
+ changedFiles,
4013
+ branchName: saved.kind === "pushed" || saved.kind === "committed" ? saved.branch ?? null : null,
4014
+ reportPosted,
3518
4015
  }
3519
4016
  }
3520
4017
 
4018
+ // How often a running run is checked against its file and time limits.
4019
+ const RUN_LIMIT_CHECK_INTERVAL_MS = 15_000
3521
4020
  // How long (ms) before we consider a run stuck and mark it error
3522
4021
  const RUN_TIMEOUT_MS = Number(args["run-timeout-ms"] || process.env.RUNNER_RUN_TIMEOUT_MS || 4 * 60 * 60 * 1000) // 4 hour default
3523
4022
  const CLAIM_ERROR_BACKOFF_MS = Number(args["claim-error-backoff-ms"] || process.env.RUNNER_CLAIM_ERROR_BACKOFF_MS || 5000)
@@ -3558,6 +4057,7 @@ async function cleanupFinishedCheckouts(state, logger) {
3558
4057
  if (!issue || !["done", "canceled"].includes(issue.status) || issue.branch_name !== branch) continue
3559
4058
  const dirty = (await execCommandSafe("git", ["-C", worktree, "status", "--porcelain"], worktree))?.stdout?.trim()
3560
4059
  if (dirty) continue
4060
+ if (await holdsUnpushedStoppedWork(worktree)) continue
3561
4061
  const gone = await execCommandSafe("git", ["-C", root, "worktree", "remove", worktree], root)
3562
4062
  if (!gone) continue
3563
4063
  // Squash merges leave the branch "unmerged" to git, hence -D.
@@ -3608,6 +4108,7 @@ async function main() {
3608
4108
  local_executor: state.local_executor,
3609
4109
  },
3610
4110
  })
4111
+ if (!cloudState) getRunnerLogger(state).info("runner.registration", describeRegistration(state), { meta: { name: state.name, labels: state.labels ?? [] } })
3611
4112
 
3612
4113
  if (registerOnly) return
3613
4114
 
@@ -3632,17 +4133,47 @@ async function main() {
3632
4133
  let lastCheckoutCleanup = 0
3633
4134
  let currentControlState = "active"
3634
4135
 
4136
+ // An admin disconnected this runner: it stops its agents and waits without
4137
+ // calling the server. Exiting would only have the service manager restart it.
4138
+ let revoked = false
4139
+ function markRevoked(error) {
4140
+ if (revoked) return
4141
+ revoked = true
4142
+ getRunnerLogger(state).error("runner.revoked", REVOKED_MESSAGE, undefined, error)
4143
+ stopAllRuns("this runner was disconnected from the workspace")
4144
+ }
4145
+
4146
+ // An admin stopped this runner: the server pauses it and cancel-requests its
4147
+ // runs; stopping the agents here too means they end now, not at the next log flush.
4148
+ let stopRequested = false
4149
+ function syncStopRequest(payload) {
4150
+ if (!payload || typeof payload.stop_requested !== "boolean" || payload.stop_requested === stopRequested) return
4151
+ stopRequested = payload.stop_requested
4152
+ if (stopRequested) {
4153
+ getRunnerLogger(state).warn("runner.stopped", "An admin stopped this runner: ending its runs and taking no new work until it is resumed")
4154
+ stopAllRuns("an admin stopped this runner")
4155
+ } else {
4156
+ getRunnerLogger(state).info("runner.resumed", "This runner was resumed and takes work again")
4157
+ }
4158
+ }
4159
+
3635
4160
  async function withRunnerAuth(operation) {
3636
4161
  try {
3637
4162
  return await operation(state)
3638
4163
  } catch (error) {
3639
- if (!isInvalidRunnerCredentialsError(error) || !workspaceApiKey) {
4164
+ if (isRevokedError(error)) markRevoked(error)
4165
+ if (revoked || !isInvalidRunnerCredentialsError(error) || !workspaceApiKey) {
3640
4166
  throw error
3641
4167
  }
3642
4168
 
3643
4169
  getRunnerLogger(state).warn("runner.credentials.invalid", "Runner credentials were rejected; re-registering", undefined, error)
3644
4170
  clearState()
3645
- state = await registerRunner()
4171
+ // Revoking clears the secret, so its first sign is "invalid
4172
+ // credentials"; registering again is then refused as revoked.
4173
+ state = await registerRunner().catch((registerError) => {
4174
+ if (isRevokedError(registerError)) markRevoked(registerError)
4175
+ throw registerError
4176
+ })
3646
4177
  getRunnerLogger(state).info("runner.reregistered", "Runner re-registered", {
3647
4178
  meta: {
3648
4179
  local_executor: state.local_executor,
@@ -3656,6 +4187,7 @@ async function main() {
3656
4187
  try {
3657
4188
  const payload = await withRunnerAuth((runnerState) => heartbeat(runnerState, status))
3658
4189
  currentControlState = payload?.control_state || currentControlState
4190
+ syncStopRequest(payload)
3659
4191
  return payload
3660
4192
  } catch {
3661
4193
  return null
@@ -3715,7 +4247,8 @@ async function main() {
3715
4247
  else if (event === "completed") triageLogger().info("runner.job.complete", "Triage job completed", jobMeta(details.job, { duration_ms: details.durationMs, confidence: details.confidence }))
3716
4248
  else if (event === "rejected") triageLogger().error("runner.job.error", `Triage result rejected (${details.status})`, jobMeta(details.job, { duration_ms: details.durationMs }))
3717
4249
  else if (event === "failed") triageLogger().error("runner.job.error", details.message, jobMeta(details.job, { duration_ms: details.durationMs }))
3718
- else if (event === "auth_failed") triageLogger().warn("runner.triage.auth_failed", `Triage polling stopped: the server rejected this runner's key (${details.status})`)
4250
+ else if (event === "auth_failed") triageLogger().error("runner.triage.auth_failed", `Triage is paused: the server rejected this runner's key (${details.status}). Retrying every ${Math.round(details.retryMs / 60000)} minutes; run \`npx pacework login\` if it keeps failing.`)
4251
+ else if (event === "auth_recovered") triageLogger().info("runner.triage.auth_recovered", "Triage resumed: the server accepts this runner's key again")
3719
4252
  },
3720
4253
  }).catch((error) => triageLogger().warn("runner.triage.stopped", "Triage polling stopped", undefined, error))
3721
4254
  } else {
@@ -3724,6 +4257,10 @@ async function main() {
3724
4257
  }
3725
4258
 
3726
4259
  while (!shuttingDown) {
4260
+ if (revoked) {
4261
+ await sleep(60_000)
4262
+ continue
4263
+ }
3727
4264
  const runnerStatus = activeRuns.size > 0 ? "busy" : "online"
3728
4265
  await heartbeatAndSync(runnerStatus)
3729
4266
  let claimFailed = false
@@ -3742,16 +4279,18 @@ async function main() {
3742
4279
  })
3743
4280
  }
3744
4281
 
3745
- // Paused for this machine's usage limit: its subscription takes nothing
3746
- // until the reset. Work paid with a key or token is not limited by it, so
3747
- // ask once a minute — the server offers only that kind while paused.
3748
- const pausedFor = usageLimitPausedUntil - Date.now()
3749
- if (pausedFor > 0) {
4282
+ // Every executor here is at its plan's limit: the machine's subscription
4283
+ // takes nothing until the reset. Work paid with a key or token is not
4284
+ // limited by it, so ask once a minute; the server offers only that kind
4285
+ // while paused, and says when the pause ends or an admin clears it.
4286
+ // With another executor still free, keep claiming as usual.
4287
+ const pausedFor = (usageLimitPause?.until ?? 0) - Date.now()
4288
+ if (pausedFor > 0 && pauseCoversEveryExecutor(usageLimitPause)) {
3750
4289
  await sleep(Math.min(pausedFor, 60_000))
3751
4290
  if (shuttingDown) continue
3752
4291
  }
3753
4292
 
3754
- while (!shuttingDown && activeRuns.size < maxParallelRuns && launchedRuns < maxRuns) {
4293
+ while (!shuttingDown && !revoked && !stopRequested && activeRuns.size < maxParallelRuns && launchedRuns < maxRuns) {
3755
4294
  let claim = null
3756
4295
  try {
3757
4296
  claim = await withRunnerAuth((runnerState) => claimNext({ ...runnerState }, activeRuns.size))
@@ -3766,17 +4305,8 @@ async function main() {
3766
4305
  break
3767
4306
  }
3768
4307
  currentControlState = claim.control_state || currentControlState
3769
- if (claim.backpressure === "backoff" && claim.available_after) {
3770
- // The server holds this runner back (a usage limit reported earlier,
3771
- // possibly before a restart). Wait it out instead of asking again.
3772
- const until = new Date(claim.available_after).getTime()
3773
- if (until > Date.now() && until > usageLimitPausedUntil) {
3774
- usageLimitPausedUntil = until
3775
- getRunnerLogger(state).info("runner.paused", "Paused until the usage limit resets", {
3776
- meta: { until: new Date(until).toISOString() },
3777
- })
3778
- }
3779
- }
4308
+ syncStopRequest(claim)
4309
+ syncUsageLimitPause(claim, getRunnerLogger(state))
3780
4310
  if (!claim.run) break
3781
4311
 
3782
4312
  const run = claim.run
@@ -3813,6 +4343,8 @@ async function main() {
3813
4343
  () => run.run_kind === "planner" ? runPlannerExecutor(run, state) : runExecutor(run, state),
3814
4344
  )
3815
4345
  .catch(async (error) => {
4346
+ // The run's own handler already reported and logged it.
4347
+ if (error?.runFailureReported) return
3816
4348
  // A failure before the executor spawns (worktree setup, repo
3817
4349
  // resolution) never reaches runExecutor's own handler, so the run
3818
4350
  // would sit in "running" until the 4-hour timeout. Mark it failed.
@@ -3830,6 +4362,7 @@ async function main() {
3830
4362
  })
3831
4363
  .finally(() => {
3832
4364
  activeRuns.delete(run.id)
4365
+ adminStoppers.delete(run.id)
3833
4366
  })
3834
4367
 
3835
4368
  activeRuns.set(run.id, runPromise)