pacework 1.2.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/pacework-runner-ctl.mjs +11 -12
- package/bin/pacework-runner.mjs +628 -95
- package/lib/automation-limits.mjs +23 -0
- package/lib/git-lock.mjs +83 -0
- package/lib/pr-base-branch.mjs +84 -0
- package/lib/push-retry.mjs +108 -0
- package/lib/run-limits.mjs +302 -0
- package/lib/runner-control.mjs +55 -0
- package/lib/skills.mjs +130 -0
- package/lib/triage-worker.mjs +29 -4
- package/lib/usage-limit.mjs +23 -0
- package/lib/version.mjs +27 -0
- package/package.json +1 -1
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What an automation's run may do (PWK-72), or null for other runs. Older
|
|
3
|
+
* servers send nothing, and the run keeps the agent's usual tools.
|
|
4
|
+
*/
|
|
5
|
+
export function automationLimits(run) {
|
|
6
|
+
const caps = run?.automation_capabilities
|
|
7
|
+
if (!caps || typeof caps !== "object") return null
|
|
8
|
+
return {
|
|
9
|
+
web: caps.web_access === true,
|
|
10
|
+
code: caps.code_access === "read" || caps.code_access === "write" ? caps.code_access : "none",
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Claude Code permission rules that keep an automation's run to its web and code access. */
|
|
15
|
+
export function automationDenyRules(run) {
|
|
16
|
+
const limits = automationLimits(run)
|
|
17
|
+
if (!limits) return []
|
|
18
|
+
const deny = []
|
|
19
|
+
if (!limits.web) deny.push("WebFetch", "WebSearch")
|
|
20
|
+
if (limits.code === "none") deny.push("Read", "Glob", "Grep", "Edit", "Write", "NotebookEdit", "Bash")
|
|
21
|
+
if (limits.code === "read") deny.push("Edit", "Write", "NotebookEdit", "Bash")
|
|
22
|
+
return deny
|
|
23
|
+
}
|
package/lib/git-lock.mjs
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process"
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Preparing checkouts for runs that start at the same moment.
|
|
5
|
+
*
|
|
6
|
+
* Several runs in one repository each add a worktree, fetch and create a
|
|
7
|
+
* branch. Git guards .git/config, refs and the index with lock files, so two
|
|
8
|
+
* of those at once fail with "could not lock config file .git/config: File
|
|
9
|
+
* exists" and the run errors before the agent starts. Within one runner the
|
|
10
|
+
* setup now takes turns per repository; a lock held by another process (a
|
|
11
|
+
* second runner, a person's git command) is waited out and retried. Pushing
|
|
12
|
+
* takes the same turn: it rewrites the shared remote URL around the push.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/** Waits between attempts when git reports a held lock: five tries over about 10 seconds. */
|
|
16
|
+
export const GIT_LOCK_RETRY_DELAYS_MS = [500, 1_500, 3_000, 5_000]
|
|
17
|
+
|
|
18
|
+
const LOCK_PATTERNS = [
|
|
19
|
+
/could not lock config file/i,
|
|
20
|
+
/unable to write upstream branch configuration/i,
|
|
21
|
+
/Unable to create '[^']*\.lock': File exists/i,
|
|
22
|
+
/cannot lock ref/i,
|
|
23
|
+
/another git process seems to be running/i,
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
/** Whether a git failure is a lock collision that waiting would clear. */
|
|
27
|
+
export function isGitLockError(error) {
|
|
28
|
+
const text = error instanceof Error ? error.message : String(error ?? "")
|
|
29
|
+
return LOCK_PATTERNS.some((pattern) => pattern.test(text))
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const queues = new Map()
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* The key for a repository's lock: its shared .git directory, which every
|
|
36
|
+
* worktree of it writes to (config, refs). Falls back to the path itself.
|
|
37
|
+
*/
|
|
38
|
+
export function repoLockKey(dir) {
|
|
39
|
+
try {
|
|
40
|
+
return execFileSync("git", ["-C", dir, "rev-parse", "--path-format=absolute", "--git-common-dir"], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim() || dir
|
|
41
|
+
} catch {
|
|
42
|
+
return dir
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** Run `fn` after every earlier call for the same repository has finished. */
|
|
47
|
+
export function withRepoLock(repo, fn) {
|
|
48
|
+
const previous = queues.get(repo) ?? Promise.resolve()
|
|
49
|
+
const result = previous.then(fn, fn)
|
|
50
|
+
// The next caller waits for this one to settle, whether it failed or not.
|
|
51
|
+
const settled = result.then(() => undefined, () => undefined)
|
|
52
|
+
queues.set(repo, settled)
|
|
53
|
+
settled.then(() => { if (queues.get(repo) === settled) queues.delete(repo) })
|
|
54
|
+
return result
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Run `fn`, retrying while git reports a held lock. `fn` must be safe to run
|
|
59
|
+
* again after a partial failure. A collision that outlasts the retries is
|
|
60
|
+
* rethrown with a message that says what happened.
|
|
61
|
+
*/
|
|
62
|
+
export async function retryGitLock(fn, { delays = GIT_LOCK_RETRY_DELAYS_MS, sleep = defaultSleep, onRetry = () => {} } = {}) {
|
|
63
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
64
|
+
try {
|
|
65
|
+
return await fn()
|
|
66
|
+
} catch (error) {
|
|
67
|
+
if (!isGitLockError(error)) throw error
|
|
68
|
+
if (attempt >= delays.length) {
|
|
69
|
+
const detail = error instanceof Error ? error.message : String(error)
|
|
70
|
+
throw new Error(
|
|
71
|
+
`Another git process kept the repository locked while this run's checkout was prepared (${attempt + 1} tries). ` +
|
|
72
|
+
`This is a timing collision, not a problem with the issue: re-run the agent.\n${detail}`,
|
|
73
|
+
)
|
|
74
|
+
}
|
|
75
|
+
onRetry(attempt + 1, error)
|
|
76
|
+
await sleep(delays[attempt])
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function defaultSleep(ms) {
|
|
82
|
+
return new Promise((resolve) => setTimeout(resolve, ms))
|
|
83
|
+
}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Choose the branch an agent's pull request is opened against.
|
|
3
|
+
*
|
|
4
|
+
* The agent's branch is cut from origin/HEAD, so the pull request must target the
|
|
5
|
+
* repository's base branch. The branch the runner's own checkout happens to be on
|
|
6
|
+
* says nothing about that: using it opened pull requests against stale feature
|
|
7
|
+
* branches, where they were merged and never reached the real base branch.
|
|
8
|
+
*
|
|
9
|
+
* Order: the base branch configured for the issue's project or team, the
|
|
10
|
+
* repository's default branch on GitHub, the clone's origin/HEAD, then "main".
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { execFile } from "node:child_process"
|
|
14
|
+
|
|
15
|
+
export const FALLBACK_BASE_BRANCH = "main"
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* @param {object} sources
|
|
19
|
+
* @param {string|null} [sources.configured] base branch configured in pacework
|
|
20
|
+
* @param {() => Promise<string|null>} [sources.githubDefault] the repository's default branch on GitHub
|
|
21
|
+
* @param {() => Promise<string|null>} [sources.originHead] the clone's origin/HEAD, without "origin/"
|
|
22
|
+
* @returns {Promise<{ branch: string, source: "configured"|"github"|"origin_head"|"fallback", errors: string[] }>}
|
|
23
|
+
* `errors` lists lookups that failed on the way, so the run log can say why a
|
|
24
|
+
* later step was used.
|
|
25
|
+
*/
|
|
26
|
+
export async function resolvePrBaseBranch({ configured = null, githubDefault, originHead } = {}) {
|
|
27
|
+
const errors = []
|
|
28
|
+
const configuredBranch = cleanBranch(configured)
|
|
29
|
+
if (configuredBranch) return { branch: configuredBranch, source: "configured", errors }
|
|
30
|
+
|
|
31
|
+
for (const [source, lookup] of [["github", githubDefault], ["origin_head", originHead]]) {
|
|
32
|
+
if (typeof lookup !== "function") continue
|
|
33
|
+
try {
|
|
34
|
+
const branch = cleanBranch(await lookup())
|
|
35
|
+
if (branch) return { branch, source, errors }
|
|
36
|
+
} catch (error) {
|
|
37
|
+
errors.push(`${source}: ${error?.message || error}`)
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return { branch: FALLBACK_BASE_BRANCH, source: "fallback", errors }
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/** The repository's default branch from `GET /repos/{owner}/{repo}`. */
|
|
44
|
+
export async function fetchGithubDefaultBranch(githubToken, repo, fetchImpl = fetch) {
|
|
45
|
+
if (!githubToken || !repo?.owner || !repo?.repo) return null
|
|
46
|
+
const res = await fetchImpl(`https://api.github.com/repos/${repo.owner}/${repo.repo}`, {
|
|
47
|
+
headers: {
|
|
48
|
+
Authorization: `Bearer ${githubToken}`,
|
|
49
|
+
Accept: "application/vnd.github+json",
|
|
50
|
+
"X-GitHub-Api-Version": "2022-11-28",
|
|
51
|
+
},
|
|
52
|
+
})
|
|
53
|
+
if (!res.ok) throw new Error(`GitHub repository lookup failed (${res.status})`)
|
|
54
|
+
const payload = await res.json()
|
|
55
|
+
return payload?.default_branch ?? null
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The branch the clone's origin/HEAD points at, e.g. "main" for origin/main. */
|
|
59
|
+
export function readOriginHeadBranch(repoRoot) {
|
|
60
|
+
return new Promise((resolve, reject) => {
|
|
61
|
+
if (!repoRoot) {
|
|
62
|
+
resolve(null)
|
|
63
|
+
return
|
|
64
|
+
}
|
|
65
|
+
execFile(
|
|
66
|
+
"git",
|
|
67
|
+
["-C", repoRoot, "symbolic-ref", "--quiet", "--short", "refs/remotes/origin/HEAD"],
|
|
68
|
+
(error, stdout) => {
|
|
69
|
+
if (error) {
|
|
70
|
+
// Exit code 1 with --quiet: origin/HEAD is not set in this clone.
|
|
71
|
+
if (error.code === 1) resolve(null)
|
|
72
|
+
else reject(error)
|
|
73
|
+
return
|
|
74
|
+
}
|
|
75
|
+
resolve(String(stdout).trim().replace(/^origin\//, "") || null)
|
|
76
|
+
},
|
|
77
|
+
)
|
|
78
|
+
})
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function cleanBranch(value) {
|
|
82
|
+
const branch = typeof value === "string" ? value.trim() : ""
|
|
83
|
+
return branch || null
|
|
84
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Getting finished work to GitHub when GitHub is having a bad moment.
|
|
3
|
+
*
|
|
4
|
+
* A single 500 on push used to fail the whole run: the commit stayed on the
|
|
5
|
+
* runner machine, the issue was paused, and the only way out was a new agent
|
|
6
|
+
* run that redid and re-paid for the work. Temporary failures are now retried,
|
|
7
|
+
* and a push that still fails leaves a note next to the checkout so the next
|
|
8
|
+
* run of the same issue pushes the commit instead of running the agent again.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import fs from "node:fs"
|
|
12
|
+
import path from "node:path"
|
|
13
|
+
|
|
14
|
+
/** Waits between attempts: four tries over about 45 seconds. */
|
|
15
|
+
export const PUSH_RETRY_DELAYS_MS = [3_000, 10_000, 30_000]
|
|
16
|
+
|
|
17
|
+
const TRANSIENT_PATTERNS = [
|
|
18
|
+
// Status codes only where git or the GitHub API states one: request IDs are
|
|
19
|
+
// hex and can contain any three digits.
|
|
20
|
+
/returned error: (5\d\d|429)|\((5\d\d|429)\)|\b(5\d\d|429) (internal|bad|service|gateway|too many)/i,
|
|
21
|
+
/internal server error|bad gateway|service unavailable|gateway time-?out/i,
|
|
22
|
+
/timed? ?out|timeout/i,
|
|
23
|
+
/connection (reset|refused|closed)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|socket hang up/i,
|
|
24
|
+
/could not resolve host|failed to connect|unable to access/i,
|
|
25
|
+
/RPC failed|early EOF|unexpected disconnect|the remote end hung up|fetch failed/i,
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
/** Errors that are about the request, not GitHub's health, and will fail again. */
|
|
29
|
+
const PERMANENT_PATTERNS = [
|
|
30
|
+
/returned error: (401|403|404)|\((401|403|404|422)\)/,
|
|
31
|
+
/authentication failed|permission .* denied|repository not found|invalid username or token/i,
|
|
32
|
+
/stale info|non-fast-forward|fetch first|protected branch|pre-receive hook declined/i,
|
|
33
|
+
]
|
|
34
|
+
|
|
35
|
+
/** True when a git or GitHub failure is worth trying again in a few seconds. */
|
|
36
|
+
export function isTransientGitHubFailure(message) {
|
|
37
|
+
const text = String(message ?? "")
|
|
38
|
+
if (PERMANENT_PATTERNS.some((pattern) => pattern.test(text))) return false
|
|
39
|
+
return TRANSIENT_PATTERNS.some((pattern) => pattern.test(text))
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Runs `attempt` until it succeeds, a failure is not transient, or the delays
|
|
44
|
+
* run out. The last error is rethrown with `attempts` set on it.
|
|
45
|
+
*/
|
|
46
|
+
export async function retryTransient(attempt, {
|
|
47
|
+
delaysMs = PUSH_RETRY_DELAYS_MS,
|
|
48
|
+
isTransient = (error) => isTransientGitHubFailure(error instanceof Error ? error.message : String(error)),
|
|
49
|
+
sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
50
|
+
onRetry = () => {},
|
|
51
|
+
} = {}) {
|
|
52
|
+
for (let index = 0; ; index += 1) {
|
|
53
|
+
try {
|
|
54
|
+
return await attempt(index + 1)
|
|
55
|
+
} catch (error) {
|
|
56
|
+
if (index >= delaysMs.length || !isTransient(error)) {
|
|
57
|
+
if (error && typeof error === "object") error.attempts = index + 1
|
|
58
|
+
throw error
|
|
59
|
+
}
|
|
60
|
+
onRetry({ attempt: index + 1, delayMs: delaysMs[index], error })
|
|
61
|
+
await sleep(delaysMs[index])
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* A git failure as a person can read it: without the command line (it holds the
|
|
68
|
+
* local path and a token-bearing remote), with any token masked, and cut at the
|
|
69
|
+
* end rather than the start so it never begins mid-word ("emote: ...").
|
|
70
|
+
*/
|
|
71
|
+
export function describeGitFailure(message, max = 600) {
|
|
72
|
+
const lines = String(message ?? "")
|
|
73
|
+
.replace(/x-access-token:[^@\s]+@/g, "x-access-token:***@")
|
|
74
|
+
.split(/\r?\n/)
|
|
75
|
+
.map((line) => line.trim())
|
|
76
|
+
.filter(Boolean)
|
|
77
|
+
if (/^git\s.*failed with code/.test(lines[0] ?? "") && lines.length > 1) lines.shift()
|
|
78
|
+
const text = lines.join(" ").replace(/\s+/g, " ").trim()
|
|
79
|
+
return text.length <= max ? text : `${text.slice(0, max - 1).trimEnd()}…`
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const PENDING_FILE = "pending-push.json"
|
|
83
|
+
|
|
84
|
+
/** Notes that this issue's checkout holds a finished commit GitHub has not received. */
|
|
85
|
+
export function writePendingPush(issueDir, pending) {
|
|
86
|
+
fs.writeFileSync(path.join(issueDir, PENDING_FILE), JSON.stringify({ ...pending, saved_at: new Date().toISOString() }, null, 2))
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* The note for this issue, if the checkout is still at the commit it names.
|
|
91
|
+
* Any other state means someone or something moved on, so the note is dropped.
|
|
92
|
+
*/
|
|
93
|
+
export function readPendingPush(issueDir, { taskId, branch, head }) {
|
|
94
|
+
const file = path.join(issueDir, PENDING_FILE)
|
|
95
|
+
let pending
|
|
96
|
+
try {
|
|
97
|
+
pending = JSON.parse(fs.readFileSync(file, "utf8"))
|
|
98
|
+
} catch {
|
|
99
|
+
return null
|
|
100
|
+
}
|
|
101
|
+
if (pending?.task_id === taskId && pending?.branch === branch && pending?.head && pending.head === head) return pending
|
|
102
|
+
clearPendingPush(issueDir)
|
|
103
|
+
return null
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function clearPendingPush(issueDir) {
|
|
107
|
+
fs.rmSync(path.join(issueDir, PENDING_FILE), { force: true })
|
|
108
|
+
}
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Limits on one agent run: how many files it may change, what it may cost and
|
|
3
|
+
* how long it may take.
|
|
4
|
+
*
|
|
5
|
+
* The runner checks them while the AI tool is working and stops the run as soon
|
|
6
|
+
* as one is crossed. Checking only after the tool had finished meant a run that
|
|
7
|
+
* was far too big still spent its full time and money first.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { execFile } from "node:child_process"
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* How the error message of a run stopped by a limit starts. The server reads it
|
|
14
|
+
* to tell a tripwire stop apart from other failures and never retries one, so
|
|
15
|
+
* keep it in step with TRIPWIRE_MARKER in backend control-plane/runner-complete.helpers.ts.
|
|
16
|
+
*/
|
|
17
|
+
export const TRIPWIRE_MARKER = "TRIPWIRE ACTIVATED"
|
|
18
|
+
|
|
19
|
+
/** Trailer on the commit that saves a stopped run's work. Cleanup keeps such a checkout until it is pushed. */
|
|
20
|
+
export const STOPPED_BY_TRIPWIRE_TRAILER = "Pacework-Stopped-By: tripwire"
|
|
21
|
+
|
|
22
|
+
export const DEFAULT_RUN_LIMITS = Object.freeze({
|
|
23
|
+
maxChangedFiles: 20,
|
|
24
|
+
maxCostUsd: 5,
|
|
25
|
+
maxMinutes: 60,
|
|
26
|
+
})
|
|
27
|
+
|
|
28
|
+
/** Where each limit can be set: a runner flag, an environment variable, `limits` in the runner profile. */
|
|
29
|
+
export const RUN_LIMIT_SETTINGS = Object.freeze({
|
|
30
|
+
maxChangedFiles: { flag: "max-changed-files", env: "RUNNER_MAX_CHANGED_FILES", profile: "maxChangedFiles" },
|
|
31
|
+
maxCostUsd: { flag: "max-run-cost-usd", env: "RUNNER_MAX_RUN_COST_USD", profile: "maxCostUsd" },
|
|
32
|
+
maxMinutes: { flag: "max-run-minutes", env: "RUNNER_MAX_RUN_MINUTES", profile: "maxMinutes" },
|
|
33
|
+
})
|
|
34
|
+
|
|
35
|
+
const OFF_VALUES = new Set(["off", "none", "0"])
|
|
36
|
+
|
|
37
|
+
/** A limit's value: a positive number, null for "off", undefined when not set or not valid. */
|
|
38
|
+
function parseLimit(value) {
|
|
39
|
+
if (value === undefined || value === null) return undefined
|
|
40
|
+
const text = String(value).trim().toLowerCase()
|
|
41
|
+
if (!text) return undefined
|
|
42
|
+
if (OFF_VALUES.has(text)) return null
|
|
43
|
+
const number = Number(text)
|
|
44
|
+
return Number.isFinite(number) && number > 0 ? number : undefined
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The limits for runs on this machine. Each comes from the runner flag, then the
|
|
49
|
+
* environment, then `limits` in the runner profile, then the default. "off" or 0
|
|
50
|
+
* turns a limit off.
|
|
51
|
+
*/
|
|
52
|
+
export function resolveRunLimits({ args = {}, env = {}, profile = {} } = {}) {
|
|
53
|
+
const profileLimits = profile && typeof profile.limits === "object" && profile.limits ? profile.limits : {}
|
|
54
|
+
const limits = {}
|
|
55
|
+
for (const [key, setting] of Object.entries(RUN_LIMIT_SETTINGS)) {
|
|
56
|
+
const candidates = [args[setting.flag], env[setting.env], profileLimits[setting.profile]]
|
|
57
|
+
const chosen = candidates.map(parseLimit).find((value) => value !== undefined)
|
|
58
|
+
limits[key] = chosen === undefined ? DEFAULT_RUN_LIMITS[key] : chosen
|
|
59
|
+
}
|
|
60
|
+
if (limits.maxChangedFiles !== null) limits.maxChangedFiles = Math.floor(limits.maxChangedFiles)
|
|
61
|
+
return limits
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const usd = (value) => `$${Number(value).toFixed(2)}`
|
|
65
|
+
|
|
66
|
+
export function formatDuration(ms) {
|
|
67
|
+
const totalSeconds = Math.max(0, Math.round(ms / 1000))
|
|
68
|
+
const minutes = Math.floor(totalSeconds / 60)
|
|
69
|
+
const seconds = totalSeconds % 60
|
|
70
|
+
if (minutes === 0) return `${seconds}s`
|
|
71
|
+
return seconds === 0 ? `${minutes} min` : `${minutes} min ${seconds}s`
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* The limits in words, for the run log, the issue and the agent's prompt.
|
|
76
|
+
* `enforced` says which of them this run can check; a limit it cannot check is
|
|
77
|
+
* named as such instead of being implied.
|
|
78
|
+
*/
|
|
79
|
+
export function describeRunLimits(limits, enforced = { files: true, cost: true, time: true }) {
|
|
80
|
+
const part = (on, value, text, offText) => (value === null ? offText : on ? text : `${text} (not checked for this run)`)
|
|
81
|
+
return [
|
|
82
|
+
part(enforced.files, limits.maxChangedFiles, `${limits.maxChangedFiles} changed files`, "no file limit"),
|
|
83
|
+
part(enforced.cost, limits.maxCostUsd, `${limits.maxCostUsd === null ? "" : usd(limits.maxCostUsd)} cost`, "no cost limit"),
|
|
84
|
+
part(enforced.time, limits.maxMinutes, `${limits.maxMinutes} minutes`, "no time limit"),
|
|
85
|
+
].join(", ")
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* The limit a run has crossed, or null. Files are checked first: a run that
|
|
90
|
+
* changes too much is the one to split, whatever it cost.
|
|
91
|
+
*
|
|
92
|
+
* @param {{ maxChangedFiles: number | null, maxCostUsd: number | null, maxMinutes: number | null }} limits
|
|
93
|
+
* Pass null for a limit that does not apply to this run.
|
|
94
|
+
* @param {{ changedFileCount?: number, costUsd?: number | null, elapsedMs?: number }} usage
|
|
95
|
+
*/
|
|
96
|
+
export function checkRunLimits(limits, usage) {
|
|
97
|
+
const files = usage.changedFileCount ?? 0
|
|
98
|
+
if (limits.maxChangedFiles !== null && files > limits.maxChangedFiles) {
|
|
99
|
+
return {
|
|
100
|
+
kind: "files",
|
|
101
|
+
limit: limits.maxChangedFiles,
|
|
102
|
+
actual: files,
|
|
103
|
+
reason: `it changed ${files} files; the limit is ${limits.maxChangedFiles}`,
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
const cost = usage.costUsd
|
|
107
|
+
if (limits.maxCostUsd !== null && typeof cost === "number" && cost >= limits.maxCostUsd) {
|
|
108
|
+
return {
|
|
109
|
+
kind: "cost",
|
|
110
|
+
limit: limits.maxCostUsd,
|
|
111
|
+
actual: cost,
|
|
112
|
+
reason: `it cost ${usd(cost)}; the limit is ${usd(limits.maxCostUsd)}`,
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
const elapsed = usage.elapsedMs ?? 0
|
|
116
|
+
if (limits.maxMinutes !== null && elapsed >= limits.maxMinutes * 60_000) {
|
|
117
|
+
return {
|
|
118
|
+
kind: "time",
|
|
119
|
+
limit: limits.maxMinutes,
|
|
120
|
+
actual: elapsed,
|
|
121
|
+
reason: `it ran for ${formatDuration(elapsed)}; the limit is ${limits.maxMinutes} minutes`,
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return null
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** The stop for an AI tool that ended the run itself at its cost limit (Claude Code's --max-budget-usd). */
|
|
128
|
+
export function costLimitStop(limit, costUsd) {
|
|
129
|
+
return {
|
|
130
|
+
kind: "cost",
|
|
131
|
+
limit,
|
|
132
|
+
actual: typeof costUsd === "number" ? costUsd : null,
|
|
133
|
+
reason: typeof costUsd === "number"
|
|
134
|
+
? `it cost ${usd(costUsd)}; the limit is ${usd(limit)}`
|
|
135
|
+
: `it reached the ${usd(limit)} cost limit`,
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** The first line of a stopped run's error message. Starts with TRIPWIRE_MARKER. */
|
|
140
|
+
export function tripwireHeadline(stop) {
|
|
141
|
+
const what = stop.kind === "files" ? "file" : stop.kind === "cost" ? "cost" : "time"
|
|
142
|
+
return `${TRIPWIRE_MARKER}: the run was stopped at its ${what} limit because ${stop.reason}.`
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* The files a run has changed, from `git status --porcelain --untracked-files=all`
|
|
147
|
+
* and `git diff --name-only` against where the run started. Counting from the
|
|
148
|
+
* run's own start, not the base branch, lets a run continue work a stopped run
|
|
149
|
+
* saved without being stopped again at once.
|
|
150
|
+
*/
|
|
151
|
+
export function parseChangedFiles(porcelain, diffNames = "") {
|
|
152
|
+
const files = new Set()
|
|
153
|
+
for (const line of String(porcelain || "").split(/\r?\n/)) {
|
|
154
|
+
if (line.length < 4) continue
|
|
155
|
+
const entry = line.slice(3)
|
|
156
|
+
const renamed = entry.indexOf(" -> ")
|
|
157
|
+
files.add((renamed >= 0 ? entry.slice(renamed + 4) : entry).trim())
|
|
158
|
+
}
|
|
159
|
+
for (const line of String(diffNames || "").split(/\r?\n/)) {
|
|
160
|
+
if (line.trim()) files.add(line.trim())
|
|
161
|
+
}
|
|
162
|
+
files.delete("")
|
|
163
|
+
return [...files].sort()
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
const REPORT_FILE_LIST_MAX = 30
|
|
167
|
+
|
|
168
|
+
/** What to tell the user about where the stopped work is. */
|
|
169
|
+
function savedWorkLines(saved) {
|
|
170
|
+
const short = saved.sha ? saved.sha.slice(0, 7) : null
|
|
171
|
+
switch (saved.kind) {
|
|
172
|
+
case "pushed":
|
|
173
|
+
return [
|
|
174
|
+
`Saved in commit \`${short}\` and pushed to branch \`${saved.branch}\` on ${saved.repo}.`,
|
|
175
|
+
"No pull request was opened, so the stopped work does not go to review on its own.",
|
|
176
|
+
]
|
|
177
|
+
case "committed":
|
|
178
|
+
return [
|
|
179
|
+
`Saved in commit \`${short}\` on local branch \`${saved.branch || "(detached)"}\` in \`${saved.worktreePath}\`.`,
|
|
180
|
+
`It was not pushed: ${saved.error || "pushing is not set up for this runner"}.`,
|
|
181
|
+
"The runner keeps this checkout until the branch is pushed.",
|
|
182
|
+
]
|
|
183
|
+
case "left_in_place":
|
|
184
|
+
return [
|
|
185
|
+
`Left as uncommitted changes in \`${saved.worktreePath}\`. ${saved.error || ""}`.trim(),
|
|
186
|
+
"The runner does not clean up this folder.",
|
|
187
|
+
]
|
|
188
|
+
case "uncommitted":
|
|
189
|
+
return [
|
|
190
|
+
`Could not be committed (${saved.error || "unknown error"}). The changes are still uncommitted in \`${saved.worktreePath}\`.`,
|
|
191
|
+
"The runner does not remove a checkout with uncommitted changes.",
|
|
192
|
+
]
|
|
193
|
+
default:
|
|
194
|
+
return ["The run had not changed any files when it was stopped."]
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function nextStepLines(stop) {
|
|
199
|
+
const setting = RUN_LIMIT_SETTINGS[stop.kind === "files" ? "maxChangedFiles" : stop.kind === "cost" ? "maxCostUsd" : "maxMinutes"]
|
|
200
|
+
const raise = `If this size is expected, raise the limit on the runner with \`--${setting.flag}\` or \`${setting.env}\`, `
|
|
201
|
+
+ `or with \`limits.${setting.profile}\` in the repository's runner profile, then run the agent again.`
|
|
202
|
+
return [
|
|
203
|
+
"- Split the issue into smaller subtasks, or plan it first, so each run fits one reviewable change.",
|
|
204
|
+
`- ${raise}`,
|
|
205
|
+
"- A new run on this issue starts from the saved work.",
|
|
206
|
+
]
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
/**
|
|
210
|
+
* The Impact Report for a stopped run: what was stopped, why, what it had
|
|
211
|
+
* spent, and where its work is. Posted on the issue and kept with the run.
|
|
212
|
+
*/
|
|
213
|
+
export function buildImpactReport({ stop, limits, enforced, elapsedMs, costUsd, changedFiles = [], saved, executor, agentName }) {
|
|
214
|
+
const shownFiles = changedFiles.slice(0, REPORT_FILE_LIST_MAX)
|
|
215
|
+
const hidden = changedFiles.length - shownFiles.length
|
|
216
|
+
return [
|
|
217
|
+
"## Impact Report: run stopped by tripwire",
|
|
218
|
+
"",
|
|
219
|
+
`**What was stopped:** ${agentName ? `${agentName}'s ` : ""}${executor || "agent"} run, after ${formatDuration(elapsedMs ?? 0)}.`,
|
|
220
|
+
`**Why:** ${stop.reason}.`,
|
|
221
|
+
`**Limits for this run:** ${describeRunLimits(limits, enforced)}.`,
|
|
222
|
+
`**Spent:** ${typeof costUsd === "number" ? usd(costUsd) : "cost not reported"}, ${formatDuration(elapsedMs ?? 0)}, ${changedFiles.length} changed file${changedFiles.length === 1 ? "" : "s"}.`,
|
|
223
|
+
"",
|
|
224
|
+
"### Where the work is",
|
|
225
|
+
...savedWorkLines(saved).map((line) => `- ${line}`),
|
|
226
|
+
...(changedFiles.length
|
|
227
|
+
? [
|
|
228
|
+
"",
|
|
229
|
+
`### Changed files (${changedFiles.length})`,
|
|
230
|
+
...shownFiles.map((file) => `- \`${file}\``),
|
|
231
|
+
...(hidden > 0 ? [`- and ${hidden} more`] : []),
|
|
232
|
+
]
|
|
233
|
+
: []),
|
|
234
|
+
"",
|
|
235
|
+
"### What to do next",
|
|
236
|
+
...nextStepLines(stop),
|
|
237
|
+
].join("\n")
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** The stopped run's error message: the headline first, so it survives a length cut that the server reads from the start. */
|
|
241
|
+
export function tripwireErrorMessage(stop, report, max = 4000) {
|
|
242
|
+
const message = `${tripwireHeadline(stop)}\n\n${report}`
|
|
243
|
+
return message.length <= max ? message : `${message.slice(0, max - 1)}…`
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
/** Run git in `cwd`: its stdout, or null when it failed (with the error passed to `onError`). */
|
|
247
|
+
function git(cwd, args, onError) {
|
|
248
|
+
return new Promise((resolve) => {
|
|
249
|
+
execFile("git", ["-C", cwd, ...args], { maxBuffer: 64 * 1024 * 1024 }, (error, stdout, stderr) => {
|
|
250
|
+
if (error) {
|
|
251
|
+
onError?.(String(stderr || error.message).trim())
|
|
252
|
+
resolve(null)
|
|
253
|
+
return
|
|
254
|
+
}
|
|
255
|
+
resolve(stdout)
|
|
256
|
+
})
|
|
257
|
+
})
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** The files a run changed since `startSha`, committed or not, untracked included. */
|
|
261
|
+
export async function listRunChangedFiles(worktreePath, startSha) {
|
|
262
|
+
const status = await git(worktreePath, ["status", "--porcelain", "--untracked-files=all"])
|
|
263
|
+
const diff = startSha ? await git(worktreePath, ["diff", "--name-only", startSha]) : null
|
|
264
|
+
return parseChangedFiles(status, diff)
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
/**
|
|
268
|
+
* Commit everything in the checkout as a stopped run's work. Hooks are skipped:
|
|
269
|
+
* one that rejects half-finished work would lose it.
|
|
270
|
+
*
|
|
271
|
+
* @returns {Promise<{ committed: boolean, sha: string | null, error: string | null }>}
|
|
272
|
+
* `committed` is false with no error when there was nothing to commit.
|
|
273
|
+
*/
|
|
274
|
+
export async function commitStoppedWork(worktreePath, { subject, body }) {
|
|
275
|
+
const dirty = (await git(worktreePath, ["status", "--porcelain"]))?.trim()
|
|
276
|
+
if (!dirty) return { committed: false, sha: null, error: null }
|
|
277
|
+
let error = null
|
|
278
|
+
const onError = (message) => { error = message.replace(/\s+/g, " ").slice(0, 200) }
|
|
279
|
+
const added = await git(worktreePath, ["add", "-A"], onError)
|
|
280
|
+
const done = added !== null
|
|
281
|
+
&& await git(worktreePath, ["commit", "--no-verify", "-m", subject, "-m", body, "-m", STOPPED_BY_TRIPWIRE_TRAILER], onError) !== null
|
|
282
|
+
if (!done) return { committed: false, sha: null, error: error || "git commit failed" }
|
|
283
|
+
const sha = (await git(worktreePath, ["rev-parse", "HEAD"]))?.trim() || null
|
|
284
|
+
return { committed: true, sha, error: null }
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/** Whether there are commits after `startSha` (or after the base branch, when the start is unknown). */
|
|
288
|
+
export async function hasCommitsSince(worktreePath, startSha) {
|
|
289
|
+
const log = await git(worktreePath, ["log", "--oneline", startSha ? `${startSha}..HEAD` : "origin/HEAD..HEAD"])
|
|
290
|
+
return Boolean(log?.trim())
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Whether a checkout's newest commit is a stopped run's saved work that is on
|
|
295
|
+
* no remote. Removing that checkout would delete the only copy.
|
|
296
|
+
*/
|
|
297
|
+
export async function holdsUnpushedStoppedWork(worktreePath) {
|
|
298
|
+
const message = (await git(worktreePath, ["log", "-1", "--format=%B"])) ?? ""
|
|
299
|
+
if (!message.includes(STOPPED_BY_TRIPWIRE_TRAILER)) return false
|
|
300
|
+
const onRemote = (await git(worktreePath, ["branch", "-r", "--contains", "HEAD"]))?.trim()
|
|
301
|
+
return !onRemote
|
|
302
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Doing what an admin decides about this runner and its runs.
|
|
3
|
+
*
|
|
4
|
+
* - Cancel: the run is cancel-requested, or the server already ended it
|
|
5
|
+
* (force-cancel marks it errored outright). The agent is stopped and nothing
|
|
6
|
+
* is committed, pushed or opened for it.
|
|
7
|
+
* - Stop: the server pauses the runner and cancel-requests its runs; the runner
|
|
8
|
+
* takes nothing new until it is resumed.
|
|
9
|
+
* - Revoke: the runner's secret stops working and registering again is refused.
|
|
10
|
+
* The runner stops its agents and waits without calling the server, so a
|
|
11
|
+
* service manager that restarts it does not turn into a retry loop.
|
|
12
|
+
* - Name and labels: a runner started with a different name, labels, repo
|
|
13
|
+
* root, executor or capacity registers again, which updates its entry.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
export const REVOKED_MESSAGE = "This runner was disconnected from the workspace. Run `pacework start` to connect it again."
|
|
17
|
+
|
|
18
|
+
/** Run statuses that mean the server has finished with the run, whatever the runner is doing. */
|
|
19
|
+
const ENDED_RUN_STATUSES = new Set(["canceled", "error", "completed"])
|
|
20
|
+
|
|
21
|
+
/** Whether a run-log or heartbeat response says the run must stop now. */
|
|
22
|
+
export function runEndedOnServer(response) {
|
|
23
|
+
if (!response) return false
|
|
24
|
+
return Boolean(response.cancel_requested) || ENDED_RUN_STATUSES.has(response.run_status)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Whether a request failed because an admin revoked this runner. */
|
|
28
|
+
export function isRevokedError(error) {
|
|
29
|
+
const message = error instanceof Error ? error.message : String(error ?? "")
|
|
30
|
+
return /revoked this runner/i.test(message)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The registration settings that differ between what this runner registered
|
|
35
|
+
* with (its saved state) and what it was started with now.
|
|
36
|
+
*/
|
|
37
|
+
export function registrationChanges(saved, wanted) {
|
|
38
|
+
if (!saved) return []
|
|
39
|
+
const changes = []
|
|
40
|
+
const same = (a, b) => (a ?? null) === (b ?? null)
|
|
41
|
+
if (!same(saved.name, wanted.name)) changes.push("name")
|
|
42
|
+
if (!same(saved.machine_name, wanted.machine_name)) changes.push("machine")
|
|
43
|
+
if (!same(saved.repo_root, wanted.repo_root)) changes.push("repo root")
|
|
44
|
+
if (!same(saved.local_executor, wanted.local_executor)) changes.push("executor")
|
|
45
|
+
if (Number(saved.max_parallel_runs ?? 1) !== Number(wanted.max_parallel_runs ?? 1)) changes.push("max parallel runs")
|
|
46
|
+
const labels = (list) => [...new Set((list ?? []).map(String))].sort().join("\n")
|
|
47
|
+
if (labels(saved.labels) !== labels(wanted.labels)) changes.push("labels")
|
|
48
|
+
return changes
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** One line naming the runner as the server knows it. */
|
|
52
|
+
export function describeRegistration(state) {
|
|
53
|
+
const labels = state?.labels?.length ? state.labels.join(", ") : "none"
|
|
54
|
+
return `Registered as "${state?.name ?? "unnamed"}" (labels: ${labels})`
|
|
55
|
+
}
|