pacework 1.7.17 → 1.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/pacework-runner.mjs +200 -39
- package/lib/acceptance.mjs +265 -0
- package/lib/capabilities.mjs +3 -0
- package/lib/evidence.mjs +8 -2
- package/lib/run-kind.mjs +25 -0
- package/lib/shell-handlers.mjs +9 -9
- package/package.json +1 -1
package/bin/pacework-runner.mjs
CHANGED
|
@@ -9,6 +9,7 @@ import process from "node:process"
|
|
|
9
9
|
import { spawn, execFileSync } from "node:child_process"
|
|
10
10
|
import { AsyncLocalStorage } from "node:async_hooks"
|
|
11
11
|
import { createLogger } from "../lib/logger.mjs"
|
|
12
|
+
import { fileLimitAppliesTo, isReviewRun } from "../lib/run-kind.mjs"
|
|
12
13
|
import { detectCapabilities, detectPower } from "../lib/capabilities.mjs"
|
|
13
14
|
import { detectRunUsageLimit, USAGE_LIMIT_MARKER } from "../lib/usage-limit.mjs"
|
|
14
15
|
import {
|
|
@@ -42,6 +43,14 @@ import { activityForStreamEvent, createActivityTracker, createStreamReader, help
|
|
|
42
43
|
import { RUNNER_VERSION } from "../lib/version.mjs"
|
|
43
44
|
import { addBranchWorktree, catchUpWithRemote, isPushRefused, pushKeepingRemoteCommits, remoteBranchExists, switchToBranch } from "../lib/branch-sync.mjs"
|
|
44
45
|
import { closeSupersededPullRequest } from "../lib/superseded-pr.mjs"
|
|
46
|
+
import {
|
|
47
|
+
acceptanceFromAgentRecords,
|
|
48
|
+
acceptancePromptSection,
|
|
49
|
+
buildAcceptanceEvidence,
|
|
50
|
+
changedAcceptanceTests,
|
|
51
|
+
commitAcceptanceWork,
|
|
52
|
+
runAcceptanceTests,
|
|
53
|
+
} from "../lib/acceptance.mjs"
|
|
45
54
|
import { agentBaseEnv, agentDeniedReads, fenceUntrusted, unsandboxedWarning } from "../lib/agent-env.mjs"
|
|
46
55
|
import {
|
|
47
56
|
buildFixInstructions,
|
|
@@ -624,29 +633,6 @@ function buildRunnerCompletionComment(input) {
|
|
|
624
633
|
].join(" ")
|
|
625
634
|
}
|
|
626
635
|
|
|
627
|
-
/**
|
|
628
|
-
* Whether a run is a review. The server says so in `purpose`, from the issue's
|
|
629
|
-
* workflow stage or the agent's role; the skill slug is only a fallback for
|
|
630
|
-
* servers that predate it. Deciding by slug made a renamed reviewer build code.
|
|
631
|
-
*/
|
|
632
|
-
function isReviewRun(run) {
|
|
633
|
-
if (run?.purpose) return run.purpose === "review"
|
|
634
|
-
return getAgentSkillProfileSlug(run?.agent) === "reviewer"
|
|
635
|
-
}
|
|
636
|
-
|
|
637
|
-
function getAgentSkillProfileSlug(agent) {
|
|
638
|
-
if (!agent || typeof agent !== "object") return null
|
|
639
|
-
const direct = agent.skill_profile
|
|
640
|
-
if (direct && typeof direct === "object" && typeof direct.slug === "string") return direct.slug
|
|
641
|
-
const nested = agent.skill_profiles
|
|
642
|
-
if (Array.isArray(nested)) {
|
|
643
|
-
const first = nested.find((entry) => entry && typeof entry.slug === "string")
|
|
644
|
-
return first?.slug || null
|
|
645
|
-
}
|
|
646
|
-
if (nested && typeof nested === "object" && typeof nested.slug === "string") return nested.slug
|
|
647
|
-
return null
|
|
648
|
-
}
|
|
649
|
-
|
|
650
636
|
function parseReviewerDecisionSummary(summary) {
|
|
651
637
|
const text = String(summary || "").trim()
|
|
652
638
|
if (!text) return null
|
|
@@ -2114,9 +2100,8 @@ function buildPrompt(run, branch, worktreePath, options = {}) {
|
|
|
2114
2100
|
const gitAvailable = options.gitAvailable !== false
|
|
2115
2101
|
const workspaceContext = options.workspaceContext || null
|
|
2116
2102
|
const greenfieldWorkspace = workspaceContext?.is_empty_workspace === true
|
|
2117
|
-
// The agent's standing instructions
|
|
2118
|
-
|
|
2119
|
-
const skillPrompt = (agent?.instructions ?? agent?.skill_profile?.system_prompt)?.trim()
|
|
2103
|
+
// The agent's standing instructions.
|
|
2104
|
+
const skillPrompt = typeof agent?.instructions === "string" ? agent.instructions.trim() : ""
|
|
2120
2105
|
const base = skillPrompt ? `${skillPrompt}\n\n---\n\n` : ""
|
|
2121
2106
|
const codingRules = Array.isArray(run?.coding_rules) ? run.coding_rules : []
|
|
2122
2107
|
const reviewerContext = run?.reviewer_context || null
|
|
@@ -3027,10 +3012,120 @@ async function runPlannerPromptWithOllama(prompt, config) {
|
|
|
3027
3012
|
return content
|
|
3028
3013
|
}
|
|
3029
3014
|
|
|
3015
|
+
/**
|
|
3016
|
+
* Why a build cannot run the issue's acceptance tests, or null when it can: a
|
|
3017
|
+
* sandboxed run does not run commands outside its sandbox, and a checkout
|
|
3018
|
+
* without the acceptance commit is not on the acceptance branch (PWK-206).
|
|
3019
|
+
*/
|
|
3020
|
+
async function acceptanceTestsUnavailable(run, worktreePath, acceptance) {
|
|
3021
|
+
if (runSandboxed(run)) return SANDBOXED_RUN_SKIP
|
|
3022
|
+
const sha = String(acceptance.commit_sha || "")
|
|
3023
|
+
if (!/^[0-9a-f]{7,64}$/i.test(sha)) return "the server sent no acceptance commit"
|
|
3024
|
+
const git = gitIn(worktreePath)
|
|
3025
|
+
if ((await git(["merge-base", "--is-ancestor", sha, "HEAD"])).code !== 0) {
|
|
3026
|
+
return `this checkout does not contain the acceptance commit ${sha.slice(0, 7)}`
|
|
3027
|
+
}
|
|
3028
|
+
return null
|
|
3029
|
+
}
|
|
3030
|
+
|
|
3031
|
+
/** Each acceptance test's own command, in the checkout, with no credentials in its environment. */
|
|
3032
|
+
function runAcceptanceCommands(acceptance, worktreePath, shouldStop) {
|
|
3033
|
+
const checkEnv = checkEnvironment(process.env)
|
|
3034
|
+
return runAcceptanceTests(acceptance.tests, {
|
|
3035
|
+
cwd: worktreePath,
|
|
3036
|
+
shouldStop,
|
|
3037
|
+
exec: (command, options) => execLocalCheck(command, { ...options, baseEnv: checkEnv }),
|
|
3038
|
+
})
|
|
3039
|
+
}
|
|
3040
|
+
|
|
3041
|
+
/** The acceptance test files the build changed since the acceptance commit, committed or not. */
|
|
3042
|
+
async function changedAcceptanceFiles(worktreePath, acceptance) {
|
|
3043
|
+
const files = [...new Set(acceptance.tests.map((test) => test.file).filter(Boolean))]
|
|
3044
|
+
if (!files.length || !acceptance.commit_sha) return []
|
|
3045
|
+
const diff = await gitIn(worktreePath)(["diff", "--name-status", "-M", String(acceptance.commit_sha), "--", ...files])
|
|
3046
|
+
if (diff.code !== 0) return []
|
|
3047
|
+
// A renamed test shows under its new name with -- paths; list renames from the whole tree too.
|
|
3048
|
+
const renames = await gitIn(worktreePath)(["diff", "--name-status", "-M", "--diff-filter=RD", String(acceptance.commit_sha)])
|
|
3049
|
+
const seen = new Set()
|
|
3050
|
+
return changedAcceptanceTests(`${diff.stdout}\n${renames.code === 0 ? renames.stdout : ""}`, files).filter((change) => {
|
|
3051
|
+
const key = `${change.file}:${change.change}`
|
|
3052
|
+
if (seen.has(key)) return false
|
|
3053
|
+
seen.add(key)
|
|
3054
|
+
return true
|
|
3055
|
+
})
|
|
3056
|
+
}
|
|
3057
|
+
|
|
3058
|
+
/** What the issue is told when a planning run starts, ends or fails, by what the run is for. */
|
|
3059
|
+
function plannerMilestones(run) {
|
|
3060
|
+
if (run.purpose === "readiness") return { started: "Triage started: getting this issue ready for an agent.", finished: "Triage finished.", failed: "Triage" }
|
|
3061
|
+
if (run.purpose === "acceptance_tests") {
|
|
3062
|
+
return {
|
|
3063
|
+
started: `Writing acceptance tests from the criteria${run.acceptance?.branch_name ? ` on \`${run.acceptance.branch_name}\`` : ""}.`,
|
|
3064
|
+
finished: "Acceptance tests written and pushed.",
|
|
3065
|
+
failed: "Writing acceptance tests",
|
|
3066
|
+
}
|
|
3067
|
+
}
|
|
3068
|
+
return { started: "Planning started on the local runner.", finished: "Planning finished and results were posted back to the issue.", failed: "Planning" }
|
|
3069
|
+
}
|
|
3070
|
+
|
|
3071
|
+
/**
|
|
3072
|
+
* The checkout an acceptance-test run writes in (PWK-205): the issue's own
|
|
3073
|
+
* worktree on the branch the server named, with the writing-acceptance-tests
|
|
3074
|
+
* skill installed. A planning run otherwise works in the repository root and
|
|
3075
|
+
* changes nothing.
|
|
3076
|
+
*/
|
|
3077
|
+
async function prepareAcceptanceCheckout(run, state, runLogger) {
|
|
3078
|
+
const branch = run.acceptance?.branch_name
|
|
3079
|
+
if (!branch) throw new Error("The server did not name the branch the acceptance tests go on.")
|
|
3080
|
+
const { token, repo, baseBranch } = await fetchGithubToken(state, run.workspace_id, run.task?.project_id ?? null, run.task?.team_id ?? null, run.task_id ?? null)
|
|
3081
|
+
if (!token || !repo?.owner || !repo?.repo) throw new Error("No GitHub repository is connected for this issue, so acceptance tests cannot be pushed.")
|
|
3082
|
+
run.__githubToken = token
|
|
3083
|
+
run.__repo = repo
|
|
3084
|
+
run.__baseBranch = baseBranch ?? null
|
|
3085
|
+
run.__repoRoot = await resolveRunRepoRoot(run, state)
|
|
3086
|
+
run.__serverBranch = branch
|
|
3087
|
+
const prepared = await prepareWorktree(run)
|
|
3088
|
+
if (!prepared.gitAvailable || prepared.localCheckoutMode === "repo_root") {
|
|
3089
|
+
throw new Error("Acceptance tests need a git checkout of their own, and this run would work in the repository root.")
|
|
3090
|
+
}
|
|
3091
|
+
const skillInstall = installSkills(prepared.worktreePath, run.skills, { log: (line) => runLogger.info("runner.run.skills", line) })
|
|
3092
|
+
return { ...prepared, skillInstall }
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
/**
|
|
3096
|
+
* After the agent wrote the acceptance tests: check each one fails, commit them
|
|
3097
|
+
* as the acceptance commit and push the branch (lib/acceptance.mjs). Returns the
|
|
3098
|
+
* report the server records, with each criterion's tests.
|
|
3099
|
+
*/
|
|
3100
|
+
async function commitAcceptanceTests(run, checkout, { logState, shouldStop, runRoot }) {
|
|
3101
|
+
const { worktreePath, branch } = checkout
|
|
3102
|
+
// Installed skills never go into the commit.
|
|
3103
|
+
checkout.skillInstall.cleanup()
|
|
3104
|
+
const checkEnv = checkEnvironment(process.env)
|
|
3105
|
+
const report = await commitAcceptanceWork({
|
|
3106
|
+
checkout: worktreePath,
|
|
3107
|
+
criteria: run.acceptance?.criteria ?? [],
|
|
3108
|
+
identifier: run.task?.identifier ?? null,
|
|
3109
|
+
git: gitIn(worktreePath),
|
|
3110
|
+
exec: (command, options) => execLocalCheck(command, { ...options, baseEnv: checkEnv }),
|
|
3111
|
+
push: () => pushBranchToGitHub(worktreePath, branch, run.__githubToken, run.__repo),
|
|
3112
|
+
setAsidePath: path.join(runRoot, "left-out-of-acceptance-commit.patch"),
|
|
3113
|
+
// The commands come from the agent: a sandboxed run does not run them outside its sandbox.
|
|
3114
|
+
runTests: !runSandboxed(run),
|
|
3115
|
+
shouldStop,
|
|
3116
|
+
log: (line) => appendLogTail(logState, `> ${line}`),
|
|
3117
|
+
})
|
|
3118
|
+
return { branch_name: branch, ...report }
|
|
3119
|
+
}
|
|
3120
|
+
|
|
3030
3121
|
async function runPlannerExecutor(run, state) {
|
|
3031
3122
|
const runLogger = getRunLogger(state, run)
|
|
3032
3123
|
const runRoot = path.join(repoRoot, ".agent", "runs", run.id)
|
|
3033
3124
|
ensureDir(runRoot)
|
|
3125
|
+
const milestones = plannerMilestones(run)
|
|
3126
|
+
/** Where the agent works: the repository root, or an acceptance-test run's own checkout. */
|
|
3127
|
+
let cwd = repoRoot
|
|
3128
|
+
let acceptanceCheckout = null
|
|
3034
3129
|
|
|
3035
3130
|
const outputFile = path.join(runRoot, "planner-last-message.txt")
|
|
3036
3131
|
const logState = { tail: "" }
|
|
@@ -3089,28 +3184,35 @@ async function runPlannerExecutor(run, state) {
|
|
|
3089
3184
|
try {
|
|
3090
3185
|
const problem = credentialProblem(run)
|
|
3091
3186
|
if (problem) throw new Error(`Could not start: ${problem}`)
|
|
3187
|
+
if (run.purpose === "acceptance_tests") {
|
|
3188
|
+
acceptanceCheckout = await prepareAcceptanceCheckout(run, state, runLogger)
|
|
3189
|
+
cwd = acceptanceCheckout.worktreePath
|
|
3190
|
+
}
|
|
3092
3191
|
runLogger.info("runner.planner.started", "Local runner planner execution started", {
|
|
3093
3192
|
meta: {
|
|
3094
3193
|
issue_identifier: run.task?.identifier || run.task_id,
|
|
3095
3194
|
},
|
|
3096
3195
|
})
|
|
3097
|
-
appendLogTail(logState, `> preparing ${run.agent?.name || "planner"} run in ${
|
|
3196
|
+
appendLogTail(logState, `> preparing ${run.agent?.name || "planner"} run in ${cwd}`)
|
|
3098
3197
|
await flushPlannerLog().catch(() => {})
|
|
3099
3198
|
await postIssueMilestone(
|
|
3100
3199
|
runMcpKey(run, state),
|
|
3101
3200
|
run.task_id,
|
|
3102
|
-
|
|
3201
|
+
milestones.started,
|
|
3103
3202
|
"info",
|
|
3104
3203
|
)
|
|
3105
3204
|
|
|
3106
3205
|
appendLogTail(logState, "> loading workspace context and planner prompt...")
|
|
3107
3206
|
await flushPlannerLog().catch(() => {})
|
|
3108
|
-
const workspaceContext = await buildWorkspaceContext(
|
|
3207
|
+
const workspaceContext = await buildWorkspaceContext(cwd)
|
|
3109
3208
|
const plannerPayload = await preparePlannerRun(run, workspaceContext)
|
|
3110
3209
|
const staticResponse = typeof plannerPayload?.staticResponse === "string" ? plannerPayload.staticResponse.trim() : ""
|
|
3111
3210
|
const answers = clarificationAnswers(run)
|
|
3211
|
+
const skillsSection = acceptanceCheckout
|
|
3212
|
+
? skillsPromptSection((run.skills ?? []).filter((skill) => acceptanceCheckout.skillInstall.installed.includes(skill.slug)), { purpose: "build" })
|
|
3213
|
+
: ""
|
|
3112
3214
|
const prompt = typeof plannerPayload?.prompt === "string" && plannerPayload.prompt
|
|
3113
|
-
?
|
|
3215
|
+
? [plannerPayload.prompt, skillsSection, answers || ASK_WHEN_UNCLEAR].filter(Boolean).join("\n\n")
|
|
3114
3216
|
: ""
|
|
3115
3217
|
|
|
3116
3218
|
if (!staticResponse && !prompt) {
|
|
@@ -3124,7 +3226,7 @@ async function runPlannerExecutor(run, state) {
|
|
|
3124
3226
|
stdioSummary = staticResponse
|
|
3125
3227
|
appendLogTail(logState, "> using static planner response.")
|
|
3126
3228
|
appendLogTail(logState, staticResponse)
|
|
3127
|
-
} else if (useOllama) {
|
|
3229
|
+
} else if (useOllama && !acceptanceCheckout) {
|
|
3128
3230
|
runLogger.info("runner.planner.ollama", "Using Ollama for planner run", {
|
|
3129
3231
|
meta: {
|
|
3130
3232
|
model: ollamaConfig.model,
|
|
@@ -3141,19 +3243,19 @@ async function runPlannerExecutor(run, state) {
|
|
|
3141
3243
|
appendLogTail(logState, ollamaResponse)
|
|
3142
3244
|
} else {
|
|
3143
3245
|
const plannerMcpKey = runMcpKey(run, state)
|
|
3144
|
-
const execSpec = buildExecutorCommand(run,
|
|
3246
|
+
const execSpec = buildExecutorCommand(run, cwd, prompt, outputFile, plannerMcpKey)
|
|
3145
3247
|
appendLogTail(logState, `> spawning ${activeExecutor()} planner executor`)
|
|
3146
|
-
appendLogTail(logState, `> cwd: ${
|
|
3248
|
+
appendLogTail(logState, `> cwd: ${cwd}`)
|
|
3147
3249
|
appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
|
|
3148
3250
|
await flushPlannerLog().catch(() => {})
|
|
3149
3251
|
const plannerMcpServers = await fetchMcpServers(state)
|
|
3150
|
-
const execEnv = buildExecutorEnv(run,
|
|
3252
|
+
const execEnv = buildExecutorEnv(run, cwd, plannerMcpKey, plannerMcpServers)
|
|
3151
3253
|
plannerMcpBackups = setupExecutorMcpSideEffects(plannerMcpKey, plannerMcpServers)
|
|
3152
3254
|
|
|
3153
3255
|
let plannerStdout = ""
|
|
3154
3256
|
await Promise.race([runTimeoutPromise, new Promise((resolve, reject) => {
|
|
3155
3257
|
const child = spawn(execSpec.command, execSpec.args, {
|
|
3156
|
-
cwd
|
|
3258
|
+
cwd,
|
|
3157
3259
|
stdio: ["pipe", "pipe", "pipe"],
|
|
3158
3260
|
detached: process.platform !== "win32",
|
|
3159
3261
|
env: execEnv,
|
|
@@ -3214,7 +3316,12 @@ async function runPlannerExecutor(run, state) {
|
|
|
3214
3316
|
if (cancelRequested) throw new Error("Canceled by admin.")
|
|
3215
3317
|
|
|
3216
3318
|
const fileContent = fs.existsSync(outputFile) ? fs.readFileSync(outputFile, "utf8").trim() : ""
|
|
3217
|
-
|
|
3319
|
+
let plannerResponse = staticResponse || fileContent || stdioSummary.trim()
|
|
3320
|
+
// An acceptance-test run reports what it committed, not what the agent said.
|
|
3321
|
+
if (acceptanceCheckout) {
|
|
3322
|
+
const report = await commitAcceptanceTests(run, acceptanceCheckout, { logState, shouldStop: () => cancelRequested, runRoot })
|
|
3323
|
+
plannerResponse = JSON.stringify({ acceptance_tests: report })
|
|
3324
|
+
}
|
|
3218
3325
|
|
|
3219
3326
|
if (!plannerResponse) {
|
|
3220
3327
|
throw new Error("Planner run finished without a final response")
|
|
@@ -3228,7 +3335,7 @@ async function runPlannerExecutor(run, state) {
|
|
|
3228
3335
|
await postIssueMilestone(
|
|
3229
3336
|
runMcpKey(run, state),
|
|
3230
3337
|
run.task_id,
|
|
3231
|
-
|
|
3338
|
+
milestones.finished,
|
|
3232
3339
|
"success",
|
|
3233
3340
|
)
|
|
3234
3341
|
|
|
@@ -3245,7 +3352,7 @@ async function runPlannerExecutor(run, state) {
|
|
|
3245
3352
|
await postIssueMilestone(
|
|
3246
3353
|
runMcpKey(run, state),
|
|
3247
3354
|
run.task_id,
|
|
3248
|
-
|
|
3355
|
+
`${milestones.failed} ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
|
|
3249
3356
|
"warning",
|
|
3250
3357
|
)
|
|
3251
3358
|
|
|
@@ -3259,6 +3366,13 @@ async function runPlannerExecutor(run, state) {
|
|
|
3259
3366
|
clearInterval(logInterval)
|
|
3260
3367
|
if (runTimeoutHandle) clearTimeout(runTimeoutHandle)
|
|
3261
3368
|
cleanupExecutorMcp(plannerMcpBackups)
|
|
3369
|
+
if (acceptanceCheckout) {
|
|
3370
|
+
acceptanceCheckout.skillInstall?.cleanup()
|
|
3371
|
+
// The branch is on GitHub now. Its checkout goes, so the build can check the
|
|
3372
|
+
// branch out wherever its own checkout is (a sub-issue's is its parent's).
|
|
3373
|
+
const owner = run.__repoRoot || repoRoot
|
|
3374
|
+
await execCommandSafe("git", ["-C", owner, "worktree", "remove", "--force", acceptanceCheckout.worktreePath], owner).catch(() => null)
|
|
3375
|
+
}
|
|
3262
3376
|
}
|
|
3263
3377
|
}
|
|
3264
3378
|
|
|
@@ -3462,7 +3576,7 @@ async function runExecutor(run, state) {
|
|
|
3462
3576
|
}
|
|
3463
3577
|
const runLimits = resolveRunLimits({ args, env: process.env, profile: runnerProfile })
|
|
3464
3578
|
// Scouts survey the codebase and may touch many files on purpose.
|
|
3465
|
-
const fileLimitApplies = gitAvailable &&
|
|
3579
|
+
const fileLimitApplies = gitAvailable && fileLimitAppliesTo(run.agent)
|
|
3466
3580
|
// The commit the run starts from: only the files this run changes count.
|
|
3467
3581
|
const runStartSha = gitAvailable
|
|
3468
3582
|
? ((await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null)
|
|
@@ -3497,8 +3611,13 @@ async function runExecutor(run, state) {
|
|
|
3497
3611
|
(run.skills ?? []).filter((skill) => skillInstall.installed.includes(skill.slug)),
|
|
3498
3612
|
{ purpose: isReviewRun(run) ? "review" : "build" },
|
|
3499
3613
|
)
|
|
3614
|
+
// A build on an issue with acceptance tests starts from them (PWK-206).
|
|
3615
|
+
const acceptanceTests = !isReviewRun(run) && gitAvailable && Array.isArray(run.acceptance_tests?.tests) && run.acceptance_tests.tests.length
|
|
3616
|
+
? run.acceptance_tests
|
|
3617
|
+
: null
|
|
3500
3618
|
const prompt = [
|
|
3501
3619
|
buildPrompt(run, branch, worktreePath, { localCheckoutMode, gitAvailable, workspaceContext, runLimitsText, sameErrorLimit }),
|
|
3620
|
+
acceptanceTests ? acceptancePromptSection(acceptanceTests, { sandboxed: runSandboxed(run) && Boolean(run.__evidenceFile), cli: EVIDENCE_CLI }) : "",
|
|
3502
3621
|
skillsSection,
|
|
3503
3622
|
].filter(Boolean).join("\n\n")
|
|
3504
3623
|
const executorOptions = {
|
|
@@ -3568,6 +3687,9 @@ async function runExecutor(run, state) {
|
|
|
3568
3687
|
let runUsage = null
|
|
3569
3688
|
/** What the checks from the CI workflow found on this machine, for the run summary. */
|
|
3570
3689
|
let localChecksSummary = ""
|
|
3690
|
+
/** The acceptance tests run before the agent starts (PWK-206), and why they were not. */
|
|
3691
|
+
let acceptanceBefore = []
|
|
3692
|
+
let acceptanceSkipped = null
|
|
3571
3693
|
/** Evidence (PWK-183): checks the runner ran, and why it ran none. */
|
|
3572
3694
|
let runnerChecks = []
|
|
3573
3695
|
let runnerChecksSkipped = null
|
|
@@ -3731,6 +3853,15 @@ async function runExecutor(run, state) {
|
|
|
3731
3853
|
fs.writeFileSync(outputFile, smokeResult.summary)
|
|
3732
3854
|
}
|
|
3733
3855
|
} else {
|
|
3856
|
+
if (acceptanceTests) {
|
|
3857
|
+
acceptanceSkipped = await acceptanceTestsUnavailable(run, worktreePath, acceptanceTests)
|
|
3858
|
+
if (!acceptanceSkipped) {
|
|
3859
|
+
appendLogTail(logState, `> running ${acceptanceTests.tests.length} acceptance test(s) before the change; they should fail`)
|
|
3860
|
+
await flushRunnerLog().catch(() => {})
|
|
3861
|
+
acceptanceBefore = await runAcceptanceCommands(acceptanceTests, worktreePath, () => runEnded || cancelRequested)
|
|
3862
|
+
for (const entry of acceptanceBefore) appendLogTail(logState, `> acceptance test ${entry.result.status} before: ${entry.name}`)
|
|
3863
|
+
}
|
|
3864
|
+
}
|
|
3734
3865
|
appendLogTail(logState, `> spawning ${activeExecutor()} executor`)
|
|
3735
3866
|
appendLogTail(logState, `> cwd: ${worktreePath}`)
|
|
3736
3867
|
appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
|
|
@@ -3945,7 +4076,36 @@ async function runExecutor(run, state) {
|
|
|
3945
4076
|
runnerChecks = rerun.checks
|
|
3946
4077
|
runnerChecksSkipped = rerun.skipped
|
|
3947
4078
|
}
|
|
4079
|
+
// The acceptance tests again, now that the change is made (they should pass),
|
|
4080
|
+
// and whether the build changed them since the acceptance commit (PWK-206).
|
|
3948
4081
|
const agentEvidence = run.__evidenceFile ? readAgentEvidence(run.__evidenceFile) : { checks: [], notVerified: [] }
|
|
4082
|
+
let acceptanceEvidence = null
|
|
4083
|
+
if (acceptanceTests) {
|
|
4084
|
+
let after = acceptanceSkipped ? [] : await runAcceptanceCommands(acceptanceTests, worktreePath, () => runEnded || cancelRequested)
|
|
4085
|
+
let ranBy = "runner"
|
|
4086
|
+
// Sandboxed: the agent ran them inside its sandbox and recorded them.
|
|
4087
|
+
if (acceptanceSkipped === SANDBOXED_RUN_SKIP) {
|
|
4088
|
+
const recorded = acceptanceFromAgentRecords(acceptanceTests.tests, agentEvidence.checks)
|
|
4089
|
+
if (recorded.before.length) {
|
|
4090
|
+
acceptanceBefore = recorded.before
|
|
4091
|
+
after = recorded.after
|
|
4092
|
+
acceptanceSkipped = null
|
|
4093
|
+
ranBy = "agent"
|
|
4094
|
+
}
|
|
4095
|
+
}
|
|
4096
|
+
for (const entry of after) appendLogTail(logState, `> acceptance test ${entry.result.status} after: ${entry.name}`)
|
|
4097
|
+
const changed = await changedAcceptanceFiles(worktreePath, acceptanceTests)
|
|
4098
|
+
for (const change of changed) appendLogTail(logState, `> acceptance test ${change.change} by the build: ${change.file}`)
|
|
4099
|
+
acceptanceEvidence = buildAcceptanceEvidence({
|
|
4100
|
+
commit: acceptanceTests.commit_sha,
|
|
4101
|
+
tests: acceptanceTests.tests,
|
|
4102
|
+
before: acceptanceBefore,
|
|
4103
|
+
after,
|
|
4104
|
+
changed,
|
|
4105
|
+
skipped: acceptanceSkipped,
|
|
4106
|
+
ranBy,
|
|
4107
|
+
})
|
|
4108
|
+
}
|
|
3949
4109
|
// Read once; removed so it can never be committed with the work.
|
|
3950
4110
|
if (run.__evidenceFile) fs.rmSync(run.__evidenceFile, { force: true })
|
|
3951
4111
|
const runEvidence = buildEvidence({
|
|
@@ -3954,6 +4114,7 @@ async function runExecutor(run, state) {
|
|
|
3954
4114
|
checks: [...runnerChecks, ...agentEvidence.checks],
|
|
3955
4115
|
notVerified: agentEvidence.notVerified,
|
|
3956
4116
|
skippedReason: runnerChecksSkipped,
|
|
4117
|
+
acceptance: acceptanceEvidence,
|
|
3957
4118
|
})
|
|
3958
4119
|
|
|
3959
4120
|
executorPhase = "finished"
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Acceptance tests (PWK-205/206): an issue's acceptance criteria become failing
|
|
3
|
+
* tests before code is written, and the build shows them go from failing to
|
|
4
|
+
* passing without changing them.
|
|
5
|
+
*
|
|
6
|
+
* - An acceptance-test run (a planning run with purpose "acceptance_tests")
|
|
7
|
+
* writes the tests and a manifest. The runner checks each test fails, commits
|
|
8
|
+
* them as the first commit on the issue's branch and pushes it.
|
|
9
|
+
* - A build run on an issue with acceptance tests runs each one before the agent
|
|
10
|
+
* starts (it should fail) and after it finishes (it should pass), and diffs the
|
|
11
|
+
* test files against the acceptance commit. Both runs and any change to the
|
|
12
|
+
* tests go into the run's evidence and the pull request.
|
|
13
|
+
*
|
|
14
|
+
* Pure helpers, plus the steps that take an `exec` and `git` to run with.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import fs from "node:fs"
|
|
18
|
+
import path from "node:path"
|
|
19
|
+
|
|
20
|
+
/** What the agent writes at the checkout's root; never committed. */
|
|
21
|
+
export const ACCEPTANCE_MANIFEST = ".pacework-acceptance.json"
|
|
22
|
+
|
|
23
|
+
const MAX_TESTS = 50
|
|
24
|
+
const MAX_OUTPUT = 2000
|
|
25
|
+
|
|
26
|
+
const text = (value, max) => typeof value === "string" && value.trim() ? value.trim().slice(0, max) : null
|
|
27
|
+
const tail = (value, max = MAX_OUTPUT) => {
|
|
28
|
+
const out = String(value || "").trim()
|
|
29
|
+
return out.length > max ? `…${out.slice(-max)}` : out
|
|
30
|
+
}
|
|
31
|
+
const sameCriterion = (a, b) => a.replace(/\s+/g, " ").trim().toLowerCase() === b.replace(/\s+/g, " ").trim().toLowerCase()
|
|
32
|
+
|
|
33
|
+
/** A path inside the checkout, relative and without `..`; null otherwise. */
|
|
34
|
+
function insideCheckout(checkout, file) {
|
|
35
|
+
const normalized = path.normalize(file).replace(/^\.\/+/, "")
|
|
36
|
+
if (!normalized || path.isAbsolute(normalized) || normalized.split(path.sep).includes("..")) return null
|
|
37
|
+
return fs.existsSync(path.join(checkout, normalized)) ? normalized : null
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The agent's manifest, checked: each test needs a criterion, a name, a file
|
|
42
|
+
* that exists in the checkout and a command. `problems` says what was wrong;
|
|
43
|
+
* `unmapped` lists criteria with neither a test nor a reason they cannot have one.
|
|
44
|
+
*/
|
|
45
|
+
export function readAcceptanceManifest(checkout, criteria = []) {
|
|
46
|
+
const problems = []
|
|
47
|
+
let raw
|
|
48
|
+
try {
|
|
49
|
+
raw = JSON.parse(fs.readFileSync(path.join(checkout, ACCEPTANCE_MANIFEST), "utf8"))
|
|
50
|
+
} catch (error) {
|
|
51
|
+
return { tests: [], untestable: [], unmapped: [...criteria], problems: [`${ACCEPTANCE_MANIFEST} is missing or not JSON (${error?.code === "ENOENT" ? "not written" : "unreadable"})`] }
|
|
52
|
+
}
|
|
53
|
+
const tests = []
|
|
54
|
+
for (const [index, entry] of (Array.isArray(raw?.tests) ? raw.tests : []).slice(0, MAX_TESTS).entries()) {
|
|
55
|
+
const criterion = text(entry?.criterion, 500)
|
|
56
|
+
const name = text(entry?.name, 300)
|
|
57
|
+
const command = text(entry?.command, 500)
|
|
58
|
+
const file = typeof entry?.file === "string" ? insideCheckout(checkout, entry.file.trim()) : null
|
|
59
|
+
if (!criterion || !name || !command || !file) {
|
|
60
|
+
problems.push(`test ${index + 1} needs a criterion, a name, a command and a file that exists in the checkout`)
|
|
61
|
+
continue
|
|
62
|
+
}
|
|
63
|
+
tests.push({ criterion, name, file, command })
|
|
64
|
+
}
|
|
65
|
+
const untestable = (Array.isArray(raw?.untestable) ? raw.untestable : []).flatMap((entry) => {
|
|
66
|
+
const criterion = text(entry?.criterion, 500)
|
|
67
|
+
return criterion ? [{ criterion, why: text(entry?.why, 300) }] : []
|
|
68
|
+
})
|
|
69
|
+
const unmapped = criteria.filter((criterion) =>
|
|
70
|
+
!tests.some((test) => sameCriterion(test.criterion, criterion))
|
|
71
|
+
&& !untestable.some((entry) => sameCriterion(entry.criterion, criterion)))
|
|
72
|
+
if (!tests.length) problems.push("no usable test in the manifest")
|
|
73
|
+
return { tests, untestable, unmapped, problems }
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Run each test's own command. `exec(command, { cwd })` resolves
|
|
78
|
+
* { code, output, timedOut }. A test passes when its command exits 0.
|
|
79
|
+
*/
|
|
80
|
+
export async function runAcceptanceTests(tests, { cwd, exec, shouldStop = () => false }) {
|
|
81
|
+
const results = []
|
|
82
|
+
for (const test of tests) {
|
|
83
|
+
if (shouldStop()) break
|
|
84
|
+
const started = Date.now()
|
|
85
|
+
const result = await exec(test.command, { cwd })
|
|
86
|
+
results.push({
|
|
87
|
+
...test,
|
|
88
|
+
result: {
|
|
89
|
+
status: result.code === 0 && !result.timedOut ? "passed" : "failed",
|
|
90
|
+
exit_code: Number.isInteger(result.code) ? result.code : null,
|
|
91
|
+
duration_ms: Date.now() - started,
|
|
92
|
+
output: tail(result.timedOut ? `${result.output}\n(timed out)` : result.output),
|
|
93
|
+
},
|
|
94
|
+
})
|
|
95
|
+
}
|
|
96
|
+
return results
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The acceptance commit's message. */
|
|
100
|
+
export function acceptanceCommitMessage(identifier) {
|
|
101
|
+
return `Acceptance tests for ${identifier || "this issue"}`
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Test files the build changed since the acceptance commit, from
|
|
106
|
+
* `git diff --name-status -M <acceptance commit>` (working tree included):
|
|
107
|
+
* modified, deleted or renamed. Any of these is flagged for the reviewer.
|
|
108
|
+
*/
|
|
109
|
+
export function changedAcceptanceTests(nameStatus, files) {
|
|
110
|
+
const watched = new Set(files)
|
|
111
|
+
const changes = []
|
|
112
|
+
for (const line of String(nameStatus || "").split(/\r?\n/)) {
|
|
113
|
+
const parts = line.split("\t").map((part) => part.trim())
|
|
114
|
+
if (parts.length < 2 || !parts[0]) continue
|
|
115
|
+
const code = parts[0][0]
|
|
116
|
+
const from = parts[1]
|
|
117
|
+
if (!watched.has(from)) continue
|
|
118
|
+
if (code === "D") changes.push({ file: from, change: "deleted" })
|
|
119
|
+
else if (code === "R") changes.push({ file: from, change: "renamed", to: parts[2] || null })
|
|
120
|
+
else if (code === "M" || code === "T") changes.push({ file: from, change: "modified" })
|
|
121
|
+
}
|
|
122
|
+
return changes
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* The acceptance part of a build run's evidence: each test before and after the
|
|
127
|
+
* change, and the tests the build changed. `before` and `after` are the results
|
|
128
|
+
* of runAcceptanceTests; `skipped` says why they were not run.
|
|
129
|
+
*/
|
|
130
|
+
export function buildAcceptanceEvidence({ commit, tests, before = [], after = [], changed = [], skipped = null, ranBy = "runner" }) {
|
|
131
|
+
const find = (results, test) => results.find((entry) => entry.name === test.name && entry.file === test.file)?.result ?? null
|
|
132
|
+
const rows = tests.slice(0, MAX_TESTS).map((test) => {
|
|
133
|
+
const was = find(before, test)
|
|
134
|
+
const now = find(after, test)
|
|
135
|
+
return {
|
|
136
|
+
criterion: test.criterion,
|
|
137
|
+
name: test.name,
|
|
138
|
+
file: test.file,
|
|
139
|
+
command: test.command,
|
|
140
|
+
before: was ? { status: was.status, exit_code: was.exit_code } : null,
|
|
141
|
+
after: now ? { status: now.status, exit_code: now.exit_code, output: now.status === "failed" ? now.output : null } : null,
|
|
142
|
+
}
|
|
143
|
+
})
|
|
144
|
+
return {
|
|
145
|
+
commit: commit ? String(commit).slice(0, 64) : null,
|
|
146
|
+
// "agent": a sandboxed run, where the agent ran them inside its sandbox and
|
|
147
|
+
// recorded them with `pacework evidence run`; the runner did not run them itself.
|
|
148
|
+
ran_by: ranBy === "agent" ? "agent" : "runner",
|
|
149
|
+
tests: rows,
|
|
150
|
+
changed: changed.slice(0, MAX_TESTS),
|
|
151
|
+
skipped: skipped ? String(skipped).slice(0, 300) : null,
|
|
152
|
+
// Every test failed before and passes after, and none was changed.
|
|
153
|
+
failing_to_passing: rows.length > 0 && !skipped && !changed.length
|
|
154
|
+
&& rows.every((row) => row.before?.status === "failed" && row.after?.status === "passed"),
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const cell = (value) => String(value ?? "").replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()
|
|
159
|
+
const BEFORE = { failed: "failing", passed: "**passing already**" }
|
|
160
|
+
const AFTER = { failed: "**failing**", passed: "passing" }
|
|
161
|
+
|
|
162
|
+
/** The Acceptance tests section of a pull request or a run, from the evidence. */
|
|
163
|
+
export function renderAcceptanceMarkdown(acceptance, { heading = "## Acceptance tests" } = {}) {
|
|
164
|
+
if (!acceptance?.tests?.length) return ""
|
|
165
|
+
const lines = [heading]
|
|
166
|
+
if (acceptance.commit) lines.push(`Written from the issue's acceptance criteria before the change, in commit \`${cell(acceptance.commit).slice(0, 7)}\`.`)
|
|
167
|
+
lines.push("", "| Criterion | Test | Before | After |", "| --- | --- | --- | --- |")
|
|
168
|
+
for (const row of acceptance.tests) {
|
|
169
|
+
lines.push(`| ${cell(row.criterion)} | \`${cell(row.name)}\` | ${row.before ? BEFORE[row.before.status] : "not run"} | ${row.after ? AFTER[row.after.status] : "not run"} |`)
|
|
170
|
+
}
|
|
171
|
+
if (acceptance.ran_by === "agent") lines.push("", "Run by the agent inside its sandbox and recorded with `pacework evidence run`; the runner could not run them outside the sandbox.")
|
|
172
|
+
if (acceptance.skipped) lines.push("", `The runner did not run them: ${cell(acceptance.skipped)}.`)
|
|
173
|
+
if (acceptance.changed?.length) {
|
|
174
|
+
lines.push("", "> [!WARNING]", "> **The build changed acceptance tests.** They are the issue's contract: the reviewer checks each change is explained and does not weaken the test.")
|
|
175
|
+
for (const change of acceptance.changed) lines.push(`> - \`${cell(change.file)}\` ${change.change}${change.to ? ` to \`${cell(change.to)}\`` : ""}`)
|
|
176
|
+
}
|
|
177
|
+
return lines.join("\n")
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* In a sandboxed run the runner cannot run the tests, so the agent runs them
|
|
182
|
+
* through `pacework evidence run`: each test's first record is before the
|
|
183
|
+
* change, its last record after it. Returns results shaped like
|
|
184
|
+
* runAcceptanceTests', for the tests the agent recorded.
|
|
185
|
+
*/
|
|
186
|
+
export function acceptanceFromAgentRecords(tests, records) {
|
|
187
|
+
const before = [], after = []
|
|
188
|
+
for (const test of tests) {
|
|
189
|
+
const runs = (records ?? []).filter((record) => record.command && record.command.trim() === String(test.command).trim())
|
|
190
|
+
if (!runs.length) continue
|
|
191
|
+
const result = (record) => ({ status: record.status === "passed" ? "passed" : "failed", exit_code: record.exit_code ?? null, duration_ms: record.duration_ms ?? null, output: record.output ?? null })
|
|
192
|
+
before.push({ ...test, result: result(runs[0]) })
|
|
193
|
+
if (runs.length > 1) after.push({ ...test, result: result(runs[runs.length - 1]) })
|
|
194
|
+
}
|
|
195
|
+
return { before, after }
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** What a builder is told about the issue's acceptance tests (PWK-206). Empty when it has none. */
|
|
199
|
+
export function acceptancePromptSection(acceptance, { sandboxed = false, cli = "pacework" } = {}) {
|
|
200
|
+
const tests = Array.isArray(acceptance?.tests) ? acceptance.tests : []
|
|
201
|
+
if (!tests.length) return ""
|
|
202
|
+
return [
|
|
203
|
+
"## Acceptance tests",
|
|
204
|
+
`This issue's acceptance criteria are already tests, committed${acceptance.commit_sha ? ` in \`${String(acceptance.commit_sha).slice(0, 7)}\`` : ""} on this branch before any code. Start from them:`,
|
|
205
|
+
"1. Run each one and see it fail.",
|
|
206
|
+
"2. Make the change that makes them pass, without breaking other tests.",
|
|
207
|
+
"3. Run them again and see them pass.",
|
|
208
|
+
"Do not change, skip, weaken or delete them. If one is wrong, leave it as it is and say why in your report: the runner compares them with the acceptance commit and flags any change for the reviewer.",
|
|
209
|
+
...(sandboxed ? [`This run is sandboxed, so the runner cannot run them for you: run each one through \`${cli} evidence run -- <its command>\` before you change anything, and again at the end. The evidence on the pull request is built from those records.`] : []),
|
|
210
|
+
"",
|
|
211
|
+
...tests.map((test) => `- ${test.criterion}: \`${test.name}\` in \`${test.file}\` (run: \`${test.command}\`)`),
|
|
212
|
+
].join("\n")
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* After the agent wrote the acceptance tests (PWK-205): read its manifest, run
|
|
217
|
+
* each test (it should fail), commit only the test files as the acceptance
|
|
218
|
+
* commit, set aside anything else the agent changed, and push. Returns the
|
|
219
|
+
* report the server records. `git(args)` resolves { code, stdout, stderr };
|
|
220
|
+
* `exec` runs a test command as runAcceptanceTests expects; `runTests` false
|
|
221
|
+
* skips running them (a sandboxed run). Changes that are not tests are saved
|
|
222
|
+
* as a patch at `setAsidePath` and taken out of the checkout.
|
|
223
|
+
*/
|
|
224
|
+
export async function commitAcceptanceWork({ checkout, criteria = [], identifier = null, git, exec, push, setAsidePath, runTests = true, shouldStop = () => false, log = () => {} }) {
|
|
225
|
+
const manifest = readAcceptanceManifest(checkout, criteria)
|
|
226
|
+
fs.rmSync(path.join(checkout, ACCEPTANCE_MANIFEST), { force: true })
|
|
227
|
+
if (!manifest.tests.length) throw new Error(`No acceptance tests were written: ${manifest.problems.join("; ")}`)
|
|
228
|
+
const before = runTests ? await runAcceptanceTests(manifest.tests, { cwd: checkout, exec, shouldStop }) : []
|
|
229
|
+
for (const entry of before) log(`acceptance test ${entry.result.status} before any code: ${entry.name}`)
|
|
230
|
+
const files = [...new Set(manifest.tests.map((test) => test.file))]
|
|
231
|
+
const added = await git(["add", "--", ...files])
|
|
232
|
+
if (added.code !== 0) throw new Error(`Could not stage the acceptance tests: ${added.stderr}`)
|
|
233
|
+
if (!(await git(["diff", "--cached", "--name-only"])).stdout.trim()) {
|
|
234
|
+
throw new Error("The manifest names tests that are already on the branch unchanged; nothing new to commit.")
|
|
235
|
+
}
|
|
236
|
+
const committed = await git(["commit", "-m", acceptanceCommitMessage(identifier)])
|
|
237
|
+
if (committed.code !== 0) throw new Error(`Could not commit the acceptance tests: ${committed.stderr}`)
|
|
238
|
+
const sha = (await git(["rev-parse", "HEAD"])).stdout.trim()
|
|
239
|
+
// Anything else the agent changed stays out of the commit: saved as a patch in
|
|
240
|
+
// the run's folder (not the repository's stash, which a person's checkout shares)
|
|
241
|
+
// and taken out of the checkout the build starts from.
|
|
242
|
+
const leftOver = (await git(["status", "--porcelain"])).stdout.trim()
|
|
243
|
+
if (leftOver) {
|
|
244
|
+
await git(["add", "-A"])
|
|
245
|
+
const patch = await git(["diff", "--cached", "--binary"])
|
|
246
|
+
if (setAsidePath) {
|
|
247
|
+
fs.mkdirSync(path.dirname(setAsidePath), { recursive: true })
|
|
248
|
+
fs.writeFileSync(setAsidePath, patch.stdout)
|
|
249
|
+
}
|
|
250
|
+
await git(["reset", "--hard", "HEAD"])
|
|
251
|
+
log(`set aside ${leftOver.split("\n").length} changed path(s) that are not acceptance tests${setAsidePath ? ` in ${setAsidePath}` : ""}`)
|
|
252
|
+
}
|
|
253
|
+
await push()
|
|
254
|
+
return {
|
|
255
|
+
commit_sha: sha,
|
|
256
|
+
tests: manifest.tests.map((test) => ({
|
|
257
|
+
...test,
|
|
258
|
+
before: before.find((entry) => entry.name === test.name && entry.file === test.file)?.result ?? null,
|
|
259
|
+
})),
|
|
260
|
+
untestable: manifest.untestable,
|
|
261
|
+
unmapped: manifest.unmapped,
|
|
262
|
+
tests_run: runTests,
|
|
263
|
+
set_aside: leftOver ? leftOver.split("\n").length : 0,
|
|
264
|
+
}
|
|
265
|
+
}
|
package/lib/capabilities.mjs
CHANGED
|
@@ -54,6 +54,9 @@ export function detectCapabilities({ repoRoot, executors = [], exec = run, platf
|
|
|
54
54
|
if (exec("git", ["--version"])) tools.push("git")
|
|
55
55
|
if (exec("docker", ["version", "--format", "{{.Server.Version}}"])) tools.push("docker")
|
|
56
56
|
if (canSandbox({ platform, executors, exec })) tools.push("sandbox")
|
|
57
|
+
// This runner writes acceptance tests in a checkout of their own (PWK-205); an
|
|
58
|
+
// older one would write them into the repository root, so it is never given them.
|
|
59
|
+
tools.push("acceptance-tests")
|
|
57
60
|
capabilities.tools = tools
|
|
58
61
|
return capabilities
|
|
59
62
|
}
|
package/lib/evidence.mjs
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
import fs from "node:fs"
|
|
15
15
|
import path from "node:path"
|
|
16
|
+
import { renderAcceptanceMarkdown } from "./acceptance.mjs"
|
|
16
17
|
|
|
17
18
|
export const EVIDENCE_VERSION = 1
|
|
18
19
|
/** The file in the checkout where `pacework evidence` writes; excluded from commits. */
|
|
@@ -87,9 +88,11 @@ export function readAgentEvidence(file) {
|
|
|
87
88
|
|
|
88
89
|
/**
|
|
89
90
|
* The run's evidence record. `role` is "build" or "review"; `rerun` is what a
|
|
90
|
-
* reviewer's runner ran again on the reviewed commit.
|
|
91
|
+
* reviewer's runner ran again on the reviewed commit. `acceptance` is the
|
|
92
|
+
* issue's acceptance tests before and after the change (PWK-206), from
|
|
93
|
+
* buildAcceptanceEvidence.
|
|
91
94
|
*/
|
|
92
|
-
export function buildEvidence({ role = "build", commit = null, checks = [], notVerified = [], skippedReason = null }) {
|
|
95
|
+
export function buildEvidence({ role = "build", commit = null, checks = [], notVerified = [], skippedReason = null, acceptance = null }) {
|
|
93
96
|
const items = checks.slice(0, MAX_ITEMS)
|
|
94
97
|
const not_verified = [...notVerified]
|
|
95
98
|
if (skippedReason) not_verified.unshift({ what: "The runner's own checks", why: skippedReason })
|
|
@@ -101,6 +104,7 @@ export function buildEvidence({ role = "build", commit = null, checks = [], notV
|
|
|
101
104
|
items,
|
|
102
105
|
not_verified: not_verified.slice(0, 20),
|
|
103
106
|
production_like: items.some((item) => item.environment === "production-like" && item.status === "passed"),
|
|
107
|
+
...(acceptance ? { acceptance } : {}),
|
|
104
108
|
}
|
|
105
109
|
}
|
|
106
110
|
|
|
@@ -139,6 +143,8 @@ export function renderEvidenceMarkdown(evidence, { heading = "## Evidence" } = {
|
|
|
139
143
|
lines.push("", "**Not verified**")
|
|
140
144
|
for (const entry of notVerified) lines.push(`- ${cell(entry.what)}${entry.why ? `: ${cell(entry.why)}` : ""}`)
|
|
141
145
|
}
|
|
146
|
+
const acceptance = renderAcceptanceMarkdown(evidence?.acceptance, { heading: "### Acceptance tests" })
|
|
147
|
+
if (acceptance) lines.push("", acceptance)
|
|
142
148
|
return lines.join("\n")
|
|
143
149
|
}
|
|
144
150
|
|
package/lib/run-kind.mjs
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What kind of run this is, from what the server says about it: the run's
|
|
3
|
+
* `purpose` (its workflow stage) first, then the agent's role.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
/** The slug of the agent's role, or null when it has none. */
|
|
7
|
+
export function agentRoleSlug(agent) {
|
|
8
|
+
const slug = agent && typeof agent === "object" ? agent.role?.slug : null
|
|
9
|
+
return typeof slug === "string" && slug ? slug : null
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Whether a run is a review. The server says so in `purpose`; the agent's role
|
|
14
|
+
* decides only when a run carries no purpose. Deciding by a name alone made a
|
|
15
|
+
* renamed reviewer build code.
|
|
16
|
+
*/
|
|
17
|
+
export function isReviewRun(run) {
|
|
18
|
+
if (run?.purpose) return run.purpose === "review"
|
|
19
|
+
return agentRoleSlug(run?.agent) === "reviewer"
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
/** Scouts survey the codebase and may touch many files on purpose. */
|
|
23
|
+
export function fileLimitAppliesTo(agent) {
|
|
24
|
+
return agentRoleSlug(agent) !== "scout"
|
|
25
|
+
}
|
package/lib/shell-handlers.mjs
CHANGED
|
@@ -204,7 +204,7 @@ export function createShellHandlers({
|
|
|
204
204
|
out.log(cmd("issues", "List issues (open / active / all / in_progress …)"))
|
|
205
205
|
out.log(cmd("mine", "Issues assigned to you"))
|
|
206
206
|
out.log(cmd("issue <id>", "View issue detail, description, comments"))
|
|
207
|
-
out.log(cmd("agents", "List agents and their
|
|
207
|
+
out.log(cmd("agents", "List agents and their roles"))
|
|
208
208
|
out.log(cmd("dashboard", "Orchestration status — runs, queue, review"))
|
|
209
209
|
out.log(cmd("logtail <run-id>","Stream live log of a running job"))
|
|
210
210
|
out.log(cmd("open [id]", "Open workspace (or specific issue) in browser"))
|
|
@@ -785,20 +785,20 @@ export function createShellHandlers({
|
|
|
785
785
|
const { agents } = await res.json()
|
|
786
786
|
if (!agents?.length) { out.log(` \x1b[2mNo agents configured.\x1b[0m\n`); break }
|
|
787
787
|
|
|
788
|
-
const
|
|
789
|
-
const COL_NAME = 22,
|
|
790
|
-
const hr = ` ${"─".repeat(COL_NAME +
|
|
788
|
+
const ROLE_COLOR = { reviewer:"\x1b[33m", implementer:"\x1b[34m", planner:"\x1b[35m" }
|
|
789
|
+
const COL_NAME = 22, COL_ROLE = 16, COL_EXE = 14
|
|
790
|
+
const hr = ` ${"─".repeat(COL_NAME + COL_ROLE + COL_EXE + 4)}`
|
|
791
791
|
out.log(`\n\x1b[1mAgents\x1b[0m \x1b[2m(${agents.length})\x1b[0m`)
|
|
792
792
|
out.log(hr)
|
|
793
|
-
out.log(` \x1b[2m${"NAME".padEnd(COL_NAME)} ${"
|
|
793
|
+
out.log(` \x1b[2m${"NAME".padEnd(COL_NAME)} ${"ROLE".padEnd(COL_ROLE)} ${"EXECUTOR".padEnd(COL_EXE)}\x1b[0m`)
|
|
794
794
|
out.log(hr)
|
|
795
795
|
for (const a of agents) {
|
|
796
|
-
const
|
|
797
|
-
const
|
|
796
|
+
const role = (a.role_name ?? a.role_slug ?? "—").toLowerCase()
|
|
797
|
+
const roleCol = ROLE_COLOR[a.role_slug] ?? "\x1b[2m"
|
|
798
798
|
const name = (a.name ?? a.slug ?? a.id.slice(0,8)).slice(0, COL_NAME).padEnd(COL_NAME)
|
|
799
|
-
const
|
|
799
|
+
const roleStr = role.slice(0, COL_ROLE).padEnd(COL_ROLE)
|
|
800
800
|
const exe = executorLabel(a.local_executor ?? "").slice(0, COL_EXE).padEnd(COL_EXE)
|
|
801
|
-
out.log(` \x1b[1m${name}\x1b[0m ${
|
|
801
|
+
out.log(` \x1b[1m${name}\x1b[0m ${roleCol}${roleStr}\x1b[0m \x1b[2m${exe}\x1b[0m`)
|
|
802
802
|
}
|
|
803
803
|
out.log(hr)
|
|
804
804
|
out.log(` \x1b[2mTip: /run <issue-id> to dispatch with agent picker\x1b[0m\n`)
|