pacework 1.7.18 → 1.7.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/pacework-runner.mjs +196 -12
- package/lib/acceptance.mjs +265 -0
- package/lib/capabilities.mjs +3 -0
- package/lib/evidence.mjs +8 -2
- package/package.json +1 -1
package/bin/pacework-runner.mjs
CHANGED
|
@@ -43,6 +43,14 @@ import { activityForStreamEvent, createActivityTracker, createStreamReader, help
|
|
|
43
43
|
import { RUNNER_VERSION } from "../lib/version.mjs"
|
|
44
44
|
import { addBranchWorktree, catchUpWithRemote, isPushRefused, pushKeepingRemoteCommits, remoteBranchExists, switchToBranch } from "../lib/branch-sync.mjs"
|
|
45
45
|
import { closeSupersededPullRequest } from "../lib/superseded-pr.mjs"
|
|
46
|
+
import {
|
|
47
|
+
acceptanceFromAgentRecords,
|
|
48
|
+
acceptancePromptSection,
|
|
49
|
+
buildAcceptanceEvidence,
|
|
50
|
+
changedAcceptanceTests,
|
|
51
|
+
commitAcceptanceWork,
|
|
52
|
+
runAcceptanceTests,
|
|
53
|
+
} from "../lib/acceptance.mjs"
|
|
46
54
|
import { agentBaseEnv, agentDeniedReads, fenceUntrusted, unsandboxedWarning } from "../lib/agent-env.mjs"
|
|
47
55
|
import {
|
|
48
56
|
buildFixInstructions,
|
|
@@ -3004,10 +3012,120 @@ async function runPlannerPromptWithOllama(prompt, config) {
|
|
|
3004
3012
|
return content
|
|
3005
3013
|
}
|
|
3006
3014
|
|
|
3015
|
+
/**
|
|
3016
|
+
* Why a build cannot run the issue's acceptance tests, or null when it can: a
|
|
3017
|
+
* sandboxed run does not run commands outside its sandbox, and a checkout
|
|
3018
|
+
* without the acceptance commit is not on the acceptance branch (PWK-206).
|
|
3019
|
+
*/
|
|
3020
|
+
async function acceptanceTestsUnavailable(run, worktreePath, acceptance) {
|
|
3021
|
+
if (runSandboxed(run)) return SANDBOXED_RUN_SKIP
|
|
3022
|
+
const sha = String(acceptance.commit_sha || "")
|
|
3023
|
+
if (!/^[0-9a-f]{7,64}$/i.test(sha)) return "the server sent no acceptance commit"
|
|
3024
|
+
const git = gitIn(worktreePath)
|
|
3025
|
+
if ((await git(["merge-base", "--is-ancestor", sha, "HEAD"])).code !== 0) {
|
|
3026
|
+
return `this checkout does not contain the acceptance commit ${sha.slice(0, 7)}`
|
|
3027
|
+
}
|
|
3028
|
+
return null
|
|
3029
|
+
}
|
|
3030
|
+
|
|
3031
|
+
/** Each acceptance test's own command, in the checkout, with no credentials in its environment. */
|
|
3032
|
+
function runAcceptanceCommands(acceptance, worktreePath, shouldStop) {
|
|
3033
|
+
const checkEnv = checkEnvironment(process.env)
|
|
3034
|
+
return runAcceptanceTests(acceptance.tests, {
|
|
3035
|
+
cwd: worktreePath,
|
|
3036
|
+
shouldStop,
|
|
3037
|
+
exec: (command, options) => execLocalCheck(command, { ...options, baseEnv: checkEnv }),
|
|
3038
|
+
})
|
|
3039
|
+
}
|
|
3040
|
+
|
|
3041
|
+
/** The acceptance test files the build changed since the acceptance commit, committed or not. */
|
|
3042
|
+
async function changedAcceptanceFiles(worktreePath, acceptance) {
|
|
3043
|
+
const files = [...new Set(acceptance.tests.map((test) => test.file).filter(Boolean))]
|
|
3044
|
+
if (!files.length || !acceptance.commit_sha) return []
|
|
3045
|
+
const diff = await gitIn(worktreePath)(["diff", "--name-status", "-M", String(acceptance.commit_sha), "--", ...files])
|
|
3046
|
+
if (diff.code !== 0) return []
|
|
3047
|
+
// A renamed test shows under its new name with -- paths; list renames from the whole tree too.
|
|
3048
|
+
const renames = await gitIn(worktreePath)(["diff", "--name-status", "-M", "--diff-filter=RD", String(acceptance.commit_sha)])
|
|
3049
|
+
const seen = new Set()
|
|
3050
|
+
return changedAcceptanceTests(`${diff.stdout}\n${renames.code === 0 ? renames.stdout : ""}`, files).filter((change) => {
|
|
3051
|
+
const key = `${change.file}:${change.change}`
|
|
3052
|
+
if (seen.has(key)) return false
|
|
3053
|
+
seen.add(key)
|
|
3054
|
+
return true
|
|
3055
|
+
})
|
|
3056
|
+
}
|
|
3057
|
+
|
|
3058
|
+
/** What the issue is told when a planning run starts, ends or fails, by what the run is for. */
|
|
3059
|
+
function plannerMilestones(run) {
|
|
3060
|
+
if (run.purpose === "readiness") return { started: "Triage started: getting this issue ready for an agent.", finished: "Triage finished.", failed: "Triage" }
|
|
3061
|
+
if (run.purpose === "acceptance_tests") {
|
|
3062
|
+
return {
|
|
3063
|
+
started: `Writing acceptance tests from the criteria${run.acceptance?.branch_name ? ` on \`${run.acceptance.branch_name}\`` : ""}.`,
|
|
3064
|
+
finished: "Acceptance tests written and pushed.",
|
|
3065
|
+
failed: "Writing acceptance tests",
|
|
3066
|
+
}
|
|
3067
|
+
}
|
|
3068
|
+
return { started: "Planning started on the local runner.", finished: "Planning finished and results were posted back to the issue.", failed: "Planning" }
|
|
3069
|
+
}
|
|
3070
|
+
|
|
3071
|
+
/**
|
|
3072
|
+
* The checkout an acceptance-test run writes in (PWK-205): the issue's own
|
|
3073
|
+
* worktree on the branch the server named, with the writing-acceptance-tests
|
|
3074
|
+
* skill installed. A planning run otherwise works in the repository root and
|
|
3075
|
+
* changes nothing.
|
|
3076
|
+
*/
|
|
3077
|
+
async function prepareAcceptanceCheckout(run, state, runLogger) {
|
|
3078
|
+
const branch = run.acceptance?.branch_name
|
|
3079
|
+
if (!branch) throw new Error("The server did not name the branch the acceptance tests go on.")
|
|
3080
|
+
const { token, repo, baseBranch } = await fetchGithubToken(state, run.workspace_id, run.task?.project_id ?? null, run.task?.team_id ?? null, run.task_id ?? null)
|
|
3081
|
+
if (!token || !repo?.owner || !repo?.repo) throw new Error("No GitHub repository is connected for this issue, so acceptance tests cannot be pushed.")
|
|
3082
|
+
run.__githubToken = token
|
|
3083
|
+
run.__repo = repo
|
|
3084
|
+
run.__baseBranch = baseBranch ?? null
|
|
3085
|
+
run.__repoRoot = await resolveRunRepoRoot(run, state)
|
|
3086
|
+
run.__serverBranch = branch
|
|
3087
|
+
const prepared = await prepareWorktree(run)
|
|
3088
|
+
if (!prepared.gitAvailable || prepared.localCheckoutMode === "repo_root") {
|
|
3089
|
+
throw new Error("Acceptance tests need a git checkout of their own, and this run would work in the repository root.")
|
|
3090
|
+
}
|
|
3091
|
+
const skillInstall = installSkills(prepared.worktreePath, run.skills, { log: (line) => runLogger.info("runner.run.skills", line) })
|
|
3092
|
+
return { ...prepared, skillInstall }
|
|
3093
|
+
}
|
|
3094
|
+
|
|
3095
|
+
/**
|
|
3096
|
+
* After the agent wrote the acceptance tests: check each one fails, commit them
|
|
3097
|
+
* as the acceptance commit and push the branch (lib/acceptance.mjs). Returns the
|
|
3098
|
+
* report the server records, with each criterion's tests.
|
|
3099
|
+
*/
|
|
3100
|
+
async function commitAcceptanceTests(run, checkout, { logState, shouldStop, runRoot }) {
|
|
3101
|
+
const { worktreePath, branch } = checkout
|
|
3102
|
+
// Installed skills never go into the commit.
|
|
3103
|
+
checkout.skillInstall.cleanup()
|
|
3104
|
+
const checkEnv = checkEnvironment(process.env)
|
|
3105
|
+
const report = await commitAcceptanceWork({
|
|
3106
|
+
checkout: worktreePath,
|
|
3107
|
+
criteria: run.acceptance?.criteria ?? [],
|
|
3108
|
+
identifier: run.task?.identifier ?? null,
|
|
3109
|
+
git: gitIn(worktreePath),
|
|
3110
|
+
exec: (command, options) => execLocalCheck(command, { ...options, baseEnv: checkEnv }),
|
|
3111
|
+
push: () => pushBranchToGitHub(worktreePath, branch, run.__githubToken, run.__repo),
|
|
3112
|
+
setAsidePath: path.join(runRoot, "left-out-of-acceptance-commit.patch"),
|
|
3113
|
+
// The commands come from the agent: a sandboxed run does not run them outside its sandbox.
|
|
3114
|
+
runTests: !runSandboxed(run),
|
|
3115
|
+
shouldStop,
|
|
3116
|
+
log: (line) => appendLogTail(logState, `> ${line}`),
|
|
3117
|
+
})
|
|
3118
|
+
return { branch_name: branch, ...report }
|
|
3119
|
+
}
|
|
3120
|
+
|
|
3007
3121
|
async function runPlannerExecutor(run, state) {
|
|
3008
3122
|
const runLogger = getRunLogger(state, run)
|
|
3009
3123
|
const runRoot = path.join(repoRoot, ".agent", "runs", run.id)
|
|
3010
3124
|
ensureDir(runRoot)
|
|
3125
|
+
const milestones = plannerMilestones(run)
|
|
3126
|
+
/** Where the agent works: the repository root, or an acceptance-test run's own checkout. */
|
|
3127
|
+
let cwd = repoRoot
|
|
3128
|
+
let acceptanceCheckout = null
|
|
3011
3129
|
|
|
3012
3130
|
const outputFile = path.join(runRoot, "planner-last-message.txt")
|
|
3013
3131
|
const logState = { tail: "" }
|
|
@@ -3066,28 +3184,35 @@ async function runPlannerExecutor(run, state) {
|
|
|
3066
3184
|
try {
|
|
3067
3185
|
const problem = credentialProblem(run)
|
|
3068
3186
|
if (problem) throw new Error(`Could not start: ${problem}`)
|
|
3187
|
+
if (run.purpose === "acceptance_tests") {
|
|
3188
|
+
acceptanceCheckout = await prepareAcceptanceCheckout(run, state, runLogger)
|
|
3189
|
+
cwd = acceptanceCheckout.worktreePath
|
|
3190
|
+
}
|
|
3069
3191
|
runLogger.info("runner.planner.started", "Local runner planner execution started", {
|
|
3070
3192
|
meta: {
|
|
3071
3193
|
issue_identifier: run.task?.identifier || run.task_id,
|
|
3072
3194
|
},
|
|
3073
3195
|
})
|
|
3074
|
-
appendLogTail(logState, `> preparing ${run.agent?.name || "planner"} run in ${
|
|
3196
|
+
appendLogTail(logState, `> preparing ${run.agent?.name || "planner"} run in ${cwd}`)
|
|
3075
3197
|
await flushPlannerLog().catch(() => {})
|
|
3076
3198
|
await postIssueMilestone(
|
|
3077
3199
|
runMcpKey(run, state),
|
|
3078
3200
|
run.task_id,
|
|
3079
|
-
|
|
3201
|
+
milestones.started,
|
|
3080
3202
|
"info",
|
|
3081
3203
|
)
|
|
3082
3204
|
|
|
3083
3205
|
appendLogTail(logState, "> loading workspace context and planner prompt...")
|
|
3084
3206
|
await flushPlannerLog().catch(() => {})
|
|
3085
|
-
const workspaceContext = await buildWorkspaceContext(
|
|
3207
|
+
const workspaceContext = await buildWorkspaceContext(cwd)
|
|
3086
3208
|
const plannerPayload = await preparePlannerRun(run, workspaceContext)
|
|
3087
3209
|
const staticResponse = typeof plannerPayload?.staticResponse === "string" ? plannerPayload.staticResponse.trim() : ""
|
|
3088
3210
|
const answers = clarificationAnswers(run)
|
|
3211
|
+
const skillsSection = acceptanceCheckout
|
|
3212
|
+
? skillsPromptSection((run.skills ?? []).filter((skill) => acceptanceCheckout.skillInstall.installed.includes(skill.slug)), { purpose: "build" })
|
|
3213
|
+
: ""
|
|
3089
3214
|
const prompt = typeof plannerPayload?.prompt === "string" && plannerPayload.prompt
|
|
3090
|
-
?
|
|
3215
|
+
? [plannerPayload.prompt, skillsSection, answers || ASK_WHEN_UNCLEAR].filter(Boolean).join("\n\n")
|
|
3091
3216
|
: ""
|
|
3092
3217
|
|
|
3093
3218
|
if (!staticResponse && !prompt) {
|
|
@@ -3101,7 +3226,7 @@ async function runPlannerExecutor(run, state) {
|
|
|
3101
3226
|
stdioSummary = staticResponse
|
|
3102
3227
|
appendLogTail(logState, "> using static planner response.")
|
|
3103
3228
|
appendLogTail(logState, staticResponse)
|
|
3104
|
-
} else if (useOllama) {
|
|
3229
|
+
} else if (useOllama && !acceptanceCheckout) {
|
|
3105
3230
|
runLogger.info("runner.planner.ollama", "Using Ollama for planner run", {
|
|
3106
3231
|
meta: {
|
|
3107
3232
|
model: ollamaConfig.model,
|
|
@@ -3118,19 +3243,19 @@ async function runPlannerExecutor(run, state) {
|
|
|
3118
3243
|
appendLogTail(logState, ollamaResponse)
|
|
3119
3244
|
} else {
|
|
3120
3245
|
const plannerMcpKey = runMcpKey(run, state)
|
|
3121
|
-
const execSpec = buildExecutorCommand(run,
|
|
3246
|
+
const execSpec = buildExecutorCommand(run, cwd, prompt, outputFile, plannerMcpKey)
|
|
3122
3247
|
appendLogTail(logState, `> spawning ${activeExecutor()} planner executor`)
|
|
3123
|
-
appendLogTail(logState, `> cwd: ${
|
|
3248
|
+
appendLogTail(logState, `> cwd: ${cwd}`)
|
|
3124
3249
|
appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
|
|
3125
3250
|
await flushPlannerLog().catch(() => {})
|
|
3126
3251
|
const plannerMcpServers = await fetchMcpServers(state)
|
|
3127
|
-
const execEnv = buildExecutorEnv(run,
|
|
3252
|
+
const execEnv = buildExecutorEnv(run, cwd, plannerMcpKey, plannerMcpServers)
|
|
3128
3253
|
plannerMcpBackups = setupExecutorMcpSideEffects(plannerMcpKey, plannerMcpServers)
|
|
3129
3254
|
|
|
3130
3255
|
let plannerStdout = ""
|
|
3131
3256
|
await Promise.race([runTimeoutPromise, new Promise((resolve, reject) => {
|
|
3132
3257
|
const child = spawn(execSpec.command, execSpec.args, {
|
|
3133
|
-
cwd
|
|
3258
|
+
cwd,
|
|
3134
3259
|
stdio: ["pipe", "pipe", "pipe"],
|
|
3135
3260
|
detached: process.platform !== "win32",
|
|
3136
3261
|
env: execEnv,
|
|
@@ -3191,7 +3316,12 @@ async function runPlannerExecutor(run, state) {
|
|
|
3191
3316
|
if (cancelRequested) throw new Error("Canceled by admin.")
|
|
3192
3317
|
|
|
3193
3318
|
const fileContent = fs.existsSync(outputFile) ? fs.readFileSync(outputFile, "utf8").trim() : ""
|
|
3194
|
-
|
|
3319
|
+
let plannerResponse = staticResponse || fileContent || stdioSummary.trim()
|
|
3320
|
+
// An acceptance-test run reports what it committed, not what the agent said.
|
|
3321
|
+
if (acceptanceCheckout) {
|
|
3322
|
+
const report = await commitAcceptanceTests(run, acceptanceCheckout, { logState, shouldStop: () => cancelRequested, runRoot })
|
|
3323
|
+
plannerResponse = JSON.stringify({ acceptance_tests: report })
|
|
3324
|
+
}
|
|
3195
3325
|
|
|
3196
3326
|
if (!plannerResponse) {
|
|
3197
3327
|
throw new Error("Planner run finished without a final response")
|
|
@@ -3205,7 +3335,7 @@ async function runPlannerExecutor(run, state) {
|
|
|
3205
3335
|
await postIssueMilestone(
|
|
3206
3336
|
runMcpKey(run, state),
|
|
3207
3337
|
run.task_id,
|
|
3208
|
-
|
|
3338
|
+
milestones.finished,
|
|
3209
3339
|
"success",
|
|
3210
3340
|
)
|
|
3211
3341
|
|
|
@@ -3222,7 +3352,7 @@ async function runPlannerExecutor(run, state) {
|
|
|
3222
3352
|
await postIssueMilestone(
|
|
3223
3353
|
runMcpKey(run, state),
|
|
3224
3354
|
run.task_id,
|
|
3225
|
-
|
|
3355
|
+
`${milestones.failed} ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
|
|
3226
3356
|
"warning",
|
|
3227
3357
|
)
|
|
3228
3358
|
|
|
@@ -3236,6 +3366,13 @@ async function runPlannerExecutor(run, state) {
|
|
|
3236
3366
|
clearInterval(logInterval)
|
|
3237
3367
|
if (runTimeoutHandle) clearTimeout(runTimeoutHandle)
|
|
3238
3368
|
cleanupExecutorMcp(plannerMcpBackups)
|
|
3369
|
+
if (acceptanceCheckout) {
|
|
3370
|
+
acceptanceCheckout.skillInstall?.cleanup()
|
|
3371
|
+
// The branch is on GitHub now. Its checkout goes, so the build can check the
|
|
3372
|
+
// branch out wherever its own checkout is (a sub-issue's is its parent's).
|
|
3373
|
+
const owner = run.__repoRoot || repoRoot
|
|
3374
|
+
await execCommandSafe("git", ["-C", owner, "worktree", "remove", "--force", acceptanceCheckout.worktreePath], owner).catch(() => null)
|
|
3375
|
+
}
|
|
3239
3376
|
}
|
|
3240
3377
|
}
|
|
3241
3378
|
|
|
@@ -3474,8 +3611,13 @@ async function runExecutor(run, state) {
|
|
|
3474
3611
|
(run.skills ?? []).filter((skill) => skillInstall.installed.includes(skill.slug)),
|
|
3475
3612
|
{ purpose: isReviewRun(run) ? "review" : "build" },
|
|
3476
3613
|
)
|
|
3614
|
+
// A build on an issue with acceptance tests starts from them (PWK-206).
|
|
3615
|
+
const acceptanceTests = !isReviewRun(run) && gitAvailable && Array.isArray(run.acceptance_tests?.tests) && run.acceptance_tests.tests.length
|
|
3616
|
+
? run.acceptance_tests
|
|
3617
|
+
: null
|
|
3477
3618
|
const prompt = [
|
|
3478
3619
|
buildPrompt(run, branch, worktreePath, { localCheckoutMode, gitAvailable, workspaceContext, runLimitsText, sameErrorLimit }),
|
|
3620
|
+
acceptanceTests ? acceptancePromptSection(acceptanceTests, { sandboxed: runSandboxed(run) && Boolean(run.__evidenceFile), cli: EVIDENCE_CLI }) : "",
|
|
3479
3621
|
skillsSection,
|
|
3480
3622
|
].filter(Boolean).join("\n\n")
|
|
3481
3623
|
const executorOptions = {
|
|
@@ -3545,6 +3687,9 @@ async function runExecutor(run, state) {
|
|
|
3545
3687
|
let runUsage = null
|
|
3546
3688
|
/** What the checks from the CI workflow found on this machine, for the run summary. */
|
|
3547
3689
|
let localChecksSummary = ""
|
|
3690
|
+
/** The acceptance tests run before the agent starts (PWK-206), and why they were not. */
|
|
3691
|
+
let acceptanceBefore = []
|
|
3692
|
+
let acceptanceSkipped = null
|
|
3548
3693
|
/** Evidence (PWK-183): checks the runner ran, and why it ran none. */
|
|
3549
3694
|
let runnerChecks = []
|
|
3550
3695
|
let runnerChecksSkipped = null
|
|
@@ -3708,6 +3853,15 @@ async function runExecutor(run, state) {
|
|
|
3708
3853
|
fs.writeFileSync(outputFile, smokeResult.summary)
|
|
3709
3854
|
}
|
|
3710
3855
|
} else {
|
|
3856
|
+
if (acceptanceTests) {
|
|
3857
|
+
acceptanceSkipped = await acceptanceTestsUnavailable(run, worktreePath, acceptanceTests)
|
|
3858
|
+
if (!acceptanceSkipped) {
|
|
3859
|
+
appendLogTail(logState, `> running ${acceptanceTests.tests.length} acceptance test(s) before the change; they should fail`)
|
|
3860
|
+
await flushRunnerLog().catch(() => {})
|
|
3861
|
+
acceptanceBefore = await runAcceptanceCommands(acceptanceTests, worktreePath, () => runEnded || cancelRequested)
|
|
3862
|
+
for (const entry of acceptanceBefore) appendLogTail(logState, `> acceptance test ${entry.result.status} before: ${entry.name}`)
|
|
3863
|
+
}
|
|
3864
|
+
}
|
|
3711
3865
|
appendLogTail(logState, `> spawning ${activeExecutor()} executor`)
|
|
3712
3866
|
appendLogTail(logState, `> cwd: ${worktreePath}`)
|
|
3713
3867
|
appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
|
|
@@ -3922,7 +4076,36 @@ async function runExecutor(run, state) {
|
|
|
3922
4076
|
runnerChecks = rerun.checks
|
|
3923
4077
|
runnerChecksSkipped = rerun.skipped
|
|
3924
4078
|
}
|
|
4079
|
+
// The acceptance tests again, now that the change is made (they should pass),
|
|
4080
|
+
// and whether the build changed them since the acceptance commit (PWK-206).
|
|
3925
4081
|
const agentEvidence = run.__evidenceFile ? readAgentEvidence(run.__evidenceFile) : { checks: [], notVerified: [] }
|
|
4082
|
+
let acceptanceEvidence = null
|
|
4083
|
+
if (acceptanceTests) {
|
|
4084
|
+
let after = acceptanceSkipped ? [] : await runAcceptanceCommands(acceptanceTests, worktreePath, () => runEnded || cancelRequested)
|
|
4085
|
+
let ranBy = "runner"
|
|
4086
|
+
// Sandboxed: the agent ran them inside its sandbox and recorded them.
|
|
4087
|
+
if (acceptanceSkipped === SANDBOXED_RUN_SKIP) {
|
|
4088
|
+
const recorded = acceptanceFromAgentRecords(acceptanceTests.tests, agentEvidence.checks)
|
|
4089
|
+
if (recorded.before.length) {
|
|
4090
|
+
acceptanceBefore = recorded.before
|
|
4091
|
+
after = recorded.after
|
|
4092
|
+
acceptanceSkipped = null
|
|
4093
|
+
ranBy = "agent"
|
|
4094
|
+
}
|
|
4095
|
+
}
|
|
4096
|
+
for (const entry of after) appendLogTail(logState, `> acceptance test ${entry.result.status} after: ${entry.name}`)
|
|
4097
|
+
const changed = await changedAcceptanceFiles(worktreePath, acceptanceTests)
|
|
4098
|
+
for (const change of changed) appendLogTail(logState, `> acceptance test ${change.change} by the build: ${change.file}`)
|
|
4099
|
+
acceptanceEvidence = buildAcceptanceEvidence({
|
|
4100
|
+
commit: acceptanceTests.commit_sha,
|
|
4101
|
+
tests: acceptanceTests.tests,
|
|
4102
|
+
before: acceptanceBefore,
|
|
4103
|
+
after,
|
|
4104
|
+
changed,
|
|
4105
|
+
skipped: acceptanceSkipped,
|
|
4106
|
+
ranBy,
|
|
4107
|
+
})
|
|
4108
|
+
}
|
|
3926
4109
|
// Read once; removed so it can never be committed with the work.
|
|
3927
4110
|
if (run.__evidenceFile) fs.rmSync(run.__evidenceFile, { force: true })
|
|
3928
4111
|
const runEvidence = buildEvidence({
|
|
@@ -3931,6 +4114,7 @@ async function runExecutor(run, state) {
|
|
|
3931
4114
|
checks: [...runnerChecks, ...agentEvidence.checks],
|
|
3932
4115
|
notVerified: agentEvidence.notVerified,
|
|
3933
4116
|
skippedReason: runnerChecksSkipped,
|
|
4117
|
+
acceptance: acceptanceEvidence,
|
|
3934
4118
|
})
|
|
3935
4119
|
|
|
3936
4120
|
executorPhase = "finished"
|
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Acceptance tests (PWK-205/206): an issue's acceptance criteria become failing
|
|
3
|
+
* tests before code is written, and the build shows them go from failing to
|
|
4
|
+
* passing without changing them.
|
|
5
|
+
*
|
|
6
|
+
* - An acceptance-test run (a planning run with purpose "acceptance_tests")
|
|
7
|
+
* writes the tests and a manifest. The runner checks each test fails, commits
|
|
8
|
+
* them as the first commit on the issue's branch and pushes it.
|
|
9
|
+
* - A build run on an issue with acceptance tests runs each one before the agent
|
|
10
|
+
* starts (it should fail) and after it finishes (it should pass), and diffs the
|
|
11
|
+
* test files against the acceptance commit. Both runs and any change to the
|
|
12
|
+
* tests go into the run's evidence and the pull request.
|
|
13
|
+
*
|
|
14
|
+
* Pure helpers, plus the steps that take an `exec` and `git` to run with.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import fs from "node:fs"
|
|
18
|
+
import path from "node:path"
|
|
19
|
+
|
|
20
|
+
/** What the agent writes at the checkout's root; never committed. */
|
|
21
|
+
export const ACCEPTANCE_MANIFEST = ".pacework-acceptance.json"
|
|
22
|
+
|
|
23
|
+
const MAX_TESTS = 50
|
|
24
|
+
const MAX_OUTPUT = 2000
|
|
25
|
+
|
|
26
|
+
const text = (value, max) => typeof value === "string" && value.trim() ? value.trim().slice(0, max) : null
|
|
27
|
+
const tail = (value, max = MAX_OUTPUT) => {
|
|
28
|
+
const out = String(value || "").trim()
|
|
29
|
+
return out.length > max ? `…${out.slice(-max)}` : out
|
|
30
|
+
}
|
|
31
|
+
const sameCriterion = (a, b) => a.replace(/\s+/g, " ").trim().toLowerCase() === b.replace(/\s+/g, " ").trim().toLowerCase()
|
|
32
|
+
|
|
33
|
+
/** A path inside the checkout, relative and without `..`; null otherwise. */
|
|
34
|
+
function insideCheckout(checkout, file) {
|
|
35
|
+
const normalized = path.normalize(file).replace(/^\.\/+/, "")
|
|
36
|
+
if (!normalized || path.isAbsolute(normalized) || normalized.split(path.sep).includes("..")) return null
|
|
37
|
+
return fs.existsSync(path.join(checkout, normalized)) ? normalized : null
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The agent's manifest, checked: each test needs a criterion, a name, a file
|
|
42
|
+
* that exists in the checkout and a command. `problems` says what was wrong;
|
|
43
|
+
* `unmapped` lists criteria with neither a test nor a reason they cannot have one.
|
|
44
|
+
*/
|
|
45
|
+
export function readAcceptanceManifest(checkout, criteria = []) {
|
|
46
|
+
const problems = []
|
|
47
|
+
let raw
|
|
48
|
+
try {
|
|
49
|
+
raw = JSON.parse(fs.readFileSync(path.join(checkout, ACCEPTANCE_MANIFEST), "utf8"))
|
|
50
|
+
} catch (error) {
|
|
51
|
+
return { tests: [], untestable: [], unmapped: [...criteria], problems: [`${ACCEPTANCE_MANIFEST} is missing or not JSON (${error?.code === "ENOENT" ? "not written" : "unreadable"})`] }
|
|
52
|
+
}
|
|
53
|
+
const tests = []
|
|
54
|
+
for (const [index, entry] of (Array.isArray(raw?.tests) ? raw.tests : []).slice(0, MAX_TESTS).entries()) {
|
|
55
|
+
const criterion = text(entry?.criterion, 500)
|
|
56
|
+
const name = text(entry?.name, 300)
|
|
57
|
+
const command = text(entry?.command, 500)
|
|
58
|
+
const file = typeof entry?.file === "string" ? insideCheckout(checkout, entry.file.trim()) : null
|
|
59
|
+
if (!criterion || !name || !command || !file) {
|
|
60
|
+
problems.push(`test ${index + 1} needs a criterion, a name, a command and a file that exists in the checkout`)
|
|
61
|
+
continue
|
|
62
|
+
}
|
|
63
|
+
tests.push({ criterion, name, file, command })
|
|
64
|
+
}
|
|
65
|
+
const untestable = (Array.isArray(raw?.untestable) ? raw.untestable : []).flatMap((entry) => {
|
|
66
|
+
const criterion = text(entry?.criterion, 500)
|
|
67
|
+
return criterion ? [{ criterion, why: text(entry?.why, 300) }] : []
|
|
68
|
+
})
|
|
69
|
+
const unmapped = criteria.filter((criterion) =>
|
|
70
|
+
!tests.some((test) => sameCriterion(test.criterion, criterion))
|
|
71
|
+
&& !untestable.some((entry) => sameCriterion(entry.criterion, criterion)))
|
|
72
|
+
if (!tests.length) problems.push("no usable test in the manifest")
|
|
73
|
+
return { tests, untestable, unmapped, problems }
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Run each test's own command. `exec(command, { cwd })` resolves
|
|
78
|
+
* { code, output, timedOut }. A test passes when its command exits 0.
|
|
79
|
+
*/
|
|
80
|
+
export async function runAcceptanceTests(tests, { cwd, exec, shouldStop = () => false }) {
|
|
81
|
+
const results = []
|
|
82
|
+
for (const test of tests) {
|
|
83
|
+
if (shouldStop()) break
|
|
84
|
+
const started = Date.now()
|
|
85
|
+
const result = await exec(test.command, { cwd })
|
|
86
|
+
results.push({
|
|
87
|
+
...test,
|
|
88
|
+
result: {
|
|
89
|
+
status: result.code === 0 && !result.timedOut ? "passed" : "failed",
|
|
90
|
+
exit_code: Number.isInteger(result.code) ? result.code : null,
|
|
91
|
+
duration_ms: Date.now() - started,
|
|
92
|
+
output: tail(result.timedOut ? `${result.output}\n(timed out)` : result.output),
|
|
93
|
+
},
|
|
94
|
+
})
|
|
95
|
+
}
|
|
96
|
+
return results
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The acceptance commit's message. */
|
|
100
|
+
export function acceptanceCommitMessage(identifier) {
|
|
101
|
+
return `Acceptance tests for ${identifier || "this issue"}`
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Test files the build changed since the acceptance commit, from
|
|
106
|
+
* `git diff --name-status -M <acceptance commit>` (working tree included):
|
|
107
|
+
* modified, deleted or renamed. Any of these is flagged for the reviewer.
|
|
108
|
+
*/
|
|
109
|
+
export function changedAcceptanceTests(nameStatus, files) {
|
|
110
|
+
const watched = new Set(files)
|
|
111
|
+
const changes = []
|
|
112
|
+
for (const line of String(nameStatus || "").split(/\r?\n/)) {
|
|
113
|
+
const parts = line.split("\t").map((part) => part.trim())
|
|
114
|
+
if (parts.length < 2 || !parts[0]) continue
|
|
115
|
+
const code = parts[0][0]
|
|
116
|
+
const from = parts[1]
|
|
117
|
+
if (!watched.has(from)) continue
|
|
118
|
+
if (code === "D") changes.push({ file: from, change: "deleted" })
|
|
119
|
+
else if (code === "R") changes.push({ file: from, change: "renamed", to: parts[2] || null })
|
|
120
|
+
else if (code === "M" || code === "T") changes.push({ file: from, change: "modified" })
|
|
121
|
+
}
|
|
122
|
+
return changes
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* The acceptance part of a build run's evidence: each test before and after the
|
|
127
|
+
* change, and the tests the build changed. `before` and `after` are the results
|
|
128
|
+
* of runAcceptanceTests; `skipped` says why they were not run.
|
|
129
|
+
*/
|
|
130
|
+
export function buildAcceptanceEvidence({ commit, tests, before = [], after = [], changed = [], skipped = null, ranBy = "runner" }) {
|
|
131
|
+
const find = (results, test) => results.find((entry) => entry.name === test.name && entry.file === test.file)?.result ?? null
|
|
132
|
+
const rows = tests.slice(0, MAX_TESTS).map((test) => {
|
|
133
|
+
const was = find(before, test)
|
|
134
|
+
const now = find(after, test)
|
|
135
|
+
return {
|
|
136
|
+
criterion: test.criterion,
|
|
137
|
+
name: test.name,
|
|
138
|
+
file: test.file,
|
|
139
|
+
command: test.command,
|
|
140
|
+
before: was ? { status: was.status, exit_code: was.exit_code } : null,
|
|
141
|
+
after: now ? { status: now.status, exit_code: now.exit_code, output: now.status === "failed" ? now.output : null } : null,
|
|
142
|
+
}
|
|
143
|
+
})
|
|
144
|
+
return {
|
|
145
|
+
commit: commit ? String(commit).slice(0, 64) : null,
|
|
146
|
+
// "agent": a sandboxed run, where the agent ran them inside its sandbox and
|
|
147
|
+
// recorded them with `pacework evidence run`; the runner did not run them itself.
|
|
148
|
+
ran_by: ranBy === "agent" ? "agent" : "runner",
|
|
149
|
+
tests: rows,
|
|
150
|
+
changed: changed.slice(0, MAX_TESTS),
|
|
151
|
+
skipped: skipped ? String(skipped).slice(0, 300) : null,
|
|
152
|
+
// Every test failed before and passes after, and none was changed.
|
|
153
|
+
failing_to_passing: rows.length > 0 && !skipped && !changed.length
|
|
154
|
+
&& rows.every((row) => row.before?.status === "failed" && row.after?.status === "passed"),
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
const cell = (value) => String(value ?? "").replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()
|
|
159
|
+
const BEFORE = { failed: "failing", passed: "**passing already**" }
|
|
160
|
+
const AFTER = { failed: "**failing**", passed: "passing" }
|
|
161
|
+
|
|
162
|
+
/** The Acceptance tests section of a pull request or a run, from the evidence. */
|
|
163
|
+
export function renderAcceptanceMarkdown(acceptance, { heading = "## Acceptance tests" } = {}) {
|
|
164
|
+
if (!acceptance?.tests?.length) return ""
|
|
165
|
+
const lines = [heading]
|
|
166
|
+
if (acceptance.commit) lines.push(`Written from the issue's acceptance criteria before the change, in commit \`${cell(acceptance.commit).slice(0, 7)}\`.`)
|
|
167
|
+
lines.push("", "| Criterion | Test | Before | After |", "| --- | --- | --- | --- |")
|
|
168
|
+
for (const row of acceptance.tests) {
|
|
169
|
+
lines.push(`| ${cell(row.criterion)} | \`${cell(row.name)}\` | ${row.before ? BEFORE[row.before.status] : "not run"} | ${row.after ? AFTER[row.after.status] : "not run"} |`)
|
|
170
|
+
}
|
|
171
|
+
if (acceptance.ran_by === "agent") lines.push("", "Run by the agent inside its sandbox and recorded with `pacework evidence run`; the runner could not run them outside the sandbox.")
|
|
172
|
+
if (acceptance.skipped) lines.push("", `The runner did not run them: ${cell(acceptance.skipped)}.`)
|
|
173
|
+
if (acceptance.changed?.length) {
|
|
174
|
+
lines.push("", "> [!WARNING]", "> **The build changed acceptance tests.** They are the issue's contract: the reviewer checks each change is explained and does not weaken the test.")
|
|
175
|
+
for (const change of acceptance.changed) lines.push(`> - \`${cell(change.file)}\` ${change.change}${change.to ? ` to \`${cell(change.to)}\`` : ""}`)
|
|
176
|
+
}
|
|
177
|
+
return lines.join("\n")
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* In a sandboxed run the runner cannot run the tests, so the agent runs them
|
|
182
|
+
* through `pacework evidence run`: each test's first record is before the
|
|
183
|
+
* change, its last record after it. Returns results shaped like
|
|
184
|
+
* runAcceptanceTests', for the tests the agent recorded.
|
|
185
|
+
*/
|
|
186
|
+
export function acceptanceFromAgentRecords(tests, records) {
|
|
187
|
+
const before = [], after = []
|
|
188
|
+
for (const test of tests) {
|
|
189
|
+
const runs = (records ?? []).filter((record) => record.command && record.command.trim() === String(test.command).trim())
|
|
190
|
+
if (!runs.length) continue
|
|
191
|
+
const result = (record) => ({ status: record.status === "passed" ? "passed" : "failed", exit_code: record.exit_code ?? null, duration_ms: record.duration_ms ?? null, output: record.output ?? null })
|
|
192
|
+
before.push({ ...test, result: result(runs[0]) })
|
|
193
|
+
if (runs.length > 1) after.push({ ...test, result: result(runs[runs.length - 1]) })
|
|
194
|
+
}
|
|
195
|
+
return { before, after }
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/** What a builder is told about the issue's acceptance tests (PWK-206). Empty when it has none. */
|
|
199
|
+
export function acceptancePromptSection(acceptance, { sandboxed = false, cli = "pacework" } = {}) {
|
|
200
|
+
const tests = Array.isArray(acceptance?.tests) ? acceptance.tests : []
|
|
201
|
+
if (!tests.length) return ""
|
|
202
|
+
return [
|
|
203
|
+
"## Acceptance tests",
|
|
204
|
+
`This issue's acceptance criteria are already tests, committed${acceptance.commit_sha ? ` in \`${String(acceptance.commit_sha).slice(0, 7)}\`` : ""} on this branch before any code. Start from them:`,
|
|
205
|
+
"1. Run each one and see it fail.",
|
|
206
|
+
"2. Make the change that makes them pass, without breaking other tests.",
|
|
207
|
+
"3. Run them again and see them pass.",
|
|
208
|
+
"Do not change, skip, weaken or delete them. If one is wrong, leave it as it is and say why in your report: the runner compares them with the acceptance commit and flags any change for the reviewer.",
|
|
209
|
+
...(sandboxed ? [`This run is sandboxed, so the runner cannot run them for you: run each one through \`${cli} evidence run -- <its command>\` before you change anything, and again at the end. The evidence on the pull request is built from those records.`] : []),
|
|
210
|
+
"",
|
|
211
|
+
...tests.map((test) => `- ${test.criterion}: \`${test.name}\` in \`${test.file}\` (run: \`${test.command}\`)`),
|
|
212
|
+
].join("\n")
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/**
|
|
216
|
+
* After the agent wrote the acceptance tests (PWK-205): read its manifest, run
|
|
217
|
+
* each test (it should fail), commit only the test files as the acceptance
|
|
218
|
+
* commit, set aside anything else the agent changed, and push. Returns the
|
|
219
|
+
* report the server records. `git(args)` resolves { code, stdout, stderr };
|
|
220
|
+
* `exec` runs a test command as runAcceptanceTests expects; `runTests` false
|
|
221
|
+
* skips running them (a sandboxed run). Changes that are not tests are saved
|
|
222
|
+
* as a patch at `setAsidePath` and taken out of the checkout.
|
|
223
|
+
*/
|
|
224
|
+
export async function commitAcceptanceWork({ checkout, criteria = [], identifier = null, git, exec, push, setAsidePath, runTests = true, shouldStop = () => false, log = () => {} }) {
|
|
225
|
+
const manifest = readAcceptanceManifest(checkout, criteria)
|
|
226
|
+
fs.rmSync(path.join(checkout, ACCEPTANCE_MANIFEST), { force: true })
|
|
227
|
+
if (!manifest.tests.length) throw new Error(`No acceptance tests were written: ${manifest.problems.join("; ")}`)
|
|
228
|
+
const before = runTests ? await runAcceptanceTests(manifest.tests, { cwd: checkout, exec, shouldStop }) : []
|
|
229
|
+
for (const entry of before) log(`acceptance test ${entry.result.status} before any code: ${entry.name}`)
|
|
230
|
+
const files = [...new Set(manifest.tests.map((test) => test.file))]
|
|
231
|
+
const added = await git(["add", "--", ...files])
|
|
232
|
+
if (added.code !== 0) throw new Error(`Could not stage the acceptance tests: ${added.stderr}`)
|
|
233
|
+
if (!(await git(["diff", "--cached", "--name-only"])).stdout.trim()) {
|
|
234
|
+
throw new Error("The manifest names tests that are already on the branch unchanged; nothing new to commit.")
|
|
235
|
+
}
|
|
236
|
+
const committed = await git(["commit", "-m", acceptanceCommitMessage(identifier)])
|
|
237
|
+
if (committed.code !== 0) throw new Error(`Could not commit the acceptance tests: ${committed.stderr}`)
|
|
238
|
+
const sha = (await git(["rev-parse", "HEAD"])).stdout.trim()
|
|
239
|
+
// Anything else the agent changed stays out of the commit: saved as a patch in
|
|
240
|
+
// the run's folder (not the repository's stash, which a person's checkout shares)
|
|
241
|
+
// and taken out of the checkout the build starts from.
|
|
242
|
+
const leftOver = (await git(["status", "--porcelain"])).stdout.trim()
|
|
243
|
+
if (leftOver) {
|
|
244
|
+
await git(["add", "-A"])
|
|
245
|
+
const patch = await git(["diff", "--cached", "--binary"])
|
|
246
|
+
if (setAsidePath) {
|
|
247
|
+
fs.mkdirSync(path.dirname(setAsidePath), { recursive: true })
|
|
248
|
+
fs.writeFileSync(setAsidePath, patch.stdout)
|
|
249
|
+
}
|
|
250
|
+
await git(["reset", "--hard", "HEAD"])
|
|
251
|
+
log(`set aside ${leftOver.split("\n").length} changed path(s) that are not acceptance tests${setAsidePath ? ` in ${setAsidePath}` : ""}`)
|
|
252
|
+
}
|
|
253
|
+
await push()
|
|
254
|
+
return {
|
|
255
|
+
commit_sha: sha,
|
|
256
|
+
tests: manifest.tests.map((test) => ({
|
|
257
|
+
...test,
|
|
258
|
+
before: before.find((entry) => entry.name === test.name && entry.file === test.file)?.result ?? null,
|
|
259
|
+
})),
|
|
260
|
+
untestable: manifest.untestable,
|
|
261
|
+
unmapped: manifest.unmapped,
|
|
262
|
+
tests_run: runTests,
|
|
263
|
+
set_aside: leftOver ? leftOver.split("\n").length : 0,
|
|
264
|
+
}
|
|
265
|
+
}
|
package/lib/capabilities.mjs
CHANGED
|
@@ -54,6 +54,9 @@ export function detectCapabilities({ repoRoot, executors = [], exec = run, platf
|
|
|
54
54
|
if (exec("git", ["--version"])) tools.push("git")
|
|
55
55
|
if (exec("docker", ["version", "--format", "{{.Server.Version}}"])) tools.push("docker")
|
|
56
56
|
if (canSandbox({ platform, executors, exec })) tools.push("sandbox")
|
|
57
|
+
// This runner writes acceptance tests in a checkout of their own (PWK-205); an
|
|
58
|
+
// older one would write them into the repository root, so it is never given them.
|
|
59
|
+
tools.push("acceptance-tests")
|
|
57
60
|
capabilities.tools = tools
|
|
58
61
|
return capabilities
|
|
59
62
|
}
|
package/lib/evidence.mjs
CHANGED
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
|
|
14
14
|
import fs from "node:fs"
|
|
15
15
|
import path from "node:path"
|
|
16
|
+
import { renderAcceptanceMarkdown } from "./acceptance.mjs"
|
|
16
17
|
|
|
17
18
|
export const EVIDENCE_VERSION = 1
|
|
18
19
|
/** The file in the checkout where `pacework evidence` writes; excluded from commits. */
|
|
@@ -87,9 +88,11 @@ export function readAgentEvidence(file) {
|
|
|
87
88
|
|
|
88
89
|
/**
|
|
89
90
|
* The run's evidence record. `role` is "build" or "review"; `rerun` is what a
|
|
90
|
-
* reviewer's runner ran again on the reviewed commit.
|
|
91
|
+
* reviewer's runner ran again on the reviewed commit. `acceptance` is the
|
|
92
|
+
* issue's acceptance tests before and after the change (PWK-206), from
|
|
93
|
+
* buildAcceptanceEvidence.
|
|
91
94
|
*/
|
|
92
|
-
export function buildEvidence({ role = "build", commit = null, checks = [], notVerified = [], skippedReason = null }) {
|
|
95
|
+
export function buildEvidence({ role = "build", commit = null, checks = [], notVerified = [], skippedReason = null, acceptance = null }) {
|
|
93
96
|
const items = checks.slice(0, MAX_ITEMS)
|
|
94
97
|
const not_verified = [...notVerified]
|
|
95
98
|
if (skippedReason) not_verified.unshift({ what: "The runner's own checks", why: skippedReason })
|
|
@@ -101,6 +104,7 @@ export function buildEvidence({ role = "build", commit = null, checks = [], notV
|
|
|
101
104
|
items,
|
|
102
105
|
not_verified: not_verified.slice(0, 20),
|
|
103
106
|
production_like: items.some((item) => item.environment === "production-like" && item.status === "passed"),
|
|
107
|
+
...(acceptance ? { acceptance } : {}),
|
|
104
108
|
}
|
|
105
109
|
}
|
|
106
110
|
|
|
@@ -139,6 +143,8 @@ export function renderEvidenceMarkdown(evidence, { heading = "## Evidence" } = {
|
|
|
139
143
|
lines.push("", "**Not verified**")
|
|
140
144
|
for (const entry of notVerified) lines.push(`- ${cell(entry.what)}${entry.why ? `: ${cell(entry.why)}` : ""}`)
|
|
141
145
|
}
|
|
146
|
+
const acceptance = renderAcceptanceMarkdown(evidence?.acceptance, { heading: "### Acceptance tests" })
|
|
147
|
+
if (acceptance) lines.push("", acceptance)
|
|
142
148
|
return lines.join("\n")
|
|
143
149
|
}
|
|
144
150
|
|