pacework 1.7.18 → 1.7.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -43,6 +43,14 @@ import { activityForStreamEvent, createActivityTracker, createStreamReader, help
43
43
  import { RUNNER_VERSION } from "../lib/version.mjs"
44
44
  import { addBranchWorktree, catchUpWithRemote, isPushRefused, pushKeepingRemoteCommits, remoteBranchExists, switchToBranch } from "../lib/branch-sync.mjs"
45
45
  import { closeSupersededPullRequest } from "../lib/superseded-pr.mjs"
46
+ import {
47
+ acceptanceFromAgentRecords,
48
+ acceptancePromptSection,
49
+ buildAcceptanceEvidence,
50
+ changedAcceptanceTests,
51
+ commitAcceptanceWork,
52
+ runAcceptanceTests,
53
+ } from "../lib/acceptance.mjs"
46
54
  import { agentBaseEnv, agentDeniedReads, fenceUntrusted, unsandboxedWarning } from "../lib/agent-env.mjs"
47
55
  import {
48
56
  buildFixInstructions,
@@ -3004,10 +3012,120 @@ async function runPlannerPromptWithOllama(prompt, config) {
3004
3012
  return content
3005
3013
  }
3006
3014
 
3015
+ /**
3016
+ * Why a build cannot run the issue's acceptance tests, or null when it can: a
3017
+ * sandboxed run does not run commands outside its sandbox, and a checkout
3018
+ * without the acceptance commit is not on the acceptance branch (PWK-206).
3019
+ */
3020
+ async function acceptanceTestsUnavailable(run, worktreePath, acceptance) {
3021
+ if (runSandboxed(run)) return SANDBOXED_RUN_SKIP
3022
+ const sha = String(acceptance.commit_sha || "")
3023
+ if (!/^[0-9a-f]{7,64}$/i.test(sha)) return "the server sent no acceptance commit"
3024
+ const git = gitIn(worktreePath)
3025
+ if ((await git(["merge-base", "--is-ancestor", sha, "HEAD"])).code !== 0) {
3026
+ return `this checkout does not contain the acceptance commit ${sha.slice(0, 7)}`
3027
+ }
3028
+ return null
3029
+ }
3030
+
3031
+ /** Each acceptance test's own command, in the checkout, with no credentials in its environment. */
3032
+ function runAcceptanceCommands(acceptance, worktreePath, shouldStop) {
3033
+ const checkEnv = checkEnvironment(process.env)
3034
+ return runAcceptanceTests(acceptance.tests, {
3035
+ cwd: worktreePath,
3036
+ shouldStop,
3037
+ exec: (command, options) => execLocalCheck(command, { ...options, baseEnv: checkEnv }),
3038
+ })
3039
+ }
3040
+
3041
+ /** The acceptance test files the build changed since the acceptance commit, committed or not. */
3042
+ async function changedAcceptanceFiles(worktreePath, acceptance) {
3043
+ const files = [...new Set(acceptance.tests.map((test) => test.file).filter(Boolean))]
3044
+ if (!files.length || !acceptance.commit_sha) return []
3045
+ const diff = await gitIn(worktreePath)(["diff", "--name-status", "-M", String(acceptance.commit_sha), "--", ...files])
3046
+ if (diff.code !== 0) return []
3047
+ // A renamed test shows under its new name with -- paths; list renames from the whole tree too.
3048
+ const renames = await gitIn(worktreePath)(["diff", "--name-status", "-M", "--diff-filter=RD", String(acceptance.commit_sha)])
3049
+ const seen = new Set()
3050
+ return changedAcceptanceTests(`${diff.stdout}\n${renames.code === 0 ? renames.stdout : ""}`, files).filter((change) => {
3051
+ const key = `${change.file}:${change.change}`
3052
+ if (seen.has(key)) return false
3053
+ seen.add(key)
3054
+ return true
3055
+ })
3056
+ }
3057
+
3058
+ /** What the issue is told when a planning run starts, ends or fails, by what the run is for. */
3059
+ function plannerMilestones(run) {
3060
+ if (run.purpose === "readiness") return { started: "Triage started: getting this issue ready for an agent.", finished: "Triage finished.", failed: "Triage" }
3061
+ if (run.purpose === "acceptance_tests") {
3062
+ return {
3063
+ started: `Writing acceptance tests from the criteria${run.acceptance?.branch_name ? ` on \`${run.acceptance.branch_name}\`` : ""}.`,
3064
+ finished: "Acceptance tests written and pushed.",
3065
+ failed: "Writing acceptance tests",
3066
+ }
3067
+ }
3068
+ return { started: "Planning started on the local runner.", finished: "Planning finished and results were posted back to the issue.", failed: "Planning" }
3069
+ }
3070
+
3071
+ /**
3072
+ * The checkout an acceptance-test run writes in (PWK-205): the issue's own
3073
+ * worktree on the branch the server named, with the writing-acceptance-tests
3074
+ * skill installed. A planning run otherwise works in the repository root and
3075
+ * changes nothing.
3076
+ */
3077
+ async function prepareAcceptanceCheckout(run, state, runLogger) {
3078
+ const branch = run.acceptance?.branch_name
3079
+ if (!branch) throw new Error("The server did not name the branch the acceptance tests go on.")
3080
+ const { token, repo, baseBranch } = await fetchGithubToken(state, run.workspace_id, run.task?.project_id ?? null, run.task?.team_id ?? null, run.task_id ?? null)
3081
+ if (!token || !repo?.owner || !repo?.repo) throw new Error("No GitHub repository is connected for this issue, so acceptance tests cannot be pushed.")
3082
+ run.__githubToken = token
3083
+ run.__repo = repo
3084
+ run.__baseBranch = baseBranch ?? null
3085
+ run.__repoRoot = await resolveRunRepoRoot(run, state)
3086
+ run.__serverBranch = branch
3087
+ const prepared = await prepareWorktree(run)
3088
+ if (!prepared.gitAvailable || prepared.localCheckoutMode === "repo_root") {
3089
+ throw new Error("Acceptance tests need a git checkout of their own, and this run would work in the repository root.")
3090
+ }
3091
+ const skillInstall = installSkills(prepared.worktreePath, run.skills, { log: (line) => runLogger.info("runner.run.skills", line) })
3092
+ return { ...prepared, skillInstall }
3093
+ }
3094
+
3095
+ /**
3096
+ * After the agent wrote the acceptance tests: check each one fails, commit them
3097
+ * as the acceptance commit and push the branch (lib/acceptance.mjs). Returns the
3098
+ * report the server records, with each criterion's tests.
3099
+ */
3100
+ async function commitAcceptanceTests(run, checkout, { logState, shouldStop, runRoot }) {
3101
+ const { worktreePath, branch } = checkout
3102
+ // Installed skills never go into the commit.
3103
+ checkout.skillInstall.cleanup()
3104
+ const checkEnv = checkEnvironment(process.env)
3105
+ const report = await commitAcceptanceWork({
3106
+ checkout: worktreePath,
3107
+ criteria: run.acceptance?.criteria ?? [],
3108
+ identifier: run.task?.identifier ?? null,
3109
+ git: gitIn(worktreePath),
3110
+ exec: (command, options) => execLocalCheck(command, { ...options, baseEnv: checkEnv }),
3111
+ push: () => pushBranchToGitHub(worktreePath, branch, run.__githubToken, run.__repo),
3112
+ setAsidePath: path.join(runRoot, "left-out-of-acceptance-commit.patch"),
3113
+ // The commands come from the agent: a sandboxed run does not run them outside its sandbox.
3114
+ runTests: !runSandboxed(run),
3115
+ shouldStop,
3116
+ log: (line) => appendLogTail(logState, `> ${line}`),
3117
+ })
3118
+ return { branch_name: branch, ...report }
3119
+ }
3120
+
3007
3121
  async function runPlannerExecutor(run, state) {
3008
3122
  const runLogger = getRunLogger(state, run)
3009
3123
  const runRoot = path.join(repoRoot, ".agent", "runs", run.id)
3010
3124
  ensureDir(runRoot)
3125
+ const milestones = plannerMilestones(run)
3126
+ /** Where the agent works: the repository root, or an acceptance-test run's own checkout. */
3127
+ let cwd = repoRoot
3128
+ let acceptanceCheckout = null
3011
3129
 
3012
3130
  const outputFile = path.join(runRoot, "planner-last-message.txt")
3013
3131
  const logState = { tail: "" }
@@ -3066,28 +3184,35 @@ async function runPlannerExecutor(run, state) {
3066
3184
  try {
3067
3185
  const problem = credentialProblem(run)
3068
3186
  if (problem) throw new Error(`Could not start: ${problem}`)
3187
+ if (run.purpose === "acceptance_tests") {
3188
+ acceptanceCheckout = await prepareAcceptanceCheckout(run, state, runLogger)
3189
+ cwd = acceptanceCheckout.worktreePath
3190
+ }
3069
3191
  runLogger.info("runner.planner.started", "Local runner planner execution started", {
3070
3192
  meta: {
3071
3193
  issue_identifier: run.task?.identifier || run.task_id,
3072
3194
  },
3073
3195
  })
3074
- appendLogTail(logState, `> preparing ${run.agent?.name || "planner"} run in ${repoRoot}`)
3196
+ appendLogTail(logState, `> preparing ${run.agent?.name || "planner"} run in ${cwd}`)
3075
3197
  await flushPlannerLog().catch(() => {})
3076
3198
  await postIssueMilestone(
3077
3199
  runMcpKey(run, state),
3078
3200
  run.task_id,
3079
- "Planning started on the local runner.",
3201
+ milestones.started,
3080
3202
  "info",
3081
3203
  )
3082
3204
 
3083
3205
  appendLogTail(logState, "> loading workspace context and planner prompt...")
3084
3206
  await flushPlannerLog().catch(() => {})
3085
- const workspaceContext = await buildWorkspaceContext(repoRoot)
3207
+ const workspaceContext = await buildWorkspaceContext(cwd)
3086
3208
  const plannerPayload = await preparePlannerRun(run, workspaceContext)
3087
3209
  const staticResponse = typeof plannerPayload?.staticResponse === "string" ? plannerPayload.staticResponse.trim() : ""
3088
3210
  const answers = clarificationAnswers(run)
3211
+ const skillsSection = acceptanceCheckout
3212
+ ? skillsPromptSection((run.skills ?? []).filter((skill) => acceptanceCheckout.skillInstall.installed.includes(skill.slug)), { purpose: "build" })
3213
+ : ""
3089
3214
  const prompt = typeof plannerPayload?.prompt === "string" && plannerPayload.prompt
3090
- ? `${plannerPayload.prompt}\n\n${answers || ASK_WHEN_UNCLEAR}`
3215
+ ? [plannerPayload.prompt, skillsSection, answers || ASK_WHEN_UNCLEAR].filter(Boolean).join("\n\n")
3091
3216
  : ""
3092
3217
 
3093
3218
  if (!staticResponse && !prompt) {
@@ -3101,7 +3226,7 @@ async function runPlannerExecutor(run, state) {
3101
3226
  stdioSummary = staticResponse
3102
3227
  appendLogTail(logState, "> using static planner response.")
3103
3228
  appendLogTail(logState, staticResponse)
3104
- } else if (useOllama) {
3229
+ } else if (useOllama && !acceptanceCheckout) {
3105
3230
  runLogger.info("runner.planner.ollama", "Using Ollama for planner run", {
3106
3231
  meta: {
3107
3232
  model: ollamaConfig.model,
@@ -3118,19 +3243,19 @@ async function runPlannerExecutor(run, state) {
3118
3243
  appendLogTail(logState, ollamaResponse)
3119
3244
  } else {
3120
3245
  const plannerMcpKey = runMcpKey(run, state)
3121
- const execSpec = buildExecutorCommand(run, repoRoot, prompt, outputFile, plannerMcpKey)
3246
+ const execSpec = buildExecutorCommand(run, cwd, prompt, outputFile, plannerMcpKey)
3122
3247
  appendLogTail(logState, `> spawning ${activeExecutor()} planner executor`)
3123
- appendLogTail(logState, `> cwd: ${repoRoot}`)
3248
+ appendLogTail(logState, `> cwd: ${cwd}`)
3124
3249
  appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
3125
3250
  await flushPlannerLog().catch(() => {})
3126
3251
  const plannerMcpServers = await fetchMcpServers(state)
3127
- const execEnv = buildExecutorEnv(run, repoRoot, plannerMcpKey, plannerMcpServers)
3252
+ const execEnv = buildExecutorEnv(run, cwd, plannerMcpKey, plannerMcpServers)
3128
3253
  plannerMcpBackups = setupExecutorMcpSideEffects(plannerMcpKey, plannerMcpServers)
3129
3254
 
3130
3255
  let plannerStdout = ""
3131
3256
  await Promise.race([runTimeoutPromise, new Promise((resolve, reject) => {
3132
3257
  const child = spawn(execSpec.command, execSpec.args, {
3133
- cwd: repoRoot,
3258
+ cwd,
3134
3259
  stdio: ["pipe", "pipe", "pipe"],
3135
3260
  detached: process.platform !== "win32",
3136
3261
  env: execEnv,
@@ -3191,7 +3316,12 @@ async function runPlannerExecutor(run, state) {
3191
3316
  if (cancelRequested) throw new Error("Canceled by admin.")
3192
3317
 
3193
3318
  const fileContent = fs.existsSync(outputFile) ? fs.readFileSync(outputFile, "utf8").trim() : ""
3194
- const plannerResponse = staticResponse || fileContent || stdioSummary.trim()
3319
+ let plannerResponse = staticResponse || fileContent || stdioSummary.trim()
3320
+ // An acceptance-test run reports what it committed, not what the agent said.
3321
+ if (acceptanceCheckout) {
3322
+ const report = await commitAcceptanceTests(run, acceptanceCheckout, { logState, shouldStop: () => cancelRequested, runRoot })
3323
+ plannerResponse = JSON.stringify({ acceptance_tests: report })
3324
+ }
3195
3325
 
3196
3326
  if (!plannerResponse) {
3197
3327
  throw new Error("Planner run finished without a final response")
@@ -3205,7 +3335,7 @@ async function runPlannerExecutor(run, state) {
3205
3335
  await postIssueMilestone(
3206
3336
  runMcpKey(run, state),
3207
3337
  run.task_id,
3208
- "Planning finished and results were posted back to the issue.",
3338
+ milestones.finished,
3209
3339
  "success",
3210
3340
  )
3211
3341
 
@@ -3222,7 +3352,7 @@ async function runPlannerExecutor(run, state) {
3222
3352
  await postIssueMilestone(
3223
3353
  runMcpKey(run, state),
3224
3354
  run.task_id,
3225
- `Planning ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
3355
+ `${milestones.failed} ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
3226
3356
  "warning",
3227
3357
  )
3228
3358
 
@@ -3236,6 +3366,13 @@ async function runPlannerExecutor(run, state) {
3236
3366
  clearInterval(logInterval)
3237
3367
  if (runTimeoutHandle) clearTimeout(runTimeoutHandle)
3238
3368
  cleanupExecutorMcp(plannerMcpBackups)
3369
+ if (acceptanceCheckout) {
3370
+ acceptanceCheckout.skillInstall?.cleanup()
3371
+ // The branch is on GitHub now. Its checkout goes, so the build can check the
3372
+ // branch out wherever its own checkout is (a sub-issue's is its parent's).
3373
+ const owner = run.__repoRoot || repoRoot
3374
+ await execCommandSafe("git", ["-C", owner, "worktree", "remove", "--force", acceptanceCheckout.worktreePath], owner).catch(() => null)
3375
+ }
3239
3376
  }
3240
3377
  }
3241
3378
 
@@ -3474,8 +3611,13 @@ async function runExecutor(run, state) {
3474
3611
  (run.skills ?? []).filter((skill) => skillInstall.installed.includes(skill.slug)),
3475
3612
  { purpose: isReviewRun(run) ? "review" : "build" },
3476
3613
  )
3614
+ // A build on an issue with acceptance tests starts from them (PWK-206).
3615
+ const acceptanceTests = !isReviewRun(run) && gitAvailable && Array.isArray(run.acceptance_tests?.tests) && run.acceptance_tests.tests.length
3616
+ ? run.acceptance_tests
3617
+ : null
3477
3618
  const prompt = [
3478
3619
  buildPrompt(run, branch, worktreePath, { localCheckoutMode, gitAvailable, workspaceContext, runLimitsText, sameErrorLimit }),
3620
+ acceptanceTests ? acceptancePromptSection(acceptanceTests, { sandboxed: runSandboxed(run) && Boolean(run.__evidenceFile), cli: EVIDENCE_CLI }) : "",
3479
3621
  skillsSection,
3480
3622
  ].filter(Boolean).join("\n\n")
3481
3623
  const executorOptions = {
@@ -3545,6 +3687,9 @@ async function runExecutor(run, state) {
3545
3687
  let runUsage = null
3546
3688
  /** What the checks from the CI workflow found on this machine, for the run summary. */
3547
3689
  let localChecksSummary = ""
3690
+ /** The acceptance tests run before the agent starts (PWK-206), and why they were not. */
3691
+ let acceptanceBefore = []
3692
+ let acceptanceSkipped = null
3548
3693
  /** Evidence (PWK-183): checks the runner ran, and why it ran none. */
3549
3694
  let runnerChecks = []
3550
3695
  let runnerChecksSkipped = null
@@ -3708,6 +3853,15 @@ async function runExecutor(run, state) {
3708
3853
  fs.writeFileSync(outputFile, smokeResult.summary)
3709
3854
  }
3710
3855
  } else {
3856
+ if (acceptanceTests) {
3857
+ acceptanceSkipped = await acceptanceTestsUnavailable(run, worktreePath, acceptanceTests)
3858
+ if (!acceptanceSkipped) {
3859
+ appendLogTail(logState, `> running ${acceptanceTests.tests.length} acceptance test(s) before the change; they should fail`)
3860
+ await flushRunnerLog().catch(() => {})
3861
+ acceptanceBefore = await runAcceptanceCommands(acceptanceTests, worktreePath, () => runEnded || cancelRequested)
3862
+ for (const entry of acceptanceBefore) appendLogTail(logState, `> acceptance test ${entry.result.status} before: ${entry.name}`)
3863
+ }
3864
+ }
3711
3865
  appendLogTail(logState, `> spawning ${activeExecutor()} executor`)
3712
3866
  appendLogTail(logState, `> cwd: ${worktreePath}`)
3713
3867
  appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
@@ -3922,7 +4076,36 @@ async function runExecutor(run, state) {
3922
4076
  runnerChecks = rerun.checks
3923
4077
  runnerChecksSkipped = rerun.skipped
3924
4078
  }
4079
+ // The acceptance tests again, now that the change is made (they should pass),
4080
+ // and whether the build changed them since the acceptance commit (PWK-206).
3925
4081
  const agentEvidence = run.__evidenceFile ? readAgentEvidence(run.__evidenceFile) : { checks: [], notVerified: [] }
4082
+ let acceptanceEvidence = null
4083
+ if (acceptanceTests) {
4084
+ let after = acceptanceSkipped ? [] : await runAcceptanceCommands(acceptanceTests, worktreePath, () => runEnded || cancelRequested)
4085
+ let ranBy = "runner"
4086
+ // Sandboxed: the agent ran them inside its sandbox and recorded them.
4087
+ if (acceptanceSkipped === SANDBOXED_RUN_SKIP) {
4088
+ const recorded = acceptanceFromAgentRecords(acceptanceTests.tests, agentEvidence.checks)
4089
+ if (recorded.before.length) {
4090
+ acceptanceBefore = recorded.before
4091
+ after = recorded.after
4092
+ acceptanceSkipped = null
4093
+ ranBy = "agent"
4094
+ }
4095
+ }
4096
+ for (const entry of after) appendLogTail(logState, `> acceptance test ${entry.result.status} after: ${entry.name}`)
4097
+ const changed = await changedAcceptanceFiles(worktreePath, acceptanceTests)
4098
+ for (const change of changed) appendLogTail(logState, `> acceptance test ${change.change} by the build: ${change.file}`)
4099
+ acceptanceEvidence = buildAcceptanceEvidence({
4100
+ commit: acceptanceTests.commit_sha,
4101
+ tests: acceptanceTests.tests,
4102
+ before: acceptanceBefore,
4103
+ after,
4104
+ changed,
4105
+ skipped: acceptanceSkipped,
4106
+ ranBy,
4107
+ })
4108
+ }
3926
4109
  // Read once; removed so it can never be committed with the work.
3927
4110
  if (run.__evidenceFile) fs.rmSync(run.__evidenceFile, { force: true })
3928
4111
  const runEvidence = buildEvidence({
@@ -3931,6 +4114,7 @@ async function runExecutor(run, state) {
3931
4114
  checks: [...runnerChecks, ...agentEvidence.checks],
3932
4115
  notVerified: agentEvidence.notVerified,
3933
4116
  skippedReason: runnerChecksSkipped,
4117
+ acceptance: acceptanceEvidence,
3934
4118
  })
3935
4119
 
3936
4120
  executorPhase = "finished"
@@ -0,0 +1,265 @@
1
+ /**
2
+ * Acceptance tests (PWK-205/206): an issue's acceptance criteria become failing
3
+ * tests before code is written, and the build shows them go from failing to
4
+ * passing without changing them.
5
+ *
6
+ * - An acceptance-test run (a planning run with purpose "acceptance_tests")
7
+ * writes the tests and a manifest. The runner checks each test fails, commits
8
+ * them as the first commit on the issue's branch and pushes it.
9
+ * - A build run on an issue with acceptance tests runs each one before the agent
10
+ * starts (it should fail) and after it finishes (it should pass), and diffs the
11
+ * test files against the acceptance commit. Both runs and any change to the
12
+ * tests go into the run's evidence and the pull request.
13
+ *
14
+ * Pure helpers, plus the steps that take an `exec` and `git` to run with.
15
+ */
16
+
17
+ import fs from "node:fs"
18
+ import path from "node:path"
19
+
20
+ /** What the agent writes at the checkout's root; never committed. */
21
+ export const ACCEPTANCE_MANIFEST = ".pacework-acceptance.json"
22
+
23
+ const MAX_TESTS = 50
24
+ const MAX_OUTPUT = 2000
25
+
26
+ const text = (value, max) => typeof value === "string" && value.trim() ? value.trim().slice(0, max) : null
27
+ const tail = (value, max = MAX_OUTPUT) => {
28
+ const out = String(value || "").trim()
29
+ return out.length > max ? `…${out.slice(-max)}` : out
30
+ }
31
+ const sameCriterion = (a, b) => a.replace(/\s+/g, " ").trim().toLowerCase() === b.replace(/\s+/g, " ").trim().toLowerCase()
32
+
33
+ /** A path inside the checkout, relative and without `..`; null otherwise. */
34
+ function insideCheckout(checkout, file) {
35
+ const normalized = path.normalize(file).replace(/^\.\/+/, "")
36
+ if (!normalized || path.isAbsolute(normalized) || normalized.split(path.sep).includes("..")) return null
37
+ return fs.existsSync(path.join(checkout, normalized)) ? normalized : null
38
+ }
39
+
40
+ /**
41
+ * The agent's manifest, checked: each test needs a criterion, a name, a file
42
+ * that exists in the checkout and a command. `problems` says what was wrong;
43
+ * `unmapped` lists criteria with neither a test nor a reason they cannot have one.
44
+ */
45
+ export function readAcceptanceManifest(checkout, criteria = []) {
46
+ const problems = []
47
+ let raw
48
+ try {
49
+ raw = JSON.parse(fs.readFileSync(path.join(checkout, ACCEPTANCE_MANIFEST), "utf8"))
50
+ } catch (error) {
51
+ return { tests: [], untestable: [], unmapped: [...criteria], problems: [`${ACCEPTANCE_MANIFEST} is missing or not JSON (${error?.code === "ENOENT" ? "not written" : "unreadable"})`] }
52
+ }
53
+ const tests = []
54
+ for (const [index, entry] of (Array.isArray(raw?.tests) ? raw.tests : []).slice(0, MAX_TESTS).entries()) {
55
+ const criterion = text(entry?.criterion, 500)
56
+ const name = text(entry?.name, 300)
57
+ const command = text(entry?.command, 500)
58
+ const file = typeof entry?.file === "string" ? insideCheckout(checkout, entry.file.trim()) : null
59
+ if (!criterion || !name || !command || !file) {
60
+ problems.push(`test ${index + 1} needs a criterion, a name, a command and a file that exists in the checkout`)
61
+ continue
62
+ }
63
+ tests.push({ criterion, name, file, command })
64
+ }
65
+ const untestable = (Array.isArray(raw?.untestable) ? raw.untestable : []).flatMap((entry) => {
66
+ const criterion = text(entry?.criterion, 500)
67
+ return criterion ? [{ criterion, why: text(entry?.why, 300) }] : []
68
+ })
69
+ const unmapped = criteria.filter((criterion) =>
70
+ !tests.some((test) => sameCriterion(test.criterion, criterion))
71
+ && !untestable.some((entry) => sameCriterion(entry.criterion, criterion)))
72
+ if (!tests.length) problems.push("no usable test in the manifest")
73
+ return { tests, untestable, unmapped, problems }
74
+ }
75
+
76
+ /**
77
+ * Run each test's own command. `exec(command, { cwd })` resolves
78
+ * { code, output, timedOut }. A test passes when its command exits 0.
79
+ */
80
+ export async function runAcceptanceTests(tests, { cwd, exec, shouldStop = () => false }) {
81
+ const results = []
82
+ for (const test of tests) {
83
+ if (shouldStop()) break
84
+ const started = Date.now()
85
+ const result = await exec(test.command, { cwd })
86
+ results.push({
87
+ ...test,
88
+ result: {
89
+ status: result.code === 0 && !result.timedOut ? "passed" : "failed",
90
+ exit_code: Number.isInteger(result.code) ? result.code : null,
91
+ duration_ms: Date.now() - started,
92
+ output: tail(result.timedOut ? `${result.output}\n(timed out)` : result.output),
93
+ },
94
+ })
95
+ }
96
+ return results
97
+ }
98
+
99
+ /** The acceptance commit's message. */
100
+ export function acceptanceCommitMessage(identifier) {
101
+ return `Acceptance tests for ${identifier || "this issue"}`
102
+ }
103
+
104
+ /**
105
+ * Test files the build changed since the acceptance commit, from
106
+ * `git diff --name-status -M <acceptance commit>` (working tree included):
107
+ * modified, deleted or renamed. Any of these is flagged for the reviewer.
108
+ */
109
+ export function changedAcceptanceTests(nameStatus, files) {
110
+ const watched = new Set(files)
111
+ const changes = []
112
+ for (const line of String(nameStatus || "").split(/\r?\n/)) {
113
+ const parts = line.split("\t").map((part) => part.trim())
114
+ if (parts.length < 2 || !parts[0]) continue
115
+ const code = parts[0][0]
116
+ const from = parts[1]
117
+ if (!watched.has(from)) continue
118
+ if (code === "D") changes.push({ file: from, change: "deleted" })
119
+ else if (code === "R") changes.push({ file: from, change: "renamed", to: parts[2] || null })
120
+ else if (code === "M" || code === "T") changes.push({ file: from, change: "modified" })
121
+ }
122
+ return changes
123
+ }
124
+
125
+ /**
126
+ * The acceptance part of a build run's evidence: each test before and after the
127
+ * change, and the tests the build changed. `before` and `after` are the results
128
+ * of runAcceptanceTests; `skipped` says why they were not run.
129
+ */
130
+ export function buildAcceptanceEvidence({ commit, tests, before = [], after = [], changed = [], skipped = null, ranBy = "runner" }) {
131
+ const find = (results, test) => results.find((entry) => entry.name === test.name && entry.file === test.file)?.result ?? null
132
+ const rows = tests.slice(0, MAX_TESTS).map((test) => {
133
+ const was = find(before, test)
134
+ const now = find(after, test)
135
+ return {
136
+ criterion: test.criterion,
137
+ name: test.name,
138
+ file: test.file,
139
+ command: test.command,
140
+ before: was ? { status: was.status, exit_code: was.exit_code } : null,
141
+ after: now ? { status: now.status, exit_code: now.exit_code, output: now.status === "failed" ? now.output : null } : null,
142
+ }
143
+ })
144
+ return {
145
+ commit: commit ? String(commit).slice(0, 64) : null,
146
+ // "agent": a sandboxed run, where the agent ran them inside its sandbox and
147
+ // recorded them with `pacework evidence run`; the runner did not run them itself.
148
+ ran_by: ranBy === "agent" ? "agent" : "runner",
149
+ tests: rows,
150
+ changed: changed.slice(0, MAX_TESTS),
151
+ skipped: skipped ? String(skipped).slice(0, 300) : null,
152
+ // Every test failed before and passes after, and none was changed.
153
+ failing_to_passing: rows.length > 0 && !skipped && !changed.length
154
+ && rows.every((row) => row.before?.status === "failed" && row.after?.status === "passed"),
155
+ }
156
+ }
157
+
158
+ const cell = (value) => String(value ?? "").replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()
159
+ const BEFORE = { failed: "failing", passed: "**passing already**" }
160
+ const AFTER = { failed: "**failing**", passed: "passing" }
161
+
162
+ /** The Acceptance tests section of a pull request or a run, from the evidence. */
163
+ export function renderAcceptanceMarkdown(acceptance, { heading = "## Acceptance tests" } = {}) {
164
+ if (!acceptance?.tests?.length) return ""
165
+ const lines = [heading]
166
+ if (acceptance.commit) lines.push(`Written from the issue's acceptance criteria before the change, in commit \`${cell(acceptance.commit).slice(0, 7)}\`.`)
167
+ lines.push("", "| Criterion | Test | Before | After |", "| --- | --- | --- | --- |")
168
+ for (const row of acceptance.tests) {
169
+ lines.push(`| ${cell(row.criterion)} | \`${cell(row.name)}\` | ${row.before ? BEFORE[row.before.status] : "not run"} | ${row.after ? AFTER[row.after.status] : "not run"} |`)
170
+ }
171
+ if (acceptance.ran_by === "agent") lines.push("", "Run by the agent inside its sandbox and recorded with `pacework evidence run`; the runner could not run them outside the sandbox.")
172
+ if (acceptance.skipped) lines.push("", `The runner did not run them: ${cell(acceptance.skipped)}.`)
173
+ if (acceptance.changed?.length) {
174
+ lines.push("", "> [!WARNING]", "> **The build changed acceptance tests.** They are the issue's contract: the reviewer checks each change is explained and does not weaken the test.")
175
+ for (const change of acceptance.changed) lines.push(`> - \`${cell(change.file)}\` ${change.change}${change.to ? ` to \`${cell(change.to)}\`` : ""}`)
176
+ }
177
+ return lines.join("\n")
178
+ }
179
+
180
+ /**
181
+ * In a sandboxed run the runner cannot run the tests, so the agent runs them
182
+ * through `pacework evidence run`: each test's first record is before the
183
+ * change, its last record after it. Returns results shaped like
184
+ * runAcceptanceTests', for the tests the agent recorded.
185
+ */
186
+ export function acceptanceFromAgentRecords(tests, records) {
187
+ const before = [], after = []
188
+ for (const test of tests) {
189
+ const runs = (records ?? []).filter((record) => record.command && record.command.trim() === String(test.command).trim())
190
+ if (!runs.length) continue
191
+ const result = (record) => ({ status: record.status === "passed" ? "passed" : "failed", exit_code: record.exit_code ?? null, duration_ms: record.duration_ms ?? null, output: record.output ?? null })
192
+ before.push({ ...test, result: result(runs[0]) })
193
+ if (runs.length > 1) after.push({ ...test, result: result(runs[runs.length - 1]) })
194
+ }
195
+ return { before, after }
196
+ }
197
+
198
+ /** What a builder is told about the issue's acceptance tests (PWK-206). Empty when it has none. */
199
+ export function acceptancePromptSection(acceptance, { sandboxed = false, cli = "pacework" } = {}) {
200
+ const tests = Array.isArray(acceptance?.tests) ? acceptance.tests : []
201
+ if (!tests.length) return ""
202
+ return [
203
+ "## Acceptance tests",
204
+ `This issue's acceptance criteria are already tests, committed${acceptance.commit_sha ? ` in \`${String(acceptance.commit_sha).slice(0, 7)}\`` : ""} on this branch before any code. Start from them:`,
205
+ "1. Run each one and see it fail.",
206
+ "2. Make the change that makes them pass, without breaking other tests.",
207
+ "3. Run them again and see them pass.",
208
+ "Do not change, skip, weaken or delete them. If one is wrong, leave it as it is and say why in your report: the runner compares them with the acceptance commit and flags any change for the reviewer.",
209
+ ...(sandboxed ? [`This run is sandboxed, so the runner cannot run them for you: run each one through \`${cli} evidence run -- <its command>\` before you change anything, and again at the end. The evidence on the pull request is built from those records.`] : []),
210
+ "",
211
+ ...tests.map((test) => `- ${test.criterion}: \`${test.name}\` in \`${test.file}\` (run: \`${test.command}\`)`),
212
+ ].join("\n")
213
+ }
214
+
215
+ /**
216
+ * After the agent wrote the acceptance tests (PWK-205): read its manifest, run
217
+ * each test (it should fail), commit only the test files as the acceptance
218
+ * commit, set aside anything else the agent changed, and push. Returns the
219
+ * report the server records. `git(args)` resolves { code, stdout, stderr };
220
+ * `exec` runs a test command as runAcceptanceTests expects; `runTests` false
221
+ * skips running them (a sandboxed run). Changes that are not tests are saved
222
+ * as a patch at `setAsidePath` and taken out of the checkout.
223
+ */
224
+ export async function commitAcceptanceWork({ checkout, criteria = [], identifier = null, git, exec, push, setAsidePath, runTests = true, shouldStop = () => false, log = () => {} }) {
225
+ const manifest = readAcceptanceManifest(checkout, criteria)
226
+ fs.rmSync(path.join(checkout, ACCEPTANCE_MANIFEST), { force: true })
227
+ if (!manifest.tests.length) throw new Error(`No acceptance tests were written: ${manifest.problems.join("; ")}`)
228
+ const before = runTests ? await runAcceptanceTests(manifest.tests, { cwd: checkout, exec, shouldStop }) : []
229
+ for (const entry of before) log(`acceptance test ${entry.result.status} before any code: ${entry.name}`)
230
+ const files = [...new Set(manifest.tests.map((test) => test.file))]
231
+ const added = await git(["add", "--", ...files])
232
+ if (added.code !== 0) throw new Error(`Could not stage the acceptance tests: ${added.stderr}`)
233
+ if (!(await git(["diff", "--cached", "--name-only"])).stdout.trim()) {
234
+ throw new Error("The manifest names tests that are already on the branch unchanged; nothing new to commit.")
235
+ }
236
+ const committed = await git(["commit", "-m", acceptanceCommitMessage(identifier)])
237
+ if (committed.code !== 0) throw new Error(`Could not commit the acceptance tests: ${committed.stderr}`)
238
+ const sha = (await git(["rev-parse", "HEAD"])).stdout.trim()
239
+ // Anything else the agent changed stays out of the commit: saved as a patch in
240
+ // the run's folder (not the repository's stash, which a person's checkout shares)
241
+ // and taken out of the checkout the build starts from.
242
+ const leftOver = (await git(["status", "--porcelain"])).stdout.trim()
243
+ if (leftOver) {
244
+ await git(["add", "-A"])
245
+ const patch = await git(["diff", "--cached", "--binary"])
246
+ if (setAsidePath) {
247
+ fs.mkdirSync(path.dirname(setAsidePath), { recursive: true })
248
+ fs.writeFileSync(setAsidePath, patch.stdout)
249
+ }
250
+ await git(["reset", "--hard", "HEAD"])
251
+ log(`set aside ${leftOver.split("\n").length} changed path(s) that are not acceptance tests${setAsidePath ? ` in ${setAsidePath}` : ""}`)
252
+ }
253
+ await push()
254
+ return {
255
+ commit_sha: sha,
256
+ tests: manifest.tests.map((test) => ({
257
+ ...test,
258
+ before: before.find((entry) => entry.name === test.name && entry.file === test.file)?.result ?? null,
259
+ })),
260
+ untestable: manifest.untestable,
261
+ unmapped: manifest.unmapped,
262
+ tests_run: runTests,
263
+ set_aside: leftOver ? leftOver.split("\n").length : 0,
264
+ }
265
+ }
@@ -54,6 +54,9 @@ export function detectCapabilities({ repoRoot, executors = [], exec = run, platf
54
54
  if (exec("git", ["--version"])) tools.push("git")
55
55
  if (exec("docker", ["version", "--format", "{{.Server.Version}}"])) tools.push("docker")
56
56
  if (canSandbox({ platform, executors, exec })) tools.push("sandbox")
57
+ // This runner writes acceptance tests in a checkout of their own (PWK-205); an
58
+ // older one would write them into the repository root, so it is never given them.
59
+ tools.push("acceptance-tests")
57
60
  capabilities.tools = tools
58
61
  return capabilities
59
62
  }
package/lib/evidence.mjs CHANGED
@@ -13,6 +13,7 @@
13
13
 
14
14
  import fs from "node:fs"
15
15
  import path from "node:path"
16
+ import { renderAcceptanceMarkdown } from "./acceptance.mjs"
16
17
 
17
18
  export const EVIDENCE_VERSION = 1
18
19
  /** The file in the checkout where `pacework evidence` writes; excluded from commits. */
@@ -87,9 +88,11 @@ export function readAgentEvidence(file) {
87
88
 
88
89
  /**
89
90
  * The run's evidence record. `role` is "build" or "review"; `rerun` is what a
90
- * reviewer's runner ran again on the reviewed commit.
91
+ * reviewer's runner ran again on the reviewed commit. `acceptance` is the
92
+ * issue's acceptance tests before and after the change (PWK-206), from
93
+ * buildAcceptanceEvidence.
91
94
  */
92
- export function buildEvidence({ role = "build", commit = null, checks = [], notVerified = [], skippedReason = null }) {
95
+ export function buildEvidence({ role = "build", commit = null, checks = [], notVerified = [], skippedReason = null, acceptance = null }) {
93
96
  const items = checks.slice(0, MAX_ITEMS)
94
97
  const not_verified = [...notVerified]
95
98
  if (skippedReason) not_verified.unshift({ what: "The runner's own checks", why: skippedReason })
@@ -101,6 +104,7 @@ export function buildEvidence({ role = "build", commit = null, checks = [], notV
101
104
  items,
102
105
  not_verified: not_verified.slice(0, 20),
103
106
  production_like: items.some((item) => item.environment === "production-like" && item.status === "passed"),
107
+ ...(acceptance ? { acceptance } : {}),
104
108
  }
105
109
  }
106
110
 
@@ -139,6 +143,8 @@ export function renderEvidenceMarkdown(evidence, { heading = "## Evidence" } = {
139
143
  lines.push("", "**Not verified**")
140
144
  for (const entry of notVerified) lines.push(`- ${cell(entry.what)}${entry.why ? `: ${cell(entry.why)}` : ""}`)
141
145
  }
146
+ const acceptance = renderAcceptanceMarkdown(evidence?.acceptance, { heading: "### Acceptance tests" })
147
+ if (acceptance) lines.push("", acceptance)
142
148
  return lines.join("\n")
143
149
  }
144
150
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pacework",
3
- "version": "1.7.18",
3
+ "version": "1.7.19",
4
4
  "description": "pacework CLI: connect this machine as a runner for pacework AI agents",
5
5
  "type": "module",
6
6
  "bin": {