pacework 1.7.17 → 1.7.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,6 +9,7 @@ import process from "node:process"
9
9
  import { spawn, execFileSync } from "node:child_process"
10
10
  import { AsyncLocalStorage } from "node:async_hooks"
11
11
  import { createLogger } from "../lib/logger.mjs"
12
+ import { fileLimitAppliesTo, isReviewRun } from "../lib/run-kind.mjs"
12
13
  import { detectCapabilities, detectPower } from "../lib/capabilities.mjs"
13
14
  import { detectRunUsageLimit, USAGE_LIMIT_MARKER } from "../lib/usage-limit.mjs"
14
15
  import {
@@ -42,6 +43,14 @@ import { activityForStreamEvent, createActivityTracker, createStreamReader, help
42
43
  import { RUNNER_VERSION } from "../lib/version.mjs"
43
44
  import { addBranchWorktree, catchUpWithRemote, isPushRefused, pushKeepingRemoteCommits, remoteBranchExists, switchToBranch } from "../lib/branch-sync.mjs"
44
45
  import { closeSupersededPullRequest } from "../lib/superseded-pr.mjs"
46
+ import {
47
+ acceptanceFromAgentRecords,
48
+ acceptancePromptSection,
49
+ buildAcceptanceEvidence,
50
+ changedAcceptanceTests,
51
+ commitAcceptanceWork,
52
+ runAcceptanceTests,
53
+ } from "../lib/acceptance.mjs"
45
54
  import { agentBaseEnv, agentDeniedReads, fenceUntrusted, unsandboxedWarning } from "../lib/agent-env.mjs"
46
55
  import {
47
56
  buildFixInstructions,
@@ -624,29 +633,6 @@ function buildRunnerCompletionComment(input) {
624
633
  ].join(" ")
625
634
  }
626
635
 
627
- /**
628
- * Whether a run is a review. The server says so in `purpose`, from the issue's
629
- * workflow stage or the agent's role; the skill slug is only a fallback for
630
- * servers that predate it. Deciding by slug made a renamed reviewer build code.
631
- */
632
- function isReviewRun(run) {
633
- if (run?.purpose) return run.purpose === "review"
634
- return getAgentSkillProfileSlug(run?.agent) === "reviewer"
635
- }
636
-
637
- function getAgentSkillProfileSlug(agent) {
638
- if (!agent || typeof agent !== "object") return null
639
- const direct = agent.skill_profile
640
- if (direct && typeof direct === "object" && typeof direct.slug === "string") return direct.slug
641
- const nested = agent.skill_profiles
642
- if (Array.isArray(nested)) {
643
- const first = nested.find((entry) => entry && typeof entry.slug === "string")
644
- return first?.slug || null
645
- }
646
- if (nested && typeof nested === "object" && typeof nested.slug === "string") return nested.slug
647
- return null
648
- }
649
-
650
636
  function parseReviewerDecisionSummary(summary) {
651
637
  const text = String(summary || "").trim()
652
638
  if (!text) return null
@@ -2114,9 +2100,8 @@ function buildPrompt(run, branch, worktreePath, options = {}) {
2114
2100
  const gitAvailable = options.gitAvailable !== false
2115
2101
  const workspaceContext = options.workspaceContext || null
2116
2102
  const greenfieldWorkspace = workspaceContext?.is_empty_workspace === true
2117
- // The agent's standing instructions; servers before skill profiles were
2118
- // retired sent them as skill_profile.system_prompt.
2119
- const skillPrompt = (agent?.instructions ?? agent?.skill_profile?.system_prompt)?.trim()
2103
+ // The agent's standing instructions.
2104
+ const skillPrompt = typeof agent?.instructions === "string" ? agent.instructions.trim() : ""
2120
2105
  const base = skillPrompt ? `${skillPrompt}\n\n---\n\n` : ""
2121
2106
  const codingRules = Array.isArray(run?.coding_rules) ? run.coding_rules : []
2122
2107
  const reviewerContext = run?.reviewer_context || null
@@ -3027,10 +3012,120 @@ async function runPlannerPromptWithOllama(prompt, config) {
3027
3012
  return content
3028
3013
  }
3029
3014
 
3015
+ /**
3016
+ * Why a build cannot run the issue's acceptance tests, or null when it can: a
3017
+ * sandboxed run does not run commands outside its sandbox, and a checkout
3018
+ * without the acceptance commit is not on the acceptance branch (PWK-206).
3019
+ */
3020
+ async function acceptanceTestsUnavailable(run, worktreePath, acceptance) {
3021
+ if (runSandboxed(run)) return SANDBOXED_RUN_SKIP
3022
+ const sha = String(acceptance.commit_sha || "")
3023
+ if (!/^[0-9a-f]{7,64}$/i.test(sha)) return "the server sent no acceptance commit"
3024
+ const git = gitIn(worktreePath)
3025
+ if ((await git(["merge-base", "--is-ancestor", sha, "HEAD"])).code !== 0) {
3026
+ return `this checkout does not contain the acceptance commit ${sha.slice(0, 7)}`
3027
+ }
3028
+ return null
3029
+ }
3030
+
3031
+ /** Each acceptance test's own command, in the checkout, with no credentials in its environment. */
3032
+ function runAcceptanceCommands(acceptance, worktreePath, shouldStop) {
3033
+ const checkEnv = checkEnvironment(process.env)
3034
+ return runAcceptanceTests(acceptance.tests, {
3035
+ cwd: worktreePath,
3036
+ shouldStop,
3037
+ exec: (command, options) => execLocalCheck(command, { ...options, baseEnv: checkEnv }),
3038
+ })
3039
+ }
3040
+
3041
+ /** The acceptance test files the build changed since the acceptance commit, committed or not. */
3042
+ async function changedAcceptanceFiles(worktreePath, acceptance) {
3043
+ const files = [...new Set(acceptance.tests.map((test) => test.file).filter(Boolean))]
3044
+ if (!files.length || !acceptance.commit_sha) return []
3045
+ const diff = await gitIn(worktreePath)(["diff", "--name-status", "-M", String(acceptance.commit_sha), "--", ...files])
3046
+ if (diff.code !== 0) return []
3047
+ // A renamed test shows under its new name with -- paths; list renames from the whole tree too.
3048
+ const renames = await gitIn(worktreePath)(["diff", "--name-status", "-M", "--diff-filter=RD", String(acceptance.commit_sha)])
3049
+ const seen = new Set()
3050
+ return changedAcceptanceTests(`${diff.stdout}\n${renames.code === 0 ? renames.stdout : ""}`, files).filter((change) => {
3051
+ const key = `${change.file}:${change.change}`
3052
+ if (seen.has(key)) return false
3053
+ seen.add(key)
3054
+ return true
3055
+ })
3056
+ }
3057
+
3058
+ /** What the issue is told when a planning run starts, ends or fails, by what the run is for. */
3059
+ function plannerMilestones(run) {
3060
+ if (run.purpose === "readiness") return { started: "Triage started: getting this issue ready for an agent.", finished: "Triage finished.", failed: "Triage" }
3061
+ if (run.purpose === "acceptance_tests") {
3062
+ return {
3063
+ started: `Writing acceptance tests from the criteria${run.acceptance?.branch_name ? ` on \`${run.acceptance.branch_name}\`` : ""}.`,
3064
+ finished: "Acceptance tests written and pushed.",
3065
+ failed: "Writing acceptance tests",
3066
+ }
3067
+ }
3068
+ return { started: "Planning started on the local runner.", finished: "Planning finished and results were posted back to the issue.", failed: "Planning" }
3069
+ }
3070
+
3071
+ /**
3072
+ * The checkout an acceptance-test run writes in (PWK-205): the issue's own
3073
+ * worktree on the branch the server named, with the writing-acceptance-tests
3074
+ * skill installed. A planning run otherwise works in the repository root and
3075
+ * changes nothing.
3076
+ */
3077
+ async function prepareAcceptanceCheckout(run, state, runLogger) {
3078
+ const branch = run.acceptance?.branch_name
3079
+ if (!branch) throw new Error("The server did not name the branch the acceptance tests go on.")
3080
+ const { token, repo, baseBranch } = await fetchGithubToken(state, run.workspace_id, run.task?.project_id ?? null, run.task?.team_id ?? null, run.task_id ?? null)
3081
+ if (!token || !repo?.owner || !repo?.repo) throw new Error("No GitHub repository is connected for this issue, so acceptance tests cannot be pushed.")
3082
+ run.__githubToken = token
3083
+ run.__repo = repo
3084
+ run.__baseBranch = baseBranch ?? null
3085
+ run.__repoRoot = await resolveRunRepoRoot(run, state)
3086
+ run.__serverBranch = branch
3087
+ const prepared = await prepareWorktree(run)
3088
+ if (!prepared.gitAvailable || prepared.localCheckoutMode === "repo_root") {
3089
+ throw new Error("Acceptance tests need a git checkout of their own, and this run would work in the repository root.")
3090
+ }
3091
+ const skillInstall = installSkills(prepared.worktreePath, run.skills, { log: (line) => runLogger.info("runner.run.skills", line) })
3092
+ return { ...prepared, skillInstall }
3093
+ }
3094
+
3095
+ /**
3096
+ * After the agent wrote the acceptance tests: check each one fails, commit them
3097
+ * as the acceptance commit and push the branch (lib/acceptance.mjs). Returns the
3098
+ * report the server records, with each criterion's tests.
3099
+ */
3100
+ async function commitAcceptanceTests(run, checkout, { logState, shouldStop, runRoot }) {
3101
+ const { worktreePath, branch } = checkout
3102
+ // Installed skills never go into the commit.
3103
+ checkout.skillInstall.cleanup()
3104
+ const checkEnv = checkEnvironment(process.env)
3105
+ const report = await commitAcceptanceWork({
3106
+ checkout: worktreePath,
3107
+ criteria: run.acceptance?.criteria ?? [],
3108
+ identifier: run.task?.identifier ?? null,
3109
+ git: gitIn(worktreePath),
3110
+ exec: (command, options) => execLocalCheck(command, { ...options, baseEnv: checkEnv }),
3111
+ push: () => pushBranchToGitHub(worktreePath, branch, run.__githubToken, run.__repo),
3112
+ setAsidePath: path.join(runRoot, "left-out-of-acceptance-commit.patch"),
3113
+ // The commands come from the agent: a sandboxed run does not run them outside its sandbox.
3114
+ runTests: !runSandboxed(run),
3115
+ shouldStop,
3116
+ log: (line) => appendLogTail(logState, `> ${line}`),
3117
+ })
3118
+ return { branch_name: branch, ...report }
3119
+ }
3120
+
3030
3121
  async function runPlannerExecutor(run, state) {
3031
3122
  const runLogger = getRunLogger(state, run)
3032
3123
  const runRoot = path.join(repoRoot, ".agent", "runs", run.id)
3033
3124
  ensureDir(runRoot)
3125
+ const milestones = plannerMilestones(run)
3126
+ /** Where the agent works: the repository root, or an acceptance-test run's own checkout. */
3127
+ let cwd = repoRoot
3128
+ let acceptanceCheckout = null
3034
3129
 
3035
3130
  const outputFile = path.join(runRoot, "planner-last-message.txt")
3036
3131
  const logState = { tail: "" }
@@ -3089,28 +3184,35 @@ async function runPlannerExecutor(run, state) {
3089
3184
  try {
3090
3185
  const problem = credentialProblem(run)
3091
3186
  if (problem) throw new Error(`Could not start: ${problem}`)
3187
+ if (run.purpose === "acceptance_tests") {
3188
+ acceptanceCheckout = await prepareAcceptanceCheckout(run, state, runLogger)
3189
+ cwd = acceptanceCheckout.worktreePath
3190
+ }
3092
3191
  runLogger.info("runner.planner.started", "Local runner planner execution started", {
3093
3192
  meta: {
3094
3193
  issue_identifier: run.task?.identifier || run.task_id,
3095
3194
  },
3096
3195
  })
3097
- appendLogTail(logState, `> preparing ${run.agent?.name || "planner"} run in ${repoRoot}`)
3196
+ appendLogTail(logState, `> preparing ${run.agent?.name || "planner"} run in ${cwd}`)
3098
3197
  await flushPlannerLog().catch(() => {})
3099
3198
  await postIssueMilestone(
3100
3199
  runMcpKey(run, state),
3101
3200
  run.task_id,
3102
- "Planning started on the local runner.",
3201
+ milestones.started,
3103
3202
  "info",
3104
3203
  )
3105
3204
 
3106
3205
  appendLogTail(logState, "> loading workspace context and planner prompt...")
3107
3206
  await flushPlannerLog().catch(() => {})
3108
- const workspaceContext = await buildWorkspaceContext(repoRoot)
3207
+ const workspaceContext = await buildWorkspaceContext(cwd)
3109
3208
  const plannerPayload = await preparePlannerRun(run, workspaceContext)
3110
3209
  const staticResponse = typeof plannerPayload?.staticResponse === "string" ? plannerPayload.staticResponse.trim() : ""
3111
3210
  const answers = clarificationAnswers(run)
3211
+ const skillsSection = acceptanceCheckout
3212
+ ? skillsPromptSection((run.skills ?? []).filter((skill) => acceptanceCheckout.skillInstall.installed.includes(skill.slug)), { purpose: "build" })
3213
+ : ""
3112
3214
  const prompt = typeof plannerPayload?.prompt === "string" && plannerPayload.prompt
3113
- ? `${plannerPayload.prompt}\n\n${answers || ASK_WHEN_UNCLEAR}`
3215
+ ? [plannerPayload.prompt, skillsSection, answers || ASK_WHEN_UNCLEAR].filter(Boolean).join("\n\n")
3114
3216
  : ""
3115
3217
 
3116
3218
  if (!staticResponse && !prompt) {
@@ -3124,7 +3226,7 @@ async function runPlannerExecutor(run, state) {
3124
3226
  stdioSummary = staticResponse
3125
3227
  appendLogTail(logState, "> using static planner response.")
3126
3228
  appendLogTail(logState, staticResponse)
3127
- } else if (useOllama) {
3229
+ } else if (useOllama && !acceptanceCheckout) {
3128
3230
  runLogger.info("runner.planner.ollama", "Using Ollama for planner run", {
3129
3231
  meta: {
3130
3232
  model: ollamaConfig.model,
@@ -3141,19 +3243,19 @@ async function runPlannerExecutor(run, state) {
3141
3243
  appendLogTail(logState, ollamaResponse)
3142
3244
  } else {
3143
3245
  const plannerMcpKey = runMcpKey(run, state)
3144
- const execSpec = buildExecutorCommand(run, repoRoot, prompt, outputFile, plannerMcpKey)
3246
+ const execSpec = buildExecutorCommand(run, cwd, prompt, outputFile, plannerMcpKey)
3145
3247
  appendLogTail(logState, `> spawning ${activeExecutor()} planner executor`)
3146
- appendLogTail(logState, `> cwd: ${repoRoot}`)
3248
+ appendLogTail(logState, `> cwd: ${cwd}`)
3147
3249
  appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
3148
3250
  await flushPlannerLog().catch(() => {})
3149
3251
  const plannerMcpServers = await fetchMcpServers(state)
3150
- const execEnv = buildExecutorEnv(run, repoRoot, plannerMcpKey, plannerMcpServers)
3252
+ const execEnv = buildExecutorEnv(run, cwd, plannerMcpKey, plannerMcpServers)
3151
3253
  plannerMcpBackups = setupExecutorMcpSideEffects(plannerMcpKey, plannerMcpServers)
3152
3254
 
3153
3255
  let plannerStdout = ""
3154
3256
  await Promise.race([runTimeoutPromise, new Promise((resolve, reject) => {
3155
3257
  const child = spawn(execSpec.command, execSpec.args, {
3156
- cwd: repoRoot,
3258
+ cwd,
3157
3259
  stdio: ["pipe", "pipe", "pipe"],
3158
3260
  detached: process.platform !== "win32",
3159
3261
  env: execEnv,
@@ -3214,7 +3316,12 @@ async function runPlannerExecutor(run, state) {
3214
3316
  if (cancelRequested) throw new Error("Canceled by admin.")
3215
3317
 
3216
3318
  const fileContent = fs.existsSync(outputFile) ? fs.readFileSync(outputFile, "utf8").trim() : ""
3217
- const plannerResponse = staticResponse || fileContent || stdioSummary.trim()
3319
+ let plannerResponse = staticResponse || fileContent || stdioSummary.trim()
3320
+ // An acceptance-test run reports what it committed, not what the agent said.
3321
+ if (acceptanceCheckout) {
3322
+ const report = await commitAcceptanceTests(run, acceptanceCheckout, { logState, shouldStop: () => cancelRequested, runRoot })
3323
+ plannerResponse = JSON.stringify({ acceptance_tests: report })
3324
+ }
3218
3325
 
3219
3326
  if (!plannerResponse) {
3220
3327
  throw new Error("Planner run finished without a final response")
@@ -3228,7 +3335,7 @@ async function runPlannerExecutor(run, state) {
3228
3335
  await postIssueMilestone(
3229
3336
  runMcpKey(run, state),
3230
3337
  run.task_id,
3231
- "Planning finished and results were posted back to the issue.",
3338
+ milestones.finished,
3232
3339
  "success",
3233
3340
  )
3234
3341
 
@@ -3245,7 +3352,7 @@ async function runPlannerExecutor(run, state) {
3245
3352
  await postIssueMilestone(
3246
3353
  runMcpKey(run, state),
3247
3354
  run.task_id,
3248
- `Planning ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
3355
+ `${milestones.failed} ${cancelRequested ? "was canceled" : "failed"}: ${truncate(message.replace(/\s+/g, " "), 280)}`,
3249
3356
  "warning",
3250
3357
  )
3251
3358
 
@@ -3259,6 +3366,13 @@ async function runPlannerExecutor(run, state) {
3259
3366
  clearInterval(logInterval)
3260
3367
  if (runTimeoutHandle) clearTimeout(runTimeoutHandle)
3261
3368
  cleanupExecutorMcp(plannerMcpBackups)
3369
+ if (acceptanceCheckout) {
3370
+ acceptanceCheckout.skillInstall?.cleanup()
3371
+ // The branch is on GitHub now. Its checkout goes, so the build can check the
3372
+ // branch out wherever its own checkout is (a sub-issue's is its parent's).
3373
+ const owner = run.__repoRoot || repoRoot
3374
+ await execCommandSafe("git", ["-C", owner, "worktree", "remove", "--force", acceptanceCheckout.worktreePath], owner).catch(() => null)
3375
+ }
3262
3376
  }
3263
3377
  }
3264
3378
 
@@ -3462,7 +3576,7 @@ async function runExecutor(run, state) {
3462
3576
  }
3463
3577
  const runLimits = resolveRunLimits({ args, env: process.env, profile: runnerProfile })
3464
3578
  // Scouts survey the codebase and may touch many files on purpose.
3465
- const fileLimitApplies = gitAvailable && getAgentSkillProfileSlug(run.agent) !== "scout"
3579
+ const fileLimitApplies = gitAvailable && fileLimitAppliesTo(run.agent)
3466
3580
  // The commit the run starts from: only the files this run changes count.
3467
3581
  const runStartSha = gitAvailable
3468
3582
  ? ((await execCommandSafe("git", ["-C", worktreePath, "rev-parse", "HEAD"], worktreePath))?.stdout?.trim() || null)
@@ -3497,8 +3611,13 @@ async function runExecutor(run, state) {
3497
3611
  (run.skills ?? []).filter((skill) => skillInstall.installed.includes(skill.slug)),
3498
3612
  { purpose: isReviewRun(run) ? "review" : "build" },
3499
3613
  )
3614
+ // A build on an issue with acceptance tests starts from them (PWK-206).
3615
+ const acceptanceTests = !isReviewRun(run) && gitAvailable && Array.isArray(run.acceptance_tests?.tests) && run.acceptance_tests.tests.length
3616
+ ? run.acceptance_tests
3617
+ : null
3500
3618
  const prompt = [
3501
3619
  buildPrompt(run, branch, worktreePath, { localCheckoutMode, gitAvailable, workspaceContext, runLimitsText, sameErrorLimit }),
3620
+ acceptanceTests ? acceptancePromptSection(acceptanceTests, { sandboxed: runSandboxed(run) && Boolean(run.__evidenceFile), cli: EVIDENCE_CLI }) : "",
3502
3621
  skillsSection,
3503
3622
  ].filter(Boolean).join("\n\n")
3504
3623
  const executorOptions = {
@@ -3568,6 +3687,9 @@ async function runExecutor(run, state) {
3568
3687
  let runUsage = null
3569
3688
  /** What the checks from the CI workflow found on this machine, for the run summary. */
3570
3689
  let localChecksSummary = ""
3690
+ /** The acceptance tests run before the agent starts (PWK-206), and why they were not. */
3691
+ let acceptanceBefore = []
3692
+ let acceptanceSkipped = null
3571
3693
  /** Evidence (PWK-183): checks the runner ran, and why it ran none. */
3572
3694
  let runnerChecks = []
3573
3695
  let runnerChecksSkipped = null
@@ -3731,6 +3853,15 @@ async function runExecutor(run, state) {
3731
3853
  fs.writeFileSync(outputFile, smokeResult.summary)
3732
3854
  }
3733
3855
  } else {
3856
+ if (acceptanceTests) {
3857
+ acceptanceSkipped = await acceptanceTestsUnavailable(run, worktreePath, acceptanceTests)
3858
+ if (!acceptanceSkipped) {
3859
+ appendLogTail(logState, `> running ${acceptanceTests.tests.length} acceptance test(s) before the change; they should fail`)
3860
+ await flushRunnerLog().catch(() => {})
3861
+ acceptanceBefore = await runAcceptanceCommands(acceptanceTests, worktreePath, () => runEnded || cancelRequested)
3862
+ for (const entry of acceptanceBefore) appendLogTail(logState, `> acceptance test ${entry.result.status} before: ${entry.name}`)
3863
+ }
3864
+ }
3734
3865
  appendLogTail(logState, `> spawning ${activeExecutor()} executor`)
3735
3866
  appendLogTail(logState, `> cwd: ${worktreePath}`)
3736
3867
  appendLogTail(logState, `> command: ${previewExecutorCommand(execSpec)}`)
@@ -3945,7 +4076,36 @@ async function runExecutor(run, state) {
3945
4076
  runnerChecks = rerun.checks
3946
4077
  runnerChecksSkipped = rerun.skipped
3947
4078
  }
4079
+ // The acceptance tests again, now that the change is made (they should pass),
4080
+ // and whether the build changed them since the acceptance commit (PWK-206).
3948
4081
  const agentEvidence = run.__evidenceFile ? readAgentEvidence(run.__evidenceFile) : { checks: [], notVerified: [] }
4082
+ let acceptanceEvidence = null
4083
+ if (acceptanceTests) {
4084
+ let after = acceptanceSkipped ? [] : await runAcceptanceCommands(acceptanceTests, worktreePath, () => runEnded || cancelRequested)
4085
+ let ranBy = "runner"
4086
+ // Sandboxed: the agent ran them inside its sandbox and recorded them.
4087
+ if (acceptanceSkipped === SANDBOXED_RUN_SKIP) {
4088
+ const recorded = acceptanceFromAgentRecords(acceptanceTests.tests, agentEvidence.checks)
4089
+ if (recorded.before.length) {
4090
+ acceptanceBefore = recorded.before
4091
+ after = recorded.after
4092
+ acceptanceSkipped = null
4093
+ ranBy = "agent"
4094
+ }
4095
+ }
4096
+ for (const entry of after) appendLogTail(logState, `> acceptance test ${entry.result.status} after: ${entry.name}`)
4097
+ const changed = await changedAcceptanceFiles(worktreePath, acceptanceTests)
4098
+ for (const change of changed) appendLogTail(logState, `> acceptance test ${change.change} by the build: ${change.file}`)
4099
+ acceptanceEvidence = buildAcceptanceEvidence({
4100
+ commit: acceptanceTests.commit_sha,
4101
+ tests: acceptanceTests.tests,
4102
+ before: acceptanceBefore,
4103
+ after,
4104
+ changed,
4105
+ skipped: acceptanceSkipped,
4106
+ ranBy,
4107
+ })
4108
+ }
3949
4109
  // Read once; removed so it can never be committed with the work.
3950
4110
  if (run.__evidenceFile) fs.rmSync(run.__evidenceFile, { force: true })
3951
4111
  const runEvidence = buildEvidence({
@@ -3954,6 +4114,7 @@ async function runExecutor(run, state) {
3954
4114
  checks: [...runnerChecks, ...agentEvidence.checks],
3955
4115
  notVerified: agentEvidence.notVerified,
3956
4116
  skippedReason: runnerChecksSkipped,
4117
+ acceptance: acceptanceEvidence,
3957
4118
  })
3958
4119
 
3959
4120
  executorPhase = "finished"
@@ -0,0 +1,265 @@
1
+ /**
2
+ * Acceptance tests (PWK-205/206): an issue's acceptance criteria become failing
3
+ * tests before code is written, and the build shows them go from failing to
4
+ * passing without changing them.
5
+ *
6
+ * - An acceptance-test run (a planning run with purpose "acceptance_tests")
7
+ * writes the tests and a manifest. The runner checks each test fails, commits
8
+ * them as the first commit on the issue's branch and pushes it.
9
+ * - A build run on an issue with acceptance tests runs each one before the agent
10
+ * starts (it should fail) and after it finishes (it should pass), and diffs the
11
+ * test files against the acceptance commit. Both runs and any change to the
12
+ * tests go into the run's evidence and the pull request.
13
+ *
14
+ * Pure helpers, plus the steps that take an `exec` and `git` to run with.
15
+ */
16
+
17
+ import fs from "node:fs"
18
+ import path from "node:path"
19
+
20
+ /** What the agent writes at the checkout's root; never committed. */
21
+ export const ACCEPTANCE_MANIFEST = ".pacework-acceptance.json"
22
+
23
+ const MAX_TESTS = 50
24
+ const MAX_OUTPUT = 2000
25
+
26
+ const text = (value, max) => typeof value === "string" && value.trim() ? value.trim().slice(0, max) : null
27
+ const tail = (value, max = MAX_OUTPUT) => {
28
+ const out = String(value || "").trim()
29
+ return out.length > max ? `…${out.slice(-max)}` : out
30
+ }
31
+ const sameCriterion = (a, b) => a.replace(/\s+/g, " ").trim().toLowerCase() === b.replace(/\s+/g, " ").trim().toLowerCase()
32
+
33
+ /** A path inside the checkout, relative and without `..`; null otherwise. */
34
+ function insideCheckout(checkout, file) {
35
+ const normalized = path.normalize(file).replace(/^\.\/+/, "")
36
+ if (!normalized || path.isAbsolute(normalized) || normalized.split(path.sep).includes("..")) return null
37
+ return fs.existsSync(path.join(checkout, normalized)) ? normalized : null
38
+ }
39
+
40
+ /**
41
+ * The agent's manifest, checked: each test needs a criterion, a name, a file
42
+ * that exists in the checkout and a command. `problems` says what was wrong;
43
+ * `unmapped` lists criteria with neither a test nor a reason they cannot have one.
44
+ */
45
+ export function readAcceptanceManifest(checkout, criteria = []) {
46
+ const problems = []
47
+ let raw
48
+ try {
49
+ raw = JSON.parse(fs.readFileSync(path.join(checkout, ACCEPTANCE_MANIFEST), "utf8"))
50
+ } catch (error) {
51
+ return { tests: [], untestable: [], unmapped: [...criteria], problems: [`${ACCEPTANCE_MANIFEST} is missing or not JSON (${error?.code === "ENOENT" ? "not written" : "unreadable"})`] }
52
+ }
53
+ const tests = []
54
+ for (const [index, entry] of (Array.isArray(raw?.tests) ? raw.tests : []).slice(0, MAX_TESTS).entries()) {
55
+ const criterion = text(entry?.criterion, 500)
56
+ const name = text(entry?.name, 300)
57
+ const command = text(entry?.command, 500)
58
+ const file = typeof entry?.file === "string" ? insideCheckout(checkout, entry.file.trim()) : null
59
+ if (!criterion || !name || !command || !file) {
60
+ problems.push(`test ${index + 1} needs a criterion, a name, a command and a file that exists in the checkout`)
61
+ continue
62
+ }
63
+ tests.push({ criterion, name, file, command })
64
+ }
65
+ const untestable = (Array.isArray(raw?.untestable) ? raw.untestable : []).flatMap((entry) => {
66
+ const criterion = text(entry?.criterion, 500)
67
+ return criterion ? [{ criterion, why: text(entry?.why, 300) }] : []
68
+ })
69
+ const unmapped = criteria.filter((criterion) =>
70
+ !tests.some((test) => sameCriterion(test.criterion, criterion))
71
+ && !untestable.some((entry) => sameCriterion(entry.criterion, criterion)))
72
+ if (!tests.length) problems.push("no usable test in the manifest")
73
+ return { tests, untestable, unmapped, problems }
74
+ }
75
+
76
+ /**
77
+ * Run each test's own command. `exec(command, { cwd })` resolves
78
+ * { code, output, timedOut }. A test passes when its command exits 0.
79
+ */
80
+ export async function runAcceptanceTests(tests, { cwd, exec, shouldStop = () => false }) {
81
+ const results = []
82
+ for (const test of tests) {
83
+ if (shouldStop()) break
84
+ const started = Date.now()
85
+ const result = await exec(test.command, { cwd })
86
+ results.push({
87
+ ...test,
88
+ result: {
89
+ status: result.code === 0 && !result.timedOut ? "passed" : "failed",
90
+ exit_code: Number.isInteger(result.code) ? result.code : null,
91
+ duration_ms: Date.now() - started,
92
+ output: tail(result.timedOut ? `${result.output}\n(timed out)` : result.output),
93
+ },
94
+ })
95
+ }
96
+ return results
97
+ }
98
+
99
+ /** The acceptance commit's message. */
100
+ export function acceptanceCommitMessage(identifier) {
101
+ return `Acceptance tests for ${identifier || "this issue"}`
102
+ }
103
+
104
+ /**
105
+ * Test files the build changed since the acceptance commit, from
106
+ * `git diff --name-status -M <acceptance commit>` (working tree included):
107
+ * modified, deleted or renamed. Any of these is flagged for the reviewer.
108
+ */
109
+ export function changedAcceptanceTests(nameStatus, files) {
110
+ const watched = new Set(files)
111
+ const changes = []
112
+ for (const line of String(nameStatus || "").split(/\r?\n/)) {
113
+ const parts = line.split("\t").map((part) => part.trim())
114
+ if (parts.length < 2 || !parts[0]) continue
115
+ const code = parts[0][0]
116
+ const from = parts[1]
117
+ if (!watched.has(from)) continue
118
+ if (code === "D") changes.push({ file: from, change: "deleted" })
119
+ else if (code === "R") changes.push({ file: from, change: "renamed", to: parts[2] || null })
120
+ else if (code === "M" || code === "T") changes.push({ file: from, change: "modified" })
121
+ }
122
+ return changes
123
+ }
124
+
125
+ /**
126
+ * The acceptance part of a build run's evidence: each test before and after the
127
+ * change, and the tests the build changed. `before` and `after` are the results
128
+ * of runAcceptanceTests; `skipped` says why they were not run.
129
+ */
130
+ export function buildAcceptanceEvidence({ commit, tests, before = [], after = [], changed = [], skipped = null, ranBy = "runner" }) {
131
+ const find = (results, test) => results.find((entry) => entry.name === test.name && entry.file === test.file)?.result ?? null
132
+ const rows = tests.slice(0, MAX_TESTS).map((test) => {
133
+ const was = find(before, test)
134
+ const now = find(after, test)
135
+ return {
136
+ criterion: test.criterion,
137
+ name: test.name,
138
+ file: test.file,
139
+ command: test.command,
140
+ before: was ? { status: was.status, exit_code: was.exit_code } : null,
141
+ after: now ? { status: now.status, exit_code: now.exit_code, output: now.status === "failed" ? now.output : null } : null,
142
+ }
143
+ })
144
+ return {
145
+ commit: commit ? String(commit).slice(0, 64) : null,
146
+ // "agent": a sandboxed run, where the agent ran them inside its sandbox and
147
+ // recorded them with `pacework evidence run`; the runner did not run them itself.
148
+ ran_by: ranBy === "agent" ? "agent" : "runner",
149
+ tests: rows,
150
+ changed: changed.slice(0, MAX_TESTS),
151
+ skipped: skipped ? String(skipped).slice(0, 300) : null,
152
+ // Every test failed before and passes after, and none was changed.
153
+ failing_to_passing: rows.length > 0 && !skipped && !changed.length
154
+ && rows.every((row) => row.before?.status === "failed" && row.after?.status === "passed"),
155
+ }
156
+ }
157
+
158
+ const cell = (value) => String(value ?? "").replace(/\|/g, "\\|").replace(/\s+/g, " ").trim()
159
+ const BEFORE = { failed: "failing", passed: "**passing already**" }
160
+ const AFTER = { failed: "**failing**", passed: "passing" }
161
+
162
+ /** The Acceptance tests section of a pull request or a run, from the evidence. */
163
+ export function renderAcceptanceMarkdown(acceptance, { heading = "## Acceptance tests" } = {}) {
164
+ if (!acceptance?.tests?.length) return ""
165
+ const lines = [heading]
166
+ if (acceptance.commit) lines.push(`Written from the issue's acceptance criteria before the change, in commit \`${cell(acceptance.commit).slice(0, 7)}\`.`)
167
+ lines.push("", "| Criterion | Test | Before | After |", "| --- | --- | --- | --- |")
168
+ for (const row of acceptance.tests) {
169
+ lines.push(`| ${cell(row.criterion)} | \`${cell(row.name)}\` | ${row.before ? BEFORE[row.before.status] : "not run"} | ${row.after ? AFTER[row.after.status] : "not run"} |`)
170
+ }
171
+ if (acceptance.ran_by === "agent") lines.push("", "Run by the agent inside its sandbox and recorded with `pacework evidence run`; the runner could not run them outside the sandbox.")
172
+ if (acceptance.skipped) lines.push("", `The runner did not run them: ${cell(acceptance.skipped)}.`)
173
+ if (acceptance.changed?.length) {
174
+ lines.push("", "> [!WARNING]", "> **The build changed acceptance tests.** They are the issue's contract: the reviewer checks each change is explained and does not weaken the test.")
175
+ for (const change of acceptance.changed) lines.push(`> - \`${cell(change.file)}\` ${change.change}${change.to ? ` to \`${cell(change.to)}\`` : ""}`)
176
+ }
177
+ return lines.join("\n")
178
+ }
179
+
180
+ /**
181
+ * In a sandboxed run the runner cannot run the tests, so the agent runs them
182
+ * through `pacework evidence run`: each test's first record is before the
183
+ * change, its last record after it. Returns results shaped like
184
+ * runAcceptanceTests', for the tests the agent recorded.
185
+ */
186
+ export function acceptanceFromAgentRecords(tests, records) {
187
+ const before = [], after = []
188
+ for (const test of tests) {
189
+ const runs = (records ?? []).filter((record) => record.command && record.command.trim() === String(test.command).trim())
190
+ if (!runs.length) continue
191
+ const result = (record) => ({ status: record.status === "passed" ? "passed" : "failed", exit_code: record.exit_code ?? null, duration_ms: record.duration_ms ?? null, output: record.output ?? null })
192
+ before.push({ ...test, result: result(runs[0]) })
193
+ if (runs.length > 1) after.push({ ...test, result: result(runs[runs.length - 1]) })
194
+ }
195
+ return { before, after }
196
+ }
197
+
198
+ /** What a builder is told about the issue's acceptance tests (PWK-206). Empty when it has none. */
199
+ export function acceptancePromptSection(acceptance, { sandboxed = false, cli = "pacework" } = {}) {
200
+ const tests = Array.isArray(acceptance?.tests) ? acceptance.tests : []
201
+ if (!tests.length) return ""
202
+ return [
203
+ "## Acceptance tests",
204
+ `This issue's acceptance criteria are already tests, committed${acceptance.commit_sha ? ` in \`${String(acceptance.commit_sha).slice(0, 7)}\`` : ""} on this branch before any code. Start from them:`,
205
+ "1. Run each one and see it fail.",
206
+ "2. Make the change that makes them pass, without breaking other tests.",
207
+ "3. Run them again and see them pass.",
208
+ "Do not change, skip, weaken or delete them. If one is wrong, leave it as it is and say why in your report: the runner compares them with the acceptance commit and flags any change for the reviewer.",
209
+ ...(sandboxed ? [`This run is sandboxed, so the runner cannot run them for you: run each one through \`${cli} evidence run -- <its command>\` before you change anything, and again at the end. The evidence on the pull request is built from those records.`] : []),
210
+ "",
211
+ ...tests.map((test) => `- ${test.criterion}: \`${test.name}\` in \`${test.file}\` (run: \`${test.command}\`)`),
212
+ ].join("\n")
213
+ }
214
+
215
+ /**
216
+ * After the agent wrote the acceptance tests (PWK-205): read its manifest, run
217
+ * each test (it should fail), commit only the test files as the acceptance
218
+ * commit, set aside anything else the agent changed, and push. Returns the
219
+ * report the server records. `git(args)` resolves { code, stdout, stderr };
220
+ * `exec` runs a test command as runAcceptanceTests expects; `runTests` false
221
+ * skips running them (a sandboxed run). Changes that are not tests are saved
222
+ * as a patch at `setAsidePath` and taken out of the checkout.
223
+ */
224
+ export async function commitAcceptanceWork({ checkout, criteria = [], identifier = null, git, exec, push, setAsidePath, runTests = true, shouldStop = () => false, log = () => {} }) {
225
+ const manifest = readAcceptanceManifest(checkout, criteria)
226
+ fs.rmSync(path.join(checkout, ACCEPTANCE_MANIFEST), { force: true })
227
+ if (!manifest.tests.length) throw new Error(`No acceptance tests were written: ${manifest.problems.join("; ")}`)
228
+ const before = runTests ? await runAcceptanceTests(manifest.tests, { cwd: checkout, exec, shouldStop }) : []
229
+ for (const entry of before) log(`acceptance test ${entry.result.status} before any code: ${entry.name}`)
230
+ const files = [...new Set(manifest.tests.map((test) => test.file))]
231
+ const added = await git(["add", "--", ...files])
232
+ if (added.code !== 0) throw new Error(`Could not stage the acceptance tests: ${added.stderr}`)
233
+ if (!(await git(["diff", "--cached", "--name-only"])).stdout.trim()) {
234
+ throw new Error("The manifest names tests that are already on the branch unchanged; nothing new to commit.")
235
+ }
236
+ const committed = await git(["commit", "-m", acceptanceCommitMessage(identifier)])
237
+ if (committed.code !== 0) throw new Error(`Could not commit the acceptance tests: ${committed.stderr}`)
238
+ const sha = (await git(["rev-parse", "HEAD"])).stdout.trim()
239
+ // Anything else the agent changed stays out of the commit: saved as a patch in
240
+ // the run's folder (not the repository's stash, which a person's checkout shares)
241
+ // and taken out of the checkout the build starts from.
242
+ const leftOver = (await git(["status", "--porcelain"])).stdout.trim()
243
+ if (leftOver) {
244
+ await git(["add", "-A"])
245
+ const patch = await git(["diff", "--cached", "--binary"])
246
+ if (setAsidePath) {
247
+ fs.mkdirSync(path.dirname(setAsidePath), { recursive: true })
248
+ fs.writeFileSync(setAsidePath, patch.stdout)
249
+ }
250
+ await git(["reset", "--hard", "HEAD"])
251
+ log(`set aside ${leftOver.split("\n").length} changed path(s) that are not acceptance tests${setAsidePath ? ` in ${setAsidePath}` : ""}`)
252
+ }
253
+ await push()
254
+ return {
255
+ commit_sha: sha,
256
+ tests: manifest.tests.map((test) => ({
257
+ ...test,
258
+ before: before.find((entry) => entry.name === test.name && entry.file === test.file)?.result ?? null,
259
+ })),
260
+ untestable: manifest.untestable,
261
+ unmapped: manifest.unmapped,
262
+ tests_run: runTests,
263
+ set_aside: leftOver ? leftOver.split("\n").length : 0,
264
+ }
265
+ }
@@ -54,6 +54,9 @@ export function detectCapabilities({ repoRoot, executors = [], exec = run, platf
54
54
  if (exec("git", ["--version"])) tools.push("git")
55
55
  if (exec("docker", ["version", "--format", "{{.Server.Version}}"])) tools.push("docker")
56
56
  if (canSandbox({ platform, executors, exec })) tools.push("sandbox")
57
+ // This runner writes acceptance tests in a checkout of their own (PWK-205); an
58
+ // older one would write them into the repository root, so it is never given them.
59
+ tools.push("acceptance-tests")
57
60
  capabilities.tools = tools
58
61
  return capabilities
59
62
  }
package/lib/evidence.mjs CHANGED
@@ -13,6 +13,7 @@
13
13
 
14
14
  import fs from "node:fs"
15
15
  import path from "node:path"
16
+ import { renderAcceptanceMarkdown } from "./acceptance.mjs"
16
17
 
17
18
  export const EVIDENCE_VERSION = 1
18
19
  /** The file in the checkout where `pacework evidence` writes; excluded from commits. */
@@ -87,9 +88,11 @@ export function readAgentEvidence(file) {
87
88
 
88
89
  /**
89
90
  * The run's evidence record. `role` is "build" or "review"; `rerun` is what a
90
- * reviewer's runner ran again on the reviewed commit.
91
+ * reviewer's runner ran again on the reviewed commit. `acceptance` is the
92
+ * issue's acceptance tests before and after the change (PWK-206), from
93
+ * buildAcceptanceEvidence.
91
94
  */
92
- export function buildEvidence({ role = "build", commit = null, checks = [], notVerified = [], skippedReason = null }) {
95
+ export function buildEvidence({ role = "build", commit = null, checks = [], notVerified = [], skippedReason = null, acceptance = null }) {
93
96
  const items = checks.slice(0, MAX_ITEMS)
94
97
  const not_verified = [...notVerified]
95
98
  if (skippedReason) not_verified.unshift({ what: "The runner's own checks", why: skippedReason })
@@ -101,6 +104,7 @@ export function buildEvidence({ role = "build", commit = null, checks = [], notV
101
104
  items,
102
105
  not_verified: not_verified.slice(0, 20),
103
106
  production_like: items.some((item) => item.environment === "production-like" && item.status === "passed"),
107
+ ...(acceptance ? { acceptance } : {}),
104
108
  }
105
109
  }
106
110
 
@@ -139,6 +143,8 @@ export function renderEvidenceMarkdown(evidence, { heading = "## Evidence" } = {
139
143
  lines.push("", "**Not verified**")
140
144
  for (const entry of notVerified) lines.push(`- ${cell(entry.what)}${entry.why ? `: ${cell(entry.why)}` : ""}`)
141
145
  }
146
+ const acceptance = renderAcceptanceMarkdown(evidence?.acceptance, { heading: "### Acceptance tests" })
147
+ if (acceptance) lines.push("", acceptance)
142
148
  return lines.join("\n")
143
149
  }
144
150
 
@@ -0,0 +1,25 @@
1
+ /**
2
+ * What kind of run this is, from what the server says about it: the run's
3
+ * `purpose` (its workflow stage) first, then the agent's role.
4
+ */
5
+
6
+ /** The slug of the agent's role, or null when it has none. */
7
+ export function agentRoleSlug(agent) {
8
+ const slug = agent && typeof agent === "object" ? agent.role?.slug : null
9
+ return typeof slug === "string" && slug ? slug : null
10
+ }
11
+
12
+ /**
13
+ * Whether a run is a review. The server says so in `purpose`; the agent's role
14
+ * decides only when a run carries no purpose. Deciding by a name alone made a
15
+ * renamed reviewer build code.
16
+ */
17
+ export function isReviewRun(run) {
18
+ if (run?.purpose) return run.purpose === "review"
19
+ return agentRoleSlug(run?.agent) === "reviewer"
20
+ }
21
+
22
+ /** Scouts survey the codebase and may touch many files on purpose. */
23
+ export function fileLimitAppliesTo(agent) {
24
+ return agentRoleSlug(agent) !== "scout"
25
+ }
@@ -204,7 +204,7 @@ export function createShellHandlers({
204
204
  out.log(cmd("issues", "List issues (open / active / all / in_progress …)"))
205
205
  out.log(cmd("mine", "Issues assigned to you"))
206
206
  out.log(cmd("issue <id>", "View issue detail, description, comments"))
207
- out.log(cmd("agents", "List agents and their skill profiles"))
207
+ out.log(cmd("agents", "List agents and their roles"))
208
208
  out.log(cmd("dashboard", "Orchestration status — runs, queue, review"))
209
209
  out.log(cmd("logtail <run-id>","Stream live log of a running job"))
210
210
  out.log(cmd("open [id]", "Open workspace (or specific issue) in browser"))
@@ -785,20 +785,20 @@ export function createShellHandlers({
785
785
  const { agents } = await res.json()
786
786
  if (!agents?.length) { out.log(` \x1b[2mNo agents configured.\x1b[0m\n`); break }
787
787
 
788
- const SKILL_COLOR = { reviewer:"\x1b[33m", implementer:"\x1b[34m", planner:"\x1b[35m" }
789
- const COL_NAME = 22, COL_SKILL = 16, COL_EXE = 14
790
- const hr = ` ${"─".repeat(COL_NAME + COL_SKILL + COL_EXE + 4)}`
788
+ const ROLE_COLOR = { reviewer:"\x1b[33m", implementer:"\x1b[34m", planner:"\x1b[35m" }
789
+ const COL_NAME = 22, COL_ROLE = 16, COL_EXE = 14
790
+ const hr = ` ${"─".repeat(COL_NAME + COL_ROLE + COL_EXE + 4)}`
791
791
  out.log(`\n\x1b[1mAgents\x1b[0m \x1b[2m(${agents.length})\x1b[0m`)
792
792
  out.log(hr)
793
- out.log(` \x1b[2m${"NAME".padEnd(COL_NAME)} ${"SKILL".padEnd(COL_SKILL)} ${"EXECUTOR".padEnd(COL_EXE)}\x1b[0m`)
793
+ out.log(` \x1b[2m${"NAME".padEnd(COL_NAME)} ${"ROLE".padEnd(COL_ROLE)} ${"EXECUTOR".padEnd(COL_EXE)}\x1b[0m`)
794
794
  out.log(hr)
795
795
  for (const a of agents) {
796
- const skill = (a.skill_profile_label ?? a.skill_profile_slug ?? "—").toLowerCase()
797
- const skillCol = SKILL_COLOR[a.skill_profile_slug] ?? "\x1b[2m"
796
+ const role = (a.role_name ?? a.role_slug ?? "—").toLowerCase()
797
+ const roleCol = ROLE_COLOR[a.role_slug] ?? "\x1b[2m"
798
798
  const name = (a.name ?? a.slug ?? a.id.slice(0,8)).slice(0, COL_NAME).padEnd(COL_NAME)
799
- const skillStr = skill.slice(0, COL_SKILL).padEnd(COL_SKILL)
799
+ const roleStr = role.slice(0, COL_ROLE).padEnd(COL_ROLE)
800
800
  const exe = executorLabel(a.local_executor ?? "").slice(0, COL_EXE).padEnd(COL_EXE)
801
- out.log(` \x1b[1m${name}\x1b[0m ${skillCol}${skillStr}\x1b[0m \x1b[2m${exe}\x1b[0m`)
801
+ out.log(` \x1b[1m${name}\x1b[0m ${roleCol}${roleStr}\x1b[0m \x1b[2m${exe}\x1b[0m`)
802
802
  }
803
803
  out.log(hr)
804
804
  out.log(` \x1b[2mTip: /run <issue-id> to dispatch with agent picker\x1b[0m\n`)
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "pacework",
3
- "version": "1.7.17",
3
+ "version": "1.7.19",
4
4
  "description": "pacework CLI: connect this machine as a runner for pacework AI agents",
5
5
  "type": "module",
6
6
  "bin": {