@vibe-agent-toolkit/agent-skills 0.2.0-rc.2 → 0.2.0-rc.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (194) hide show
  1. package/dist/files-config.d.ts +34 -1
  2. package/dist/files-config.d.ts.map +1 -1
  3. package/dist/files-config.js +280 -36
  4. package/dist/files-config.js.map +1 -1
  5. package/dist/fs-attribution.d.ts +45 -0
  6. package/dist/fs-attribution.d.ts.map +1 -0
  7. package/dist/fs-attribution.js +57 -0
  8. package/dist/fs-attribution.js.map +1 -0
  9. package/dist/index.d.ts +5 -4
  10. package/dist/index.d.ts.map +1 -1
  11. package/dist/index.js +8 -2
  12. package/dist/index.js.map +1 -1
  13. package/dist/inventory/detectors/declared-but-missing.d.ts +1 -1
  14. package/dist/inventory/detectors/declared-but-missing.d.ts.map +1 -1
  15. package/dist/inventory/detectors/marketplace-source-missing.d.ts +1 -1
  16. package/dist/inventory/detectors/marketplace-source-missing.d.ts.map +1 -1
  17. package/dist/inventory/detectors/present-but-undeclared.d.ts +1 -1
  18. package/dist/inventory/detectors/present-but-undeclared.d.ts.map +1 -1
  19. package/dist/inventory/detectors/reference-target-missing.d.ts +1 -1
  20. package/dist/inventory/detectors/reference-target-missing.d.ts.map +1 -1
  21. package/dist/inventory/index.d.ts +1 -1
  22. package/dist/inventory/index.d.ts.map +1 -1
  23. package/dist/inventory/index.js +1 -1
  24. package/dist/inventory/index.js.map +1 -1
  25. package/dist/inventory/serialize.d.ts +4 -4
  26. package/dist/inventory/serialize.d.ts.map +1 -1
  27. package/dist/inventory/serialize.js +6 -6
  28. package/dist/inventory/serialize.js.map +1 -1
  29. package/dist/post-build-checks.d.ts +1 -1
  30. package/dist/post-build-checks.d.ts.map +1 -1
  31. package/dist/post-build-checks.js.map +1 -1
  32. package/dist/projection/skill-extent.d.ts +247 -0
  33. package/dist/projection/skill-extent.d.ts.map +1 -0
  34. package/dist/projection/skill-extent.js +558 -0
  35. package/dist/projection/skill-extent.js.map +1 -0
  36. package/dist/schemas/agent-skill-frontmatter.js +1 -1
  37. package/dist/schemas/agent-skill-frontmatter.js.map +1 -1
  38. package/dist/skill-packager.d.ts +34 -4
  39. package/dist/skill-packager.d.ts.map +1 -1
  40. package/dist/skill-packager.js +113 -23
  41. package/dist/skill-packager.js.map +1 -1
  42. package/dist/skill-source/git-clone.d.ts.map +1 -1
  43. package/dist/skill-source/git-clone.js +9 -8
  44. package/dist/skill-source/git-clone.js.map +1 -1
  45. package/dist/skill-test/baseline-delta.d.ts +1575 -0
  46. package/dist/skill-test/baseline-delta.d.ts.map +1 -0
  47. package/dist/skill-test/baseline-delta.js +272 -0
  48. package/dist/skill-test/baseline-delta.js.map +1 -0
  49. package/dist/skill-test/baseline-integrity.d.ts +944 -0
  50. package/dist/skill-test/baseline-integrity.d.ts.map +1 -0
  51. package/dist/skill-test/baseline-integrity.js +3083 -0
  52. package/dist/skill-test/baseline-integrity.js.map +1 -0
  53. package/dist/skill-test/eval-executor.d.ts +19 -3
  54. package/dist/skill-test/eval-executor.d.ts.map +1 -1
  55. package/dist/skill-test/eval-executor.js +3 -3
  56. package/dist/skill-test/eval-executor.js.map +1 -1
  57. package/dist/skill-test/eval-fragment.d.ts +80 -6
  58. package/dist/skill-test/eval-fragment.d.ts.map +1 -1
  59. package/dist/skill-test/eval-fragment.js +61 -7
  60. package/dist/skill-test/eval-fragment.js.map +1 -1
  61. package/dist/skill-test/eval-grader.d.ts.map +1 -1
  62. package/dist/skill-test/eval-grader.js +143 -7
  63. package/dist/skill-test/eval-grader.js.map +1 -1
  64. package/dist/skill-test/eval-inputs.d.ts +107 -6
  65. package/dist/skill-test/eval-inputs.d.ts.map +1 -1
  66. package/dist/skill-test/eval-inputs.js +173 -16
  67. package/dist/skill-test/eval-inputs.js.map +1 -1
  68. package/dist/skill-test/eval-lint.d.ts +13 -0
  69. package/dist/skill-test/eval-lint.d.ts.map +1 -1
  70. package/dist/skill-test/eval-lint.js +17 -4
  71. package/dist/skill-test/eval-lint.js.map +1 -1
  72. package/dist/skill-test/executor-prompt.d.ts +13 -5
  73. package/dist/skill-test/executor-prompt.d.ts.map +1 -1
  74. package/dist/skill-test/executor-prompt.js +8 -5
  75. package/dist/skill-test/executor-prompt.js.map +1 -1
  76. package/dist/skill-test/exit-codes.d.ts +53 -16
  77. package/dist/skill-test/exit-codes.d.ts.map +1 -1
  78. package/dist/skill-test/exit-codes.js +123 -40
  79. package/dist/skill-test/exit-codes.js.map +1 -1
  80. package/dist/skill-test/fragment-merge.d.ts +1 -18
  81. package/dist/skill-test/fragment-merge.d.ts.map +1 -1
  82. package/dist/skill-test/fragment-merge.js +63 -4
  83. package/dist/skill-test/fragment-merge.js.map +1 -1
  84. package/dist/skill-test/grader-prompt.d.ts +27 -7
  85. package/dist/skill-test/grader-prompt.d.ts.map +1 -1
  86. package/dist/skill-test/grader-prompt.js +253 -61
  87. package/dist/skill-test/grader-prompt.js.map +1 -1
  88. package/dist/skill-test/grader-text.d.ts +145 -0
  89. package/dist/skill-test/grader-text.d.ts.map +1 -0
  90. package/dist/skill-test/grader-text.js +410 -0
  91. package/dist/skill-test/grader-text.js.map +1 -0
  92. package/dist/skill-test/grading-adapter.d.ts +32 -0
  93. package/dist/skill-test/grading-adapter.d.ts.map +1 -1
  94. package/dist/skill-test/grading-adapter.js +50 -3
  95. package/dist/skill-test/grading-adapter.js.map +1 -1
  96. package/dist/skill-test/grading-schema.d.ts +160 -5
  97. package/dist/skill-test/grading-schema.d.ts.map +1 -1
  98. package/dist/skill-test/grading-schema.js +66 -7
  99. package/dist/skill-test/grading-schema.js.map +1 -1
  100. package/dist/skill-test/index.d.ts +1 -1
  101. package/dist/skill-test/index.d.ts.map +1 -1
  102. package/dist/skill-test/index.js +1 -1
  103. package/dist/skill-test/index.js.map +1 -1
  104. package/dist/skill-test/lock.d.ts +56 -0
  105. package/dist/skill-test/lock.d.ts.map +1 -1
  106. package/dist/skill-test/lock.js +75 -2
  107. package/dist/skill-test/lock.js.map +1 -1
  108. package/dist/skill-test/pipeline.d.ts +16 -0
  109. package/dist/skill-test/pipeline.d.ts.map +1 -1
  110. package/dist/skill-test/pipeline.js +16 -9
  111. package/dist/skill-test/pipeline.js.map +1 -1
  112. package/dist/skill-test/preflight.d.ts.map +1 -1
  113. package/dist/skill-test/preflight.js +58 -14
  114. package/dist/skill-test/preflight.js.map +1 -1
  115. package/dist/skill-test/run-harness.d.ts +578 -35
  116. package/dist/skill-test/run-harness.d.ts.map +1 -1
  117. package/dist/skill-test/run-harness.js +1278 -147
  118. package/dist/skill-test/run-harness.js.map +1 -1
  119. package/dist/skill-test/staging.d.ts +8 -2
  120. package/dist/skill-test/staging.d.ts.map +1 -1
  121. package/dist/skill-test/staging.js +8 -4
  122. package/dist/skill-test/staging.js.map +1 -1
  123. package/dist/skill-test/tool-eval-schema.d.ts +6 -6
  124. package/dist/test-input.d.ts +1 -1
  125. package/dist/test-input.d.ts.map +1 -1
  126. package/dist/validators/agent-instruction-presence.d.ts +1 -1
  127. package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
  128. package/dist/validators/bundled-resource-link-detection.d.ts +1 -1
  129. package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
  130. package/dist/validators/bundled-resource-link-detection.js +1 -1
  131. package/dist/validators/bundled-resource-link-detection.js.map +1 -1
  132. package/dist/validators/compat-detectors.d.ts +1 -1
  133. package/dist/validators/compat-detectors.d.ts.map +1 -1
  134. package/dist/validators/compat-detectors.js +1 -1
  135. package/dist/validators/compat-detectors.js.map +1 -1
  136. package/dist/validators/cross-skill-dependency-detection.d.ts +1 -1
  137. package/dist/validators/cross-skill-dependency-detection.d.ts.map +1 -1
  138. package/dist/validators/cross-skill-dependency-detection.js +1 -1
  139. package/dist/validators/cross-skill-dependency-detection.js.map +1 -1
  140. package/dist/validators/description-style-detection.d.ts +1 -1
  141. package/dist/validators/description-style-detection.d.ts.map +1 -1
  142. package/dist/validators/description-style-detection.js +1 -1
  143. package/dist/validators/description-style-detection.js.map +1 -1
  144. package/dist/validators/frontmatter-validation.d.ts +1 -1
  145. package/dist/validators/frontmatter-validation.d.ts.map +1 -1
  146. package/dist/validators/frontmatter-validation.js +1 -1
  147. package/dist/validators/frontmatter-validation.js.map +1 -1
  148. package/dist/validators/imperative-body-detection.d.ts +1 -1
  149. package/dist/validators/imperative-body-detection.d.ts.map +1 -1
  150. package/dist/validators/imperative-body-detection.js +1 -1
  151. package/dist/validators/imperative-body-detection.js.map +1 -1
  152. package/dist/validators/index.d.ts +1 -1
  153. package/dist/validators/index.d.ts.map +1 -1
  154. package/dist/validators/kebab-case-detection.d.ts +1 -1
  155. package/dist/validators/kebab-case-detection.d.ts.map +1 -1
  156. package/dist/validators/kebab-case-detection.js +1 -1
  157. package/dist/validators/kebab-case-detection.js.map +1 -1
  158. package/dist/validators/marketplace-validator.js +1 -1
  159. package/dist/validators/marketplace-validator.js.map +1 -1
  160. package/dist/validators/packaging-validator.d.ts +38 -5
  161. package/dist/validators/packaging-validator.d.ts.map +1 -1
  162. package/dist/validators/packaging-validator.js +87 -10
  163. package/dist/validators/packaging-validator.js.map +1 -1
  164. package/dist/validators/plugin-hosted-shape.d.ts +1 -1
  165. package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
  166. package/dist/validators/plugin-hosted-shape.js +1 -1
  167. package/dist/validators/plugin-hosted-shape.js.map +1 -1
  168. package/dist/validators/plugin-recommended-fields.d.ts +1 -1
  169. package/dist/validators/plugin-recommended-fields.d.ts.map +1 -1
  170. package/dist/validators/plugin-recommended-fields.js +1 -1
  171. package/dist/validators/plugin-recommended-fields.js.map +1 -1
  172. package/dist/validators/registry-validator.js +1 -1
  173. package/dist/validators/registry-validator.js.map +1 -1
  174. package/dist/validators/rule-engine/rule-engine.d.ts +1 -1
  175. package/dist/validators/rule-engine/rule-engine.d.ts.map +1 -1
  176. package/dist/validators/rule-engine/rule-engine.js +20 -9
  177. package/dist/validators/rule-engine/rule-engine.js.map +1 -1
  178. package/dist/validators/skill-validator.d.ts.map +1 -1
  179. package/dist/validators/skill-validator.js +21 -3
  180. package/dist/validators/skill-validator.js.map +1 -1
  181. package/dist/validators/types.d.ts +1 -1
  182. package/dist/validators/types.d.ts.map +1 -1
  183. package/dist/validators/unified-validator.js +1 -1
  184. package/dist/validators/unified-validator.js.map +1 -1
  185. package/dist/validators/validation-rules.d.ts +1 -1
  186. package/dist/validators/validation-rules.d.ts.map +1 -1
  187. package/dist/validators/walker-to-issues.d.ts +7 -3
  188. package/dist/validators/walker-to-issues.d.ts.map +1 -1
  189. package/dist/validators/walker-to-issues.js +37 -8
  190. package/dist/validators/walker-to-issues.js.map +1 -1
  191. package/dist/walk-link-graph.d.ts.map +1 -1
  192. package/dist/walk-link-graph.js +36 -2
  193. package/dist/walk-link-graph.js.map +1 -1
  194. package/package.json +7 -7
@@ -11,16 +11,18 @@
11
11
  * → release lock → return result
12
12
  */
13
13
  import { randomBytes } from 'node:crypto';
14
- import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
14
+ import { existsSync, lstatSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
15
15
  import { homedir } from 'node:os';
16
16
  import { basename, dirname } from 'node:path';
17
17
  import { fileURLToPath } from 'node:url';
18
18
  import { getToolVersion, killAllActiveClaudeChildren, mkdirSyncReal, normalizedTmpdir, probeAuthStatus, resolveAssetReference, safeExecResult, safePath, toForwardSlash, } from '@vibe-agent-toolkit/utils';
19
19
  import { resolveSkillSource } from '../skill-source/resolve-skill-source.js';
20
+ import { BaselineArtifactSchema, computeBaselineDelta, formatBaselineDeltaLine, } from './baseline-delta.js';
21
+ import { activeContaminationSignals, armExpectationSkew, detectBaselineContamination, scrubControlArmEnv, skillContentNeedles, summarizeBaselineIntegrity, } from './baseline-integrity.js';
20
22
  import { assembleChildEnv, assertKnownEnvTokens, computeEnvTokens, resolveInjectEnv } from './declared-env.js';
21
23
  import { runExecutorForEval } from './eval-executor.js';
22
24
  import { runGraderForEval } from './eval-grader.js';
23
- import { EvalInputError, parseEvalSuite, stageEvalWorkspaces } from './eval-inputs.js';
25
+ import { armDirSegment, EvalInputError, parseEvalSuite, stageEvalWorkspaces, } from './eval-inputs.js';
24
26
  import { lintEvalExpectations, lintToolExpectationExecutables } from './eval-lint.js';
25
27
  import { DEFAULT_EVALS_SUBPATH } from './eval-suite-isolation.js';
26
28
  import { writeEvalsTemplate } from './evals-template.js';
@@ -28,11 +30,12 @@ import { BootstrapNeededError, DuplicateStagedSkillError, InternalHarnessError,
28
30
  import { mergeFragmentsToFriction, mergeFragmentsToGrading, mergeFragmentsToToolEval } from './fragment-merge.js';
29
31
  import { FrictionReportSchema } from './friction-schema.js';
30
32
  import { DEFAULT_CONCURRENCY, DEFAULT_GRADER_MODEL } from './grader-model.js';
33
+ import { sanitizeGraderText } from './grader-text.js';
31
34
  import { reconcileGrading } from './grading-adapter.js';
32
35
  import { GradingReportSchema } from './grading-schema.js';
33
- import { assertSafeHarnessRoot, assertSafeWorkdir, prepareHarnessRoot, resolveHarnessRoot } from './harness-location.js';
36
+ import { assertSafeHarnessRoot, assertSafeWorkdir, HarnessLocationError, prepareHarnessRoot, resolveHarnessRoot, } from './harness-location.js';
34
37
  import { acquireHarnessLock, installSignalCleanup } from './lock.js';
35
- import { runPipeline } from './pipeline.js';
38
+ import { RateLimitSignal, runPipeline } from './pipeline.js';
36
39
  import { detectPluginLayout } from './plugin-layout.js';
37
40
  import { runPreflight } from './preflight.js';
38
41
  import { descriptorToSource, stageHarness } from './staging.js';
@@ -182,56 +185,98 @@ export function buildResolveCtx(harnessRoot, repoRoot) {
182
185
  };
183
186
  }
184
187
  /**
185
- * Map of flags to dummy values used when probing for flag support.
188
+ * Match a flag name as a whole token in `claude --help` output.
186
189
  *
187
- * Each value is chosen so value-validation (e.g., enum checks) accepts it
188
- * before `--help` short-circuits the session. The empty string `''` for
189
- * `--setting-sources` is intentional — an empty comma-separated list is valid.
190
+ * The boundary matters: a bare `includes('--plugin-dir')` also matches
191
+ * `--plugin-dirs` or `--plugin-dir-cache`, and vat's own flag names are prefixes
192
+ * of each other (`--max-turns` / `--max-turns-per-eval` would collide). Neither
193
+ * side of the token may be a letter or a dash.
190
194
  */
191
- const FLAG_DUMMY_VALUES = {
192
- '--plugin-dir': '.',
193
- '--setting-sources': '',
194
- '--output-format': 'stream-json',
195
- '--permission-mode': 'bypassPermissions',
196
- '--max-turns': '1',
197
- '--max-budget-usd': '1',
198
- };
195
+ export function helpTextDeclaresFlag(helpText, flag) {
196
+ // A scan rather than a built regex: the flag is data (it comes from a list a
197
+ // future edit will extend), and building a pattern from data is both a lint
198
+ // error here and the kind of thing that silently changes meaning when someone
199
+ // adds a flag containing a metacharacter.
200
+ const isTokenChar = (ch) => ch !== undefined && (ch === '-' || ch === '_' || /[a-z0-9]/i.test(ch));
201
+ for (let from = helpText.indexOf(flag); from !== -1; from = helpText.indexOf(flag, from + 1)) {
202
+ if (!isTokenChar(helpText[from - 1]) && !isTokenChar(helpText[from + flag.length]))
203
+ return true;
204
+ }
205
+ return false;
206
+ }
199
207
  /**
200
- * Return a sensible dummy value for a given CLI flag so that value-validation
201
- * doesn't reject the argument before `--help` can short-circuit the session.
208
+ * A flag name no `claude` will ever accept. It is probed alongside the real ones
209
+ * as a NEGATIVE CONTROL: if the probe reports this as supported, the probe is
210
+ * broken and its answer about every other flag is worthless.
202
211
  */
203
- export function flagDummyValueFor(flag) {
204
- return FLAG_DUMMY_VALUES[flag] ?? '1';
205
- }
212
+ export const FLAG_PROBE_SENTINEL = '--vat-probe-flag-that-cannot-exist';
206
213
  /**
207
- * Build a token-free flag-parse probe.
214
+ * Build a token-free flag-support probe from `claude --help`.
208
215
  *
209
- * For each flag, runs `claude <flag> <dummyValue> --help` via safeExecResult.
210
- * Exit 0 means claude's parser accepts the flag; `--help` short-circuits before
211
- * any session or tokens are consumed.
216
+ * ⚠️ This REPLACES an exit-code probe (`claude <flag> <dummy> --help`, exit 0 ⇒
217
+ * supported) that could not discriminate at all. Verified against claude 2.x:
218
+ * `claude --no-such-flag-xyz 1 --help` exits **0**, because `--help` short-circuits
219
+ * before argument validation. Every one of preflight's `flag <name>` checks
220
+ * therefore reported "supported" unconditionally — including for flags that do not
221
+ * exist — so the gate that is supposed to stop vat spawning with an unsupported
222
+ * flag was decorative. (The same command without `--help` exits 1, but running it
223
+ * would start a real session and bill tokens, which is why `--help` was there.)
212
224
  *
213
- * The dummy value is always included in the args, even when it is an empty
214
- * string — some flags (e.g., `--setting-sources`) require a value argument,
215
- * so omitting it causes exit 1 before `--help` can be reached.
225
+ * Help-text matching does discriminate, and it is still token-free: one `--help`
226
+ * invocation for the whole run instead of one spawn per flag.
216
227
  *
217
- * The probe is a closure — it is NOT spawned at module scope, only at call time
218
- * inside runPreflight.
228
+ * The probe carries its own negative control ({@link FLAG_PROBE_SENTINEL}). If a
229
+ * future `claude` prints something that matches the sentinel — or `--help` fails
230
+ * outright — the probe reports EVERY flag unsupported rather than every flag
231
+ * supported. A probe that cannot tell must fail closed and be seen; the defect it
232
+ * replaces failed open and was invisible.
219
233
  */
220
- function buildFlagParseProbe() {
221
- return (flag) => {
222
- const dummy = flagDummyValueFor(flag);
223
- const result = safeExecResult('claude', [flag, dummy, '--help'], {
224
- stdio: 'pipe',
225
- encoding: 'utf8',
226
- timeout: 15_000,
227
- });
228
- return result.success;
229
- };
234
+ export function buildFlagParseProbe(runHelp = defaultClaudeHelp) {
235
+ const helpText = runHelp();
236
+ // No help output, or a sentinel "match" ⇒ the probe is not trustworthy.
237
+ const usable = helpText !== null && !helpTextDeclaresFlag(helpText, FLAG_PROBE_SENTINEL);
238
+ return (flag) => usable && helpTextDeclaresFlag(helpText ?? '', flag);
239
+ }
240
+ /** Run `claude --help` once, returning its combined output (null if unreachable). */
241
+ function defaultClaudeHelp() {
242
+ const result = safeExecResult('claude', ['--help'], {
243
+ stdio: 'pipe',
244
+ encoding: 'utf8',
245
+ timeout: 15_000,
246
+ });
247
+ if (!result.success)
248
+ return null;
249
+ // `stdout`/`stderr` are typed `Buffer | string`; `encoding: 'utf8'` makes them
250
+ // strings, and `String()` is correct either way. Both streams are read because
251
+ // a CLI is free to print its usage to either.
252
+ return `${String(result.stdout)}\n${String(result.stderr)}`;
230
253
  }
254
+ /**
255
+ * ⚠️ KNOWN OPEN — `costEstimate` here is DEAD, in three independent ways, and none
256
+ * of them is fixable from inside this function:
257
+ *
258
+ * 1. `evalCount: 1` is a placeholder. The suite is not parsed until Step 5.5, which
259
+ * runs AFTER preflight, so the real count does not exist yet. Making it real means
260
+ * either moving preflight after the suite parse (it currently gates that parse) or
261
+ * adding a second reader of `evals.json` — and two readers of the answer key is
262
+ * precisely the drift this module refuses elsewhere.
263
+ * 2. `renderPreflightSummary` returns only FAILING checks, and it is called only
264
+ * inside the `if (!preflightResult.passed)` branch — so on a passing run the
265
+ * estimate never reaches a terminal at all.
266
+ * 3. Preflight runs after the `--dry-run` short-circuit, so a dry run never gets here.
267
+ *
268
+ * The consequence is that {@link buildDryRunSummary}'s spawn-count line is the ONLY
269
+ * pre-spend number an operator ever sees. Fix the count before making the check
270
+ * reachable: a reachable estimate that says "1 evals × 2 config" is worse than a
271
+ * silent one, because it is quotable.
272
+ */
231
273
  export function buildPreflightInput(evalsPath, pluginDirs, opts, knobs) {
232
274
  const costEstimate = {
233
275
  evalCount: 1,
234
- configurations: 1,
276
+ // `configurations` IS the A/B dimension: `--baseline` runs every eval under two
277
+ // configurations (skill declared, skill withheld). Correct even while
278
+ // `evalCount` is a placeholder, and it is the half that does not need the suite.
279
+ configurations: opts.baseline === true ? 2 : 1,
235
280
  runsPerQuery: 1,
236
281
  maxBudgetUsd: knobs.maxBudgetUsd,
237
282
  };
@@ -273,13 +318,31 @@ export function renderPreflightSummary(checks) {
273
318
  export function isAcknowledged(opts) {
274
319
  return opts.dryRun === true || opts.acknowledgedRunsSkillCode === true;
275
320
  }
321
+ /**
322
+ * Most friction lines vat will put on an operator's terminal in one run. The
323
+ * full set is always on disk; a grader that emits thousands of items must not
324
+ * be able to scroll the run's real output away.
325
+ */
326
+ const MAX_FRICTION_LINES = 50;
276
327
  /**
277
328
  * Format a friction report for human consumption — one line per entry as
278
329
  * `[<severity>] <category>: <message>`. Pure; returns the empty string for no
279
330
  * entries so the caller can skip emitting anything.
331
+ *
332
+ * `severity` and `category` are closed enums vat owns, so they are safe to
333
+ * interpolate. `message` is GRADER text and is sanitized here even though
334
+ * {@link parseEvalFragment} already sanitized it on the way in — this is a
335
+ * second, independent boundary, not a redundant one: {@link emitFrictionReport}
336
+ * re-reads `friction.json` from the harness results dir, which same-uid skill
337
+ * code can reach and rewrite after vat wrote it.
280
338
  */
281
339
  export function formatFrictionReport(items) {
282
- return items.map(i => `[${i.severity}] ${i.category}: ${i.message}`).join('\n');
340
+ const shown = items.slice(0, MAX_FRICTION_LINES);
341
+ const lines = shown.map(i => `[${i.severity}] ${i.category}: ${sanitizeGraderText(i.message)}`);
342
+ if (items.length > shown.length) {
343
+ lines.push(`... and ${items.length - shown.length} more (full list in friction.json)`);
344
+ }
345
+ return lines.join('\n');
283
346
  }
284
347
  /**
285
348
  * Read the run's friction.json (if present + valid) and echo a concise report to
@@ -309,6 +372,18 @@ function emitFrictionReport(frictionPath) {
309
372
  process.stderr.write(`\nPackaging friction (${parsed.data.items.length}):\n`);
310
373
  process.stderr.write(formatFrictionReport(parsed.data.items) + '\n');
311
374
  }
375
+ /**
376
+ * Write the `--baseline` delta line and ⚠️ banner to stderr, if this run produced
377
+ * one. A no-op on a non-baseline run.
378
+ *
379
+ * A function rather than an `if` at the call site purely to keep the orchestrator
380
+ * inside its cognitive-complexity budget; the ORDER it is called in is the part
381
+ * that matters, and that is documented where it is called.
382
+ */
383
+ function emitBaselineReport(report) {
384
+ if (report !== undefined)
385
+ process.stderr.write(report);
386
+ }
312
387
  /**
313
388
  * Name any optional companions (`--with-optional`/config `optional:`) that were
314
389
  * skipped because their source could not be resolved — never silently, so the
@@ -419,24 +494,66 @@ export function resolveEvalSuitePath(input) {
419
494
  return undefined;
420
495
  return safePath.join(input.holdDir, basename(input.evalsSubpath));
421
496
  }
497
+ /**
498
+ * The per-run executor workspace root: `<tmp>/vat-skill-test-ws-<dirToken>/`.
499
+ *
500
+ * Deliberately OUTSIDE `harnessRoot`, alongside {@link resolveGraderOutDir} and
501
+ * {@link resolveEvalSuiteHoldDir}. The executor's cwd lives under here, and the
502
+ * harness root is where vat stages runnable copies of the skill (`staged/` and
503
+ * the assembled plugin dir). While the two were parent and child, the skill-absent
504
+ * arm of a `--baseline` run sat one `ls ..` away from the treatment it was
505
+ * supposed to be denied — the control could reach vat's own copy without ever
506
+ * leaving its working directory.
507
+ *
508
+ * Named by an unpredictable token, like the other vat-only dirs, and removed by
509
+ * the same cleanup. Pure (derives a path only).
510
+ */
511
+ export function resolveWorkspacesRoot(dirToken) {
512
+ return safePath.join(normalizedTmpdir(), `vat-skill-test-ws-${dirToken}`);
513
+ }
422
514
  /** Parse the staged eval suite and materialize each eval's input `files` into
423
- * `<harnessRoot>/workspaces/<id>/`. Returns the workspaces root, the parsed
424
- * {@link EvalSuite} (so the eval loop has the entries without re-reading), and the
425
- * declared eval count (derived from the suite). The dir is wiped first so a reused
426
- * harness root cannot leak a prior run's inputs. Throws {@link EvalInputError}
427
- * (mapped by the caller to exit 2) on a bad suite or a missing input file. */
428
- export function stageWorkspacesForRun(evalsPath, harnessRoot) {
429
- const workspacesRoot = safePath.joinUnderRoot(harnessRoot, 'workspaces');
515
+ * `<workspacesRoot>/<arm>/<id>/`, once per arm this run will execute. Returns the
516
+ * workspaces root, the parsed {@link EvalSuite} (so the eval loop has the entries
517
+ * without re-reading), and the declared eval count (derived from the suite). The
518
+ * dir is wiped first so a reused root cannot leak a prior run's inputs. Throws
519
+ * {@link EvalInputError} (mapped by the caller to exit 2) on a bad suite or a
520
+ * missing input file. */
521
+ /**
522
+ * Mint this run's opaque per-arm workspace directory segments.
523
+ *
524
+ * Independent random tokens, NOT the arm names and not derived from one another:
525
+ * the segment ends up in the executor's cwd, which the prompt states verbatim, so
526
+ * a name like `without` unblinds the control arm outright — and a name like `with`
527
+ * hands it a guessable path to the treatment arm's live output one directory over.
528
+ * 8 bytes is ample for "not guessable within a run that lasts minutes"; these are
529
+ * blinding/anti-enumeration tokens, not secrets protecting data at rest.
530
+ */
531
+ export function mintArmWorkspaceDirs(baseline) {
532
+ const token = () => randomBytes(8).toString('hex');
533
+ return baseline ? { with: token(), without: token() } : { with: token() };
534
+ }
535
+ export function stageWorkspacesForRun(evalsPath, workspacesRoot, armDirs) {
430
536
  rmSync(workspacesRoot, { recursive: true, force: true });
431
537
  // 0700, matching the harness root. These dirs hold each eval's declared input
432
538
  // files, which with an out-of-tree suite may be data that was never in the repo
433
539
  // at all. Inheriting the umask (0755) left them readable by any local user the
434
540
  // moment `--out` relocated the harness root out from under its 0700 parent.
435
541
  mkdirSyncReal(workspacesRoot, { recursive: true, mode: 0o700 });
542
+ // The same shared-tmp hardening the harness root gets. `mkdirSync(recursive)`
543
+ // on an existing path neither throws nor chmods, so without this an attacker
544
+ // winning the race between the rmSync and the mkdir owns the executor's working
545
+ // directory and nothing ever re-checks. Cheap: the leaf sits directly under the
546
+ // trusted tmp boundary, so the ancestry walk degrades to one lstat + stat.
547
+ assertSafeHarnessRoot(workspacesRoot, process.getuid?.() ?? -1);
436
548
  // eslint-disable-next-line security/detect-non-literal-fs-filename -- evalsPath is our staged-subject path
437
549
  const suite = parseEvalSuite(readFileSync(evalsPath, 'utf-8'));
438
550
  return {
439
- workspacesRoot: stageEvalWorkspaces({ suite, evalsDir: dirname(evalsPath), workspacesRoot }),
551
+ workspacesRoot: stageEvalWorkspaces({
552
+ suite,
553
+ evalsDir: dirname(evalsPath),
554
+ workspacesRoot,
555
+ armDirs,
556
+ }),
440
557
  declaredEvalCount: suite.evals.length,
441
558
  suite,
442
559
  };
@@ -453,9 +570,9 @@ function buildModelFlag(model) {
453
570
  * Keeping the try/catch in a private helper avoids inflating the orchestrator's
454
571
  * cognitive complexity.
455
572
  */
456
- function attemptStageWorkspaces(evalsPath, harnessRoot) {
573
+ function attemptStageWorkspaces(evalsPath, harnessRoot, workspacesRoot, armDirs) {
457
574
  try {
458
- return stageWorkspacesForRun(evalsPath, harnessRoot);
575
+ return stageWorkspacesForRun(evalsPath, workspacesRoot, armDirs);
459
576
  }
460
577
  catch (e) {
461
578
  if (e instanceof EvalInputError) {
@@ -576,6 +693,38 @@ export function buildStaleDistWarningLines(roleLabel) {
576
693
  * Always includes the assembled spawn command, the staged-manifest entry count +
577
694
  * fingerprint, and the provenance.json path so a stale tree is visible at a glance.
578
695
  */
696
+ /** `s` unless the count is exactly one — the plural the two spawn-plan lines share. */
697
+ function plural(count) {
698
+ return count === 1 ? '' : 's';
699
+ }
700
+ /**
701
+ * The two lines that say WHAT A REAL RUN WOULD COST, in PAIRS — not evals.
702
+ *
703
+ * `--baseline` runs every eval on both arms and each arm is a full executor→grader
704
+ * pair, so the spawn count is `evals × arms` and the `claude` session count is twice
705
+ * that. Reporting the eval count as the pair count understated a baseline run by
706
+ * exactly 2x: a 3-eval suite previewed as "3 evals as executor→grader spawn pairs"
707
+ * and then launched 6 pairs / 12 sessions.
708
+ *
709
+ * The arm count is spelled out rather than folded silently into the number, because
710
+ * "6 pairs" for a 3-eval suite reads as a bug until you are told why. And the budget
711
+ * flag is labelled PER SPAWN with the ceiling multiplied out, because the name
712
+ * `--max-budget-usd` reads as a run total — under `--baseline` that misreading
713
+ * under-provisions by a factor of four times the eval count.
714
+ */
715
+ function buildSpawnPlanLines(input) {
716
+ const pairs = input.evalCount * (input.baseline ? 2 : 1);
717
+ const sessions = pairs * 2;
718
+ const armNote = input.baseline
719
+ ? ` (${input.evalCount} eval${plural(input.evalCount)} × 2 arms — --baseline runs each eval with AND without the skill)`
720
+ : '';
721
+ return [
722
+ `[dry-run] Would run ${pairs} executor→grader spawn pair${plural(pairs)}${armNote} ` +
723
+ `at concurrency ${input.concurrency} — ${sessions} claude session${plural(sessions)} in total.`,
724
+ `[dry-run] --max-budget-usd is PER SPAWN ($${input.maxBudgetUsd}), not per run: worst case ≈ ` +
725
+ `$${(sessions * input.maxBudgetUsd).toFixed(2)} across those ${sessions} sessions.`,
726
+ ];
727
+ }
579
728
  export function buildDryRunSummary(input) {
580
729
  const lines = [];
581
730
  if (input.wouldBuild) {
@@ -605,17 +754,37 @@ export function buildDryRunSummary(input) {
605
754
  // it is labelled rather than left to be taken at face value.
606
755
  const stagedFromUnbuiltTree = input.wouldBuild && input.dryRunStagedExistingDist !== undefined;
607
756
  const provisional = stagedFromUnbuiltTree ? ' (PROVISIONAL — not rebuilt)' : '';
608
- lines.push(`[dry-run] Would run ${input.evalCount} eval${input.evalCount === 1 ? '' : 's'} as executor→grader spawn ` +
609
- `pair${input.evalCount === 1 ? '' : 's'} at concurrency ${input.concurrency}.`, `[dry-run] Executor ${input.modelFlag}; grader model ${input.graderModel} (prompt via stdin).`, `[dry-run] Staged manifest: ${count} entr${count === 1 ? 'y' : 'ies'} | ` +
610
- `fingerprint: ${input.provenanceFingerprint}${provisional}`, `[dry-run] Provenance: ${input.provenancePath}`);
757
+ lines.push(...buildSpawnPlanLines(input), `[dry-run] Executor ${input.modelFlag}; grader model ${input.graderModel} (prompt via stdin).`, `[dry-run] Staged manifest: ${count} entr${count === 1 ? 'y' : 'ies'} | ` +
758
+ `fingerprint: ${input.provenanceFingerprint}${provisional}`, `[dry-run] Provenance would be written to: ${input.provenancePath}`);
611
759
  return lines.join('\n');
612
760
  }
613
761
  /**
614
- * Remove the harness directory after a run so staged untrusted skill bytes and
615
- * prompts do not accumulate in OS tmp. No-op when the user asked to keep it, when
616
- * the dir is a user-supplied location (`--out`/`--workdir`), or when it is already
617
- * gone. Idempotent and never throws — it runs from a `finally`, so it must not
618
- * mask the run's real outcome.
762
+ * The one child of the harness root that cleanup NEVER removes. Cleanup exists to
763
+ * evict staged untrusted skill bytes and prompts from OS tmp; `results/` is the
764
+ * opposite — it is the run's product, written solely by vat.
765
+ */
766
+ export const RETAINED_RESULTS_DIRNAME = 'results';
767
+ /**
768
+ * Remove the harness directory's staged contents after a run so untrusted skill
769
+ * bytes and prompts do not accumulate in OS tmp. No-op when the user asked to keep
770
+ * it, when the dir is a user-supplied location (`--out`/`--workdir`), or when it is
771
+ * already gone. Idempotent and never throws — it runs from a `finally`, so it must
772
+ * not mask the run's real outcome.
773
+ *
774
+ * `results/` SURVIVES. It holds `grading.json`, `friction.json`, `tool-eval.json`
775
+ * and — for a `--baseline` run — `baseline.json`, i.e. every artifact the command's
776
+ * own help text tells the operator to go read. Removing the whole root on the
777
+ * DEFAULT invocation (no `--out`, no `--workdir`, no `--keep`) deleted those files
778
+ * inside this function's own `finally`, before `run.ts` had printed a single line:
779
+ * the operator was handed a `Harness:` path to a directory that no longer existed.
780
+ * That is the invocation `vat-skill-testing.md`'s copy-paste example uses.
781
+ *
782
+ * The retention is deliberately UNCONDITIONAL rather than `--baseline`-only: every
783
+ * run's verdict detail lives in the same directory, and a rule that depends on a
784
+ * flag is a rule that will drift away from the flag. The cost is bounded — the
785
+ * harness root is a deterministic function of the subject set, so a re-run reuses
786
+ * (and {@link wipeStaleArtifacts} clears) the same `results/`, rather than
787
+ * accumulating one per run.
619
788
  *
620
789
  * SAFETY: re-asserts the root is not a symlink immediately before removal (via
621
790
  * `lstat`, which does NOT follow the link). A root swapped to a symlink between
@@ -635,7 +804,20 @@ export function cleanupHarness(harnessRoot, opts) {
635
804
  // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
636
805
  if (lstatSync(harnessRoot).isSymbolicLink())
637
806
  return;
638
- rmSync(harnessRoot, { recursive: true, force: true });
807
+ // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
808
+ const entries = readdirSync(harnessRoot);
809
+ // Nothing to retain — a run that ended before Step 7 (preflight refusal, lock
810
+ // failure, a throw during staging) has no results/, so leaving an empty 0700
811
+ // dir behind in tmp would be pure litter. Remove the root outright.
812
+ if (!entries.includes(RETAINED_RESULTS_DIRNAME)) {
813
+ rmSync(harnessRoot, { recursive: true, force: true });
814
+ return;
815
+ }
816
+ for (const entry of entries) {
817
+ if (entry === RETAINED_RESULTS_DIRNAME)
818
+ continue;
819
+ rmSync(safePath.joinUnderRoot(harnessRoot, entry), { recursive: true, force: true });
820
+ }
639
821
  }
640
822
  catch {
641
823
  // Swallow: a failed cleanup is not a run failure.
@@ -699,14 +881,43 @@ export function resolveGraderOutDir(dirToken) {
699
881
  return safePath.join(normalizedTmpdir(), `vat-skill-grade-${dirToken}`);
700
882
  }
701
883
  /**
702
- * The executor working directory for one eval: its staged input workspace
703
- * `<workspacesRoot>/<id>` when the eval declares input `files`, else undefined
704
- * (the executor then defaults to the staged subject dir). Pure + unit-testable.
884
+ * The executor working directory for one eval ON ONE ARM: its staged input
885
+ * workspace `<workspacesRoot>/<arm>/<id>`, ALWAYS — empty when the eval declares
886
+ * no input `files` (see {@link stageEvalWorkspaces}).
887
+ *
888
+ * This used to return undefined for a file-less eval, which made the executor
889
+ * fall back to running inside the staged subject dir. That fallback was the
890
+ * quiet half of the `--baseline` control defect: the skill-absent arm's cwd
891
+ * was the skill.
892
+ *
893
+ * The per-arm segment closes the OTHER half, found a round later: without it both
894
+ * arms of one eval shared a single writable directory AND ran concurrently, so
895
+ * the control arm could read the treatment arm's output files and answer from
896
+ * them. Keyed on the eval id alone this is a pure function of the eval; the arm
897
+ * is what makes the two runs independent.
898
+ *
899
+ * The segment is an OPAQUE per-run token, never the arm's name — this path is
900
+ * quoted verbatim to the executor as its working directory, so `…/without/e1`
901
+ * told the control arm it was the control, and `../with/e1` told it where to look.
902
+ * See {@link mintArmWorkspaceDirs}. Pure + unit-testable.
903
+ */
904
+ export function resolvePerEvalWorkspaceDir(entry, workspacesRoot, arm, armDirs) {
905
+ return safePath.joinUnderRoot(armWorkspaceRoot(workspacesRoot, armDirs, arm), String(entry.id));
906
+ }
907
+ /**
908
+ * `<workspacesRoot>/<that arm's opaque segment>` — ONE arm's staged workspace root,
909
+ * derived in exactly one place.
910
+ *
911
+ * The consolidation is the point, not the line count. Every consumer that wants an
912
+ * arm root wants this join, and a consumer that takes the result as a bare `string`
913
+ * has no way to tell an arm root from the workspaces ROOT one level above it:
914
+ * handing {@link resolveSkillContentNeedles} `workspacesRoot` instead of
915
+ * `<workspacesRoot>/<withSegment>` type-checked, ran, and left all 2270 unit tests
916
+ * green while every fixture exclusion silently missed. Callers now pass the two
917
+ * inputs and let this derive the path.
705
918
  */
706
- export function resolvePerEvalWorkspaceDir(entry, workspacesRoot) {
707
- if (entry.files === undefined || entry.files.length === 0)
708
- return undefined;
709
- return safePath.joinUnderRoot(workspacesRoot, String(entry.id));
919
+ function armWorkspaceRoot(workspacesRoot, armDirs, arm) {
920
+ return safePath.joinUnderRoot(workspacesRoot, armDirSegment(armDirs, arm));
710
921
  }
711
922
  /**
712
923
  * Best-effort removal of a vat-only tmp dir that lives OUTSIDE the harness root —
@@ -714,6 +925,36 @@ export function resolvePerEvalWorkspaceDir(entry, workspacesRoot) {
714
925
  * cleanup, where masking the run's real outcome would be worse than a leftover 0700
715
926
  * tmp dir) and refuses to follow a symlinked root.
716
927
  */
928
+ /**
929
+ * Run one cleanup step, swallowing whatever it throws.
930
+ *
931
+ * The discipline every other step in the harness `finally` already follows, stated
932
+ * once: a cleanup failure is not a run failure, and letting one out REPLACES an
933
+ * already-good result — verdict computed, artifacts written, summary composed — with
934
+ * an error and exit 1.
935
+ *
936
+ * ⚠️ A CALL-SITE GUARD IS THE WEAK FORM, and this helper is a last resort rather
937
+ * than the pattern to copy. It is invisible to the compiler and to the test suite:
938
+ * reverting either of its two original call sites to a bare call left the unit suite
939
+ * AND the integration suite green, so nothing but review stood between the fix and
940
+ * its own silent removal. `lock.release()` was therefore made non-throwing at its own
941
+ * definition instead (see {@link acquireHarnessLock}), which is a contract the type
942
+ * carries and no edit here can undo. That leaves ONE caller —
943
+ * `killAllActiveClaudeChildren`, which lives in `@vibe-agent-toolkit/utils` and is
944
+ * only contractually non-throwing as long as that package says so.
945
+ *
946
+ * The failure is reported on stderr rather than silently dropped: a lockfile that
947
+ * could not be removed will break the NEXT run with a "busy" error, and an operator
948
+ * who saw nothing here has no way to connect the two.
949
+ */
950
+ export function swallowCleanupFailure(step) {
951
+ try {
952
+ step();
953
+ }
954
+ catch (err) {
955
+ process.stderr.write(`warning: harness cleanup step failed (the run's result stands): ${err instanceof Error ? err.message : String(err)}\n`);
956
+ }
957
+ }
717
958
  export function removeVatOnlyDir(dir) {
718
959
  if (dir === undefined)
719
960
  return;
@@ -737,34 +978,108 @@ export function removeVatOnlyDir(dir) {
737
978
  * absent). Grader fragments are written under a PER-ARM subdir of the vat-only
738
979
  * grader dir so a WITH and WITHOUT run of the same eval id cannot collide.
739
980
  *
740
- * Throws propagate: an executor/grader {@link InternalHarnessError} (timeout,
741
- * stall, spawn error, grader failure/missing-fragment) fails the whole run
742
- * (exit 1); a RateLimitSignal is retried by the pipeline. An executor CLEAN
743
- * failure is NOT thrown — its transcript flows into the grader, whose failing
981
+ * Throws, and the caller ({@link runEvalWorker}) decides what a throw costs — see
982
+ * its docblock for why that depends on the arm. An executor CLEAN failure is NOT
983
+ * thrown on either arm: its transcript flows into the grader, whose failing
744
984
  * fragment surfaces as an eval failure (exit 4 via the verdict), never exit 1.
745
985
  */
746
- async function runEvalWorker(item, ctx) {
986
+ /**
987
+ * The executor env for ONE eval.
988
+ *
989
+ * `workspaceDir` is threaded to the token resolver ONLY when the eval actually
990
+ * declares input `files`, even though every eval now HAS a workspace. The two are
991
+ * different questions: the workspace is where the executor runs, while
992
+ * `${fixturesDir}` promises a staged `fixtures/` directory that EXISTS — and
993
+ * `stageEvalWorkspaces` creates that only for evals declaring files. Passing it
994
+ * unconditionally silently downgraded `UnresolvableEnvTokenError` (a loud exit 2
995
+ * naming the exact fix) into a dead path the skill discovers at runtime and is
996
+ * then blamed for.
997
+ */
998
+ function resolveExecutorEnvForEval(item, ctx, workspaceDir) {
999
+ if (ctx.perEvalEnv === undefined)
1000
+ return ctx.executorEnv;
1001
+ const declaresFiles = item.entry.files !== undefined && item.entry.files.length > 0;
1002
+ const { input, logged } = ctx.perEvalEnv;
1003
+ const env = resolveDeclaredChildEnv({
1004
+ ...input,
1005
+ ...(declaresFiles ? { workspaceDir } : {}),
1006
+ quiet: logged.done,
1007
+ }).env;
1008
+ logged.done = true;
1009
+ return env;
1010
+ }
1011
+ /**
1012
+ * Scrub the control arm's env and SAY what it lost.
1013
+ *
1014
+ * The arms are meant to differ in the skill and nothing else, so any var withheld
1015
+ * from one side is a confound. A control arm silently spawned without `PATH` or
1016
+ * without its declared fixtures still runs — it just does worse, which reports as
1017
+ * skill lift. That failure is invisible unless the withholding is announced, and
1018
+ * `dropped` was previously computed and thrown away.
1019
+ */
1020
+ function scrubAndReportControlArmEnv(env, harnessRoot, evalId) {
1021
+ // `[]` because vat does not declare model env vars today — `resolveAuth` is
1022
+ // called with no `modelVars`, so none is ever forwarded and the exemption set
1023
+ // has nothing extra to hold. The parameter is REQUIRED rather than defaulted so
1024
+ // that stops being a silent assumption: the day vat forwards a model var, this
1025
+ // line fails to compile and someone decides, instead of the scrub quietly
1026
+ // dropping it and running the control arm on a different model.
1027
+ const { env: scrubbed, droppedForbiddenKey, droppedNamingRoot, retainedLeaks } = scrubControlArmEnv(env, harnessRoot, []);
1028
+ // Two rules, two reasons, two lines. One merged line said "naming the harness
1029
+ // root" about both, which is false for every rule-1 drop — those go by NAME and
1030
+ // their value may name nothing in the harness at all.
1031
+ const withheld = [
1032
+ [droppedForbiddenKey, 'by name, whatever their value'],
1033
+ [droppedNamingRoot, 'because their value names the harness root'],
1034
+ ];
1035
+ for (const [names, why] of withheld) {
1036
+ if (names.length === 0)
1037
+ continue;
1038
+ process.stderr.write(`control arm (${evalId}): withheld ${names.length} env var(s) ${why}: ${names.join(', ')}\n`);
1039
+ }
1040
+ if (retainedLeaks.length > 0) {
1041
+ process.stderr.write(`⚠️ control arm (${evalId}): ${retainedLeaks.join(', ')} name(s) the harness root but are required to run, so they were NOT withheld. ` +
1042
+ `The control arm can read the harness through them — move --out/--workdir off this path before trusting the delta.\n`);
1043
+ }
1044
+ return scrubbed;
1045
+ }
1046
+ async function gradeOneArm(item, ctx) {
747
1047
  const evalId = String(item.entry.id);
748
- const workspaceDir = resolvePerEvalWorkspaceDir(item.entry, ctx.workspacesRoot);
1048
+ const workspaceDir = resolvePerEvalWorkspaceDir(item.entry, ctx.workspacesRoot, item.arm, ctx.armDirs);
749
1049
  const onProgress = (chunk) => { process.stderr.write(chunk); };
750
- // Resolve the declared `env` against THIS eval's workspace. An eval that
751
- // declares no input files has no workspace, so a value using ${fixturesDir}
752
- // throws UnresolvableEnvTokenError (exit 2) instead of injecting a dead path.
753
- let executorEnv = ctx.executorEnv;
754
- if (ctx.perEvalEnv !== undefined) {
755
- const { input, logged } = ctx.perEvalEnv;
756
- executorEnv = resolveDeclaredChildEnv({
757
- ...input,
758
- ...(workspaceDir === undefined ? {} : { workspaceDir }),
759
- quiet: logged.done,
760
- }).env;
761
- logged.done = true;
762
- }
1050
+ // Resolve the declared `env` against THIS eval's workspace.
1051
+ //
1052
+ // `workspaceDir` is threaded to the token resolver ONLY when the eval actually
1053
+ // declares input `files`, even though every eval now HAS a workspace. The two
1054
+ // are different questions: the workspace is where the executor runs, while
1055
+ // `${fixturesDir}` promises a staged `fixtures/` directory that exists — and
1056
+ // `stageEvalWorkspaces` creates that only for evals declaring files. Passing it
1057
+ // unconditionally silently downgraded `UnresolvableEnvTokenError` (a loud exit 2
1058
+ // naming the exact fix) into a dead path the skill discovers at runtime and gets
1059
+ // blamed for.
1060
+ let executorEnv = resolveExecutorEnvForEval(item, ctx, workspaceDir);
1061
+ // The skill-absent arm is not told where the subject is staged. Handing it that
1062
+ // path would defeat the control outright: the staged dir holds the SKILL.md AND
1063
+ // any executable the skill ships, so a control arm given the path can recover
1064
+ // the entire treatment with one `cat`. Withholding the plugin dir while naming
1065
+ // the staged dir in the prompt was the defect this arm's whole value rests on.
1066
+ const isWithArm = item.arm === 'with';
1067
+ // ...and the same reasoning applied to the ENVIRONMENT, which is the channel the
1068
+ // first round of this fix missed entirely: the run assembles ONE env and both
1069
+ // arms were handed it verbatim, `CLAUDE_PLUGIN_ROOT` included. See
1070
+ // scrubControlArmEnv — prompt, argv, cwd and env are four channels, not three.
1071
+ //
1072
+ // Report what the control arm lost. The arms are supposed to differ in the skill
1073
+ // and nothing else, so any var withheld from one side is a confound the operator
1074
+ // has to be able to see — silently spawning a degraded control is how a harness
1075
+ // manufactures the lift it is meant to measure.
1076
+ if (!isWithArm)
1077
+ executorEnv = scrubAndReportControlArmEnv(executorEnv, ctx.harnessRoot, evalId);
763
1078
  const outcome = await runExecutorForEval({
764
1079
  evalId,
765
1080
  task: item.entry.prompt,
766
- subjectStagedDir: ctx.subjectStagedDir,
767
- ...(workspaceDir === undefined ? {} : { workspaceDir }),
1081
+ ...(isWithArm ? { subjectStagedDir: ctx.subjectStagedDir } : {}),
1082
+ workspaceDir,
768
1083
  pluginDirs: item.arm === 'without' ? [] : ctx.pluginDirs,
769
1084
  env: executorEnv,
770
1085
  ...(ctx.model === undefined ? {} : { model: ctx.model }),
@@ -778,7 +1093,6 @@ async function runEvalWorker(item, ctx) {
778
1093
  recordSessionCost(ctx.costAccumulator, outcome.parsed.result?.totalCostUsd);
779
1094
  // Tool expectations are about the SKILL's tools, so they ride the WITH arm ONLY —
780
1095
  // the WITHOUT (skill-absent) arm has no skill, hence nothing to judge tools against.
781
- const isWithArm = item.arm === 'with';
782
1096
  const fragment = await runGraderForEval({
783
1097
  evalId,
784
1098
  transcript: outcome.transcript,
@@ -803,7 +1117,384 @@ async function runEvalWorker(item, ctx) {
803
1117
  onProgress,
804
1118
  costSink: (usd) => recordSessionCost(ctx.costAccumulator, usd),
805
1119
  });
806
- return { ...fragment, arm: item.arm };
1120
+ const graded = withoutGraderContamination(fragment);
1121
+ if (isWithArm)
1122
+ return { ...graded, arm: item.arm };
1123
+ return { ...graded, arm: item.arm, ...baselineContaminationFor(outcome.transcript, ctx, evalId) };
1124
+ }
1125
+ /**
1126
+ * Run one work item and decide what its failure costs the RUN.
1127
+ *
1128
+ * TREATMENT arm: rethrow. `grading.json` is the treatment arm, the verdict is
1129
+ * computed from it, and a run that cannot produce one has produced nothing — there
1130
+ * is no partial result worth writing, so the throw propagates through `runPipeline`
1131
+ * to `mapErrorToExitCode` exactly as before (exit 1).
1132
+ *
1133
+ * CONTROL arm: record and continue. Every failure mode of this arm — executor spawn
1134
+ * error, stall, wall-timeout, grader non-zero exit, grader wrote no fragment,
1135
+ * fragment is not valid JSON, nonce mismatch, and (newest) a grader that returned a
1136
+ * different number of expectations than the eval declares — used to propagate out of
1137
+ * `runPipeline` and annihilate the treatment work already completed and billed. A
1138
+ * two-eval probe was measured doing precisely that: both treatment executors ran,
1139
+ * both treatment graders ran, the control arm's executor timed out, the run threw,
1140
+ * and `results/` was left holding `provenance.json` and nothing else. The operator
1141
+ * paid for the whole run and got no grading.json, no baseline.json, and an error
1142
+ * that did not even say which arm had died.
1143
+ *
1144
+ * Recording it instead costs the DELTA, which is the only thing the control arm
1145
+ * feeds, and the delta already has a truthful way to be absent (`null`). The
1146
+ * treatment artifacts are written, the failure is named on stderr as it happens and
1147
+ * again in `baselineIntegrity.controlArmFailures`, and `armExpectationSkew` sees an
1148
+ * eval graded on one arm only and withholds the number.
1149
+ *
1150
+ * A {@link RateLimitSignal} is rethrown on BOTH arms — it is not a failure, it is the
1151
+ * pipeline's retry protocol, and swallowing it on the control arm would convert every
1152
+ * transient rate limit into a permanently dead control arm. The worker CANNOT tell a
1153
+ * first 429 from the last one; only the pipeline knows the budget is spent, which is
1154
+ * what {@link rateLimitExhaustedOutcome} is wired to handle.
1155
+ */
1156
+ async function runEvalWorker(item, ctx) {
1157
+ try {
1158
+ return { fragment: await gradeOneArm(item, ctx) };
1159
+ }
1160
+ catch (err) {
1161
+ if (err instanceof RateLimitSignal || item.arm === 'with')
1162
+ throw nameTheArm(err, item.arm);
1163
+ return recordControlFailure(item, err);
1164
+ }
1165
+ }
1166
+ /**
1167
+ * The control arm ran out of rate-limit retries — record it and let the run live.
1168
+ *
1169
+ * The retryable case is handled correctly by {@link runEvalWorker} and must stay that
1170
+ * way; this is the EXHAUSTED case, which used to be indistinguishable from it. A
1171
+ * control executor rate-limited on every attempt threw after five retries, taking
1172
+ * down a run in which both treatment executors and both treatment graders had already
1173
+ * completed and been billed, leaving `results/` holding `provenance.json` and nothing
1174
+ * else — the exact failure {@link runEvalWorker} exists to prevent, reached by the one
1175
+ * error class it had to let through.
1176
+ *
1177
+ * The treatment arm still hard-fails: there is no treatment result to salvage, which
1178
+ * is the same asymmetry {@link EvalWorkOutcome} documents.
1179
+ *
1180
+ * Wired as the pipeline's `onRetriesExhausted`, and `--baseline` — which doubles the
1181
+ * spawn count — is the run most likely to need it.
1182
+ */
1183
+ export function rateLimitExhaustedOutcome(item, error) {
1184
+ if (item.arm === 'with')
1185
+ throw nameTheArm(error, item.arm);
1186
+ return recordControlFailure(item, error);
1187
+ }
1188
+ /**
1189
+ * Turn one control-arm error into a recorded failure the run carries past. Shared by
1190
+ * the worker's catch and the exhausted-retries handler so both report identically —
1191
+ * an operator must not be able to tell which layer noticed.
1192
+ */
1193
+ function recordControlFailure(item, err) {
1194
+ const named = nameTheArm(err, item.arm);
1195
+ // `detail` is schema-required non-empty: an error that stringifies to nothing
1196
+ // would otherwise fail the artifact write and take down the run this whole
1197
+ // branch exists to keep alive.
1198
+ const detail = (named instanceof Error ? named.message : String(named)) ||
1199
+ `[${ARM_LABEL[item.arm]}] failed for eval "${String(item.entry.id)}" with a non-descriptive error`;
1200
+ process.stderr.write(`\n⚠️ ${detail}\n` +
1201
+ ' The control arm for this eval is unusable, so the --baseline delta will be WITHHELD. ' +
1202
+ 'The treatment arm is unaffected and the run continues — its grading is what you paid for.\n');
1203
+ return { controlFailure: { evalId: String(item.entry.id), detail } };
1204
+ }
1205
+ /**
1206
+ * Fragment fields VAT derives itself and therefore never accepts FROM a grader.
1207
+ *
1208
+ * Both are computed from the executor transcript's structure, which the grader is
1209
+ * shown as opaque text and never asked about — so a value in either from a grader
1210
+ * is not merely redundant, it is a claim the grader is in no position to make.
1211
+ */
1212
+ const VAT_ATTACHED_FRAGMENT_KEYS = ['contamination', 'degraded'];
1213
+ /**
1214
+ * Strip the fields VAT attaches itself, before VAT attaches them.
1215
+ *
1216
+ * The grader's only input is the executor transcript, which untrusted skill code
1217
+ * controls — so a prompt injection there could otherwise fabricate a
1218
+ * `BASELINE CONTAMINATED` verdict carrying attacker-chosen strings into an
1219
+ * operator-facing artifact, or (via `degraded`) claim a scan degraded that did not.
1220
+ * Spreading VAT's value last is NOT sufficient: it is absent on a clean run, so it
1221
+ * overwrites nothing — which is exactly why the omission direction matters more
1222
+ * than the invention one for `degraded`. A grader that omits it cannot re-hide a
1223
+ * degradation VAT detected, because VAT's value is spread after this strip; a
1224
+ * grader that INVENTS it would otherwise stamp a blind-scan warning on a run that
1225
+ * scanned properly, and an operator who learns to ignore that warning stops reading
1226
+ * the real ones.
1227
+ */
1228
+ export function withoutGraderContamination(fragment) {
1229
+ if (VAT_ATTACHED_FRAGMENT_KEYS.every((key) => fragment[key] === undefined))
1230
+ return fragment;
1231
+ const copy = { ...fragment };
1232
+ for (const key of VAT_ATTACHED_FRAGMENT_KEYS)
1233
+ delete copy[key];
1234
+ return copy;
1235
+ }
1236
+ /**
1237
+ * How each arm is named in every message that can only be about one of them.
1238
+ *
1239
+ * Spelled out rather than `'with'`/`'without'`: the internal names are a preposition
1240
+ * apart and the operator-facing question is "which half of my experiment died", so
1241
+ * `Executor timed out for eval "e1"` — the actual text a destroyed run used to print
1242
+ * — named neither the arm nor the fact that the OTHER arm had already succeeded and
1243
+ * been thrown away.
1244
+ */
1245
+ const ARM_LABEL = {
1246
+ with: 'treatment arm (skill available)',
1247
+ without: 'control arm (skill withheld)',
1248
+ };
1249
+ /**
1250
+ * Prefix a worker error's message with the arm it came from, IN PLACE.
1251
+ *
1252
+ * Mutating rather than re-wrapping is deliberate. `mapErrorToExitCode` dispatches on
1253
+ * the error's CLASS, and so does the run's own regression test for a forged grader
1254
+ * nonce (`rejects.toThrow(GradingNonceError)`); wrapping every failure in a fresh
1255
+ * `InternalHarnessError` to get a better message would flatten
1256
+ * `EvalFragmentError`/`GradingNonceError`/`RateLimitSignal` into one class and lose
1257
+ * both. The stack is preserved for the same reason.
1258
+ *
1259
+ * Idempotent — a worker error passes through here exactly once, but a retried item
1260
+ * must not accumulate prefixes.
1261
+ */
1262
+ function nameTheArm(err, arm) {
1263
+ const prefix = `[${ARM_LABEL[arm]}] `;
1264
+ if (err instanceof Error && !err.message.startsWith(prefix))
1265
+ err.message = `${prefix}${err.message}`;
1266
+ return err;
1267
+ }
1268
+ /**
1269
+ * Lift the run's content needles off the STAGED SKILL.md.
1270
+ *
1271
+ * Staged, not source: the staged copy is byte-identical to what the treatment arm
1272
+ * is given, so a needle taken from it is exactly the text an ambient copy of the
1273
+ * same build would carry.
1274
+ *
1275
+ * The exclusion set is EVERY channel through which vat itself hands the arm text:
1276
+ * the suite's prompts, its `expected_output`, its expectations, and the CONTENTS
1277
+ * of the input `files` fixtures vat stages into the arm's workspace and instructs
1278
+ * it to operate on. An adopter who quotes a sentence of their SKILL.md through any
1279
+ * of the four would otherwise stamp every run contaminated, including the arm that
1280
+ * merely read the input vat gave it — the skill's words arriving THROUGH vat are
1281
+ * not the arm reaching the skill. The fixture channel is the one that was missed,
1282
+ * and it is the worst of the four to get wrong: the summary attached to that
1283
+ * verdict tells the operator to go uninstall an ambient copy of the plugin that
1284
+ * does not exist.
1285
+ *
1286
+ * Fixtures are read from the STAGED workspace rather than the suite directory, for
1287
+ * the same reason the SKILL.md is: that is byte-for-byte what the arm was handed.
1288
+ *
1289
+ * Returns `[]` when the file cannot be read. A missing SKILL.md is not this
1290
+ * function's error to raise (staging has already validated the subject), and an
1291
+ * unarmed signal is reported as unarmed rather than claimed as clean. An
1292
+ * unreadable FIXTURE is skipped for the same reason, and errs the safe way: a
1293
+ * fixture we could not exclude can only make the signal noisier, never blinder.
1294
+ */
1295
+ export function resolveSkillContentNeedles(subjectStagedDir, evals,
1296
+ /**
1297
+ * Where the staged fixtures live, as the two inputs an arm root is DERIVED from
1298
+ * rather than as the pre-joined path itself — see {@link armWorkspaceRoot}. Taking
1299
+ * a bare `string` here let the call site pass `workspacesRoot` (one level too high,
1300
+ * so every `<id>/` fixture lookup missed) with the whole unit suite still green.
1301
+ *
1302
+ * The `with` arm, because the fixture BYTES are what matter and both arms are
1303
+ * staged from the same source — and `with` is the only arm every run mints.
1304
+ */
1305
+ workspaces) {
1306
+ const stagedWorkspaceRoot = armWorkspaceRoot(workspaces.workspacesRoot, workspaces.armDirs, 'with');
1307
+ const skillMdPath = safePath.join(subjectStagedDir, 'SKILL.md');
1308
+ let markdown;
1309
+ try {
1310
+ // eslint-disable-next-line security/detect-non-literal-fs-filename -- path derived from vat's own staged dir
1311
+ markdown = readFileSync(skillMdPath, 'utf8');
1312
+ }
1313
+ catch {
1314
+ return [];
1315
+ }
1316
+ const excluded = evals
1317
+ .flatMap((entry) => [
1318
+ entry.prompt,
1319
+ entry.expected_output ?? '',
1320
+ ...entry.expectations,
1321
+ ...readStagedFixtures(stagedWorkspaceRoot, entry),
1322
+ ])
1323
+ .join('\n');
1324
+ return skillContentNeedles(markdown, excluded);
1325
+ }
1326
+ /**
1327
+ * The text of one eval's staged input files. Unreadable/binary fixtures are skipped.
1328
+ *
1329
+ * A declared `files[]` entry is legitimately a DIRECTORY — staging does `existsSync`
1330
+ * then a recursive `cpSync` — so this walks one. It used to call `readFileSync` on
1331
+ * the declared path, get `EISDIR`, and skip it: the directory's whole contents were
1332
+ * in the arm's cwd and absent from the exclusion set, and two runs differing ONLY in
1333
+ * whether the same bytes were declared as a file or as their parent directory
1334
+ * reported `contaminated: false` and `contaminated: true` (with a `skill-content`
1335
+ * hit, and a summary telling the operator to uninstall an ambient plugin copy that
1336
+ * does not exist).
1337
+ *
1338
+ * The `catch`'s old claim that skipping "errs the safe way: noisier, never blinder"
1339
+ * is BACKWARDS for this signal. Over-reporting is the harm here, because a false
1340
+ * `contaminated: true` is actioned by discarding a run that was fine. A fixture that
1341
+ * cannot be read is still skipped — there is nothing else to do with it — but that
1342
+ * is a cost, not a safety margin.
1343
+ */
1344
+ function readStagedFixtures(stagedWorkspaceRoot, entry) {
1345
+ const texts = [];
1346
+ for (const rel of entry.files ?? []) {
1347
+ const staged = safePath.joinUnderRoot(safePath.joinUnderRoot(stagedWorkspaceRoot, String(entry.id)), rel);
1348
+ collectFixtureText(staged, texts);
1349
+ }
1350
+ return texts;
1351
+ }
1352
+ /** Depth cap on the fixture walk — a symlink loop must not hang the run's setup. */
1353
+ const MAX_FIXTURE_WALK_DEPTH = 32;
1354
+ /** Append `path`'s text, recursing when it is a directory. Never throws. */
1355
+ function collectFixtureText(path, into, depth = 0) {
1356
+ if (depth > MAX_FIXTURE_WALK_DEPTH)
1357
+ return;
1358
+ let entries;
1359
+ try {
1360
+ // `withFileTypes` on the DIRECTORY is one syscall for the whole level, and
1361
+ // `lstat`-shaped: a symlinked entry reports as a link rather than as whatever it
1362
+ // points at, so the walk stays inside the bytes vat actually staged.
1363
+ // eslint-disable-next-line security/detect-non-literal-fs-filename -- path is contained under vat's own staged workspace
1364
+ entries = readdirSync(path, { withFileTypes: true });
1365
+ }
1366
+ catch {
1367
+ // Not a directory (the common case: a plain file fixture), or unreadable.
1368
+ try {
1369
+ // eslint-disable-next-line security/detect-non-literal-fs-filename -- path is contained under vat's own staged workspace
1370
+ into.push(readFileSync(path, 'utf8'));
1371
+ }
1372
+ catch {
1373
+ // Unreadable or binary — nothing to exclude, and not this function's error to raise.
1374
+ }
1375
+ return;
1376
+ }
1377
+ for (const child of entries) {
1378
+ if (!child.isFile() && !child.isDirectory())
1379
+ continue;
1380
+ collectFixtureText(safePath.joinUnderRoot(path, child.name), into, depth + 1);
1381
+ }
1382
+ }
1383
+ /**
1384
+ * Build the detector input for the skill-absent arm. ONE builder, so the hits
1385
+ * written per eval and the `signals` list stamped on the run-level integrity
1386
+ * block are derived from the same values — a second construction site is how the
1387
+ * block would come to claim a detector the scan never actually ran.
1388
+ *
1389
+ * `transcript` and `evalId` are supplied per call; everything else is per-run.
1390
+ *
1391
+ * Takes a NARROWED context rather than the whole {@link EvalRunContext} so the
1392
+ * builder is directly unit-testable — the detector honours `armCwd` (dropping it
1393
+ * there kills three tests) while the wiring that SUPPLIES it was droppable green,
1394
+ * and a dropped `armCwd` anchors every relative climb one directory too high, where
1395
+ * no needle can see it.
1396
+ *
1397
+ * `evalId` is what turns the arm's workspace ROOT into the arm's actual starting
1398
+ * CWD (`<armWorkspaceDir>/<evalId>` — see {@link resolvePerEvalWorkspaceDir}).
1399
+ * The detector needs both and they are one directory apart: containment is judged
1400
+ * against the root (the arm's whole tree is its own), while the cwd walk must
1401
+ * start exactly where the executor did, or every relative climb in the transcript
1402
+ * resolves one level too high and lands where no needle can see it.
1403
+ *
1404
+ * `evalId` is REQUIRED here, and the signals-only caller gets its own entry point
1405
+ * ({@link buildContaminationSignalsInput}) rather than sharing an optional third
1406
+ * argument. An optional parameter is a pin the compiler cannot hold: dropping the
1407
+ * argument at the SCAN call site left all 2270 unit tests green and was caught only
1408
+ * by the integration suite — which this repo's fast implementer loop explicitly says
1409
+ * not to run. That is the same shape as the dropped `armCwd` that anchored the whole
1410
+ * walk one directory too high, also green.
1411
+ */
1412
+ export function buildContaminationInput(transcript, ctx, evalId) {
1413
+ return contaminationInput(transcript, ctx, evalId);
1414
+ }
1415
+ /**
1416
+ * The same input built for the "which detectors are ARMED?" question, which is asked
1417
+ * once per run with no transcript and no eval — so it can honestly supply neither,
1418
+ * and `armCwd` is correctly absent rather than accidentally dropped.
1419
+ */
1420
+ export function buildContaminationSignalsInput(ctx) {
1421
+ return contaminationInput('', ctx, undefined);
1422
+ }
1423
+ function contaminationInput(transcript, ctx, evalId) {
1424
+ const armRoot = ctx.armDirs.without === undefined || ctx.armDirs.without === ''
1425
+ ? undefined
1426
+ : armWorkspaceRoot(ctx.workspacesRoot, ctx.armDirs, 'without');
1427
+ return {
1428
+ transcript,
1429
+ harnessRoot: ctx.harnessRoot,
1430
+ // Only ever called for the skill-absent arm, so the sibling is `with` — the
1431
+ // treatment arm's live working directory, one `ls ..` away and containing no
1432
+ // harness path for the needles above to find.
1433
+ siblingArmDir: armWorkspaceRoot(ctx.workspacesRoot, ctx.armDirs, 'with'),
1434
+ // The control arm's OWN root, the exact counterpart of the line above. Its
1435
+ // absolute path is stated in the arm's own prompt, so the arm reuses it
1436
+ // constantly; without this the executable-name signal reads every mention of
1437
+ // the arm's own scratch script as a reach into the skill.
1438
+ //
1439
+ // Conditional because this builder also serves `buildContaminationSignalsInput`,
1440
+ // which runs on EVERY run — and a non-baseline run mints no `without` arm, so
1441
+ // `armDirSegment` would (correctly) throw rather than invent a segment.
1442
+ ...(armRoot === undefined ? {} : { armWorkspaceDir: armRoot }),
1443
+ // ...and the eval's own subdirectory under it, which is where the executor was
1444
+ // actually spawned. Absent without an evalId, in which case the walk anchors
1445
+ // at the root — one level high, but the best anchor available.
1446
+ ...(armRoot === undefined || evalId === undefined
1447
+ ? {}
1448
+ : { armCwd: safePath.joinUnderRoot(armRoot, evalId) }),
1449
+ // VAT's private tmp dirs: the held eval suite (the `expected_output` answer
1450
+ // key) and the grader dir (the run's integrity nonce). Both are SIBLINGS of
1451
+ // the arm's cwd under the OS temp dir — two hops or one `$TMPDIR` expansion
1452
+ // away — and contain no harness path, so neither the harness needles nor the
1453
+ // sibling-arm needles can see them. The workspaces root is deliberately NOT
1454
+ // in this list: it is the arm's own legitimate cwd.
1455
+ vatPrivateDirs: [ctx.evalSuiteHoldDir, ctx.graderOutDir],
1456
+ skillContentNeedles: ctx.skillContentNeedles,
1457
+ // The skill-relative PATH (`scripts/csvsum.py`), not the extension-stripped
1458
+ // `name`. An ambient copy of the skill is a COPY, so it reproduces the skill's
1459
+ // internal layout under whatever root it sits at and still matches; the control
1460
+ // arm's own `/tmp/summary.txt` no longer does. `howInvoked` is a command string,
1461
+ // not a path, so it is deliberately NOT used here.
1462
+ ...(ctx.declaredExecutables === undefined
1463
+ ? {}
1464
+ : { executablePaths: ctx.declaredExecutables.map((e) => e.path) }),
1465
+ };
1466
+ }
1467
+ function baselineContaminationFor(transcript, ctx, evalId) {
1468
+ const { hits, degraded } = detectBaselineContamination(buildContaminationInput(transcript, ctx, evalId));
1469
+ // A degraded scan writes `contaminated: false` with exactly the same bytes as one
1470
+ // that walked the transcript properly, so silence here would be the "quietly wrong
1471
+ // measurement" this whole module exists to prevent. It goes to BOTH surfaces, and
1472
+ // neither substitutes for the other:
1473
+ // stderr reaches the operator watching the run, and the fragment carries it to
1474
+ // `baselineIntegrity.degraded` in baseline.json, which is what a reader has
1475
+ // months later when the stderr scrollback is gone.
1476
+ //
1477
+ // The `evalId` is stamped HERE rather than by the detector: the detector is pure
1478
+ // and scans one transcript with no idea which eval it belongs to, and an
1479
+ // unattributed degradation in a run-level list is unactionable.
1480
+ if (degraded !== undefined) {
1481
+ process.stderr.write(`⚠️ control arm (${evalId}): contamination scan DEGRADED (${degraded.reason}) — ${degraded.detail}. ` +
1482
+ 'A clean verdict from this eval is weaker evidence than a clean structured scan: the walk ' +
1483
+ 'either stopped resolving paths from this point on, or (for an unparseable transcript) fell ' +
1484
+ 'back to flat text matching, which both over- and under-reports.\n');
1485
+ }
1486
+ return {
1487
+ ...(hits.length === 0 ? {} : { contamination: hits }),
1488
+ ...(degraded === undefined ? {} : { degraded: { ...degraded, evalId } }),
1489
+ };
1490
+ }
1491
+ /**
1492
+ * Which detectors this run armed, for the `signals` field on the integrity block.
1493
+ * Derived from {@link buildContaminationInput} with an empty transcript — the
1494
+ * armed set is a property of the run's paths, never of any one transcript.
1495
+ */
1496
+ function contaminationSignalsFor(ctx) {
1497
+ return activeContaminationSignals(buildContaminationSignalsInput(ctx));
807
1498
  }
808
1499
  /**
809
1500
  * Run evals TIER by TIER (ascending), bounded-parallel WITHIN each tier, with a
@@ -820,9 +1511,16 @@ async function runEvalWorker(item, ctx) {
820
1511
  async function runEvalsTiered(input) {
821
1512
  const groups = groupEvalsByTier(input.evals);
822
1513
  const fragments = [];
1514
+ const controlFailures = [];
823
1515
  for (const [index, group] of groups.entries()) {
824
- const tierFragments = await input.runTier(buildEvalWorkItems(group.evals, input.baseline));
1516
+ const outcomes = await input.runTier(buildEvalWorkItems(group.evals, input.baseline));
1517
+ const tierFragments = outcomes.flatMap((o) => (o.fragment === undefined ? [] : [o.fragment]));
1518
+ controlFailures.push(...outcomes.flatMap((o) => (o.controlFailure === undefined ? [] : [o.controlFailure])));
825
1519
  fragments.push(...tierFragments);
1520
+ // A control-arm failure deliberately does NOT gate. The gate asks whether the
1521
+ // SKILL failed a cheap expectation before spending on expensive ones, and the
1522
+ // control arm is not the skill — treating its executor timeout as a gating
1523
+ // failure would stop a run whose treatment tier passed everything.
826
1524
  const withArm = tierFragments.filter((f) => f.arm !== 'without');
827
1525
  if (!shouldGateAfterTier(withArm))
828
1526
  continue;
@@ -833,9 +1531,9 @@ async function runEvalsTiered(input) {
833
1531
  // Legibility (required): name the skipped tiers on stderr so a fail-fast run is
834
1532
  // never mistaken for a smaller passing suite. stdout stays machine-readable.
835
1533
  process.stderr.write(formatSkippedTiersSummary(skipped) + '\n');
836
- return { fragments, skipped };
1534
+ return { fragments, controlFailures, skipped };
837
1535
  }
838
- return { fragments };
1536
+ return { fragments, controlFailures };
839
1537
  }
840
1538
  /** Resolve the run's grading/friction/baseline/tool-eval artifact paths under
841
1539
  * `resultsDir`. Single source of truth for the filenames (used by the
@@ -858,10 +1556,52 @@ export function resolveArtifactPaths(resultsDir) {
858
1556
  * (`force: true`) — a missing file is fine.
859
1557
  */
860
1558
  export function wipeStaleArtifacts(paths) {
861
- rmSync(paths.gradingOut, { force: true });
862
- rmSync(paths.frictionOut, { force: true });
863
- rmSync(paths.baselineOut, { force: true });
864
- rmSync(paths.toolEvalOut, { force: true });
1559
+ for (const path of [paths.gradingOut, paths.frictionOut, paths.baselineOut, paths.toolEvalOut]) {
1560
+ rmSync(path, { force: true });
1561
+ // ...and the quarantined copy a PRIOR run's failed gate set aside. It is just as
1562
+ // stale, and left alone `results/` accumulates one `.rejected` per broken run
1563
+ // with nothing to say which is the newest.
1564
+ rmSync(rejectedArtifactPath(path), { force: true });
1565
+ }
1566
+ }
1567
+ /**
1568
+ * Create `results/` and write this run's provenance — unless this is a DRY RUN, in
1569
+ * which case it does nothing at all.
1570
+ *
1571
+ * NOTHING under `results/` may be created, written, or removed by a dry run. The
1572
+ * harness root is a deterministic function of the subject, so `results/` here holds
1573
+ * the PREVIOUS run's output: creating the dir, rewriting `provenance.json`, and (at
1574
+ * the call site) wiping the stale artifacts meant "what would this cost next time?"
1575
+ * destroyed the run the operator was about to read. Measured twice: a real
1576
+ * `--baseline` run left `[baseline, friction, grading, provenance, tool-eval].json`
1577
+ * and a subsequent `--dry-run` against the same subject left `[provenance.json]`.
1578
+ *
1579
+ * A function rather than an `if` at the call site because the orchestrator is at its
1580
+ * cognitive-complexity ceiling; the RULE is what matters and it is written here.
1581
+ */
1582
+ function openResultsDir(input) {
1583
+ if (input.dryRun)
1584
+ return;
1585
+ // 0700, matching the harness root. `grading.json` quotes the executor transcript
1586
+ // verbatim in each expectation's evidence, so whatever the skill read out of its
1587
+ // input files ends up here as text — and unlike the workspaces, results/ SURVIVES
1588
+ // `--keep` by design.
1589
+ mkdirSyncReal(input.resultsDir, { recursive: true, mode: 0o700 });
1590
+ // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
1591
+ writeFileSync(input.provenancePath, JSON.stringify(input.provenance, null, 2) + '\n', 'utf-8');
1592
+ }
1593
+ /**
1594
+ * Where a results artifact goes when it fails the fail-closed gate.
1595
+ *
1596
+ * The gate throws and `results/` is deliberately RETAINED, so the file that failed
1597
+ * used to sit there under its authoritative name — the one artifact whose schema
1598
+ * violation is BY DEFINITION a harness bug, indistinguishable in a CI archive from a
1599
+ * good one until the next run's wipe. Renamed rather than deleted: the bytes are what
1600
+ * a maintainer needs in order to see which refine tripped, and a name ending in
1601
+ * `.rejected` cannot be mistaken for the artifact a reader came for.
1602
+ */
1603
+ export function rejectedArtifactPath(path) {
1604
+ return `${path}.rejected`;
865
1605
  }
866
1606
  /**
867
1607
  * Merge the run's grader fragments and WRITE the run artifacts (vat is the SOLE
@@ -875,9 +1615,118 @@ export function wipeStaleArtifacts(paths) {
875
1615
  * can compute the COMPOSITE verdict. Every fragment's per-run nonce is re-verified
876
1616
  * inside {@link mergeFragmentsToGrading}.
877
1617
  */
878
- function writeRunArtifactsAndReconcile(fragments, runNonce, paths) {
1618
+ /**
1619
+ * Collect per-eval baseline-integrity findings from the WITHOUT-arm fragments.
1620
+ * Fragments with no `contamination` are clean and contribute nothing. Pure.
1621
+ */
1622
+ /**
1623
+ * What one arm graded, per eval — the SINGLE derivation behind both the parity check
1624
+ * ({@link armExpectationSkew}, which reads `total`) and the A/B delta (which reads
1625
+ * `passed`). Deriving those separately would let the two blocks in `baseline.json`
1626
+ * disagree about the same arm. Pure.
1627
+ *
1628
+ * Exported for its own unit test, not for reuse. Setting `passed` to `total` here
1629
+ * failed zero unit tests and three integration ones: every unit fixture graded
1630
+ * everything it declared, so the two fields were numerically identical in all of
1631
+ * them and the ONE thing this function exists to keep distinct was unpinned.
1632
+ */
1633
+ export function gradedCounts(fragments) {
1634
+ return fragments.map((f) => ({
1635
+ evalId: f.evalId,
1636
+ passed: f.expectations.filter((e) => e.passed).length,
1637
+ total: f.expectations.length,
1638
+ }));
1639
+ }
1640
+ /**
1641
+ * Both per-eval integrity channels the WITHOUT-arm fragments carry, lifted in ONE
1642
+ * pass: what the scan FOUND (`findings`) and how well it was able to look
1643
+ * (`degraded`).
1644
+ *
1645
+ * Together, not separately, because they are read together: `contaminated: false`
1646
+ * means "looked properly and found nothing" only when `degraded` is empty, and a
1647
+ * caller that collects one and forgets the other publishes the first half of that
1648
+ * sentence as if it were the whole of it. That is not hypothetical — the degradation
1649
+ * was detected, schema'd, summarised and rendered, and then never collected, so
1650
+ * every `baseline.json` shipped claiming a clean scan it had not necessarily run.
1651
+ */
1652
+ function collectBaselineFindings(withoutArm) {
1653
+ return {
1654
+ findings: withoutArm
1655
+ .filter((f) => f.contamination !== undefined && f.contamination.length > 0)
1656
+ .map((f) => ({ evalId: f.evalId, hits: f.contamination })),
1657
+ degraded: withoutArm.flatMap((f) => (f.degraded === undefined ? [] : [f.degraded])),
1658
+ };
1659
+ }
1660
+ /**
1661
+ * Restate a {@link SkippedEvalsSummary} as the delta block's truncation record.
1662
+ *
1663
+ * A separate shape rather than embedding the tier summary wholesale: the delta cares
1664
+ * about WHAT IS MISSING FROM ITS TOTALS, which is the flat set of evals that never
1665
+ * ran on either arm, while `SkippedEvalsSummary` is organised for the tier reporter.
1666
+ * Pure.
1667
+ */
1668
+ function deltaTruncationFor(skipped) {
1669
+ if (skipped === undefined || skipped.totalSkipped === 0)
1670
+ return null;
1671
+ return {
1672
+ gatedByTier: skipped.gatedByTier,
1673
+ firstSkippedTier: skipped.firstSkippedTier,
1674
+ totalSkipped: skipped.totalSkipped,
1675
+ evalIds: skipped.tiers.flatMap((tier) => tier.evalIds),
1676
+ };
1677
+ }
1678
+ /**
1679
+ * The stderr report for a `--baseline` run: the delta line, then the ⚠️ banner when
1680
+ * the run is unusable. Pure — the caller decides WHEN it goes out, which is the
1681
+ * whole point (see the call site).
1682
+ *
1683
+ * Composed on EVERY baseline run, including a withheld one: "the delta is
1684
+ * unavailable, and here is why" is the report an operator needs, and a run that
1685
+ * silently prints nothing is indistinguishable from a vat that still does not
1686
+ * compute a delta at all. The delta line comes FIRST so an unusable run reads as the
1687
+ * number followed by the caveat that explains it — the banner is the only line that
1688
+ * says the number means nothing, so it must never be the one scrolled away.
1689
+ *
1690
+ * ⚠️ IT IS NOT LITERALLY LAST ON THE SCREEN, and this docblock used to say it was.
1691
+ * The CLI prints `Harness:`, `Results:`, `Workspaces:` and `Summary:` AFTER the
1692
+ * harness returns, so 2–4 lines always follow the banner. What this ordering buys is
1693
+ * that the banner is not separated from its delta by up to ~1250 rows of friction —
1694
+ * a bounded, short tail after it, not none. Moving those CLI lines above the harness
1695
+ * report is the CLI's call, not this module's.
1696
+ *
1697
+ * THREE conditions have to hold before the caveat is dropped, not two. A DEGRADED
1698
+ * scan is neither contaminated nor incomparable, so a two-clause gate printed
1699
+ * `Baseline delta: +0 (with skill: 2/2, without skill: 2/2).` and exited 0 while
1700
+ * `baseline.json` carried `degraded: [...]` and an `integrity.summary` ending in a
1701
+ * `⚠️ DEGRADED SCAN: …` sentence that was composed, written to the artifact, and
1702
+ * never emitted anywhere. Everything this module says about telling "checked and clean"
1703
+ * apart from "checked with the blunt instrument" was true of the artifact and false
1704
+ * of the terminal — and the terminal is the only surface an operator reads.
1705
+ *
1706
+ * A CONTAMINATED run still reports its number, deliberately. Contamination does not
1707
+ * make the subtraction illegal — both arms were graded to the same depth, so the
1708
+ * arithmetic is sound; what it invalidates is INTERPRETING that number as skill
1709
+ * lift, which is precisely what the ⚠️ line says ("The reported delta is NOT a
1710
+ * measure of skill lift"), and what `vat-skill-testing.md` turns into the
1711
+ * instruction "discard the delta". Withholding it here would (a) leave that sentence
1712
+ * referring to a number that was never reported, and (b) spend `null` — which means
1713
+ * "these arms cannot be subtracted", a different failure with a different remedy —
1714
+ * on a run where they can be. A contaminated delta is also diagnostic in its own
1715
+ * right: it collapses toward zero, which is corroboration, not noise.
1716
+ */
1717
+ export function formatBaselineReport(delta, integrity) {
1718
+ const line = `\n${formatBaselineDeltaLine(delta)}\n`;
1719
+ // Any of the three makes the delta unusable-as-stated, and all three are only
1720
+ // visible on stderr — nobody opens baseline.json unprompted.
1721
+ if (!integrity.contaminated && integrity.comparable && integrity.degraded.length === 0)
1722
+ return line;
1723
+ return `${line}\n⚠️ ${integrity.summary}\n\n`;
1724
+ }
1725
+ function writeRunArtifactsAndReconcile(input) {
1726
+ const { fragments, runNonce, paths, signals, controlFailures } = input;
879
1727
  const { withArm, withoutArm } = partitionFragmentsByArm(fragments);
880
- const grading = mergeFragmentsToGrading(withArm, runNonce);
1728
+ let baselineReport;
1729
+ const grading = mergeFragmentsToGrading(withArm, runNonce, 'with');
881
1730
  // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
882
1731
  writeFileSync(paths.gradingOut, JSON.stringify(grading, null, 2) + '\n', 'utf-8');
883
1732
  const friction = mergeFragmentsToFriction(fragments);
@@ -888,12 +1737,104 @@ function writeRunArtifactsAndReconcile(fragments, runNonce, paths) {
888
1737
  const toolEval = mergeFragmentsToToolEval(withArm);
889
1738
  // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
890
1739
  writeFileSync(paths.toolEvalOut, JSON.stringify(toolEval, null, 2) + '\n', 'utf-8');
891
- if (withoutArm.length > 0) {
892
- const baseline = mergeFragmentsToGrading(withoutArm, runNonce);
1740
+ // `controlFailures` is the second disjunct, and it is load-bearing: when EVERY
1741
+ // control eval died there are no WITHOUT-arm fragments at all, and gating on the
1742
+ // fragment count alone would write no baseline.json — so the one run that most
1743
+ // needs to say "the control arm did not happen" would say nothing, exactly like a
1744
+ // run where `--baseline` was never passed. `mergeFragmentsToGrading` over an empty
1745
+ // arm yields a zero/zero report, which is the honest shape for it.
1746
+ if (withoutArm.length > 0 || controlFailures.length > 0) {
1747
+ const baseline = mergeFragmentsToGrading(withoutArm, runNonce, 'without');
1748
+ // Stamp the integrity verdict onto baseline.json (GradingReportSchema is
1749
+ // .passthrough(), so the extra field is contract-legal). Written on EVERY
1750
+ // baseline run, clean or not: a reader must be able to tell "checked and
1751
+ // clean" from "produced before this check existed", and only an
1752
+ // unconditional field does that.
1753
+ // Parity BEFORE the verdict prose, because it decides whether the number the
1754
+ // prose is about means anything. Deliberately NOT a throw: the control arm's
1755
+ // grader misbehaving must not discard a perfectly good treatment result — it
1756
+ // must make the DELTA say, in writing, that it cannot be subtracted.
1757
+ // ONE derivation per arm, feeding BOTH the parity check and the delta. The
1758
+ // whole reason `ArmEvalGrade` carries `passed` alongside `total` is that these
1759
+ // two blocks must provably be talking about the same numbers — re-deriving the
1760
+ // counts for the delta would let `baselineIntegrity.skew` and `baselineDelta`
1761
+ // disagree about one run, and a reader holding two contradicting blocks has no
1762
+ // way to tell which one to believe.
1763
+ const withGrades = gradedCounts(withArm);
1764
+ const withoutGrades = gradedCounts(withoutArm);
1765
+ const skew = armExpectationSkew(withGrades, withoutGrades);
1766
+ const { findings, degraded } = collectBaselineFindings(withoutArm);
1767
+ const integrity = summarizeBaselineIntegrity({
1768
+ findings,
1769
+ // WHAT WAS SCANNED, not what was armed. `signals` is derived from the run's
1770
+ // PATHS, so it is fully populated on a run where every control eval died before
1771
+ // producing a transcript — and the shipped documentation teaches `signals` as
1772
+ // exactly the discriminator between a clean verdict and a blind one ("an empty
1773
+ // list means nothing was looking"). A control-timeout run wrote
1774
+ // `"contaminated": false` alongside `checked by: harness-path, sibling-arm,
1775
+ // vat-private-dir, skill-content` having scanned zero transcripts: the list was
1776
+ // full and nothing was looking. With no control fragment there is no
1777
+ // observation, so there is no detector to claim.
1778
+ signals: withoutArm.length === 0 ? [] : signals,
1779
+ // The same fact `signals` is gated on, stated as a number rather than inferred
1780
+ // from an empty list — it gates the VERDICT PROSE, which `signals` cannot reach.
1781
+ // Without it the summary opened "No skill-absent eval was observed reaching the
1782
+ // skill. The A/B delta is interpretable as instruction lift", which is a positive
1783
+ // claim about an observation that never happened, with the correction third.
1784
+ // Required, not defaulted, for the same reason `signals` is: a default lets every
1785
+ // caller overclaim in the "clean" direction, which is where it gets believed.
1786
+ // Control evals whose transcript actually reached the scanner — NOT evals declared.
1787
+ observedEvals: withoutArm.length,
1788
+ skew,
1789
+ degraded,
1790
+ controlArmFailures: controlFailures,
1791
+ });
1792
+ // The SUBTRACTION `--baseline` exists to perform, and until this it was never
1793
+ // performed anywhere: vat wrote two same-shaped artifacts and left the operator
1794
+ // to do the arithmetic by hand while the docs said it "reports the delta".
1795
+ // `skew` is passed rather than re-checked so the withholding decision has
1796
+ // exactly one author.
1797
+ // `truncated` rides along rather than withholding: fail-fast stops BOTH arms at
1798
+ // the same tier boundary, so the arithmetic over the tiers that ran is sound —
1799
+ // what it changes is the SCOPE of the claim. Without it, a tier-0 failure in an
1800
+ // 8-eval suite printed `+1 (with skill: 2/2, without skill: 1/2)`, which is
1801
+ // byte-identical to a complete 2-eval suite. And this is the COMMON case, not a
1802
+ // corner: `--baseline` exists to measure a skill that may not be helping, so a
1803
+ // WITH-arm failure — the thing that fires the gate — is the expected state of an
1804
+ // interesting baseline run.
1805
+ const delta = computeBaselineDelta({
1806
+ withArm: withGrades,
1807
+ withoutArm: withoutGrades,
1808
+ skew,
1809
+ controlArmFailures: controlFailures,
1810
+ truncated: deltaTruncationFor(input.skipped),
1811
+ });
1812
+ // A SIBLING of `baselineIntegrity`, not a field inside it, because they answer
1813
+ // different questions: integrity answers "may I subtract these two arms?", and
1814
+ // the delta IS the subtraction. Burying the number inside the block that
1815
+ // qualifies it would also mean a reader who wants only the lift has to
1816
+ // understand the contamination vocabulary first. (GradingReportSchema is
1817
+ // .passthrough(), so both extra keys are contract-legal — see the comment above.)
893
1818
  // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
894
- writeFileSync(paths.baselineOut, JSON.stringify(baseline, null, 2) + '\n', 'utf-8');
1819
+ writeFileSync(paths.baselineOut, JSON.stringify({ ...baseline, baselineIntegrity: integrity, baselineDelta: delta }, null, 2) + '\n', 'utf-8');
1820
+ // RETURNED, not written. The delta line and the ⚠️ banner must not be separated
1821
+ // from each other by the friction report: `emitFrictionReport` runs from the
1822
+ // harness `finally`, long after this point, and its two caps (50 lines × 2000
1823
+ // chars) allow 101,245 characters — about 1250 rows of an 80-column terminal —
1824
+ // between the banner and the prompt. The banner is the only line that says the
1825
+ // number above it is meaningless, so pushing it off-screen is the same failure as
1826
+ // not printing it. Handing the text back lets the caller emit friction FIRST and
1827
+ // this after it; a total-output budget on friction would only shorten the
1828
+ // distance, not end it. (The banner is not the LAST line of the process's output
1829
+ // — the CLI adds `Harness:`/`Results:`/`Workspaces:`/`Summary:` after the harness
1830
+ // returns. That tail is 2–4 lines and bounded, which is the point.)
1831
+ baselineReport = formatBaselineReport(delta, integrity);
895
1832
  }
896
- return { verdict: reconcileGrading(grading), toolEval };
1833
+ return {
1834
+ verdict: reconcileGrading(grading),
1835
+ toolEval,
1836
+ ...(baselineReport === undefined ? {} : { baselineReport }),
1837
+ };
897
1838
  }
898
1839
  /**
899
1840
  * The COMPOSITE run verdict (issue #145 Phase T): the run passes only when BOTH
@@ -976,25 +1917,75 @@ function assertVatWroteArtifact(path, validate, label) {
976
1917
  raw = JSON.parse(readFileSync(path, 'utf-8'));
977
1918
  }
978
1919
  catch (err) {
979
- throw new InternalHarnessError(`vat-written ${label} at ${path} is not valid JSON (harness bug): ${err instanceof Error ? err.message : String(err)}`);
1920
+ throw new InternalHarnessError(`vat-written ${label} at ${path} is not valid JSON (harness bug)${quarantineArtifact(path)}: ` +
1921
+ `${err instanceof Error ? err.message : String(err)}`);
980
1922
  }
981
1923
  try {
982
1924
  validate(raw);
983
1925
  }
984
1926
  catch (err) {
985
- throw new InternalHarnessError(`vat-written ${label} at ${path} failed its schema (harness bug): ${err instanceof Error ? err.message : String(err)}`);
1927
+ throw new InternalHarnessError(`vat-written ${label} at ${path} failed its schema (harness bug)${quarantineArtifact(path)}: ` +
1928
+ `${err instanceof Error ? err.message : String(err)}`);
1929
+ }
1930
+ }
1931
+ /**
1932
+ * Move a gate-failing artifact aside and say where it went. See
1933
+ * {@link rejectedArtifactPath}.
1934
+ *
1935
+ * Best-effort and never throws: this runs on the way OUT of a run that is already
1936
+ * failing, and a cleanup that replaces the real diagnosis with a rename error would
1937
+ * be strictly worse than a file left in place. If the rename fails the file is
1938
+ * removed instead — an absent artifact reads as "vat wrote nothing", which is at
1939
+ * least not a false claim of authority.
1940
+ */
1941
+ function quarantineArtifact(path) {
1942
+ const rejected = rejectedArtifactPath(path);
1943
+ try {
1944
+ // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
1945
+ renameSync(path, rejected);
1946
+ return `; moved aside to ${rejected} so it is not read as authoritative`;
1947
+ }
1948
+ catch {
1949
+ try {
1950
+ rmSync(path, { force: true });
1951
+ return '; removed so it is not read as authoritative';
1952
+ }
1953
+ catch {
1954
+ return '';
1955
+ }
986
1956
  }
987
1957
  }
988
1958
  /**
989
1959
  * Post-merge D2 gate over the vat-written results artifacts (vat is SOLE writer):
990
- * grading.json, friction.json, and tool-eval.json must each exist + parse + validate.
991
- * A separate, explicit check (NOT folded into any prompt-invariant linter) — fail-closed
992
- * and safe because vat wrote every one of these itself just above.
1960
+ * grading.json, friction.json, tool-eval.json — and, on a baseline run, baseline.json
1961
+ * — must each exist + parse + validate. A separate, explicit check (NOT folded into
1962
+ * any prompt-invariant linter) — fail-closed and safe because vat wrote every one of
1963
+ * these itself just above.
1964
+ *
1965
+ * `baseline.json` was EXEMPT from this gate for as long as the gate has existed, and
1966
+ * it is the artifact the exemption fits worst: on a `--baseline` run it is the only
1967
+ * durable record of the thing the command sells, and this branch has already been
1968
+ * bitten once by a default run deleting `results/` out from under it. Its validation
1969
+ * is also the strictest of the four — {@link BaselineArtifactSchema} carries the
1970
+ * `passed <= total` and `delta === with − without` refines, so a merge that produced
1971
+ * a self-contradicting number fails the run here instead of being written down.
1972
+ *
1973
+ * `baselineRun` gates it rather than "does the file exist", deliberately: an
1974
+ * existence check would pass on the one failure that matters (vat asked for a
1975
+ * baseline and wrote nothing), which is exactly what a fail-CLOSED gate must not do.
1976
+ * A non-baseline run writes no baseline.json at all and is not asked about one.
993
1977
  */
994
- function assertVatWroteArtifacts(paths) {
1978
+ export function assertVatWroteArtifacts(paths, baselineRun) {
995
1979
  assertVatWroteArtifact(paths.gradingOut, (raw) => { GradingReportSchema.parse(raw); }, 'grading.json');
996
1980
  assertVatWroteArtifact(paths.frictionOut, (raw) => { FrictionReportSchema.parse(raw); }, 'friction.json');
997
1981
  assertVatWroteArtifact(paths.toolEvalOut, (raw) => { ToolEvalReportSchema.parse(raw); }, 'tool-eval.json');
1982
+ if (!baselineRun)
1983
+ return;
1984
+ assertVatWroteArtifact(paths.baselineOut, (raw) => {
1985
+ // Both halves: the grading report it IS, and the two blocks vat stamps onto it.
1986
+ GradingReportSchema.parse(raw);
1987
+ BaselineArtifactSchema.parse(raw);
1988
+ }, 'baseline.json');
998
1989
  }
999
1990
  // ---------------------------------------------------------------------------
1000
1991
  // Main orchestrator
@@ -1003,7 +1994,23 @@ function assertVatWroteArtifacts(paths) {
1003
1994
  * Domain orchestrator for `vat skill test run`. Pure of process.exit — all
1004
1995
  * exit-code decisions live in the caller (run.ts).
1005
1996
  */
1006
- export async function runSkillTestHarness(opts) {
1997
+ /**
1998
+ * Resolve WHERE this run lives, and whether the harness owns that location.
1999
+ *
2000
+ * `--out` names the harness root exactly; `--workdir` names the BASE the root is
2001
+ * derived under. They are mutually exclusive: passing both used to silently
2002
+ * discard `--workdir` — the run went entirely to `--out` and the `--workdir` path
2003
+ * was never even created — so an operator using the pair to separate the
2004
+ * executor's cwd from the staged trees got no separation and no warning.
2005
+ *
2006
+ * `harnessCreated` is true only for the derived-under-OS-tmp case; an explicit
2007
+ * `--out` or `--workdir` is a location the user owns and is never auto-removed.
2008
+ */
2009
+ export function resolveHarnessLocation(opts) {
2010
+ if (opts.out !== undefined && opts.workdir !== undefined) {
2011
+ throw new HarnessLocationError('--out and --workdir are mutually exclusive: --out names the harness root exactly, ' +
2012
+ '--workdir names the base it is derived under. Pass one.');
2013
+ }
1007
2014
  // §7 workdir safety: refuse a --workdir whose ancestry contains CLAUDE.md/.claude
1008
2015
  // BEFORE deriving the harness root from it (defense in depth with --setting-sources "").
1009
2016
  if (opts.workdir !== undefined) {
@@ -1024,11 +2031,19 @@ export async function runSkillTestHarness(opts) {
1024
2031
  // caught, because the walk inspects `$HOME/proj` and stops only at `$HOME` itself.
1025
2032
  assertSafeWorkdir(opts.workdir, homedir());
1026
2033
  }
1027
- const harnessRoot = opts.out ?? resolveHarnessRoot([opts.subject], opts.workdir);
1028
- // Only auto-remove the dir the harness itself created under OS tmp. An explicit
1029
- // --out (exact dir) or --workdir (user-chosen base) is a location the user owns;
1030
- // treat it like --keep and never delete it.
1031
- const harnessCreated = opts.out === undefined && opts.workdir === undefined;
2034
+ return {
2035
+ // `--out` is RESOLVED to an absolute path, not taken verbatim. A relative
2036
+ // `--out ./h` used to be stored as `./h` and then used as a contamination
2037
+ // needle — a two-character string that matches almost any transcript, so every
2038
+ // eval reported contaminated and the operator was told to discard a clean run.
2039
+ harnessRoot: opts.out === undefined
2040
+ ? resolveHarnessRoot([opts.subject], opts.workdir)
2041
+ : safePath.resolve(opts.out),
2042
+ harnessCreated: opts.out === undefined && opts.workdir === undefined,
2043
+ };
2044
+ }
2045
+ export async function runSkillTestHarness(opts) {
2046
+ const { harnessRoot, harnessCreated } = resolveHarnessLocation(opts);
1032
2047
  const repoRoot = opts.repoRoot ?? harnessRoot;
1033
2048
  // Two distinct questions, deliberately not one value. `evalsRef` is what the
1034
2049
  // adopter ASKED FOR — a `test.evals`/`--evals` value, or `undefined` for the
@@ -1071,6 +2086,31 @@ export async function runSkillTestHarness(opts) {
1071
2086
  // when the subject carried no suite, which is the common case) so the staging call
1072
2087
  // stays branch-free; removed by the same cleanup as the grader dir.
1073
2088
  const evalSuiteHoldDir = resolveEvalSuiteHoldDir(randomBytes(16).toString('hex'));
2089
+ // The executor's per-eval working directories. Like the grader and hold dirs
2090
+ // these live OUTSIDE harnessRoot — see resolveWorkspacesRoot for why that
2091
+ // separation is load-bearing for --baseline — so they need their own cleanup.
2092
+ const workspacesRoot = resolveWorkspacesRoot(randomBytes(16).toString('hex'));
2093
+ // Whether those workspaces OUTLIVE the run. `--keep` is the ONLY thing that
2094
+ // retains them.
2095
+ //
2096
+ // Deliberately NOT mirrored on `cleanupHarness`'s rule (keep OR user-owned
2097
+ // location). That rule exists because `--out`/`--workdir` name a directory the
2098
+ // USER owns, so vat has no business deleting it. Since these moved out of the
2099
+ // harness root they live under OS tmp, which the user did not choose and does
2100
+ // not manage — carrying the rule across would orphan a
2101
+ // `vat-skill-test-ws-<token>` dir on every `--out` run, forever. The location
2102
+ // changed, so the retention policy has to change with it.
2103
+ //
2104
+ // Decided ONCE, here, because two consumers ask the question — cleanup (delete
2105
+ // or not) and the returned `workspacesPath` (report or not) — and a second copy
2106
+ // of the predicate would drift, which is exactly how the result came to name a
2107
+ // directory cleanup had already removed.
2108
+ const retainWorkspaces = opts.keep === true;
2109
+ // The opaque per-arm segments beneath it. Minted here, with the roots, so the
2110
+ // same values reach staging, the executor's cwd, and the contamination
2111
+ // detector's sibling-arm needle — one source of truth for a value whose whole
2112
+ // job is to be unguessable and to carry no meaning.
2113
+ const armDirs = mintArmWorkspaceDirs(opts.baseline === true);
1074
2114
  const cleanup = () => {
1075
2115
  // Reap any still-in-flight executor/grader children FIRST. On the concurrent
1076
2116
  // error path `Promise.all` rejects without cancelling its siblings, and a
@@ -1080,7 +2120,13 @@ export async function runSkillTestHarness(opts) {
1080
2120
  // NOTE: the registry is a module-level singleton, so this assumes one run per
1081
2121
  // process (true for the CLI). A library embedding two concurrent runs in one
1082
2122
  // process would need per-run child scoping instead.
1083
- killAllActiveClaudeChildren();
2123
+ //
2124
+ // `cleanupHarness`, `removeVatOnlyDir` and `lock.release()` each swallow
2125
+ // explicitly at their own definition, for the stated reason that cleanup "must
2126
+ // not mask the run's real outcome". `killAllActiveClaudeChildren` lives in
2127
+ // another package, so its guard is applied here — see {@link swallowCleanupFailure}
2128
+ // for why a call-site guard is the weaker of the two forms.
2129
+ swallowCleanupFailure(killAllActiveClaudeChildren);
1084
2130
  lock.release();
1085
2131
  cleanupHarness(harnessRoot, { keep: opts.keep === true, created: harnessCreated });
1086
2132
  removeVatOnlyDir(graderOutDir);
@@ -1088,6 +2134,10 @@ export async function runSkillTestHarness(opts) {
1088
2134
  // --keep, which retains the harness dir for inspection but has no business
1089
2135
  // retaining the key), and never conditionally.
1090
2136
  removeVatOnlyDir(evalSuiteHoldDir);
2137
+ // Per-eval workspaces are the executor's cwd; `retainWorkspaces` (minted with
2138
+ // the root above) is the single author of whether they survive.
2139
+ if (!retainWorkspaces)
2140
+ removeVatOnlyDir(workspacesRoot);
1091
2141
  };
1092
2142
  const removeSignalCleanup = installSignalCleanup({ onSignal: cleanup });
1093
2143
  // Hoisted so the finally can surface packaging friction even when a throw
@@ -1097,6 +2147,23 @@ export async function runSkillTestHarness(opts) {
1097
2147
  // once merging succeeded; on a pre-merge throw the finally simply finds no
1098
2148
  // friction.json and emits nothing (a no-op).
1099
2149
  let frictionOut;
2150
+ /**
2151
+ * Emit the friction report AT MOST ONCE per run.
2152
+ *
2153
+ * There are now two call sites, and they exist for different reasons that must
2154
+ * not become two reports: the happy path calls it deliberately BEFORE the
2155
+ * baseline delta/banner (so ~1250 rows of friction cannot come between the two —
2156
+ * see the call site), and the `finally` calls it for the throw paths, where no
2157
+ * happy-path call ever happened. A flag rather than a "did we already return normally?"
2158
+ * test, because the throw can land anywhere between the two.
2159
+ */
2160
+ let frictionEmitted = false;
2161
+ const emitFrictionOnce = () => {
2162
+ if (frictionEmitted)
2163
+ return;
2164
+ frictionEmitted = true;
2165
+ emitFrictionReport(frictionOut);
2166
+ };
1100
2167
  try {
1101
2168
  // Step 3: Stage the harness FIRST — the subject's own evals/evals.json lands
1102
2169
  // inside its staged dir, so we must stage before we can locate it. The subject's
@@ -1104,6 +2171,19 @@ export async function runSkillTestHarness(opts) {
1104
2171
  // arrives via opts.subjectSource; build no longer happens in-domain here.
1105
2172
  const resolveCtx = buildResolveCtx(harnessRoot, repoRoot);
1106
2173
  const items = buildStageItems(opts, repoRoot);
2174
+ // Hold a copy of the subject's suite ONLY when this run would have to read it
2175
+ // from there. `resolveEvalSuitePath` prefers the AUTHORED source and falls back
2176
+ // to the hold dir, so when the authored copy exists the held one is never read
2177
+ // — and staging was writing it anyway, putting a second copy of the
2178
+ // `expected_output` answer key in the shared OS temp dir on the common path,
2179
+ // for nothing. The hold dir is 0700 and outside the executor's `--add-dir`
2180
+ // sandbox, but that is Claude's permission model, not an OS boundary: skill
2181
+ // code runs as the same uid and `ls $TMPDIR` finds it. The cheapest answer to
2182
+ // "can the executor reach the answer key" is not to write it there.
2183
+ // Same predicate as `resolveEvalSuitePath`'s rule 1, so the two cannot disagree
2184
+ // about which copy this run uses.
2185
+ // eslint-disable-next-line security/detect-non-literal-fs-filename -- authored source path resolved from opts
2186
+ const authoredSuiteExists = existsSync(resolveScaffoldEvalsPath(opts, repoRoot, evalsRef));
1107
2187
  const stageResult = await stageHarness({
1108
2188
  harnessRoot,
1109
2189
  items,
@@ -1111,7 +2191,7 @@ export async function runSkillTestHarness(opts) {
1111
2191
  ctx: resolveCtx,
1112
2192
  currentUid,
1113
2193
  evalsSubpath,
1114
- evalSuiteHoldDir,
2194
+ ...(authoredSuiteExists ? {} : { evalSuiteHoldDir }),
1115
2195
  });
1116
2196
  const { pluginDirs, subjectStagedDir, subjectPluginRoot, skippedOptional } = stageResult;
1117
2197
  emitSkippedOptionalWarning(skippedOptional);
@@ -1147,10 +2227,13 @@ export async function runSkillTestHarness(opts) {
1147
2227
  // Step 5.5: Parse the eval suite and stage per-eval input workspaces. The
1148
2228
  // parsed suite is threaded on so the eval loop has the entries without
1149
2229
  // re-reading; declaredEvalCount is derived from it (suite.evals.length).
1150
- const workspaceStageResult = attemptStageWorkspaces(evalsPath, harnessRoot);
2230
+ const workspaceStageResult = attemptStageWorkspaces(evalsPath, harnessRoot, workspacesRoot, armDirs);
1151
2231
  if ('exitCode' in workspaceStageResult)
1152
2232
  return workspaceStageResult;
1153
- const { workspacesRoot, declaredEvalCount, suite } = workspaceStageResult;
2233
+ // `workspacesRoot` is resolved above (outside the try, so cleanup can reach it
2234
+ // on a signal); staging returns it unchanged, so only the derived values are
2235
+ // destructured here.
2236
+ const { declaredEvalCount, suite } = workspaceStageResult;
1154
2237
  emitEvalLintWarnings(suite.evals, (opts.declaredExecutables ?? []).map((e) => e.name));
1155
2238
  // Step 6: Enforce the §12 security ack (must pass --i-understand-this-runs-skill-code).
1156
2239
  // Only enforced when not a dry-run and not already acknowledged.
@@ -1165,11 +2248,7 @@ export async function runSkillTestHarness(opts) {
1165
2248
  // results/ — grading.json/friction.json/baseline.json come from the merged
1166
2249
  // grader fragments below, never from the (untrusted) model.
1167
2250
  const resultsDir = safePath.joinUnderRoot(harnessRoot, 'results');
1168
- // 0700, matching the harness root. `grading.json` quotes the executor
1169
- // transcript verbatim in each expectation's evidence, so whatever the skill
1170
- // read out of its input files ends up here as text — and unlike the
1171
- // workspaces, results/ SURVIVES `--keep` by design.
1172
- mkdirSyncReal(resultsDir, { recursive: true, mode: 0o700 });
2251
+ const isDryRun = opts.dryRun === true;
1173
2252
  // Record what was actually staged & tested (the subject identity + the
1174
2253
  // staged manifest fingerprint + per-entry content hashes) so a run is
1175
2254
  // auditable: which source was resolved, and whether it was rebuilt by run.ts.
@@ -1180,17 +2259,16 @@ export async function runSkillTestHarness(opts) {
1180
2259
  rebuilt: opts.rebuilt === true,
1181
2260
  };
1182
2261
  const provenancePath = safePath.join(resultsDir, 'provenance.json');
1183
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
1184
- writeFileSync(provenancePath, JSON.stringify(provenance, null, 2) + '\n', 'utf-8');
2262
+ openResultsDir({ dryRun: isDryRun, resultsDir, provenancePath, provenance });
1185
2263
  process.stderr.write(`Provenance: ${provenance.fingerprint}\n`);
1186
- // Resolve the results/ artifact paths and WIPE any prior run's artifacts up
1187
- // front: harnessRoot is deterministic and may be reused (--keep/--out, or after
1188
- // a crash before cleanup), so a stale grading/friction/baseline.json must never
1189
- // leak into — or be echoed by the finally on — this run. `frictionOut` is
1190
- // hoisted so the finally can echo THIS run's friction (and only after the wipe).
2264
+ // Resolved here, WIPED after the dry-run short-circuit below. harnessRoot is
2265
+ // deterministic and may be reused (--keep/--out, or after a crash before
2266
+ // cleanup), so a stale grading/friction/baseline.json must never leak into — or
2267
+ // be echoed by the finally on — this run; but the wipe is a REAL run's
2268
+ // prerogative, and running it here put it ahead of the dry-run return and of
2269
+ // every remaining pre-pipeline failure point (Step 7.5 token resolution among
2270
+ // them, which is how a FAILING dry run also took the prior artifacts with it).
1191
2271
  const artifacts = resolveArtifactPaths(resultsDir);
1192
- wipeStaleArtifacts(artifacts);
1193
- frictionOut = artifacts.frictionOut;
1194
2272
  // Step 7.5: Resolve the executor's declared test env (Features A + B). Token
1195
2273
  // resolution can hard-fail (exit 2) on an unknown ${token}; do it before the
1196
2274
  // dry-run short-circuit so a dry run validates interpolation too. resolvedAuth
@@ -1216,8 +2294,9 @@ export async function runSkillTestHarness(opts) {
1216
2294
  const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
1217
2295
  const modelFlag = buildModelFlag(knobs.model);
1218
2296
  process.stderr.write(`Model: ${knobs.model ?? '(claude default)'} | grader: ${graderModel}\n`);
1219
- // Step 8: Dry-run short-circuit — return assembled info without spawning.
1220
- if (opts.dryRun === true) {
2297
+ // Step 8: Dry-run short-circuit — return assembled info without spawning, and
2298
+ // without having touched results/ (see Step 7).
2299
+ if (isDryRun) {
1221
2300
  return {
1222
2301
  harnessPath: harnessRoot,
1223
2302
  exitCode: SkillTestExitCode.Ok,
@@ -1231,11 +2310,20 @@ export async function runSkillTestHarness(opts) {
1231
2310
  provenanceEntryCount: provenance.entries.length,
1232
2311
  modelFlag,
1233
2312
  evalCount: declaredEvalCount,
2313
+ // The one fact that doubles the spawn count, and the one the preview was
2314
+ // blind to — `opts.baseline` was in scope here the whole time.
2315
+ baseline: opts.baseline === true,
1234
2316
  concurrency,
1235
2317
  graderModel,
2318
+ maxBudgetUsd: knobs.maxBudgetUsd,
1236
2319
  }),
1237
2320
  };
1238
2321
  }
2322
+ // Past the dry-run return, so this is a REAL run: evict any prior run's
2323
+ // artifacts before writing our own. `frictionOut` is assigned WITH the wipe, so
2324
+ // the `finally` can only ever echo THIS run's friction.
2325
+ wipeStaleArtifacts(artifacts);
2326
+ frictionOut = artifacts.frictionOut;
1239
2327
  // Step 9: Run the vat-owned executor→grader pipeline. ONE secret per-run nonce
1240
2328
  // is stamped into every grader prompt (via stdin) and re-verified per fragment
1241
2329
  // on merge — this is what distinguishes a fragment produced by a grader WE
@@ -1258,8 +2346,15 @@ export async function runSkillTestHarness(opts) {
1258
2346
  const evalCtx = {
1259
2347
  subjectStagedDir,
1260
2348
  workspacesRoot,
2349
+ armDirs,
2350
+ harnessRoot,
1261
2351
  pluginDirs,
1262
2352
  graderOutDir,
2353
+ evalSuiteHoldDir,
2354
+ skillContentNeedles: resolveSkillContentNeedles(subjectStagedDir, suite.evals, {
2355
+ workspacesRoot,
2356
+ armDirs,
2357
+ }),
1263
2358
  runNonce,
1264
2359
  graderModel,
1265
2360
  costAccumulator,
@@ -1283,17 +2378,24 @@ export async function runSkillTestHarness(opts) {
1283
2378
  // higher (more expensive) tiers are SKIPPED (never graded, never passed), so a
1284
2379
  // broken foundational expectation stops the run from spending on hard tiers.
1285
2380
  // Each tier is one runPipeline: bounded-parallel, retrying a RateLimitSignal
1286
- // per item. An InternalHarnessError thrown by any executor/grader (timeout/
1287
- // stall/spawn-error, grader failure, missing fragment, nonce mismatch)
1288
- // propagates OUT unhandled → mapErrorToExitCode → exit 1; a spawn or grader
1289
- // break is never laundered into a pass/fail verdict (R1 no-laundering).
1290
- const { fragments, skipped } = await runEvalsTiered({
2381
+ // per item. An InternalHarnessError thrown by a TREATMENT-arm executor/grader
2382
+ // (timeout/stall/spawn-error, grader failure, missing fragment, nonce mismatch,
2383
+ // declared-count mismatch) propagates OUT unhandled → mapErrorToExitCode →
2384
+ // exit 1; a spawn or grader break is never laundered into a pass/fail verdict
2385
+ // (R1 no-laundering). The same break on the CONTROL arm is recorded and the run
2386
+ // continues — see runEvalWorker for why the two arms are not symmetric.
2387
+ const { fragments, controlFailures, skipped } = await runEvalsTiered({
1291
2388
  evals: suite.evals,
1292
2389
  baseline: opts.baseline === true,
1293
2390
  runTier: (items) => runPipeline({
1294
2391
  items,
1295
2392
  concurrency,
1296
2393
  worker: (item) => runEvalWorker(item, evalCtx),
2394
+ // The worker rethrows every RateLimitSignal so the pipeline can retry; this
2395
+ // is where an EXHAUSTED one stops being a retry and becomes an outcome. Only
2396
+ // the pipeline knows the budget is spent (see rateLimitExhaustedOutcome).
2397
+ onRetriesExhausted: (item, _index, error) => rateLimitExhaustedOutcome(item, error),
2398
+ ...(opts.rateLimitBackoffMs === undefined ? {} : { onRateLimit: opts.rateLimitBackoffMs }),
1297
2399
  }),
1298
2400
  });
1299
2401
  // Step 10: vat merges the grader fragments and writes grading.json/friction.json/
@@ -1301,11 +2403,32 @@ export async function runSkillTestHarness(opts) {
1301
2403
  // authoritative prose-expectation verdict from the WITH-arm per-expectation
1302
2404
  // `passed` flags — NOT any self-reported summary. An executor CLEAN failure
1303
2405
  // reaches here as a FAILing fragment → composite verdict → exit 4.
1304
- const { verdict, toolEval } = writeRunArtifactsAndReconcile(fragments, runNonce, artifacts);
2406
+ const { verdict, toolEval, baselineReport } = writeRunArtifactsAndReconcile({
2407
+ fragments,
2408
+ runNonce,
2409
+ paths: artifacts,
2410
+ signals: contaminationSignalsFor(evalCtx),
2411
+ controlFailures,
2412
+ // `skipped` was in scope here and simply not passed, which is how the delta
2413
+ // block and its printed line came to describe a truncated run as a complete one.
2414
+ skipped,
2415
+ });
1305
2416
  // D2 fail-closed gate: vat is the SOLE writer of results/, so a missing/unparseable/
1306
- // invalid grading.json, friction.json, or tool-eval.json after the merge is a HARNESS
1307
- // bug (exit 1), never a skill fault. A SEPARATE explicit check, not a prompt invariant.
1308
- assertVatWroteArtifacts(artifacts);
2417
+ // invalid grading.json, friction.json, tool-eval.json — or, on a baseline run,
2418
+ // baseline.json — after the merge is a HARNESS bug (exit 1), never a skill fault.
2419
+ // A SEPARATE explicit check, not a prompt invariant.
2420
+ assertVatWroteArtifacts(artifacts, opts.baseline === true);
2421
+ // Friction FIRST, then the baseline report — the ORDER is the point. Friction is
2422
+ // capped at 50 lines of 2000 characters, so emitting it after the ⚠️ contaminated
2423
+ // banner (which is what the `finally` used to do) could put ~1250 terminal rows
2424
+ // between the operator and the only line saying the delta above is meaningless.
2425
+ // Emitting it here leaves nothing of vat's own between the delta and its caveat;
2426
+ // `emitFrictionOnce` keeps the `finally` call a no-op so a normal run reports it
2427
+ // exactly once. The CLI still appends `Harness:`/`Results:`/`Workspaces:`/
2428
+ // `Summary:` after the harness returns, so the banner is last-but-2-to-4, not
2429
+ // last — a bounded tail, unlike the friction report.
2430
+ emitFrictionOnce();
2431
+ emitBaselineReport(baselineReport);
1309
2432
  // Composite verdict: the run passes only when BOTH output expectations AND every
1310
2433
  // tool-expectation verdict passed — so an output all-pass where a `mustRun` never
1311
2434
  // ran yields FAIL → exit 4. Tool verdicts stay in tool-eval.json (C2); the composite
@@ -1316,6 +2439,12 @@ export async function runSkillTestHarness(opts) {
1316
2439
  const summary = buildRunSummaryWithSkips(verdict, toolEval, allPassed, skipped, costAccumulator);
1317
2440
  return {
1318
2441
  harnessPath: harnessRoot,
2442
+ // Reported ONLY when the workspaces outlive the run. The `finally` below
2443
+ // runs cleanup BEFORE this value ever reaches a caller, so reporting it
2444
+ // unconditionally handed the operator a path to a directory that no longer
2445
+ // existed. Same predicate as cleanup's, by construction.
2446
+ ...(retainWorkspaces ? { workspacesPath: workspacesRoot } : {}),
2447
+ resultsPath: resultsDir,
1319
2448
  exitCode: verdictExitCode(allPassed, opts.tolerateEvalFailure === true),
1320
2449
  summary,
1321
2450
  };
@@ -1324,8 +2453,10 @@ export async function runSkillTestHarness(opts) {
1324
2453
  // Surface any packaging-fidelity friction VAT merged into friction.json to
1325
2454
  // STDERR BEFORE cleanup removes the harness dir. Best-effort + stderr-only
1326
2455
  // (stdout stays machine-readable); a no-op when the path is unassigned (a
1327
- // pre-merge throw) or friction.json is absent or empty.
1328
- emitFrictionReport(frictionOut);
2456
+ // pre-merge throw) or friction.json is absent or empty. On a run that reached
2457
+ // the verdict this is ALREADY a no-op — the happy path emits it earlier, ahead
2458
+ // of the baseline banner — so what is left here is the throw paths.
2459
+ emitFrictionOnce();
1329
2460
  // Remove the signal handlers first (no listener leak across runs), then run
1330
2461
  // the same cleanup: release the lock, remove the harness dir, and remove the
1331
2462
  // vat-only grader dir (which lives outside harnessRoot).