@vibe-agent-toolkit/agent-skills 0.1.42 → 0.2.0-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (322) hide show
  1. package/README.md +5 -5
  2. package/dist/builder.d.ts.map +1 -1
  3. package/dist/builder.js +7 -17
  4. package/dist/builder.js.map +1 -1
  5. package/dist/content-type-routing.d.ts +32 -0
  6. package/dist/content-type-routing.d.ts.map +1 -1
  7. package/dist/content-type-routing.js +23 -0
  8. package/dist/content-type-routing.js.map +1 -1
  9. package/dist/files-config.d.ts +34 -1
  10. package/dist/files-config.d.ts.map +1 -1
  11. package/dist/files-config.js +277 -39
  12. package/dist/files-config.js.map +1 -1
  13. package/dist/fs-attribution.d.ts +45 -0
  14. package/dist/fs-attribution.d.ts.map +1 -0
  15. package/dist/fs-attribution.js +57 -0
  16. package/dist/fs-attribution.js.map +1 -0
  17. package/dist/import.d.ts.map +1 -1
  18. package/dist/import.js +0 -4
  19. package/dist/import.js.map +1 -1
  20. package/dist/index.d.ts +13 -8
  21. package/dist/index.d.ts.map +1 -1
  22. package/dist/index.js +25 -7
  23. package/dist/index.js.map +1 -1
  24. package/dist/inventory/detectors/declared-but-missing.d.ts +1 -1
  25. package/dist/inventory/detectors/declared-but-missing.d.ts.map +1 -1
  26. package/dist/inventory/detectors/marketplace-source-missing.d.ts +1 -1
  27. package/dist/inventory/detectors/marketplace-source-missing.d.ts.map +1 -1
  28. package/dist/inventory/detectors/present-but-undeclared.d.ts +1 -1
  29. package/dist/inventory/detectors/present-but-undeclared.d.ts.map +1 -1
  30. package/dist/inventory/detectors/reference-target-missing.d.ts +1 -1
  31. package/dist/inventory/detectors/reference-target-missing.d.ts.map +1 -1
  32. package/dist/inventory/index.d.ts +1 -1
  33. package/dist/inventory/index.d.ts.map +1 -1
  34. package/dist/inventory/index.js +1 -1
  35. package/dist/inventory/index.js.map +1 -1
  36. package/dist/inventory/serialize.d.ts +4 -4
  37. package/dist/inventory/serialize.d.ts.map +1 -1
  38. package/dist/inventory/serialize.js +6 -6
  39. package/dist/inventory/serialize.js.map +1 -1
  40. package/dist/inventory/types.d.ts +8 -0
  41. package/dist/inventory/types.d.ts.map +1 -1
  42. package/dist/inventory/types.js.map +1 -1
  43. package/dist/link-graph-members.d.ts +18 -0
  44. package/dist/link-graph-members.d.ts.map +1 -0
  45. package/dist/link-graph-members.js +18 -0
  46. package/dist/link-graph-members.js.map +1 -0
  47. package/dist/parsers/frontmatter-parser.d.ts.map +1 -1
  48. package/dist/parsers/frontmatter-parser.js +30 -6
  49. package/dist/parsers/frontmatter-parser.js.map +1 -1
  50. package/dist/plugin-distribution-layout.d.ts +70 -6
  51. package/dist/plugin-distribution-layout.d.ts.map +1 -1
  52. package/dist/plugin-distribution-layout.js +84 -11
  53. package/dist/plugin-distribution-layout.js.map +1 -1
  54. package/dist/post-build-checks.d.ts +38 -1
  55. package/dist/post-build-checks.d.ts.map +1 -1
  56. package/dist/post-build-checks.js +68 -17
  57. package/dist/post-build-checks.js.map +1 -1
  58. package/dist/projection/skill-extent.d.ts +247 -0
  59. package/dist/projection/skill-extent.d.ts.map +1 -0
  60. package/dist/projection/skill-extent.js +560 -0
  61. package/dist/projection/skill-extent.js.map +1 -0
  62. package/dist/schemas/agent-skill-frontmatter.js +1 -1
  63. package/dist/schemas/agent-skill-frontmatter.js.map +1 -1
  64. package/dist/schemas/allowed-tools.d.ts +31 -0
  65. package/dist/schemas/allowed-tools.d.ts.map +1 -0
  66. package/dist/schemas/allowed-tools.js +88 -0
  67. package/dist/schemas/allowed-tools.js.map +1 -0
  68. package/dist/schemas/index.d.ts +1 -0
  69. package/dist/schemas/index.d.ts.map +1 -1
  70. package/dist/schemas/index.js +1 -0
  71. package/dist/schemas/index.js.map +1 -1
  72. package/dist/schemas/installed-plugins-registry.d.ts.map +1 -1
  73. package/dist/schemas/installed-plugins-registry.js +7 -1
  74. package/dist/schemas/installed-plugins-registry.js.map +1 -1
  75. package/dist/schemas/marketplace-manifest.d.ts.map +1 -1
  76. package/dist/schemas/marketplace-manifest.js +17 -6
  77. package/dist/schemas/marketplace-manifest.js.map +1 -1
  78. package/dist/skill-identity.d.ts +18 -3
  79. package/dist/skill-identity.d.ts.map +1 -1
  80. package/dist/skill-identity.js +25 -6
  81. package/dist/skill-identity.js.map +1 -1
  82. package/dist/skill-packager.d.ts +144 -20
  83. package/dist/skill-packager.d.ts.map +1 -1
  84. package/dist/skill-packager.js +447 -105
  85. package/dist/skill-packager.js.map +1 -1
  86. package/dist/skill-source/content-hash.d.ts.map +1 -1
  87. package/dist/skill-source/content-hash.js +21 -10
  88. package/dist/skill-source/content-hash.js.map +1 -1
  89. package/dist/skill-source/fetch-cache.d.ts.map +1 -1
  90. package/dist/skill-source/fetch-cache.js +0 -5
  91. package/dist/skill-source/fetch-cache.js.map +1 -1
  92. package/dist/skill-source/git-clone.d.ts +1 -1
  93. package/dist/skill-source/git-clone.d.ts.map +1 -1
  94. package/dist/skill-source/git-clone.js +10 -10
  95. package/dist/skill-source/git-clone.js.map +1 -1
  96. package/dist/skill-source/sources/url-source.d.ts.map +1 -1
  97. package/dist/skill-source/sources/url-source.js +2 -2
  98. package/dist/skill-source/sources/url-source.js.map +1 -1
  99. package/dist/skill-source/stage.d.ts.map +1 -1
  100. package/dist/skill-source/stage.js +0 -6
  101. package/dist/skill-source/stage.js.map +1 -1
  102. package/dist/skill-test/baseline-delta.d.ts +1575 -0
  103. package/dist/skill-test/baseline-delta.d.ts.map +1 -0
  104. package/dist/skill-test/baseline-delta.js +272 -0
  105. package/dist/skill-test/baseline-delta.js.map +1 -0
  106. package/dist/skill-test/baseline-integrity.d.ts +944 -0
  107. package/dist/skill-test/baseline-integrity.d.ts.map +1 -0
  108. package/dist/skill-test/baseline-integrity.js +3090 -0
  109. package/dist/skill-test/baseline-integrity.js.map +1 -0
  110. package/dist/skill-test/build-hook.d.ts +3 -2
  111. package/dist/skill-test/build-hook.d.ts.map +1 -1
  112. package/dist/skill-test/build-hook.js +4 -5
  113. package/dist/skill-test/build-hook.js.map +1 -1
  114. package/dist/skill-test/configure-writer.js +1 -1
  115. package/dist/skill-test/configure-writer.js.map +1 -1
  116. package/dist/skill-test/declared-env.d.ts +5 -4
  117. package/dist/skill-test/declared-env.d.ts.map +1 -1
  118. package/dist/skill-test/declared-env.js +8 -9
  119. package/dist/skill-test/declared-env.js.map +1 -1
  120. package/dist/skill-test/eval-executor.d.ts +21 -5
  121. package/dist/skill-test/eval-executor.d.ts.map +1 -1
  122. package/dist/skill-test/eval-executor.js +6 -6
  123. package/dist/skill-test/eval-executor.js.map +1 -1
  124. package/dist/skill-test/eval-fragment.d.ts +106 -28
  125. package/dist/skill-test/eval-fragment.d.ts.map +1 -1
  126. package/dist/skill-test/eval-fragment.js +78 -16
  127. package/dist/skill-test/eval-fragment.js.map +1 -1
  128. package/dist/skill-test/eval-grader.d.ts +65 -7
  129. package/dist/skill-test/eval-grader.d.ts.map +1 -1
  130. package/dist/skill-test/eval-grader.js +263 -64
  131. package/dist/skill-test/eval-grader.js.map +1 -1
  132. package/dist/skill-test/eval-inputs.d.ts +110 -8
  133. package/dist/skill-test/eval-inputs.d.ts.map +1 -1
  134. package/dist/skill-test/eval-inputs.js +177 -22
  135. package/dist/skill-test/eval-inputs.js.map +1 -1
  136. package/dist/skill-test/eval-lint.d.ts +17 -4
  137. package/dist/skill-test/eval-lint.d.ts.map +1 -1
  138. package/dist/skill-test/eval-lint.js +22 -9
  139. package/dist/skill-test/eval-lint.js.map +1 -1
  140. package/dist/skill-test/eval-suite-isolation.d.ts.map +1 -1
  141. package/dist/skill-test/eval-suite-isolation.js +8 -5
  142. package/dist/skill-test/eval-suite-isolation.js.map +1 -1
  143. package/dist/skill-test/evals-template.d.ts +1 -1
  144. package/dist/skill-test/evals-template.d.ts.map +1 -1
  145. package/dist/skill-test/evals-template.js +1 -3
  146. package/dist/skill-test/evals-template.js.map +1 -1
  147. package/dist/skill-test/executor-prompt.d.ts +15 -7
  148. package/dist/skill-test/executor-prompt.d.ts.map +1 -1
  149. package/dist/skill-test/executor-prompt.js +10 -7
  150. package/dist/skill-test/executor-prompt.js.map +1 -1
  151. package/dist/skill-test/failure-reason.d.ts +113 -0
  152. package/dist/skill-test/failure-reason.d.ts.map +1 -0
  153. package/dist/skill-test/failure-reason.js +124 -0
  154. package/dist/skill-test/failure-reason.js.map +1 -0
  155. package/dist/skill-test/fragment-merge.d.ts +3 -20
  156. package/dist/skill-test/fragment-merge.d.ts.map +1 -1
  157. package/dist/skill-test/fragment-merge.js +67 -8
  158. package/dist/skill-test/fragment-merge.js.map +1 -1
  159. package/dist/skill-test/grader-prompt.d.ts +29 -9
  160. package/dist/skill-test/grader-prompt.d.ts.map +1 -1
  161. package/dist/skill-test/grader-prompt.js +264 -68
  162. package/dist/skill-test/grader-prompt.js.map +1 -1
  163. package/dist/skill-test/grader-text.d.ts +145 -0
  164. package/dist/skill-test/grader-text.d.ts.map +1 -0
  165. package/dist/skill-test/grader-text.js +410 -0
  166. package/dist/skill-test/grader-text.js.map +1 -0
  167. package/dist/skill-test/grading-adapter.d.ts +38 -2
  168. package/dist/skill-test/grading-adapter.d.ts.map +1 -1
  169. package/dist/skill-test/grading-adapter.js +57 -9
  170. package/dist/skill-test/grading-adapter.js.map +1 -1
  171. package/dist/skill-test/grading-schema.d.ts +160 -5
  172. package/dist/skill-test/grading-schema.d.ts.map +1 -1
  173. package/dist/skill-test/grading-schema.js +66 -7
  174. package/dist/skill-test/grading-schema.js.map +1 -1
  175. package/dist/skill-test/harness-location.d.ts +3 -2
  176. package/dist/skill-test/harness-location.d.ts.map +1 -1
  177. package/dist/skill-test/harness-location.js +5 -16
  178. package/dist/skill-test/harness-location.js.map +1 -1
  179. package/dist/skill-test/index.d.ts +2 -2
  180. package/dist/skill-test/index.d.ts.map +1 -1
  181. package/dist/skill-test/index.js +2 -2
  182. package/dist/skill-test/index.js.map +1 -1
  183. package/dist/skill-test/lock.d.ts +58 -1
  184. package/dist/skill-test/lock.d.ts.map +1 -1
  185. package/dist/skill-test/lock.js +78 -6
  186. package/dist/skill-test/lock.js.map +1 -1
  187. package/dist/skill-test/pipeline.d.ts +19 -1
  188. package/dist/skill-test/pipeline.d.ts.map +1 -1
  189. package/dist/skill-test/pipeline.js +21 -10
  190. package/dist/skill-test/pipeline.js.map +1 -1
  191. package/dist/skill-test/preflight.d.ts +1 -1
  192. package/dist/skill-test/preflight.d.ts.map +1 -1
  193. package/dist/skill-test/preflight.js +59 -16
  194. package/dist/skill-test/preflight.js.map +1 -1
  195. package/dist/skill-test/prompt-invariants.d.ts +4 -2
  196. package/dist/skill-test/prompt-invariants.d.ts.map +1 -1
  197. package/dist/skill-test/prompt-invariants.js +5 -4
  198. package/dist/skill-test/prompt-invariants.js.map +1 -1
  199. package/dist/skill-test/run-harness.d.ts +665 -85
  200. package/dist/skill-test/run-harness.d.ts.map +1 -1
  201. package/dist/skill-test/run-harness.js +1522 -237
  202. package/dist/skill-test/run-harness.js.map +1 -1
  203. package/dist/skill-test/staging.d.ts +20 -8
  204. package/dist/skill-test/staging.d.ts.map +1 -1
  205. package/dist/skill-test/staging.js +25 -18
  206. package/dist/skill-test/staging.js.map +1 -1
  207. package/dist/skill-test/tier-plan.d.ts +1 -1
  208. package/dist/skill-test/tier-plan.js +1 -1
  209. package/dist/skill-test/tool-eval-schema.d.ts +6 -6
  210. package/dist/skill-test/tool-eval-schema.js +2 -2
  211. package/dist/skill-test/vendor-manifest.d.ts +2 -1
  212. package/dist/skill-test/vendor-manifest.d.ts.map +1 -1
  213. package/dist/skill-test/vendor-manifest.js +10 -10
  214. package/dist/skill-test/vendor-manifest.js.map +1 -1
  215. package/dist/test-input.d.ts +59 -8
  216. package/dist/test-input.d.ts.map +1 -1
  217. package/dist/test-input.js +64 -19
  218. package/dist/test-input.js.map +1 -1
  219. package/dist/validators/agent-instruction-presence.d.ts +1 -1
  220. package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
  221. package/dist/validators/agent-instruction-presence.js +45 -2
  222. package/dist/validators/agent-instruction-presence.js.map +1 -1
  223. package/dist/validators/bundled-resource-link-detection.d.ts +6 -4
  224. package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
  225. package/dist/validators/bundled-resource-link-detection.js +39 -39
  226. package/dist/validators/bundled-resource-link-detection.js.map +1 -1
  227. package/dist/validators/compat-detectors.d.ts +18 -1
  228. package/dist/validators/compat-detectors.d.ts.map +1 -1
  229. package/dist/validators/compat-detectors.js +47 -20
  230. package/dist/validators/compat-detectors.js.map +1 -1
  231. package/dist/validators/cross-skill-dependency-detection.d.ts +1 -1
  232. package/dist/validators/cross-skill-dependency-detection.d.ts.map +1 -1
  233. package/dist/validators/cross-skill-dependency-detection.js +1 -1
  234. package/dist/validators/cross-skill-dependency-detection.js.map +1 -1
  235. package/dist/validators/description-style-detection.d.ts +1 -1
  236. package/dist/validators/description-style-detection.d.ts.map +1 -1
  237. package/dist/validators/description-style-detection.js +1 -1
  238. package/dist/validators/description-style-detection.js.map +1 -1
  239. package/dist/validators/format-detection.d.ts +3 -0
  240. package/dist/validators/format-detection.d.ts.map +1 -1
  241. package/dist/validators/format-detection.js +41 -26
  242. package/dist/validators/format-detection.js.map +1 -1
  243. package/dist/validators/frontmatter-validation.d.ts +1 -1
  244. package/dist/validators/frontmatter-validation.d.ts.map +1 -1
  245. package/dist/validators/frontmatter-validation.js +2 -2
  246. package/dist/validators/frontmatter-validation.js.map +1 -1
  247. package/dist/validators/imperative-body-detection.d.ts +1 -1
  248. package/dist/validators/imperative-body-detection.d.ts.map +1 -1
  249. package/dist/validators/imperative-body-detection.js +1 -1
  250. package/dist/validators/imperative-body-detection.js.map +1 -1
  251. package/dist/validators/index.d.ts +1 -1
  252. package/dist/validators/index.d.ts.map +1 -1
  253. package/dist/validators/kebab-case-detection.d.ts +1 -1
  254. package/dist/validators/kebab-case-detection.d.ts.map +1 -1
  255. package/dist/validators/kebab-case-detection.js +1 -1
  256. package/dist/validators/kebab-case-detection.js.map +1 -1
  257. package/dist/validators/marketplace-validator.d.ts.map +1 -1
  258. package/dist/validators/marketplace-validator.js +7 -2
  259. package/dist/validators/marketplace-validator.js.map +1 -1
  260. package/dist/validators/mcp-tool-qualification.d.ts +158 -0
  261. package/dist/validators/mcp-tool-qualification.d.ts.map +1 -0
  262. package/dist/validators/mcp-tool-qualification.js +467 -0
  263. package/dist/validators/mcp-tool-qualification.js.map +1 -0
  264. package/dist/validators/packaged-size-limit.d.ts +296 -0
  265. package/dist/validators/packaged-size-limit.d.ts.map +1 -0
  266. package/dist/validators/packaged-size-limit.js +492 -0
  267. package/dist/validators/packaged-size-limit.js.map +1 -0
  268. package/dist/validators/packaging-validator.d.ts +149 -10
  269. package/dist/validators/packaging-validator.d.ts.map +1 -1
  270. package/dist/validators/packaging-validator.js +371 -55
  271. package/dist/validators/packaging-validator.js.map +1 -1
  272. package/dist/validators/plugin-hosted-shape.d.ts +4 -2
  273. package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
  274. package/dist/validators/plugin-hosted-shape.js +6 -7
  275. package/dist/validators/plugin-hosted-shape.js.map +1 -1
  276. package/dist/validators/plugin-recommended-fields.d.ts +1 -1
  277. package/dist/validators/plugin-recommended-fields.d.ts.map +1 -1
  278. package/dist/validators/plugin-recommended-fields.js +1 -1
  279. package/dist/validators/plugin-recommended-fields.js.map +1 -1
  280. package/dist/validators/referenced-path-missing.d.ts +211 -0
  281. package/dist/validators/referenced-path-missing.d.ts.map +1 -0
  282. package/dist/validators/referenced-path-missing.js +378 -0
  283. package/dist/validators/referenced-path-missing.js.map +1 -0
  284. package/dist/validators/registry-validator.d.ts.map +1 -1
  285. package/dist/validators/registry-validator.js +1 -2
  286. package/dist/validators/registry-validator.js.map +1 -1
  287. package/dist/validators/rule-engine/index.d.ts +1 -1
  288. package/dist/validators/rule-engine/rule-context.d.ts +20 -3
  289. package/dist/validators/rule-engine/rule-context.d.ts.map +1 -1
  290. package/dist/validators/rule-engine/rule-context.js +3 -1
  291. package/dist/validators/rule-engine/rule-context.js.map +1 -1
  292. package/dist/validators/rule-engine/rule-engine.d.ts +2 -2
  293. package/dist/validators/rule-engine/rule-engine.d.ts.map +1 -1
  294. package/dist/validators/rule-engine/rule-engine.js +52 -13
  295. package/dist/validators/rule-engine/rule-engine.js.map +1 -1
  296. package/dist/validators/skill-validator.d.ts.map +1 -1
  297. package/dist/validators/skill-validator.js +91 -54
  298. package/dist/validators/skill-validator.js.map +1 -1
  299. package/dist/validators/source-only-codes.js +1 -1
  300. package/dist/validators/source-only-codes.js.map +1 -1
  301. package/dist/validators/types.d.ts +21 -1
  302. package/dist/validators/types.d.ts.map +1 -1
  303. package/dist/validators/unified-validator.d.ts.map +1 -1
  304. package/dist/validators/unified-validator.js +4 -5
  305. package/dist/validators/unified-validator.js.map +1 -1
  306. package/dist/validators/validation-rules.d.ts +4 -4
  307. package/dist/validators/validation-rules.d.ts.map +1 -1
  308. package/dist/validators/validation-rules.js +9 -38
  309. package/dist/validators/validation-rules.js.map +1 -1
  310. package/dist/validators/walker-to-issues.d.ts +9 -3
  311. package/dist/validators/walker-to-issues.d.ts.map +1 -1
  312. package/dist/validators/walker-to-issues.js +64 -19
  313. package/dist/validators/walker-to-issues.js.map +1 -1
  314. package/dist/walk-link-graph.d.ts +64 -5
  315. package/dist/walk-link-graph.d.ts.map +1 -1
  316. package/dist/walk-link-graph.js +376 -92
  317. package/dist/walk-link-graph.js.map +1 -1
  318. package/package.json +14 -22
  319. package/dist/skill-test/exit-codes.d.ts +0 -97
  320. package/dist/skill-test/exit-codes.d.ts.map +0 -1
  321. package/dist/skill-test/exit-codes.js +0 -141
  322. package/dist/skill-test/exit-codes.js.map +0 -1
@@ -3,7 +3,7 @@
3
3
  *
4
4
  * Implements the full harness sequence as a pure async function (no process.exit):
5
5
  * lock → assert safe workdir + harness root → stage → resolve staged-subject
6
- * eval path → bootstrap-check (scaffold template + exit 3 if absent)
6
+ * eval path → bootstrap-check (scaffold template + exit 2 with `Reason: bootstrap` if absent)
7
7
  * → preflight (return early exitCode 2 on failure) → ack enforcement
8
8
  * → dry-run short-circuit → build per-eval work items → run the vat-owned
9
9
  * executor→grader pipeline (bounded-parallel) → merge grader fragments →
@@ -11,27 +11,34 @@
11
11
  * → release lock → return result
12
12
  */
13
13
  import { randomBytes } from 'node:crypto';
14
- import { existsSync, lstatSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
14
+ import { existsSync, lstatSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
15
+ import { homedir } from 'node:os';
15
16
  import { basename, dirname } from 'node:path';
16
17
  import { fileURLToPath } from 'node:url';
17
- import { getToolVersion, killAllActiveClaudeChildren, mkdirSyncReal, normalizedTmpdir, probeAuthStatus, resolveAssetReference, safeExecResult, safePath, toForwardSlash, } from '@vibe-agent-toolkit/utils';
18
+ import { ExitCode } from '@vibe-agent-toolkit/schema';
19
+ import { direntKind, isPathAbsentError, mkdirSyncReal, normalizedTmpdir, prefixMessageOnce, resolveAssetReference, safePath, toForwardSlashAnyPlatform, } from '@vibe-agent-toolkit/utils';
20
+ import { getToolVersion, safeExecResult, } from '@vibe-agent-toolkit/utils/process';
21
+ import { killAllActiveClaudeChildren, probeAuthStatus, } from '@vibe-agent-toolkit/utils/skill-test';
18
22
  import { resolveSkillSource } from '../skill-source/resolve-skill-source.js';
23
+ import { BaselineArtifactSchema, computeBaselineDelta, formatBaselineDeltaLine, } from './baseline-delta.js';
24
+ import { activeContaminationSignals, armExpectationSkew, detectBaselineContamination, scrubControlArmEnv, skillContentNeedles, summarizeBaselineIntegrity, } from './baseline-integrity.js';
19
25
  import { assembleChildEnv, assertKnownEnvTokens, computeEnvTokens, resolveInjectEnv } from './declared-env.js';
20
26
  import { runExecutorForEval } from './eval-executor.js';
21
- import { runGraderForEval } from './eval-grader.js';
22
- import { EvalInputError, parseEvalSuite, stageEvalWorkspaces } from './eval-inputs.js';
27
+ import { GRADER_FRAGMENT_UNPARSEABLE, GraderFragmentUnparseableError, runGraderForEval } from './eval-grader.js';
28
+ import { armDirSegment, EvalInputError, parseEvalSuite, stageEvalWorkspaces, } from './eval-inputs.js';
23
29
  import { lintEvalExpectations, lintToolExpectationExecutables } from './eval-lint.js';
24
30
  import { DEFAULT_EVALS_SUBPATH } from './eval-suite-isolation.js';
25
31
  import { writeEvalsTemplate } from './evals-template.js';
26
- import { BootstrapNeededError, DuplicateStagedSkillError, InternalHarnessError, SkillTestExitCode, } from './exit-codes.js';
32
+ import { BootstrapNeededError, DuplicateStagedSkillError, InternalHarnessError, } from './failure-reason.js';
27
33
  import { mergeFragmentsToFriction, mergeFragmentsToGrading, mergeFragmentsToToolEval } from './fragment-merge.js';
28
34
  import { FrictionReportSchema } from './friction-schema.js';
29
35
  import { DEFAULT_CONCURRENCY, DEFAULT_GRADER_MODEL } from './grader-model.js';
36
+ import { sanitizeGraderText } from './grader-text.js';
30
37
  import { reconcileGrading } from './grading-adapter.js';
31
38
  import { GradingReportSchema } from './grading-schema.js';
32
- import { assertSafeHarnessRoot, assertSafeWorkdir, prepareHarnessRoot, resolveHarnessRoot } from './harness-location.js';
39
+ import { assertSafeHarnessRoot, assertSafeWorkdir, HarnessLocationError, prepareHarnessRoot, resolveHarnessRoot, } from './harness-location.js';
33
40
  import { acquireHarnessLock, installSignalCleanup } from './lock.js';
34
- import { runPipeline } from './pipeline.js';
41
+ import { RateLimitSignal, runPipeline } from './pipeline.js';
35
42
  import { detectPluginLayout } from './plugin-layout.js';
36
43
  import { runPreflight } from './preflight.js';
37
44
  import { descriptorToSource, stageHarness } from './staging.js';
@@ -49,7 +56,7 @@ import { verifyVendoredManifest } from './vendor-manifest.js';
49
56
  const VENDORED_SKILL_CREATOR_DIR = safePath.resolve(dirname(fileURLToPath(import.meta.url)), '../../vendor/skill-creator');
50
57
  /**
51
58
  * Absolute path to the vendored skill-creator grader rubric each per-eval grader
52
- * spawn is told to judge against (issue #145). Lives inside
59
+ * spawn is told to judge against. Lives inside
53
60
  * {@link VENDORED_SKILL_CREATOR_DIR}, whose hash manifest is verified during
54
61
  * preflight — so the rubric the grader uses is the pinned, integrity-checked copy.
55
62
  */
@@ -82,28 +89,30 @@ export const SKILL_TEST_BUILTIN_CAPS = {
82
89
  };
83
90
  /**
84
91
  * Resolve the effective PER-EVAL wall-clock timeout (ms). Each executor and
85
- * grader spawn is an independent, bounded-parallel unit (issue #145), so the
92
+ * grader spawn is an independent, bounded-parallel unit, so the
86
93
  * budget is a flat per-spawn ceiling — NOT scaled by the suite size the way the
87
94
  * old single serial run's budget was. An explicit `--timeout` (seconds)
88
95
  * wins; otherwise the flat {@link DEFAULT_TIMEOUT_MS} default applies.
89
96
  */
90
- export function resolveTimeoutMs(opts) {
97
+ function resolveTimeoutMs(opts) {
91
98
  return opts.timeout === undefined ? DEFAULT_TIMEOUT_MS : opts.timeout * 1000;
92
99
  }
93
100
  /**
94
101
  * Map an eval verdict to a process exit code. Default behavior (fail-closed): a
95
- * failing verdict escalates to EvalFailure (4) — distinct from the harness-broke
96
- * codes (1/2/3) so a CI consumer can `case $? in 0);; 4) tolerate;; *) hard fail;; esac`.
102
+ * failing verdict is `FINDINGS` — the harness ran to completion and what it
103
+ * examined did not pass, exactly as `vat skills validate` reports an
104
+ * error-severity finding — while a harness that could not run is `ERROR`, so a
105
+ * CI consumer can `case $? in 0);; 1) tolerate;; *) hard fail;; esac`.
97
106
  * When `tolerateEvalFailure` is set (interactive opt-out), a failing verdict is
98
- * downgraded to Ok (0) and the count lives only in the summary/grading.json.
107
+ * downgraded to `OK` and the count lives only in the summary/grading.json.
99
108
  */
100
109
  export function verdictExitCode(allPassed, tolerateEvalFailure) {
101
- return !allPassed && !tolerateEvalFailure ? SkillTestExitCode.EvalFailure : SkillTestExitCode.Ok;
110
+ return !allPassed && !tolerateEvalFailure ? ExitCode.FINDINGS : ExitCode.OK;
102
111
  }
103
- export function resolveStallMs(opts) {
112
+ function resolveStallMs(opts) {
104
113
  return opts.stall === undefined ? undefined : opts.stall * 1000;
105
114
  }
106
- export function resolveKnobs(opts) {
115
+ function resolveKnobs(opts) {
107
116
  const result = {
108
117
  maxTurns: opts.maxTurns ?? DEFAULT_MAX_TURNS,
109
118
  maxBudgetUsd: opts.maxBudgetUsd ?? DEFAULT_MAX_BUDGET_USD,
@@ -123,13 +132,13 @@ export function resolveKnobs(opts) {
123
132
  * sources (npm/url/vendored/workspace) have no local source tree to walk, so they
124
133
  * are always staged flat (returns undefined). undefined → flat staging.
125
134
  */
126
- export function detectItemPluginLayout(source, repoRoot) {
135
+ function detectItemPluginLayout(source, repoRoot) {
127
136
  if (!('path' in source))
128
137
  return undefined;
129
138
  const sourceDir = resolveAssetReference(source.path, repoRoot);
130
139
  return detectPluginLayout(sourceDir, existsSync) ?? undefined;
131
140
  }
132
- export function makeStageItem(name, source, repoRoot, role, optional) {
141
+ function makeStageItem(name, source, repoRoot, role, optional) {
133
142
  const pluginLayout = detectItemPluginLayout(source, repoRoot);
134
143
  return {
135
144
  name,
@@ -143,14 +152,14 @@ export function makeStageItem(name, source, repoRoot, role, optional) {
143
152
  * Build the full set of items `stageHarness` will stage: the subject, every
144
153
  * REQUIRED `--with` companion, and every OPTIONAL `--with-optional` companion.
145
154
  * Both `with` and `optional` STAGE the named companion and make it invocable —
146
- * they differ only in required-vs-optional resolution (issue #153; a `--with`
147
- * name that named no positional skill used to be silently dropped, never staged,
148
- * no manifest trace). Fail-closed on a DUPLICATE staged name (subject / `with` /
155
+ * they differ only in required-vs-optional resolution (a `--with` name that
156
+ * names no positional skill must never be silently dropped — unstaged, with no
157
+ * manifest trace). Fail-closed on a DUPLICATE staged name (subject / `with` /
149
158
  * `optional` all share one namespace): the first repeat throws
150
159
  * {@link DuplicateStagedSkillError} rather than silently letting a later item
151
160
  * clobber an earlier one under the same staged slot.
152
161
  */
153
- export function buildStageItems(opts, repoRoot) {
162
+ function buildStageItems(opts, repoRoot) {
154
163
  const items = [];
155
164
  const seen = new Set();
156
165
  const pushItem = (item) => {
@@ -173,7 +182,7 @@ export function buildStageItems(opts, repoRoot) {
173
182
  }
174
183
  return items;
175
184
  }
176
- export function buildResolveCtx(harnessRoot, repoRoot) {
185
+ function buildResolveCtx(harnessRoot, repoRoot) {
177
186
  return {
178
187
  repoRoot,
179
188
  stagingRoot: safePath.joinUnderRoot(harnessRoot, 'staged'),
@@ -181,56 +190,111 @@ export function buildResolveCtx(harnessRoot, repoRoot) {
181
190
  };
182
191
  }
183
192
  /**
184
- * Map of flags to dummy values used when probing for flag support.
193
+ * Match a flag name as a whole token in `claude --help` output.
185
194
  *
186
- * Each value is chosen so value-validation (e.g., enum checks) accepts it
187
- * before `--help` short-circuits the session. The empty string `''` for
188
- * `--setting-sources` is intentional — an empty comma-separated list is valid.
195
+ * The boundary matters: a bare `includes('--plugin-dir')` also matches
196
+ * `--plugin-dirs` or `--plugin-dir-cache`, and vat's own flag names are prefixes
197
+ * of each other (`--max-turns` / `--max-turns-per-eval` would collide). Neither
198
+ * side of the token may be a letter or a dash.
189
199
  */
190
- const FLAG_DUMMY_VALUES = {
191
- '--plugin-dir': '.',
192
- '--setting-sources': '',
193
- '--output-format': 'stream-json',
194
- '--permission-mode': 'bypassPermissions',
195
- '--max-turns': '1',
196
- '--max-budget-usd': '1',
197
- };
200
+ function helpTextDeclaresFlag(helpText, flag) {
201
+ // A scan rather than a built regex: the flag is data (it comes from a list a
202
+ // future edit will extend), and building a pattern from data is both a lint
203
+ // error here and the kind of thing that silently changes meaning when someone
204
+ // adds a flag containing a metacharacter.
205
+ const isTokenChar = (ch) => ch !== undefined && (ch === '-' || ch === '_' || /[a-z0-9]/i.test(ch));
206
+ for (let from = helpText.indexOf(flag); from !== -1; from = helpText.indexOf(flag, from + 1)) {
207
+ if (!isTokenChar(helpText[from - 1]) && !isTokenChar(helpText[from + flag.length]))
208
+ return true;
209
+ }
210
+ return false;
211
+ }
198
212
  /**
199
- * Return a sensible dummy value for a given CLI flag so that value-validation
200
- * doesn't reject the argument before `--help` can short-circuit the session.
213
+ * A flag name no `claude` will ever accept. It is probed alongside the real ones
214
+ * as a NEGATIVE CONTROL: if the probe reports this as supported, the probe is
215
+ * broken and its answer about every other flag is worthless.
201
216
  */
202
- export function flagDummyValueFor(flag) {
203
- return FLAG_DUMMY_VALUES[flag] ?? '1';
204
- }
217
+ const FLAG_PROBE_SENTINEL = '--vat-probe-flag-that-cannot-exist';
205
218
  /**
206
- * Build a token-free flag-parse probe.
219
+ * Build a token-free flag-support probe from `claude --help`.
220
+ *
221
+ * ⚠️ This REPLACES an exit-code probe (`claude <flag> <dummy> --help`, exit 0 ⇒
222
+ * supported) that could not discriminate at all. Verified against claude 2.x:
223
+ * `claude --no-such-flag-xyz 1 --help` exits **0**, because `--help` short-circuits
224
+ * before argument validation. Every one of preflight's `flag <name>` checks
225
+ * therefore reported "supported" unconditionally — including for flags that do not
226
+ * exist — so the gate that is supposed to stop vat spawning with an unsupported
227
+ * flag was decorative. (The same command without `--help` exits 1, but running it
228
+ * would start a real session and bill tokens, which is why `--help` was there.)
207
229
  *
208
- * For each flag, runs `claude <flag> <dummyValue> --help` via safeExecResult.
209
- * Exit 0 means claude's parser accepts the flag; `--help` short-circuits before
210
- * any session or tokens are consumed.
230
+ * Help-text matching does discriminate, and it is still token-free: one `--help`
231
+ * invocation for the whole run instead of one spawn per flag.
211
232
  *
212
- * The dummy value is always included in the args, even when it is an empty
213
- * string — some flags (e.g., `--setting-sources`) require a value argument,
214
- * so omitting it causes exit 1 before `--help` can be reached.
233
+ * The probe carries its own negative control ({@link FLAG_PROBE_SENTINEL}). If a
234
+ * future `claude` prints something that matches the sentinel — or `--help` fails
235
+ * outright — the probe reports EVERY flag unsupported rather than every flag
236
+ * supported. A probe that cannot tell must fail closed and be seen; the defect it
237
+ * replaces failed open and was invisible.
215
238
  *
216
- * The probe is a closure — it is NOT spawned at module scope, only at call time
217
- * inside runPreflight.
239
+ * `--help` runs on the FIRST flag query, not when the probe is built. Building
240
+ * it is part of assembling preflight's input, and a preflight that never asks
241
+ * (a stubbed one, or one that fails an earlier check) must not have already
242
+ * paid for a subprocess: with the spawn at build time every harness run cost
243
+ * one real `claude --help` — 75 ms where the CLI is installed, an instant
244
+ * ENOENT where it is not — which read as a 10× platform split in a test that
245
+ * had injected a fake spawn and stubbed preflight precisely so no process
246
+ * would start.
218
247
  */
219
- function buildFlagParseProbe() {
248
+ function buildFlagParseProbe(runHelp = defaultClaudeHelp) {
249
+ let helpText;
220
250
  return (flag) => {
221
- const dummy = flagDummyValueFor(flag);
222
- const result = safeExecResult('claude', [flag, dummy, '--help'], {
223
- stdio: 'pipe',
224
- encoding: 'utf8',
225
- timeout: 15_000,
226
- });
227
- return result.success;
251
+ if (helpText === undefined)
252
+ helpText = runHelp();
253
+ // No help output, or a sentinel "match" ⇒ the probe is not trustworthy.
254
+ const usable = helpText !== null && !helpTextDeclaresFlag(helpText, FLAG_PROBE_SENTINEL);
255
+ return usable && helpTextDeclaresFlag(helpText ?? '', flag);
228
256
  };
229
257
  }
230
- export function buildPreflightInput(evalsPath, pluginDirs, opts, knobs) {
258
+ /** Run `claude --help` once, returning its combined output (null if unreachable). */
259
+ function defaultClaudeHelp() {
260
+ const result = safeExecResult('claude', ['--help'], {
261
+ stdio: 'pipe',
262
+ encoding: 'utf8',
263
+ timeout: 15_000,
264
+ });
265
+ if (!result.success)
266
+ return null;
267
+ // `stdout`/`stderr` are typed `Buffer | string`; `encoding: 'utf8'` makes them
268
+ // strings, and `String()` is correct either way. Both streams are read because
269
+ // a CLI is free to print its usage to either.
270
+ return `${String(result.stdout)}\n${String(result.stderr)}`;
271
+ }
272
+ /**
273
+ * ⚠️ KNOWN OPEN — `costEstimate` here is DEAD, in three independent ways, and none
274
+ * of them is fixable from inside this function:
275
+ *
276
+ * 1. `evalCount: 1` is a placeholder. The suite is not parsed until Step 5.5, which
277
+ * runs AFTER preflight, so the real count does not exist yet. Making it real means
278
+ * either moving preflight after the suite parse (it currently gates that parse) or
279
+ * adding a second reader of `evals.json` — and two readers of the answer key is
280
+ * precisely the drift this module refuses elsewhere.
281
+ * 2. `renderPreflightSummary` returns only FAILING checks, and it is called only
282
+ * inside the `if (!preflightResult.passed)` branch — so on a passing run the
283
+ * estimate never reaches a terminal at all.
284
+ * 3. Preflight runs after the `--dry-run` short-circuit, so a dry run never gets here.
285
+ *
286
+ * The consequence is that {@link buildDryRunSummary}'s spawn-count line is the ONLY
287
+ * pre-spend number an operator ever sees. Fix the count before making the check
288
+ * reachable: a reachable estimate that says "1 evals × 2 config" is worse than a
289
+ * silent one, because it is quotable.
290
+ */
291
+ function buildPreflightInput(evalsPath, pluginDirs, opts, knobs) {
231
292
  const costEstimate = {
232
293
  evalCount: 1,
233
- configurations: 1,
294
+ // `configurations` IS the A/B dimension: `--baseline` runs every eval under two
295
+ // configurations (skill declared, skill withheld). Correct even while
296
+ // `evalCount` is a placeholder, and it is the half that does not need the suite.
297
+ configurations: opts.baseline === true ? 2 : 1,
234
298
  runsPerQuery: 1,
235
299
  maxBudgetUsd: knobs.maxBudgetUsd,
236
300
  };
@@ -256,7 +320,7 @@ export function buildPreflightInput(evalsPath, pluginDirs, opts, knobs) {
256
320
  preflightOpts.requireAuth = opts.requireAuth;
257
321
  return preflightOpts;
258
322
  }
259
- export function renderPreflightSummary(checks) {
323
+ function renderPreflightSummary(checks) {
260
324
  const failed = checks.filter(c => !c.passed);
261
325
  return failed.length > 0
262
326
  ? failed.map(c => ` [FAIL] ${c.name}: ${c.message}`).join('\n')
@@ -272,42 +336,79 @@ export function renderPreflightSummary(checks) {
272
336
  export function isAcknowledged(opts) {
273
337
  return opts.dryRun === true || opts.acknowledgedRunsSkillCode === true;
274
338
  }
339
+ /**
340
+ * Most friction lines vat will put on an operator's terminal in one run. The
341
+ * full set is always on disk; a grader that emits thousands of items must not
342
+ * be able to scroll the run's real output away.
343
+ */
344
+ const MAX_FRICTION_LINES = 50;
275
345
  /**
276
346
  * Format a friction report for human consumption — one line per entry as
277
347
  * `[<severity>] <category>: <message>`. Pure; returns the empty string for no
278
348
  * entries so the caller can skip emitting anything.
349
+ *
350
+ * `severity` and `category` are closed enums vat owns, so they are safe to
351
+ * interpolate. `message` is GRADER text and is sanitized here even though
352
+ * {@link parseEvalFragment} already sanitized it on the way in — this is a
353
+ * second, independent boundary, not a redundant one: {@link emitFrictionReport}
354
+ * re-reads `friction.json` from the harness results dir, which same-uid skill
355
+ * code can reach and rewrite after vat wrote it.
279
356
  */
280
357
  export function formatFrictionReport(items) {
281
- return items.map(i => `[${i.severity}] ${i.category}: ${i.message}`).join('\n');
358
+ const shown = items.slice(0, MAX_FRICTION_LINES);
359
+ const lines = shown.map(i => `[${i.severity}] ${i.category}: ${sanitizeGraderText(i.message)}`);
360
+ if (items.length > shown.length) {
361
+ lines.push(`... and ${items.length - shown.length} more (full list in friction.json)`);
362
+ }
363
+ return lines.join('\n');
282
364
  }
283
365
  /**
284
366
  * Read the run's friction.json (if present + valid) and echo a concise report to
285
367
  * STDERR so users don't miss packaging-fidelity friction VAT merged from the grader
286
- * fragments (it is otherwise only written to disk). Best-effort: a missing, unparseable, or
287
- * empty report emits nothing. Never touches stdout (which stays machine-readable).
288
- * Accepts `undefined` (a no-op) so the harness `finally` can call it unconditionally
289
- * even when a throw preempted assignment of the friction path.
368
+ * fragments (it is otherwise only written to disk). Best-effort: a missing or empty
369
+ * report emits nothing. A report that is there but cannot be read or does not parse
370
+ * emits a one-line warning instead — this runs from the harness `finally`, so it
371
+ * must not throw, but vat is the sole writer of friction.json and a file it wrote
372
+ * that it cannot read back is worth a line. Never touches stdout (which stays
373
+ * machine-readable). Accepts `undefined` (a no-op) so the harness `finally` can
374
+ * call it unconditionally even when a throw preempted assignment of the path.
290
375
  */
291
376
  function emitFrictionReport(frictionPath) {
292
377
  if (frictionPath === undefined)
293
378
  return;
294
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
295
379
  if (!existsSync(frictionPath))
296
380
  return;
297
381
  let raw;
298
382
  try {
299
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
300
383
  raw = JSON.parse(readFileSync(frictionPath, 'utf-8'));
301
384
  }
302
- catch {
385
+ catch (err) {
386
+ process.stderr.write(`warning: could not read the friction report at ${frictionPath}: ` +
387
+ `${err instanceof Error ? err.message : String(err)}\n`);
303
388
  return;
304
389
  }
305
390
  const parsed = FrictionReportSchema.safeParse(raw);
306
- if (!parsed.success || parsed.data.items.length === 0)
391
+ if (!parsed.success) {
392
+ process.stderr.write(`warning: the friction report at ${frictionPath} does not match its schema: ${parsed.error.message}\n`);
393
+ return;
394
+ }
395
+ if (parsed.data.items.length === 0)
307
396
  return;
308
397
  process.stderr.write(`\nPackaging friction (${parsed.data.items.length}):\n`);
309
398
  process.stderr.write(formatFrictionReport(parsed.data.items) + '\n');
310
399
  }
400
+ /**
401
+ * Write the `--baseline` delta line and ⚠️ banner to stderr, if this run produced
402
+ * one. A no-op on a non-baseline run.
403
+ *
404
+ * A function rather than an `if` at the call site purely to keep the orchestrator
405
+ * inside its cognitive-complexity budget; the ORDER it is called in is the part
406
+ * that matters, and that is documented where it is called.
407
+ */
408
+ function emitBaselineReport(report) {
409
+ if (report !== undefined)
410
+ process.stderr.write(report);
411
+ }
311
412
  /**
312
413
  * Name any optional companions (`--with-optional`/config `optional:`) that were
313
414
  * skipped because their source could not be resolved — never silently, so the
@@ -317,7 +418,8 @@ function emitFrictionReport(frictionPath) {
317
418
  function emitSkippedOptionalWarning(skippedOptional) {
318
419
  if (skippedOptional.length === 0)
319
420
  return;
320
- process.stderr.write(`warning: optional companion skill(s) not staged (source unresolvable): ${skippedOptional.join(', ')}\n`);
421
+ const named = skippedOptional.map((item) => `${item.name} (${item.reason})`).join(', ');
422
+ process.stderr.write(`warning: optional companion skill(s) not staged: ${named}\n`);
321
423
  }
322
424
  /**
323
425
  * Resolve the PERSISTENT location where a bootstrap scaffold should be written
@@ -326,8 +428,8 @@ function emitSkippedOptionalWarning(skippedOptional) {
326
428
  * that real source dir; otherwise we anchor under the repo root by skill name.
327
429
  */
328
430
  /** The subject skill's display name (trailing segment of the subject arg). */
329
- export function subjectSkillName(opts) {
330
- return basename(toForwardSlash(opts.subject)) || opts.subject;
431
+ function subjectSkillName(opts) {
432
+ return basename(toForwardSlashAnyPlatform(opts.subject)) || opts.subject;
331
433
  }
332
434
  /**
333
435
  * Where this run READS its eval suite from, anchored at the subject's authored
@@ -351,7 +453,7 @@ export function subjectSkillName(opts) {
351
453
  * into `<skillDir>/<path>`, which does not exist — so the run did not fail, it
352
454
  * bootstrapped a starter template at the bogus location and graded that.
353
455
  */
354
- export function resolveScaffoldEvalsPath(opts, repoRoot, evalsRef) {
456
+ function resolveScaffoldEvalsPath(opts, repoRoot, evalsRef) {
355
457
  const resolveFrom = (baseDir) => evalsRef === undefined
356
458
  ? safePath.join(baseDir, DEFAULT_EVALS_SUBPATH)
357
459
  : resolveAssetReference(evalsRef, baseDir);
@@ -370,7 +472,7 @@ export function resolveScaffoldEvalsPath(opts, repoRoot, evalsRef) {
370
472
  return resolveFrom(sourceDir);
371
473
  }
372
474
  /**
373
- * Bootstrap (exit 3) — called when the eval suite is absent everywhere. A real run
475
+ * Bootstrap (exit 2, `Reason: bootstrap`) — called when the eval suite is absent everywhere. A real run
374
476
  * writes a persistent template at the source scaffold location and reports it; a dry
375
477
  * run must never touch the filesystem, so it reports where a real run *would*
376
478
  * scaffold and writes nothing. Both surface the same exit-3 BootstrapNeededError.
@@ -391,7 +493,7 @@ function bootstrapEvalSuite(opts, repoRoot, evalsRef) {
391
493
  * is performing. Created 0700 and removed in the run's cleanup, exactly like
392
494
  * {@link resolveGraderOutDir}. Pure (derives a path only).
393
495
  */
394
- export function resolveEvalSuiteHoldDir(dirToken) {
496
+ function resolveEvalSuiteHoldDir(dirToken) {
395
497
  return safePath.join(normalizedTmpdir(), `vat-skill-evals-${dirToken}`);
396
498
  }
397
499
  /**
@@ -404,38 +506,78 @@ export function resolveEvalSuiteHoldDir(dirToken) {
404
506
  * 2. **The vat-only hold dir**, when staging harvested a suite that exists nowhere
405
507
  * else (a fetched artifact). Its layout mirrors an authored evals dir, so
406
508
  * `fixtures/` resolve relative to it unchanged.
407
- * 3. Neither → `undefined`, and the caller bootstraps a template (exit 3).
509
+ * 3. Neither → `undefined`, and the caller bootstraps a template (exit 2, `Reason: bootstrap`).
408
510
  *
409
511
  * Returns the suite's absolute path; its `dirname` is the base for each eval's
410
512
  * declared input `files`.
411
513
  */
412
- export function resolveEvalSuitePath(input) {
514
+ function resolveEvalSuitePath(input) {
413
515
  const authored = resolveScaffoldEvalsPath(input.opts, input.repoRoot, input.evalsRef);
414
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- authored source path resolved from opts
415
516
  if (existsSync(authored))
416
517
  return authored;
417
518
  if (!input.subjectEvalSuiteHeld)
418
519
  return undefined;
419
520
  return safePath.join(input.holdDir, basename(input.evalsSubpath));
420
521
  }
522
+ /**
523
+ * The per-run executor workspace root: `<tmp>/vat-skill-test-ws-<dirToken>/`.
524
+ *
525
+ * Deliberately OUTSIDE `harnessRoot`, alongside {@link resolveGraderOutDir} and
526
+ * {@link resolveEvalSuiteHoldDir}. The executor's cwd lives under here, and the
527
+ * harness root is where vat stages runnable copies of the skill (`staged/` and
528
+ * the assembled plugin dir). While the two were parent and child, the skill-absent
529
+ * arm of a `--baseline` run sat one `ls ..` away from the treatment it was
530
+ * supposed to be denied — the control could reach vat's own copy without ever
531
+ * leaving its working directory.
532
+ *
533
+ * Named by an unpredictable token, like the other vat-only dirs, and removed by
534
+ * the same cleanup. Pure (derives a path only).
535
+ */
536
+ function resolveWorkspacesRoot(dirToken) {
537
+ return safePath.join(normalizedTmpdir(), `vat-skill-test-ws-${dirToken}`);
538
+ }
421
539
  /** Parse the staged eval suite and materialize each eval's input `files` into
422
- * `<harnessRoot>/workspaces/<id>/`. Returns the workspaces root, the parsed
423
- * {@link EvalSuite} (so the eval loop has the entries without re-reading), and the
424
- * declared eval count (derived from the suite). The dir is wiped first so a reused
425
- * harness root cannot leak a prior run's inputs. Throws {@link EvalInputError}
426
- * (mapped by the caller to exit 2) on a bad suite or a missing input file. */
427
- export function stageWorkspacesForRun(evalsPath, harnessRoot) {
428
- const workspacesRoot = safePath.joinUnderRoot(harnessRoot, 'workspaces');
540
+ * `<workspacesRoot>/<arm>/<id>/`, once per arm this run will execute. Returns the
541
+ * workspaces root, the parsed {@link EvalSuite} (so the eval loop has the entries
542
+ * without re-reading), and the declared eval count (derived from the suite). The
543
+ * dir is wiped first so a reused root cannot leak a prior run's inputs. Throws
544
+ * {@link EvalInputError} (mapped by the caller to exit 2) on a bad suite or a
545
+ * missing input file. */
546
+ /**
547
+ * Mint this run's opaque per-arm workspace directory segments.
548
+ *
549
+ * Independent random tokens, NOT the arm names and not derived from one another:
550
+ * the segment ends up in the executor's cwd, which the prompt states verbatim, so
551
+ * a name like `without` unblinds the control arm outright — and a name like `with`
552
+ * hands it a guessable path to the treatment arm's live output one directory over.
553
+ * 8 bytes is ample for "not guessable within a run that lasts minutes"; these are
554
+ * blinding/anti-enumeration tokens, not secrets protecting data at rest.
555
+ */
556
+ function mintArmWorkspaceDirs(baseline) {
557
+ const token = () => randomBytes(8).toString('hex');
558
+ return baseline ? { with: token(), without: token() } : { with: token() };
559
+ }
560
+ function stageWorkspacesForRun(evalsPath, workspacesRoot, armDirs) {
429
561
  rmSync(workspacesRoot, { recursive: true, force: true });
430
562
  // 0700, matching the harness root. These dirs hold each eval's declared input
431
563
  // files, which with an out-of-tree suite may be data that was never in the repo
432
564
  // at all. Inheriting the umask (0755) left them readable by any local user the
433
565
  // moment `--out` relocated the harness root out from under its 0700 parent.
434
566
  mkdirSyncReal(workspacesRoot, { recursive: true, mode: 0o700 });
435
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- evalsPath is our staged-subject path
567
+ // The same shared-tmp hardening the harness root gets. `mkdirSync(recursive)`
568
+ // on an existing path neither throws nor chmods, so without this an attacker
569
+ // winning the race between the rmSync and the mkdir owns the executor's working
570
+ // directory and nothing ever re-checks. Cheap: the leaf sits directly under the
571
+ // trusted tmp boundary, so the ancestry walk degrades to one lstat + stat.
572
+ assertSafeHarnessRoot(workspacesRoot, process.getuid?.() ?? -1);
436
573
  const suite = parseEvalSuite(readFileSync(evalsPath, 'utf-8'));
437
574
  return {
438
- workspacesRoot: stageEvalWorkspaces({ suite, evalsDir: dirname(evalsPath), workspacesRoot }),
575
+ workspacesRoot: stageEvalWorkspaces({
576
+ suite,
577
+ evalsDir: dirname(evalsPath),
578
+ workspacesRoot,
579
+ armDirs,
580
+ }),
439
581
  declaredEvalCount: suite.evals.length,
440
582
  suite,
441
583
  };
@@ -452,15 +594,16 @@ function buildModelFlag(model) {
452
594
  * Keeping the try/catch in a private helper avoids inflating the orchestrator's
453
595
  * cognitive complexity.
454
596
  */
455
- function attemptStageWorkspaces(evalsPath, harnessRoot) {
597
+ function attemptStageWorkspaces(evalsPath, harnessRoot, workspacesRoot, armDirs) {
456
598
  try {
457
- return stageWorkspacesForRun(evalsPath, harnessRoot);
599
+ return stageWorkspacesForRun(evalsPath, workspacesRoot, armDirs);
458
600
  }
459
601
  catch (e) {
460
602
  if (e instanceof EvalInputError) {
461
603
  return {
462
604
  harnessPath: harnessRoot,
463
- exitCode: SkillTestExitCode.Preflight,
605
+ exitCode: ExitCode.ERROR,
606
+ reason: 'preflight',
464
607
  summary: `Eval input error:\n ${e.message}`,
465
608
  };
466
609
  }
@@ -470,8 +613,8 @@ function attemptStageWorkspaces(evalsPath, harnessRoot) {
470
613
  /**
471
614
  * Advisory (never-fatal) lint pass over the parsed suite: nudges authors away
472
615
  * from presence-only expectations ("mentions/includes …") that a hallucinated
473
- * or wrong-for-the-right-reason answer could still satisfy (issue #145
474
- * follow-up). Purely additive stderr noise, emitted before any spawn — never
616
+ * or wrong-for-the-right-reason answer could still satisfy.
617
+ * Purely additive stderr noise, emitted before any spawn — never
475
618
  * affects exitCode. Extracted so the Step 5.5 call site stays a single line
476
619
  * (keeps the orchestrator's cognitive complexity budget).
477
620
  */
@@ -575,6 +718,38 @@ export function buildStaleDistWarningLines(roleLabel) {
575
718
  * Always includes the assembled spawn command, the staged-manifest entry count +
576
719
  * fingerprint, and the provenance.json path so a stale tree is visible at a glance.
577
720
  */
721
+ /** `s` unless the count is exactly one — the plural the two spawn-plan lines share. */
722
+ function plural(count) {
723
+ return count === 1 ? '' : 's';
724
+ }
725
+ /**
726
+ * The two lines that say WHAT A REAL RUN WOULD COST, in PAIRS — not evals.
727
+ *
728
+ * `--baseline` runs every eval on both arms and each arm is a full executor→grader
729
+ * pair, so the spawn count is `evals × arms` and the `claude` session count is twice
730
+ * that. Reporting the eval count as the pair count understated a baseline run by
731
+ * exactly 2x: a 3-eval suite previewed as "3 evals as executor→grader spawn pairs"
732
+ * and then launched 6 pairs / 12 sessions.
733
+ *
734
+ * The arm count is spelled out rather than folded silently into the number, because
735
+ * "6 pairs" for a 3-eval suite reads as a bug until you are told why. And the budget
736
+ * flag is labelled PER SPAWN with the ceiling multiplied out, because the name
737
+ * `--max-budget-usd` reads as a run total — under `--baseline` that misreading
738
+ * under-provisions by a factor of four times the eval count.
739
+ */
740
+ function buildSpawnPlanLines(input) {
741
+ const pairs = input.evalCount * (input.baseline ? 2 : 1);
742
+ const sessions = pairs * 2;
743
+ const armNote = input.baseline
744
+ ? ` (${input.evalCount} eval${plural(input.evalCount)} × 2 arms — --baseline runs each eval with AND without the skill)`
745
+ : '';
746
+ return [
747
+ `[dry-run] Would run ${pairs} executor→grader spawn pair${plural(pairs)}${armNote} ` +
748
+ `at concurrency ${input.concurrency} — ${sessions} claude session${plural(sessions)} in total.`,
749
+ `[dry-run] --max-budget-usd is PER SPAWN ($${input.maxBudgetUsd}), not per run: worst case ≈ ` +
750
+ `$${(sessions * input.maxBudgetUsd).toFixed(2)} across those ${sessions} sessions.`,
751
+ ];
752
+ }
578
753
  export function buildDryRunSummary(input) {
579
754
  const lines = [];
580
755
  if (input.wouldBuild) {
@@ -604,41 +779,71 @@ export function buildDryRunSummary(input) {
604
779
  // it is labelled rather than left to be taken at face value.
605
780
  const stagedFromUnbuiltTree = input.wouldBuild && input.dryRunStagedExistingDist !== undefined;
606
781
  const provisional = stagedFromUnbuiltTree ? ' (PROVISIONAL — not rebuilt)' : '';
607
- lines.push(`[dry-run] Would run ${input.evalCount} eval${input.evalCount === 1 ? '' : 's'} as executor→grader spawn ` +
608
- `pair${input.evalCount === 1 ? '' : 's'} at concurrency ${input.concurrency}.`, `[dry-run] Executor ${input.modelFlag}; grader model ${input.graderModel} (prompt via stdin).`, `[dry-run] Staged manifest: ${count} entr${count === 1 ? 'y' : 'ies'} | ` +
609
- `fingerprint: ${input.provenanceFingerprint}${provisional}`, `[dry-run] Provenance: ${input.provenancePath}`);
782
+ lines.push(...buildSpawnPlanLines(input), `[dry-run] Executor ${input.modelFlag}; grader model ${input.graderModel} (prompt via stdin).`, `[dry-run] Staged manifest: ${count} entr${count === 1 ? 'y' : 'ies'} | ` +
783
+ `fingerprint: ${input.provenanceFingerprint}${provisional}`, `[dry-run] Provenance would be written to: ${input.provenancePath}`);
610
784
  return lines.join('\n');
611
785
  }
612
786
  /**
613
- * Remove the harness directory after a run so staged untrusted skill bytes and
614
- * prompts do not accumulate in OS tmp. No-op when the user asked to keep it, when
615
- * the dir is a user-supplied location (`--out`/`--workdir`), or when it is already
616
- * gone. Idempotent and never throws — it runs from a `finally`, so it must not
617
- * mask the run's real outcome.
787
+ * The one child of the harness root that cleanup NEVER removes. Cleanup exists to
788
+ * evict staged untrusted skill bytes and prompts from OS tmp; `results/` is the
789
+ * opposite — it is the run's product, written solely by vat.
790
+ */
791
+ const RETAINED_RESULTS_DIRNAME = 'results';
792
+ /**
793
+ * Remove the harness directory's staged contents after a run so untrusted skill
794
+ * bytes and prompts do not accumulate in OS tmp. No-op when the user asked to keep
795
+ * it, when the dir is a user-supplied location (`--out`/`--workdir`), or when it is
796
+ * already gone. Idempotent and never throws — it runs from a `finally`, so it must
797
+ * not mask the run's real outcome.
798
+ *
799
+ * `results/` SURVIVES. It holds `grading.json`, `friction.json`, `tool-eval.json`
800
+ * and — for a `--baseline` run — `baseline.json`, i.e. every artifact the command's
801
+ * own help text tells the operator to go read. Removing the whole root on the
802
+ * DEFAULT invocation (no `--out`, no `--workdir`, no `--keep`) deleted those files
803
+ * inside this function's own `finally`, before `run.ts` had printed a single line:
804
+ * the operator was handed a `Harness:` path to a directory that no longer existed.
805
+ * That is the invocation `vat-skill-testing.md`'s copy-paste example uses.
806
+ *
807
+ * The retention is deliberately UNCONDITIONAL rather than `--baseline`-only: every
808
+ * run's verdict detail lives in the same directory, and a rule that depends on a
809
+ * flag is a rule that will drift away from the flag. The cost is bounded — the
810
+ * harness root is a deterministic function of the subject set, so a re-run reuses
811
+ * (and {@link wipeStaleArtifacts} clears) the same `results/`, rather than
812
+ * accumulating one per run.
618
813
  *
619
814
  * SAFETY: re-asserts the root is not a symlink immediately before removal (via
620
815
  * `lstat`, which does NOT follow the link). A root swapped to a symlink between
621
816
  * the run and cleanup is left in place rather than followed — `rmSync(recursive)`
622
817
  * could otherwise delete the symlink's target outside tmp.
623
818
  */
624
- export function cleanupHarness(harnessRoot, opts) {
819
+ function cleanupHarness(harnessRoot, opts) {
625
820
  if (opts.keep || !opts.created)
626
821
  return;
627
822
  // Best-effort: cleanup runs from a `finally`, so a TOCTOU race (the dir is
628
823
  // reaped between checks) or a permission error must never throw out and mask
629
- // the run's real outcome. Worst case is a leftover 0700 tmp dir, not a failure.
630
- try {
631
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
824
+ // the run's real outcome. Worst case is a leftover 0700 tmp dir, not a failure —
825
+ // but a leftover the operator was never told about is staged untrusted bytes
826
+ // sitting in tmp with nothing to connect them to this run, so the failure is
827
+ // REPORTED on stderr ({@link swallowCleanupFailure}), not swallowed silently.
828
+ swallowCleanupFailure(() => {
632
829
  if (!existsSync(harnessRoot))
633
830
  return;
634
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived harness root
635
831
  if (lstatSync(harnessRoot).isSymbolicLink())
636
832
  return;
637
- rmSync(harnessRoot, { recursive: true, force: true });
638
- }
639
- catch {
640
- // Swallow: a failed cleanup is not a run failure.
641
- }
833
+ const entries = readdirSync(harnessRoot);
834
+ // Nothing to retain — a run that ended before Step 7 (preflight refusal, lock
835
+ // failure, a throw during staging) has no results/, so leaving an empty 0700
836
+ // dir behind in tmp would be pure litter. Remove the root outright.
837
+ if (!entries.includes(RETAINED_RESULTS_DIRNAME)) {
838
+ rmSync(harnessRoot, { recursive: true, force: true });
839
+ return;
840
+ }
841
+ for (const entry of entries) {
842
+ if (entry === RETAINED_RESULTS_DIRNAME)
843
+ continue;
844
+ rmSync(safePath.joinUnderRoot(harnessRoot, entry), { recursive: true, force: true });
845
+ }
846
+ });
642
847
  }
643
848
  /**
644
849
  * Build the per-eval work items for a set of evals (one tier's worth, or a whole
@@ -648,7 +853,7 @@ export function cleanupHarness(harnessRoot, opts) {
648
853
  * (see {@link partitionFragmentsByArm}) and never drives tier gating. Pure +
649
854
  * unit-testable.
650
855
  */
651
- export function buildEvalWorkItems(evals, baseline) {
856
+ function buildEvalWorkItems(evals, baseline) {
652
857
  const items = [];
653
858
  for (const entry of evals) {
654
859
  items.push({ entry, arm: 'with' });
@@ -663,7 +868,7 @@ export function buildEvalWorkItems(evals, baseline) {
663
868
  * fragment with no `arm` (or `arm: 'with'`) is a WITH-arm fragment. Pure +
664
869
  * unit-testable.
665
870
  */
666
- export function partitionFragmentsByArm(fragments) {
871
+ function partitionFragmentsByArm(fragments) {
667
872
  const withArm = [];
668
873
  const withoutArm = [];
669
874
  for (const fragment of fragments) {
@@ -694,18 +899,47 @@ export function partitionFragmentsByArm(fragments) {
694
899
  * container; that is tracked as a follow-up (see CHANGELOG "Security" notes) and
695
900
  * is the only thing that closes the residual race outright.
696
901
  */
697
- export function resolveGraderOutDir(dirToken) {
902
+ function resolveGraderOutDir(dirToken) {
698
903
  return safePath.join(normalizedTmpdir(), `vat-skill-grade-${dirToken}`);
699
904
  }
700
905
  /**
701
- * The executor working directory for one eval: its staged input workspace
702
- * `<workspacesRoot>/<id>` when the eval declares input `files`, else undefined
703
- * (the executor then defaults to the staged subject dir). Pure + unit-testable.
906
+ * The executor working directory for one eval ON ONE ARM: its staged input
907
+ * workspace `<workspacesRoot>/<arm>/<id>`, ALWAYS — empty when the eval declares
908
+ * no input `files` (see {@link stageEvalWorkspaces}).
909
+ *
910
+ * This used to return undefined for a file-less eval, which made the executor
911
+ * fall back to running inside the staged subject dir. That fallback was the
912
+ * quiet half of the `--baseline` control defect: the skill-absent arm's cwd
913
+ * was the skill.
914
+ *
915
+ * The per-arm segment closes the OTHER half, found a round later: without it both
916
+ * arms of one eval shared a single writable directory AND ran concurrently, so
917
+ * the control arm could read the treatment arm's output files and answer from
918
+ * them. Keyed on the eval id alone this is a pure function of the eval; the arm
919
+ * is what makes the two runs independent.
920
+ *
921
+ * The segment is an OPAQUE per-run token, never the arm's name — this path is
922
+ * quoted verbatim to the executor as its working directory, so `…/without/e1`
923
+ * told the control arm it was the control, and `../with/e1` told it where to look.
924
+ * See {@link mintArmWorkspaceDirs}. Pure + unit-testable.
704
925
  */
705
- export function resolvePerEvalWorkspaceDir(entry, workspacesRoot) {
706
- if (entry.files === undefined || entry.files.length === 0)
707
- return undefined;
708
- return safePath.joinUnderRoot(workspacesRoot, String(entry.id));
926
+ function resolvePerEvalWorkspaceDir(entry, workspacesRoot, arm, armDirs) {
927
+ return safePath.joinUnderRoot(armWorkspaceRoot(workspacesRoot, armDirs, arm), String(entry.id));
928
+ }
929
+ /**
930
+ * `<workspacesRoot>/<that arm's opaque segment>` — ONE arm's staged workspace root,
931
+ * derived in exactly one place.
932
+ *
933
+ * The consolidation is the point, not the line count. Every consumer that wants an
934
+ * arm root wants this join, and a consumer that takes the result as a bare `string`
935
+ * has no way to tell an arm root from the workspaces ROOT one level above it:
936
+ * handing {@link resolveSkillContentNeedles} `workspacesRoot` instead of
937
+ * `<workspacesRoot>/<withSegment>` type-checked, ran, and left all 2270 unit tests
938
+ * green while every fixture exclusion silently missed. Callers now pass the two
939
+ * inputs and let this derive the path.
940
+ */
941
+ function armWorkspaceRoot(workspacesRoot, armDirs, arm) {
942
+ return safePath.joinUnderRoot(workspacesRoot, armDirSegment(armDirs, arm));
709
943
  }
710
944
  /**
711
945
  * Best-effort removal of a vat-only tmp dir that lives OUTSIDE the harness root —
@@ -713,57 +947,160 @@ export function resolvePerEvalWorkspaceDir(entry, workspacesRoot) {
713
947
  * cleanup, where masking the run's real outcome would be worse than a leftover 0700
714
948
  * tmp dir) and refuses to follow a symlinked root.
715
949
  */
716
- export function removeVatOnlyDir(dir) {
950
+ /**
951
+ * Run one cleanup step, swallowing whatever it throws.
952
+ *
953
+ * The discipline every other step in the harness `finally` already follows, stated
954
+ * once: a cleanup failure is not a run failure, and letting one out REPLACES an
955
+ * already-good result — verdict computed, artifacts written, summary composed — with
956
+ * an error and exit 1.
957
+ *
958
+ * ⚠️ A CALL-SITE GUARD IS THE WEAK FORM, and this helper is a last resort rather
959
+ * than the pattern to copy. It is invisible to the compiler and to the test suite:
960
+ * reverting either of its two original call sites to a bare call left the unit suite
961
+ * AND the integration suite green, so nothing but review stood between the fix and
962
+ * its own silent removal. `lock.release()` was therefore made non-throwing at its own
963
+ * definition instead (see {@link acquireHarnessLock}), which is a contract the type
964
+ * carries and no edit here can undo. That leaves ONE caller —
965
+ * `killAllActiveClaudeChildren`, which lives in `@vibe-agent-toolkit/utils` and is
966
+ * only contractually non-throwing as long as that package says so.
967
+ *
968
+ * The failure is reported on stderr rather than silently dropped: a lockfile that
969
+ * could not be removed will break the NEXT run with a "busy" error, and an operator
970
+ * who saw nothing here has no way to connect the two.
971
+ */
972
+ function swallowCleanupFailure(step) {
973
+ try {
974
+ step();
975
+ }
976
+ catch (err) {
977
+ process.stderr.write(`warning: harness cleanup step failed (the run's result stands): ${err instanceof Error ? err.message : String(err)}\n`);
978
+ }
979
+ }
980
+ /**
981
+ * Remove one vat-only temp dir. Same discipline as {@link cleanupHarness}: a failed
982
+ * removal is not a run failure, and it is reported rather than swallowed.
983
+ */
984
+ function removeVatOnlyDir(dir) {
717
985
  if (dir === undefined)
718
986
  return;
719
- try {
720
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived vat-only tmp dir
987
+ swallowCleanupFailure(() => {
721
988
  if (!existsSync(dir))
722
989
  return;
723
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived vat-only tmp dir
724
990
  if (lstatSync(dir).isSymbolicLink())
725
991
  return;
726
992
  rmSync(dir, { recursive: true, force: true });
727
- }
728
- catch {
729
- // Swallow: a failed cleanup is not a run failure.
730
- }
993
+ });
731
994
  }
732
995
  /**
733
996
  * Run ONE work item: the blind executor spawn, then the grader spawn over its
734
- * captured transcript (issue #145). Returns the grader fragment tagged with the
997
+ * captured transcript. Returns the grader fragment tagged with the
735
998
  * item's arm. The WITHOUT arm runs the executor with `pluginDirs: []` (skill
736
999
  * absent). Grader fragments are written under a PER-ARM subdir of the vat-only
737
1000
  * grader dir so a WITH and WITHOUT run of the same eval id cannot collide.
738
1001
  *
739
- * Throws propagate: an executor/grader {@link InternalHarnessError} (timeout,
740
- * stall, spawn error, grader failure/missing-fragment) fails the whole run
741
- * (exit 1); a RateLimitSignal is retried by the pipeline. An executor CLEAN
742
- * failure is NOT thrown — its transcript flows into the grader, whose failing
743
- * fragment surfaces as an eval failure (exit 4 via the verdict), never exit 1.
1002
+ * Throws, and the caller ({@link runEvalWorker}) decides what a throw costs — see
1003
+ * its docblock for why that depends on the arm. An executor CLEAN failure is NOT
1004
+ * thrown on either arm: its transcript flows into the grader, whose failing
1005
+ * fragment surfaces as an eval failure (exit FINDINGS via the verdict), never ERROR.
744
1006
  */
745
- async function runEvalWorker(item, ctx) {
1007
+ /**
1008
+ * The executor env for ONE eval.
1009
+ *
1010
+ * `workspaceDir` is threaded to the token resolver ONLY when the eval actually
1011
+ * declares input `files`, even though every eval now HAS a workspace. The two are
1012
+ * different questions: the workspace is where the executor runs, while
1013
+ * `${fixturesDir}` promises a staged `fixtures/` directory that EXISTS — and
1014
+ * `stageEvalWorkspaces` creates that only for evals declaring files. Passing it
1015
+ * unconditionally silently downgraded `UnresolvableEnvTokenError` (a loud exit 2
1016
+ * naming the exact fix) into a dead path the skill discovers at runtime and is
1017
+ * then blamed for.
1018
+ */
1019
+ function resolveExecutorEnvForEval(item, ctx, workspaceDir) {
1020
+ if (ctx.perEvalEnv === undefined)
1021
+ return ctx.executorEnv;
1022
+ const declaresFiles = item.entry.files !== undefined && item.entry.files.length > 0;
1023
+ const { input, logged } = ctx.perEvalEnv;
1024
+ const env = resolveDeclaredChildEnv({
1025
+ ...input,
1026
+ ...(declaresFiles ? { workspaceDir } : {}),
1027
+ quiet: logged.done,
1028
+ }).env;
1029
+ logged.done = true;
1030
+ return env;
1031
+ }
1032
+ /**
1033
+ * Scrub the control arm's env and SAY what it lost.
1034
+ *
1035
+ * The arms are meant to differ in the skill and nothing else, so any var withheld
1036
+ * from one side is a confound. A control arm silently spawned without `PATH` or
1037
+ * without its declared fixtures still runs — it just does worse, which reports as
1038
+ * skill lift. That failure is invisible unless the withholding is announced, and
1039
+ * `dropped` was previously computed and thrown away.
1040
+ */
1041
+ function scrubAndReportControlArmEnv(env, harnessRoot, evalId) {
1042
+ // `[]` because vat does not declare model env vars today — `resolveAuth` is
1043
+ // called with no `modelVars`, so none is ever forwarded and the exemption set
1044
+ // has nothing extra to hold. The parameter is REQUIRED rather than defaulted so
1045
+ // that stops being a silent assumption: the day vat forwards a model var, this
1046
+ // line fails to compile and someone decides, instead of the scrub quietly
1047
+ // dropping it and running the control arm on a different model.
1048
+ const { env: scrubbed, droppedForbiddenKey, droppedNamingRoot, retainedLeaks } = scrubControlArmEnv(env, harnessRoot, []);
1049
+ // Two rules, two reasons, two lines. One merged line said "naming the harness
1050
+ // root" about both, which is false for every rule-1 drop — those go by NAME and
1051
+ // their value may name nothing in the harness at all.
1052
+ const withheld = [
1053
+ [droppedForbiddenKey, 'by name, whatever their value'],
1054
+ [droppedNamingRoot, 'because their value names the harness root'],
1055
+ ];
1056
+ for (const [names, why] of withheld) {
1057
+ if (names.length === 0)
1058
+ continue;
1059
+ process.stderr.write(`control arm (${evalId}): withheld ${names.length} env var(s) ${why}: ${names.join(', ')}\n`);
1060
+ }
1061
+ if (retainedLeaks.length > 0) {
1062
+ process.stderr.write(`⚠️ control arm (${evalId}): ${retainedLeaks.join(', ')} name(s) the harness root but are required to run, so they were NOT withheld. ` +
1063
+ `The control arm can read the harness through them — move --out/--workdir off this path before trusting the delta.\n`);
1064
+ }
1065
+ return scrubbed;
1066
+ }
1067
+ async function gradeOneArm(item, ctx) {
746
1068
  const evalId = String(item.entry.id);
747
- const workspaceDir = resolvePerEvalWorkspaceDir(item.entry, ctx.workspacesRoot);
1069
+ const workspaceDir = resolvePerEvalWorkspaceDir(item.entry, ctx.workspacesRoot, item.arm, ctx.armDirs);
748
1070
  const onProgress = (chunk) => { process.stderr.write(chunk); };
749
- // Resolve the declared `env` against THIS eval's workspace. An eval that
750
- // declares no input files has no workspace, so a value using ${fixturesDir}
751
- // throws UnresolvableEnvTokenError (exit 2) instead of injecting a dead path.
752
- let executorEnv = ctx.executorEnv;
753
- if (ctx.perEvalEnv !== undefined) {
754
- const { input, logged } = ctx.perEvalEnv;
755
- executorEnv = resolveDeclaredChildEnv({
756
- ...input,
757
- ...(workspaceDir === undefined ? {} : { workspaceDir }),
758
- quiet: logged.done,
759
- }).env;
760
- logged.done = true;
761
- }
1071
+ // Resolve the declared `env` against THIS eval's workspace.
1072
+ //
1073
+ // `workspaceDir` is threaded to the token resolver ONLY when the eval actually
1074
+ // declares input `files`, even though every eval now HAS a workspace. The two
1075
+ // are different questions: the workspace is where the executor runs, while
1076
+ // `${fixturesDir}` promises a staged `fixtures/` directory that exists — and
1077
+ // `stageEvalWorkspaces` creates that only for evals declaring files. Passing it
1078
+ // unconditionally silently downgraded `UnresolvableEnvTokenError` (a loud exit 2
1079
+ // naming the exact fix) into a dead path the skill discovers at runtime and gets
1080
+ // blamed for.
1081
+ let executorEnv = resolveExecutorEnvForEval(item, ctx, workspaceDir);
1082
+ // The skill-absent arm is not told where the subject is staged. Handing it that
1083
+ // path would defeat the control outright: the staged dir holds the SKILL.md AND
1084
+ // any executable the skill ships, so a control arm given the path can recover
1085
+ // the entire treatment with one `cat`. Withholding the plugin dir while naming
1086
+ // the staged dir in the prompt was the defect this arm's whole value rests on.
1087
+ const isWithArm = item.arm === 'with';
1088
+ // ...and the same reasoning applied to the ENVIRONMENT, which is the channel the
1089
+ // first round of this fix missed entirely: the run assembles ONE env and both
1090
+ // arms were handed it verbatim, `CLAUDE_PLUGIN_ROOT` included. See
1091
+ // scrubControlArmEnv — prompt, argv, cwd and env are four channels, not three.
1092
+ //
1093
+ // Report what the control arm lost. The arms are supposed to differ in the skill
1094
+ // and nothing else, so any var withheld from one side is a confound the operator
1095
+ // has to be able to see — silently spawning a degraded control is how a harness
1096
+ // manufactures the lift it is meant to measure.
1097
+ if (!isWithArm)
1098
+ executorEnv = scrubAndReportControlArmEnv(executorEnv, ctx.harnessRoot, evalId);
762
1099
  const outcome = await runExecutorForEval({
763
1100
  evalId,
764
1101
  task: item.entry.prompt,
765
- subjectStagedDir: ctx.subjectStagedDir,
766
- ...(workspaceDir === undefined ? {} : { workspaceDir }),
1102
+ ...(isWithArm ? { subjectStagedDir: ctx.subjectStagedDir } : {}),
1103
+ workspaceDir,
767
1104
  pluginDirs: item.arm === 'without' ? [] : ctx.pluginDirs,
768
1105
  env: executorEnv,
769
1106
  ...(ctx.model === undefined ? {} : { model: ctx.model }),
@@ -777,7 +1114,6 @@ async function runEvalWorker(item, ctx) {
777
1114
  recordSessionCost(ctx.costAccumulator, outcome.parsed.result?.totalCostUsd);
778
1115
  // Tool expectations are about the SKILL's tools, so they ride the WITH arm ONLY —
779
1116
  // the WITHOUT (skill-absent) arm has no skill, hence nothing to judge tools against.
780
- const isWithArm = item.arm === 'with';
781
1117
  const fragment = await runGraderForEval({
782
1118
  evalId,
783
1119
  transcript: outcome.transcript,
@@ -802,11 +1138,437 @@ async function runEvalWorker(item, ctx) {
802
1138
  onProgress,
803
1139
  costSink: (usd) => recordSessionCost(ctx.costAccumulator, usd),
804
1140
  });
805
- return { ...fragment, arm: item.arm };
1141
+ const graded = withoutGraderContamination(fragment);
1142
+ if (isWithArm)
1143
+ return { ...graded, arm: item.arm };
1144
+ return { ...graded, arm: item.arm, ...baselineContaminationFor(outcome.transcript, ctx, evalId) };
1145
+ }
1146
+ /**
1147
+ * Run one work item and decide what its failure costs the RUN.
1148
+ *
1149
+ * TREATMENT arm: rethrow. `grading.json` is the treatment arm, the verdict is
1150
+ * computed from it, and a run that cannot produce one has produced nothing — there
1151
+ * is no partial result worth writing, so the throw propagates through `runPipeline`
1152
+ * to `skillTestFailureReason` exactly as before (`ERROR`, reason `internal`).
1153
+ *
1154
+ * CONTROL arm: record and continue. Every failure mode of this arm — executor spawn
1155
+ * error, stall, wall-timeout, grader non-zero exit, grader wrote no fragment,
1156
+ * fragment unparseable twice, nonce mismatch, and a grader that returned a
1157
+ * different number of expectations than the eval declares — used to propagate out of
1158
+ * `runPipeline` and annihilate the treatment work already completed and billed. A
1159
+ * two-eval probe was measured doing precisely that: both treatment executors ran,
1160
+ * both treatment graders ran, the control arm's executor timed out, the run threw,
1161
+ * and `results/` was left holding `provenance.json` and nothing else. The operator
1162
+ * paid for the whole run and got no grading.json, no baseline.json, and an error
1163
+ * that did not even say which arm had died.
1164
+ *
1165
+ * Recording it instead costs the DELTA, which is the only thing the control arm
1166
+ * feeds, and the delta already has a truthful way to be absent (`null`). The
1167
+ * treatment artifacts are written, the failure is named on stderr as it happens and
1168
+ * again in `baselineIntegrity.controlArmFailures`, and `armExpectationSkew` sees an
1169
+ * eval graded on one arm only and withholds the number.
1170
+ *
1171
+ * A {@link RateLimitSignal} is rethrown on BOTH arms — it is not a failure, it is the
1172
+ * pipeline's retry protocol, and swallowing it on the control arm would convert every
1173
+ * transient rate limit into a permanently dead control arm. The worker CANNOT tell a
1174
+ * first 429 from the last one; only the pipeline knows the budget is spent, which is
1175
+ * what {@link rateLimitExhaustedOutcome} is wired to handle.
1176
+ */
1177
+ async function runEvalWorker(item, ctx) {
1178
+ try {
1179
+ return { fragment: await gradeOneArm(item, ctx) };
1180
+ }
1181
+ catch (err) {
1182
+ if (err instanceof RateLimitSignal)
1183
+ throw nameTheArm(err, item.arm);
1184
+ if (item.arm === 'with') {
1185
+ if (err instanceof GraderFragmentUnparseableError)
1186
+ return { fragment: unparseableGradingFragment(item, ctx, err) };
1187
+ throw nameTheArm(err, item.arm);
1188
+ }
1189
+ return recordControlFailure(item, err);
1190
+ }
1191
+ }
1192
+ /**
1193
+ * The TREATMENT arm's outcome for a grader that could not write valid JSON twice
1194
+ * ({@link GraderFragmentUnparseableError}): this eval FAILS, in the same fragment
1195
+ * shape a grader's own failing verdict takes, so it merges, counts and gates
1196
+ * exactly like one — `FAIL n/m`, `FINDINGS` unless `--allow-eval-failure`, the other
1197
+ * evals reported. Every declared expectation is marked failed with the reason as
1198
+ * its `evidence`; a `tool` verdict is NOT invented, because none was produced —
1199
+ * the eval already fails on the output channel, and `tool-eval.json` omitting it
1200
+ * is the truth. Not `InternalHarnessError`: the harness ran the grader twice and
1201
+ * it was the model that could not produce strict JSON.
1202
+ *
1203
+ * Treatment only. On the control arm the same error is recorded as a control
1204
+ * failure (delta withheld) — a synthesized control `fail` would read as lift.
1205
+ */
1206
+ function unparseableGradingFragment(item, ctx, err) {
1207
+ process.stderr.write(`\n⚠️ [${ARM_LABEL[item.arm]}] ${err.message}\n` +
1208
+ ` Eval "${err.evalId}" is reported as FAILED (${GRADER_FRAGMENT_UNPARSEABLE}); the run continues.\n`);
1209
+ return {
1210
+ runNonce: ctx.runNonce,
1211
+ evalId: err.evalId,
1212
+ arm: item.arm,
1213
+ expectations: item.entry.expectations.map((text) => ({
1214
+ // Suite text enters an artifact here without a grader's echo in between, so
1215
+ // it takes the same sanitizer a grader's echo would have.
1216
+ text: sanitizeGraderText(text),
1217
+ passed: false,
1218
+ evidence: `${GRADER_FRAGMENT_UNPARSEABLE}: ${err.detail}`,
1219
+ })),
1220
+ };
1221
+ }
1222
+ /**
1223
+ * The control arm ran out of rate-limit retries — record it and let the run live.
1224
+ *
1225
+ * The retryable case is handled correctly by {@link runEvalWorker} and must stay that
1226
+ * way; this is the EXHAUSTED case, which used to be indistinguishable from it. A
1227
+ * control executor rate-limited on every attempt threw after five retries, taking
1228
+ * down a run in which both treatment executors and both treatment graders had already
1229
+ * completed and been billed, leaving `results/` holding `provenance.json` and nothing
1230
+ * else — the exact failure {@link runEvalWorker} exists to prevent, reached by the one
1231
+ * error class it had to let through.
1232
+ *
1233
+ * The treatment arm still hard-fails: there is no treatment result to salvage, which
1234
+ * is the same asymmetry {@link EvalWorkOutcome} documents.
1235
+ *
1236
+ * Wired as the pipeline's `onRetriesExhausted`, and `--baseline` — which doubles the
1237
+ * spawn count — is the run most likely to need it.
1238
+ */
1239
+ function rateLimitExhaustedOutcome(item, error) {
1240
+ if (item.arm === 'with')
1241
+ throw nameTheArm(error, item.arm);
1242
+ return recordControlFailure(item, error);
1243
+ }
1244
+ /**
1245
+ * Turn one control-arm error into a recorded failure the run carries past. Shared by
1246
+ * the worker's catch and the exhausted-retries handler so both report identically —
1247
+ * an operator must not be able to tell which layer noticed.
1248
+ */
1249
+ function recordControlFailure(item, err) {
1250
+ const named = nameTheArm(err, item.arm);
1251
+ // `detail` is schema-required non-empty: an error that stringifies to nothing
1252
+ // would otherwise fail the artifact write and take down the run this whole
1253
+ // branch exists to keep alive.
1254
+ const detail = (named instanceof Error ? named.message : String(named)) ||
1255
+ `[${ARM_LABEL[item.arm]}] failed for eval "${String(item.entry.id)}" with a non-descriptive error`;
1256
+ process.stderr.write(`\n⚠️ ${detail}\n` +
1257
+ ' The control arm for this eval is unusable, so the --baseline delta will be WITHHELD. ' +
1258
+ 'The treatment arm is unaffected and the run continues — its grading is what you paid for.\n');
1259
+ return { controlFailure: { evalId: String(item.entry.id), detail } };
1260
+ }
1261
+ /**
1262
+ * Fragment fields VAT derives itself and therefore never accepts FROM a grader.
1263
+ *
1264
+ * Both are computed from the executor transcript's structure, which the grader is
1265
+ * shown as opaque text and never asked about — so a value in either from a grader
1266
+ * is not merely redundant, it is a claim the grader is in no position to make.
1267
+ */
1268
+ const VAT_ATTACHED_FRAGMENT_KEYS = ['contamination', 'degraded'];
1269
+ /**
1270
+ * Strip the fields VAT attaches itself, before VAT attaches them.
1271
+ *
1272
+ * The grader's only input is the executor transcript, which untrusted skill code
1273
+ * controls — so a prompt injection there could otherwise fabricate a
1274
+ * `BASELINE CONTAMINATED` verdict carrying attacker-chosen strings into an
1275
+ * operator-facing artifact, or (via `degraded`) claim a scan degraded that did not.
1276
+ * Spreading VAT's value last is NOT sufficient: it is absent on a clean run, so it
1277
+ * overwrites nothing — which is exactly why the omission direction matters more
1278
+ * than the invention one for `degraded`. A grader that omits it cannot re-hide a
1279
+ * degradation VAT detected, because VAT's value is spread after this strip; a
1280
+ * grader that INVENTS it would otherwise stamp a blind-scan warning on a run that
1281
+ * scanned properly, and an operator who learns to ignore that warning stops reading
1282
+ * the real ones.
1283
+ */
1284
+ function withoutGraderContamination(fragment) {
1285
+ if (VAT_ATTACHED_FRAGMENT_KEYS.every((key) => fragment[key] === undefined))
1286
+ return fragment;
1287
+ const copy = { ...fragment };
1288
+ for (const key of VAT_ATTACHED_FRAGMENT_KEYS)
1289
+ delete copy[key];
1290
+ return copy;
1291
+ }
1292
+ /**
1293
+ * How each arm is named in every message that can only be about one of them.
1294
+ *
1295
+ * Spelled out rather than `'with'`/`'without'`: the internal names are a preposition
1296
+ * apart and the operator-facing question is "which half of my experiment died", so
1297
+ * `Executor timed out for eval "e1"` — the actual text a destroyed run used to print
1298
+ * — named neither the arm nor the fact that the OTHER arm had already succeeded and
1299
+ * been thrown away.
1300
+ */
1301
+ const ARM_LABEL = {
1302
+ with: 'treatment arm (skill available)',
1303
+ without: 'control arm (skill withheld)',
1304
+ };
1305
+ /**
1306
+ * Prefix a worker error's message with the arm it came from, IN PLACE.
1307
+ *
1308
+ * Mutating rather than re-wrapping is deliberate. `skillTestFailureReason` reads the
1309
+ * error's own declared reason, and the run's regression test for a forged grader
1310
+ * nonce (`rejects.toThrow(GradingNonceError)`); wrapping every failure in a fresh
1311
+ * `InternalHarnessError` to get a better message would flatten
1312
+ * `EvalFragmentError`/`GradingNonceError`/`RateLimitSignal` into one class and lose
1313
+ * both. The stack is preserved for the same reason.
1314
+ *
1315
+ * Idempotent — a worker error passes through here exactly once, but a retried item
1316
+ * must not accumulate prefixes.
1317
+ */
1318
+ function nameTheArm(err, arm) {
1319
+ prefixMessageOnce(err, `[${ARM_LABEL[arm]}] `);
1320
+ return err;
1321
+ }
1322
+ /**
1323
+ * Lift the run's content needles off the STAGED SKILL.md.
1324
+ *
1325
+ * Staged, not source: the staged copy is byte-identical to what the treatment arm
1326
+ * is given, so a needle taken from it is exactly the text an ambient copy of the
1327
+ * same build would carry.
1328
+ *
1329
+ * The exclusion set is EVERY channel through which vat itself hands the arm text:
1330
+ * the suite's prompts, its `expected_output`, its expectations, and the CONTENTS
1331
+ * of the input `files` fixtures vat stages into the arm's workspace and instructs
1332
+ * it to operate on. An adopter who quotes a sentence of their SKILL.md through any
1333
+ * of the four would otherwise stamp every run contaminated, including the arm that
1334
+ * merely read the input vat gave it — the skill's words arriving THROUGH vat are
1335
+ * not the arm reaching the skill. The fixture channel is the one that was missed,
1336
+ * and it is the worst of the four to get wrong: the summary attached to that
1337
+ * verdict tells the operator to go uninstall an ambient copy of the plugin that
1338
+ * does not exist.
1339
+ *
1340
+ * Fixtures are read from the STAGED workspace rather than the suite directory, for
1341
+ * the same reason the SKILL.md is: that is byte-for-byte what the arm was handed.
1342
+ *
1343
+ * Returns `[]` when there is no staged SKILL.md. A missing SKILL.md is not this
1344
+ * function's error to raise (staging has already validated the subject), and an
1345
+ * unarmed signal is reported as unarmed rather than claimed as clean. A SKILL.md
1346
+ * that is THERE but refused is a different thing — `[]` would silently disarm the
1347
+ * contamination signal for the whole run — so a refusal propagates. The same split
1348
+ * applies to fixtures: see {@link collectFixtureText}.
1349
+ */
1350
+ function resolveSkillContentNeedles(subjectStagedDir, evals,
1351
+ /**
1352
+ * Where the staged fixtures live, as the two inputs an arm root is DERIVED from
1353
+ * rather than as the pre-joined path itself — see {@link armWorkspaceRoot}. Taking
1354
+ * a bare `string` here let the call site pass `workspacesRoot` (one level too high,
1355
+ * so every `<id>/` fixture lookup missed) with the whole unit suite still green.
1356
+ *
1357
+ * The `with` arm, because the fixture BYTES are what matter and both arms are
1358
+ * staged from the same source — and `with` is the only arm every run mints.
1359
+ */
1360
+ workspaces) {
1361
+ const stagedWorkspaceRoot = armWorkspaceRoot(workspaces.workspacesRoot, workspaces.armDirs, 'with');
1362
+ const skillMdPath = safePath.join(subjectStagedDir, 'SKILL.md');
1363
+ let markdown;
1364
+ try {
1365
+ markdown = readFileSync(skillMdPath, 'utf8');
1366
+ }
1367
+ catch (err) {
1368
+ if (isPathAbsentError(err))
1369
+ return [];
1370
+ throw err;
1371
+ }
1372
+ const excluded = evals
1373
+ .flatMap((entry) => [
1374
+ entry.prompt,
1375
+ entry.expected_output ?? '',
1376
+ ...entry.expectations,
1377
+ ...readStagedFixtures(stagedWorkspaceRoot, entry),
1378
+ ])
1379
+ .join('\n');
1380
+ return skillContentNeedles(markdown, excluded);
1381
+ }
1382
+ /**
1383
+ * The text of one eval's staged input files. A declared fixture that was never
1384
+ * staged is skipped; one that is there but cannot be read propagates.
1385
+ *
1386
+ * A declared `files[]` entry is legitimately a DIRECTORY — staging does `existsSync`
1387
+ * then a recursive `cpSync` — so this walks one. It used to call `readFileSync` on
1388
+ * the declared path, get `EISDIR`, and skip it: the directory's whole contents were
1389
+ * in the arm's cwd and absent from the exclusion set, and two runs differing ONLY in
1390
+ * whether the same bytes were declared as a file or as their parent directory
1391
+ * reported `contaminated: false` and `contaminated: true` (with a `skill-content`
1392
+ * hit, and a summary telling the operator to uninstall an ambient plugin copy that
1393
+ * does not exist).
1394
+ *
1395
+ * The `catch`'s old claim that skipping "errs the safe way: noisier, never blinder"
1396
+ * is BACKWARDS for this signal. Over-reporting is the harm here, because a false
1397
+ * `contaminated: true` is actioned by discarding a run that was fine. So only an
1398
+ * ABSENT fixture is skipped (staging does `existsSync` then `cpSync`, so a declared
1399
+ * file missing at the source is simply not in the workspace); a fixture that is
1400
+ * there and refused throws, because an exclusion set vat could not compute is a
1401
+ * signal vat cannot stand behind.
1402
+ */
1403
+ function readStagedFixtures(stagedWorkspaceRoot, entry) {
1404
+ const texts = [];
1405
+ for (const rel of entry.files ?? []) {
1406
+ const staged = safePath.joinUnderRoot(safePath.joinUnderRoot(stagedWorkspaceRoot, String(entry.id)), rel);
1407
+ collectFixtureText(staged, texts);
1408
+ }
1409
+ return texts;
1410
+ }
1411
+ /** Depth cap on the fixture walk — a symlink loop must not hang the run's setup. */
1412
+ const MAX_FIXTURE_WALK_DEPTH = 32;
1413
+ /**
1414
+ * Append the declared fixture's text — the file's, or every file's under it when
1415
+ * it is a directory. `lstat` first, so the ONE absence case (`ENOENT`, or `ENOTDIR`
1416
+ * for a path routed through a file) is the only thing skipped; a symlink at the
1417
+ * declared path is skipped too, for the same reason the walk below skips one.
1418
+ */
1419
+ function collectFixtureText(path, into) {
1420
+ let kind;
1421
+ try {
1422
+ kind = lstatSync(path);
1423
+ }
1424
+ catch (err) {
1425
+ if (isPathAbsentError(err))
1426
+ return;
1427
+ throw err;
1428
+ }
1429
+ if (kind.isFile())
1430
+ into.push(readFileSync(path, 'utf8'));
1431
+ else if (kind.isDirectory())
1432
+ walkFixtureDir(path, into, 0);
1433
+ }
1434
+ /** Append every file's text under `dir`, recursing. Symlinked entries are skipped. */
1435
+ function walkFixtureDir(dir, into, depth) {
1436
+ if (depth > MAX_FIXTURE_WALK_DEPTH)
1437
+ return;
1438
+ // `withFileTypes` on the DIRECTORY is one syscall for the whole level, and
1439
+ // `lstat`-shaped: a symlinked entry reports as a link rather than as whatever it
1440
+ // points at, so the walk stays inside the bytes vat actually staged.
1441
+ for (const child of readdirSync(dir, { withFileTypes: true })) {
1442
+ const childPath = safePath.joinUnderRoot(dir, child.name);
1443
+ const kind = direntKind(child);
1444
+ // A link is skipped by design (see above): the walk reads only staged bytes.
1445
+ if (kind === 'symlink')
1446
+ continue;
1447
+ if (kind === 'file')
1448
+ into.push(readFileSync(childPath, 'utf8'));
1449
+ else if (kind === 'directory')
1450
+ walkFixtureDir(childPath, into, depth + 1);
1451
+ }
1452
+ }
1453
+ /**
1454
+ * Build the detector input for the skill-absent arm. ONE builder, so the hits
1455
+ * written per eval and the `signals` list stamped on the run-level integrity
1456
+ * block are derived from the same values — a second construction site is how the
1457
+ * block would come to claim a detector the scan never actually ran.
1458
+ *
1459
+ * `transcript` and `evalId` are supplied per call; everything else is per-run.
1460
+ *
1461
+ * Takes a NARROWED context rather than the whole {@link EvalRunContext} so the
1462
+ * builder is directly unit-testable — the detector honours `armCwd` (dropping it
1463
+ * there kills three tests) while the wiring that SUPPLIES it was droppable green,
1464
+ * and a dropped `armCwd` anchors every relative climb one directory too high, where
1465
+ * no needle can see it.
1466
+ *
1467
+ * `evalId` is what turns the arm's workspace ROOT into the arm's actual starting
1468
+ * CWD (`<armWorkspaceDir>/<evalId>` — see {@link resolvePerEvalWorkspaceDir}).
1469
+ * The detector needs both and they are one directory apart: containment is judged
1470
+ * against the root (the arm's whole tree is its own), while the cwd walk must
1471
+ * start exactly where the executor did, or every relative climb in the transcript
1472
+ * resolves one level too high and lands where no needle can see it.
1473
+ *
1474
+ * `evalId` is REQUIRED here, and the signals-only caller gets its own entry point
1475
+ * ({@link buildContaminationSignalsInput}) rather than sharing an optional third
1476
+ * argument. An optional parameter is a pin the compiler cannot hold: dropping the
1477
+ * argument at the SCAN call site left all 2270 unit tests green and was caught only
1478
+ * by the integration suite — which this repo's fast implementer loop explicitly says
1479
+ * not to run. That is the same shape as the dropped `armCwd` that anchored the whole
1480
+ * walk one directory too high, also green.
1481
+ */
1482
+ function buildContaminationInput(transcript, ctx, evalId) {
1483
+ return contaminationInput(transcript, ctx, evalId);
1484
+ }
1485
+ /**
1486
+ * The same input built for the "which detectors are ARMED?" question, which is asked
1487
+ * once per run with no transcript and no eval — so it can honestly supply neither,
1488
+ * and `armCwd` is correctly absent rather than accidentally dropped.
1489
+ */
1490
+ function buildContaminationSignalsInput(ctx) {
1491
+ return contaminationInput('', ctx, undefined);
1492
+ }
1493
+ function contaminationInput(transcript, ctx, evalId) {
1494
+ const armRoot = ctx.armDirs.without === undefined || ctx.armDirs.without === ''
1495
+ ? undefined
1496
+ : armWorkspaceRoot(ctx.workspacesRoot, ctx.armDirs, 'without');
1497
+ return {
1498
+ transcript,
1499
+ harnessRoot: ctx.harnessRoot,
1500
+ // Only ever called for the skill-absent arm, so the sibling is `with` — the
1501
+ // treatment arm's live working directory, one `ls ..` away and containing no
1502
+ // harness path for the needles above to find.
1503
+ siblingArmDir: armWorkspaceRoot(ctx.workspacesRoot, ctx.armDirs, 'with'),
1504
+ // The control arm's OWN root, the exact counterpart of the line above. Its
1505
+ // absolute path is stated in the arm's own prompt, so the arm reuses it
1506
+ // constantly; without this the executable-name signal reads every mention of
1507
+ // the arm's own scratch script as a reach into the skill.
1508
+ //
1509
+ // Conditional because this builder also serves `buildContaminationSignalsInput`,
1510
+ // which runs on EVERY run — and a non-baseline run mints no `without` arm, so
1511
+ // `armDirSegment` would (correctly) throw rather than invent a segment.
1512
+ ...(armRoot === undefined ? {} : { armWorkspaceDir: armRoot }),
1513
+ // ...and the eval's own subdirectory under it, which is where the executor was
1514
+ // actually spawned. Absent without an evalId, in which case the walk anchors
1515
+ // at the root — one level high, but the best anchor available.
1516
+ ...(armRoot === undefined || evalId === undefined
1517
+ ? {}
1518
+ : { armCwd: safePath.joinUnderRoot(armRoot, evalId) }),
1519
+ // VAT's private tmp dirs: the held eval suite (the `expected_output` answer
1520
+ // key) and the grader dir (the run's integrity nonce). Both are SIBLINGS of
1521
+ // the arm's cwd under the OS temp dir — two hops or one `$TMPDIR` expansion
1522
+ // away — and contain no harness path, so neither the harness needles nor the
1523
+ // sibling-arm needles can see them. The workspaces root is deliberately NOT
1524
+ // in this list: it is the arm's own legitimate cwd.
1525
+ vatPrivateDirs: [ctx.evalSuiteHoldDir, ctx.graderOutDir],
1526
+ skillContentNeedles: ctx.skillContentNeedles,
1527
+ // The skill-relative PATH (`scripts/csvsum.py`), not the extension-stripped
1528
+ // `name`. An ambient copy of the skill is a COPY, so it reproduces the skill's
1529
+ // internal layout under whatever root it sits at and still matches; the control
1530
+ // arm's own `/tmp/summary.txt` no longer does. `howInvoked` is a command string,
1531
+ // not a path, so it is deliberately NOT used here.
1532
+ ...(ctx.declaredExecutables === undefined
1533
+ ? {}
1534
+ : { executablePaths: ctx.declaredExecutables.map((e) => e.path) }),
1535
+ };
1536
+ }
1537
+ function baselineContaminationFor(transcript, ctx, evalId) {
1538
+ const { hits, degraded } = detectBaselineContamination(buildContaminationInput(transcript, ctx, evalId));
1539
+ // A degraded scan writes `contaminated: false` with exactly the same bytes as one
1540
+ // that walked the transcript properly, so silence here would be the "quietly wrong
1541
+ // measurement" this whole module exists to prevent. It goes to BOTH surfaces, and
1542
+ // neither substitutes for the other:
1543
+ // stderr reaches the operator watching the run, and the fragment carries it to
1544
+ // `baselineIntegrity.degraded` in baseline.json, which is what a reader has
1545
+ // months later when the stderr scrollback is gone.
1546
+ //
1547
+ // The `evalId` is stamped HERE rather than by the detector: the detector is pure
1548
+ // and scans one transcript with no idea which eval it belongs to, and an
1549
+ // unattributed degradation in a run-level list is unactionable.
1550
+ if (degraded !== undefined) {
1551
+ process.stderr.write(`⚠️ control arm (${evalId}): contamination scan DEGRADED (${degraded.reason}) — ${degraded.detail}. ` +
1552
+ 'A clean verdict from this eval is weaker evidence than a clean structured scan: the walk ' +
1553
+ 'either stopped resolving paths from this point on, or (for an unparseable transcript) fell ' +
1554
+ 'back to flat text matching, which both over- and under-reports.\n');
1555
+ }
1556
+ return {
1557
+ ...(hits.length === 0 ? {} : { contamination: hits }),
1558
+ ...(degraded === undefined ? {} : { degraded: { ...degraded, evalId } }),
1559
+ };
1560
+ }
1561
+ /**
1562
+ * Which detectors this run armed, for the `signals` field on the integrity block.
1563
+ * Derived from {@link buildContaminationInput} with an empty transcript — the
1564
+ * armed set is a property of the run's paths, never of any one transcript.
1565
+ */
1566
+ function contaminationSignalsFor(ctx) {
1567
+ return activeContaminationSignals(buildContaminationSignalsInput(ctx));
806
1568
  }
807
1569
  /**
808
1570
  * Run evals TIER by TIER (ascending), bounded-parallel WITHIN each tier, with a
809
- * GATE between tiers (issue #145 Phase G). After a tier completes, apply the
1571
+ * GATE between tiers. After a tier completes, apply the
810
1572
  * default gate policy ({@link shouldGateAfterTier}) over that tier's WITH-arm
811
1573
  * fragments: if any eval in the tier did not fully pass, do NOT launch the higher
812
1574
  * (more expensive) tiers — their evals are recorded as SKIPPED (a distinct state,
@@ -819,9 +1581,16 @@ async function runEvalWorker(item, ctx) {
819
1581
  async function runEvalsTiered(input) {
820
1582
  const groups = groupEvalsByTier(input.evals);
821
1583
  const fragments = [];
1584
+ const controlFailures = [];
822
1585
  for (const [index, group] of groups.entries()) {
823
- const tierFragments = await input.runTier(buildEvalWorkItems(group.evals, input.baseline));
1586
+ const outcomes = await input.runTier(buildEvalWorkItems(group.evals, input.baseline));
1587
+ const tierFragments = outcomes.flatMap((o) => (o.fragment === undefined ? [] : [o.fragment]));
1588
+ controlFailures.push(...outcomes.flatMap((o) => (o.controlFailure === undefined ? [] : [o.controlFailure])));
824
1589
  fragments.push(...tierFragments);
1590
+ // A control-arm failure deliberately does NOT gate. The gate asks whether the
1591
+ // SKILL failed a cheap expectation before spending on expensive ones, and the
1592
+ // control arm is not the skill — treating its executor timeout as a gating
1593
+ // failure would stop a run whose treatment tier passed everything.
825
1594
  const withArm = tierFragments.filter((f) => f.arm !== 'without');
826
1595
  if (!shouldGateAfterTier(withArm))
827
1596
  continue;
@@ -832,14 +1601,14 @@ async function runEvalsTiered(input) {
832
1601
  // Legibility (required): name the skipped tiers on stderr so a fail-fast run is
833
1602
  // never mistaken for a smaller passing suite. stdout stays machine-readable.
834
1603
  process.stderr.write(formatSkippedTiersSummary(skipped) + '\n');
835
- return { fragments, skipped };
1604
+ return { fragments, controlFailures, skipped };
836
1605
  }
837
- return { fragments };
1606
+ return { fragments, controlFailures };
838
1607
  }
839
1608
  /** Resolve the run's grading/friction/baseline/tool-eval artifact paths under
840
1609
  * `resultsDir`. Single source of truth for the filenames (used by the
841
1610
  * pre-pipeline stale wipe AND the post-merge writer). */
842
- export function resolveArtifactPaths(resultsDir) {
1611
+ function resolveArtifactPaths(resultsDir) {
843
1612
  return {
844
1613
  gradingOut: safePath.join(resultsDir, 'grading.json'),
845
1614
  frictionOut: safePath.join(resultsDir, 'friction.json'),
@@ -856,11 +1625,52 @@ export function resolveArtifactPaths(resultsDir) {
856
1625
  * run's. Wiping all three up front closes that cross-run leak. Best-effort
857
1626
  * (`force: true`) — a missing file is fine.
858
1627
  */
859
- export function wipeStaleArtifacts(paths) {
860
- rmSync(paths.gradingOut, { force: true });
861
- rmSync(paths.frictionOut, { force: true });
862
- rmSync(paths.baselineOut, { force: true });
863
- rmSync(paths.toolEvalOut, { force: true });
1628
+ function wipeStaleArtifacts(paths) {
1629
+ for (const path of [paths.gradingOut, paths.frictionOut, paths.baselineOut, paths.toolEvalOut]) {
1630
+ rmSync(path, { force: true });
1631
+ // ...and the quarantined copy a PRIOR run's failed gate set aside. It is just as
1632
+ // stale, and left alone `results/` accumulates one `.rejected` per broken run
1633
+ // with nothing to say which is the newest.
1634
+ rmSync(rejectedArtifactPath(path), { force: true });
1635
+ }
1636
+ }
1637
+ /**
1638
+ * Create `results/` and write this run's provenance — unless this is a DRY RUN, in
1639
+ * which case it does nothing at all.
1640
+ *
1641
+ * NOTHING under `results/` may be created, written, or removed by a dry run. The
1642
+ * harness root is a deterministic function of the subject, so `results/` here holds
1643
+ * the PREVIOUS run's output: creating the dir, rewriting `provenance.json`, and (at
1644
+ * the call site) wiping the stale artifacts meant "what would this cost next time?"
1645
+ * destroyed the run the operator was about to read. Measured twice: a real
1646
+ * `--baseline` run left `[baseline, friction, grading, provenance, tool-eval].json`
1647
+ * and a subsequent `--dry-run` against the same subject left `[provenance.json]`.
1648
+ *
1649
+ * A function rather than an `if` at the call site because the orchestrator is at its
1650
+ * cognitive-complexity ceiling; the RULE is what matters and it is written here.
1651
+ */
1652
+ function openResultsDir(input) {
1653
+ if (input.dryRun)
1654
+ return;
1655
+ // 0700, matching the harness root. `grading.json` quotes the executor transcript
1656
+ // verbatim in each expectation's evidence, so whatever the skill read out of its
1657
+ // input files ends up here as text — and unlike the workspaces, results/ SURVIVES
1658
+ // `--keep` by design.
1659
+ mkdirSyncReal(input.resultsDir, { recursive: true, mode: 0o700 });
1660
+ writeFileSync(input.provenancePath, JSON.stringify(input.provenance, null, 2) + '\n', 'utf-8');
1661
+ }
1662
+ /**
1663
+ * Where a results artifact goes when it fails the fail-closed gate.
1664
+ *
1665
+ * The gate throws and `results/` is deliberately RETAINED, so the file that failed
1666
+ * used to sit there under its authoritative name — the one artifact whose schema
1667
+ * violation is BY DEFINITION a harness bug, indistinguishable in a CI archive from a
1668
+ * good one until the next run's wipe. Renamed rather than deleted: the bytes are what
1669
+ * a maintainer needs in order to see which refine tripped, and a name ending in
1670
+ * `.rejected` cannot be mistaken for the artifact a reader came for.
1671
+ */
1672
+ function rejectedArtifactPath(path) {
1673
+ return `${path}.rejected`;
864
1674
  }
865
1675
  /**
866
1676
  * Merge the run's grader fragments and WRITE the run artifacts (vat is the SOLE
@@ -874,49 +1684,253 @@ export function wipeStaleArtifacts(paths) {
874
1684
  * can compute the COMPOSITE verdict. Every fragment's per-run nonce is re-verified
875
1685
  * inside {@link mergeFragmentsToGrading}.
876
1686
  */
877
- function writeRunArtifactsAndReconcile(fragments, runNonce, paths) {
1687
+ /**
1688
+ * Collect per-eval baseline-integrity findings from the WITHOUT-arm fragments.
1689
+ * Fragments with no `contamination` are clean and contribute nothing. Pure.
1690
+ */
1691
+ /**
1692
+ * What one arm graded, per eval — the SINGLE derivation behind both the parity check
1693
+ * ({@link armExpectationSkew}, which reads `total`) and the A/B delta (which reads
1694
+ * `passed`). Deriving those separately would let the two blocks in `baseline.json`
1695
+ * disagree about the same arm. Pure.
1696
+ *
1697
+ * Exported for its own unit test, not for reuse. Setting `passed` to `total` here
1698
+ * failed zero unit tests and three integration ones: every unit fixture graded
1699
+ * everything it declared, so the two fields were numerically identical in all of
1700
+ * them and the ONE thing this function exists to keep distinct was unpinned.
1701
+ */
1702
+ function gradedCounts(fragments) {
1703
+ return fragments.map((f) => ({
1704
+ evalId: f.evalId,
1705
+ passed: f.expectations.filter((e) => e.passed).length,
1706
+ total: f.expectations.length,
1707
+ }));
1708
+ }
1709
+ /**
1710
+ * Both per-eval integrity channels the WITHOUT-arm fragments carry, lifted in ONE
1711
+ * pass: what the scan FOUND (`findings`) and how well it was able to look
1712
+ * (`degraded`).
1713
+ *
1714
+ * Together, not separately, because they are read together: `contaminated: false`
1715
+ * means "looked properly and found nothing" only when `degraded` is empty, and a
1716
+ * caller that collects one and forgets the other publishes the first half of that
1717
+ * sentence as if it were the whole of it. That is not hypothetical — the degradation
1718
+ * was detected, schema'd, summarised and rendered, and then never collected, so
1719
+ * every `baseline.json` shipped claiming a clean scan it had not necessarily run.
1720
+ */
1721
+ function collectBaselineFindings(withoutArm) {
1722
+ return {
1723
+ findings: withoutArm
1724
+ .filter((f) => f.contamination !== undefined && f.contamination.length > 0)
1725
+ .map((f) => ({ evalId: f.evalId, hits: f.contamination })),
1726
+ degraded: withoutArm.flatMap((f) => (f.degraded === undefined ? [] : [f.degraded])),
1727
+ };
1728
+ }
1729
+ /**
1730
+ * Restate a {@link SkippedEvalsSummary} as the delta block's truncation record.
1731
+ *
1732
+ * A separate shape rather than embedding the tier summary wholesale: the delta cares
1733
+ * about WHAT IS MISSING FROM ITS TOTALS, which is the flat set of evals that never
1734
+ * ran on either arm, while `SkippedEvalsSummary` is organised for the tier reporter.
1735
+ * Pure.
1736
+ */
1737
+ function deltaTruncationFor(skipped) {
1738
+ if (skipped === undefined || skipped.totalSkipped === 0)
1739
+ return null;
1740
+ return {
1741
+ gatedByTier: skipped.gatedByTier,
1742
+ firstSkippedTier: skipped.firstSkippedTier,
1743
+ totalSkipped: skipped.totalSkipped,
1744
+ evalIds: skipped.tiers.flatMap((tier) => tier.evalIds),
1745
+ };
1746
+ }
1747
+ /**
1748
+ * The stderr report for a `--baseline` run: the delta line, then the ⚠️ banner when
1749
+ * the run is unusable. Pure — the caller decides WHEN it goes out, which is the
1750
+ * whole point (see the call site).
1751
+ *
1752
+ * Composed on EVERY baseline run, including a withheld one: "the delta is
1753
+ * unavailable, and here is why" is the report an operator needs, and a run that
1754
+ * silently prints nothing is indistinguishable from a vat that still does not
1755
+ * compute a delta at all. The delta line comes FIRST so an unusable run reads as the
1756
+ * number followed by the caveat that explains it — the banner is the only line that
1757
+ * says the number means nothing, so it must never be the one scrolled away.
1758
+ *
1759
+ * ⚠️ IT IS NOT LITERALLY LAST ON THE SCREEN, and this docblock used to say it was.
1760
+ * The CLI prints `Harness:`, `Results:`, `Workspaces:` and `Summary:` AFTER the
1761
+ * harness returns, so 2–4 lines always follow the banner. What this ordering buys is
1762
+ * that the banner is not separated from its delta by up to ~1250 rows of friction —
1763
+ * a bounded, short tail after it, not none. Moving those CLI lines above the harness
1764
+ * report is the CLI's call, not this module's.
1765
+ *
1766
+ * THREE conditions have to hold before the caveat is dropped, not two. A DEGRADED
1767
+ * scan is neither contaminated nor incomparable, so a two-clause gate printed
1768
+ * `Baseline delta: +0 (with skill: 2/2, without skill: 2/2).` and exited 0 while
1769
+ * `baseline.json` carried `degraded: [...]` and an `integrity.summary` ending in a
1770
+ * `⚠️ DEGRADED SCAN: …` sentence that was composed, written to the artifact, and
1771
+ * never emitted anywhere. Everything this module says about telling "checked and clean"
1772
+ * apart from "checked with the blunt instrument" was true of the artifact and false
1773
+ * of the terminal — and the terminal is the only surface an operator reads.
1774
+ *
1775
+ * A CONTAMINATED run still reports its number, deliberately. Contamination does not
1776
+ * make the subtraction illegal — both arms were graded to the same depth, so the
1777
+ * arithmetic is sound; what it invalidates is INTERPRETING that number as skill
1778
+ * lift, which is precisely what the ⚠️ line says ("The reported delta is NOT a
1779
+ * measure of skill lift"), and what `vat-skill-testing.md` turns into the
1780
+ * instruction "discard the delta". Withholding it here would (a) leave that sentence
1781
+ * referring to a number that was never reported, and (b) spend `null` — which means
1782
+ * "these arms cannot be subtracted", a different failure with a different remedy —
1783
+ * on a run where they can be. A contaminated delta is also diagnostic in its own
1784
+ * right: it collapses toward zero, which is corroboration, not noise.
1785
+ */
1786
+ function formatBaselineReport(delta, integrity) {
1787
+ const line = `\n${formatBaselineDeltaLine(delta)}\n`;
1788
+ // Any of the three makes the delta unusable-as-stated, and all three are only
1789
+ // visible on stderr — nobody opens baseline.json unprompted.
1790
+ if (!integrity.contaminated && integrity.comparable && integrity.degraded.length === 0)
1791
+ return line;
1792
+ return `${line}\n⚠️ ${integrity.summary}\n\n`;
1793
+ }
1794
+ const IN_PLACE_SUBJECT_FRICTION = {
1795
+ severity: 'low',
1796
+ category: 'path-assumption',
1797
+ message: 'In-place skill (publish: false) — staged from source; links that leave the skill directory are not ' +
1798
+ 'present in the harness, so an eval that depends on them tests the skill without them.',
1799
+ };
1800
+ function writeRunArtifactsAndReconcile(input) {
1801
+ const { fragments, runNonce, paths, signals, controlFailures } = input;
878
1802
  const { withArm, withoutArm } = partitionFragmentsByArm(fragments);
879
- const grading = mergeFragmentsToGrading(withArm, runNonce);
880
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
1803
+ let baselineReport;
1804
+ const grading = mergeFragmentsToGrading(withArm, runNonce, 'with');
881
1805
  writeFileSync(paths.gradingOut, JSON.stringify(grading, null, 2) + '\n', 'utf-8');
882
- const friction = mergeFragmentsToFriction(fragments);
883
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
1806
+ const merged = mergeFragmentsToFriction(fragments);
1807
+ const friction = input.subjectInPlace ? { items: [IN_PLACE_SUBJECT_FRICTION, ...merged.items] } : merged;
884
1808
  writeFileSync(paths.frictionOut, JSON.stringify(friction, null, 2) + '\n', 'utf-8');
885
1809
  // Tool verdicts come from the WITH arm ONLY — the WITHOUT/skill-absent arm never
886
1810
  // carries toolExpectations, so its fragments have no `tool` body to merge.
887
1811
  const toolEval = mergeFragmentsToToolEval(withArm);
888
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
889
1812
  writeFileSync(paths.toolEvalOut, JSON.stringify(toolEval, null, 2) + '\n', 'utf-8');
890
- if (withoutArm.length > 0) {
891
- const baseline = mergeFragmentsToGrading(withoutArm, runNonce);
892
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
893
- writeFileSync(paths.baselineOut, JSON.stringify(baseline, null, 2) + '\n', 'utf-8');
1813
+ // `controlFailures` is the second disjunct, and it is load-bearing: when EVERY
1814
+ // control eval died there are no WITHOUT-arm fragments at all, and gating on the
1815
+ // fragment count alone would write no baseline.json — so the one run that most
1816
+ // needs to say "the control arm did not happen" would say nothing, exactly like a
1817
+ // run where `--baseline` was never passed. `mergeFragmentsToGrading` over an empty
1818
+ // arm yields a zero/zero report, which is the honest shape for it.
1819
+ if (withoutArm.length > 0 || controlFailures.length > 0) {
1820
+ const baseline = mergeFragmentsToGrading(withoutArm, runNonce, 'without');
1821
+ // Stamp the integrity verdict onto baseline.json (GradingReportSchema is
1822
+ // .passthrough(), so the extra field is contract-legal). Written on EVERY
1823
+ // baseline run, clean or not: a reader must be able to tell "checked and
1824
+ // clean" from "produced before this check existed", and only an
1825
+ // unconditional field does that.
1826
+ // Parity BEFORE the verdict prose, because it decides whether the number the
1827
+ // prose is about means anything. Deliberately NOT a throw: the control arm's
1828
+ // grader misbehaving must not discard a perfectly good treatment result — it
1829
+ // must make the DELTA say, in writing, that it cannot be subtracted.
1830
+ // ONE derivation per arm, feeding BOTH the parity check and the delta. The
1831
+ // whole reason `ArmEvalGrade` carries `passed` alongside `total` is that these
1832
+ // two blocks must provably be talking about the same numbers — re-deriving the
1833
+ // counts for the delta would let `baselineIntegrity.skew` and `baselineDelta`
1834
+ // disagree about one run, and a reader holding two contradicting blocks has no
1835
+ // way to tell which one to believe.
1836
+ const withGrades = gradedCounts(withArm);
1837
+ const withoutGrades = gradedCounts(withoutArm);
1838
+ const skew = armExpectationSkew(withGrades, withoutGrades);
1839
+ const { findings, degraded } = collectBaselineFindings(withoutArm);
1840
+ const integrity = summarizeBaselineIntegrity({
1841
+ findings,
1842
+ // WHAT WAS SCANNED, not what was armed. `signals` is derived from the run's
1843
+ // PATHS, so it is fully populated on a run where every control eval died before
1844
+ // producing a transcript — and the shipped documentation teaches `signals` as
1845
+ // exactly the discriminator between a clean verdict and a blind one ("an empty
1846
+ // list means nothing was looking"). A control-timeout run wrote
1847
+ // `"contaminated": false` alongside `checked by: harness-path, sibling-arm,
1848
+ // vat-private-dir, skill-content` having scanned zero transcripts: the list was
1849
+ // full and nothing was looking. With no control fragment there is no
1850
+ // observation, so there is no detector to claim.
1851
+ signals: withoutArm.length === 0 ? [] : signals,
1852
+ // The same fact `signals` is gated on, stated as a number rather than inferred
1853
+ // from an empty list — it gates the VERDICT PROSE, which `signals` cannot reach.
1854
+ // Without it the summary opened "No skill-absent eval was observed reaching the
1855
+ // skill. The A/B delta is interpretable as instruction lift", which is a positive
1856
+ // claim about an observation that never happened, with the correction third.
1857
+ // Required, not defaulted, for the same reason `signals` is: a default lets every
1858
+ // caller overclaim in the "clean" direction, which is where it gets believed.
1859
+ // Control evals whose transcript actually reached the scanner — NOT evals declared.
1860
+ observedEvals: withoutArm.length,
1861
+ skew,
1862
+ degraded,
1863
+ controlArmFailures: controlFailures,
1864
+ });
1865
+ // The SUBTRACTION `--baseline` exists to perform, and until this it was never
1866
+ // performed anywhere: vat wrote two same-shaped artifacts and left the operator
1867
+ // to do the arithmetic by hand while the docs said it "reports the delta".
1868
+ // `skew` is passed rather than re-checked so the withholding decision has
1869
+ // exactly one author.
1870
+ // `truncated` rides along rather than withholding: fail-fast stops BOTH arms at
1871
+ // the same tier boundary, so the arithmetic over the tiers that ran is sound —
1872
+ // what it changes is the SCOPE of the claim. Without it, a tier-0 failure in an
1873
+ // 8-eval suite printed `+1 (with skill: 2/2, without skill: 1/2)`, which is
1874
+ // byte-identical to a complete 2-eval suite. And this is the COMMON case, not a
1875
+ // corner: `--baseline` exists to measure a skill that may not be helping, so a
1876
+ // WITH-arm failure — the thing that fires the gate — is the expected state of an
1877
+ // interesting baseline run.
1878
+ const delta = computeBaselineDelta({
1879
+ withArm: withGrades,
1880
+ withoutArm: withoutGrades,
1881
+ skew,
1882
+ controlArmFailures: controlFailures,
1883
+ truncated: deltaTruncationFor(input.skipped),
1884
+ });
1885
+ // A SIBLING of `baselineIntegrity`, not a field inside it, because they answer
1886
+ // different questions: integrity answers "may I subtract these two arms?", and
1887
+ // the delta IS the subtraction. Burying the number inside the block that
1888
+ // qualifies it would also mean a reader who wants only the lift has to
1889
+ // understand the contamination vocabulary first. (GradingReportSchema is
1890
+ // .passthrough(), so both extra keys are contract-legal — see the comment above.)
1891
+ writeFileSync(paths.baselineOut, JSON.stringify({ ...baseline, baselineIntegrity: integrity, baselineDelta: delta }, null, 2) + '\n', 'utf-8');
1892
+ // RETURNED, not written. The delta line and the ⚠️ banner must not be separated
1893
+ // from each other by the friction report: `emitFrictionReport` runs from the
1894
+ // harness `finally`, long after this point, and its two caps (50 lines × 2000
1895
+ // chars) allow 101,245 characters — about 1250 rows of an 80-column terminal —
1896
+ // between the banner and the prompt. The banner is the only line that says the
1897
+ // number above it is meaningless, so pushing it off-screen is the same failure as
1898
+ // not printing it. Handing the text back lets the caller emit friction FIRST and
1899
+ // this after it; a total-output budget on friction would only shorten the
1900
+ // distance, not end it. (The banner is not the LAST line of the process's output
1901
+ // — the CLI adds `Harness:`/`Results:`/`Workspaces:`/`Summary:` after the harness
1902
+ // returns. That tail is 2–4 lines and bounded, which is the point.)
1903
+ baselineReport = formatBaselineReport(delta, integrity);
894
1904
  }
895
- return { verdict: reconcileGrading(grading), toolEval };
1905
+ return {
1906
+ verdict: reconcileGrading(grading),
1907
+ toolEval,
1908
+ ...(baselineReport === undefined ? {} : { baselineReport }),
1909
+ };
896
1910
  }
897
1911
  /**
898
- * The COMPOSITE run verdict (issue #145 Phase T): the run passes only when BOTH
1912
+ * The COMPOSITE run verdict: the run passes only when BOTH
899
1913
  * the prose-expectation grading passed AND every tool-expectation verdict passed.
900
1914
  * Tool verdicts live in tool-eval.json (a SEPARATE channel — C2); this combines
901
1915
  * the two at the exit-code layer WITHOUT mixing the channels' data. When no eval
902
1916
  * declared `toolExpectations`, `toolEval.evals` is empty and this equals
903
1917
  * `outputAllPassed`. Pure + unit-testable.
904
1918
  */
905
- export function computeCompositeVerdict(outputAllPassed, toolEval) {
1919
+ function computeCompositeVerdict(outputAllPassed, toolEval) {
906
1920
  return outputAllPassed && toolEval.evals.every((v) => v.passed);
907
1921
  }
908
1922
  /**
909
1923
  * The run's final pass/fail after cost-tiered fail-fast: the {@link
910
1924
  * computeCompositeVerdict} of the tiers that RAN, AND no tiers were skipped. A
911
1925
  * fail-fast run that gated higher tiers is NEVER a pass (skipped ≠ passed) — this
912
- * forces `false` so the exit code is EvalFailure (4), never downgraded to 0 by the
1926
+ * forces `false` so the exit code is FINDINGS, never downgraded to 0 by the
913
1927
  * composite path alone. Pure + unit-testable.
914
1928
  */
915
- export function resolveCompositeAllPassed(outputAllPassed, toolEval, skipped) {
1929
+ function resolveCompositeAllPassed(outputAllPassed, toolEval, skipped) {
916
1930
  return computeCompositeVerdict(outputAllPassed, toolEval) && skipped === undefined;
917
1931
  }
918
1932
  /** Fold one session's `total_cost_usd` into the accumulator; a non-number (mock spawn / missing result) is ignored. */
919
- export function recordSessionCost(acc, totalCostUsd) {
1933
+ function recordSessionCost(acc, totalCostUsd) {
920
1934
  if (typeof totalCostUsd !== 'number' || !Number.isFinite(totalCostUsd))
921
1935
  return;
922
1936
  acc.totalUsd += totalCostUsd;
@@ -927,7 +1941,7 @@ export function recordSessionCost(acc, totalCostUsd) {
927
1941
  * when no session reported a cost (e.g. every spawn was a test mock) so the suffix
928
1942
  * never adds noise to a run with no cost signal. Pure + unit-testable.
929
1943
  */
930
- export function formatRunCostSuffix(cost) {
1944
+ function formatRunCostSuffix(cost) {
931
1945
  if (cost === undefined || cost.sessions === 0)
932
1946
  return '';
933
1947
  return ` | ≈$${cost.totalUsd.toFixed(2)} across ${cost.sessions} session${cost.sessions === 1 ? '' : 's'}`;
@@ -939,7 +1953,7 @@ export function formatRunCostSuffix(cost) {
939
1953
  * verdict failed, a `(N tool)` suffix names how many — so a composite FAIL whose
940
1954
  * OUTPUT counts look all-green (e.g. `FAIL 3/3 (1 tool)`) is self-explaining. Pure.
941
1955
  */
942
- export function buildRunSummary(verdict, toolEval, compositeAllPassed) {
1956
+ function buildRunSummary(verdict, toolEval, compositeAllPassed) {
943
1957
  const base = `${compositeAllPassed ? 'PASS' : 'FAIL'} ${verdict.passed}/${verdict.total}`;
944
1958
  const toolFailures = toolEval.evals.filter((v) => !v.passed).length;
945
1959
  return toolFailures > 0 ? `${base} (${toolFailures} tool)` : base;
@@ -951,7 +1965,7 @@ export function buildRunSummary(verdict, toolEval, compositeAllPassed) {
951
1965
  * tiers ALWAYS reads FAIL (skipped ≠ passed forces `compositeAllPassed` false at
952
1966
  * the call site), so the base line is already FAIL when the note is present.
953
1967
  */
954
- export function buildRunSummaryWithSkips(verdict, toolEval, compositeAllPassed, skipped, cost) {
1968
+ function buildRunSummaryWithSkips(verdict, toolEval, compositeAllPassed, skipped, cost) {
955
1969
  // The spend suffix rides the verdict line (like `(N tool)`); the skipped-tiers
956
1970
  // note stays on its own following line.
957
1971
  const base = buildRunSummary(verdict, toolEval, compositeAllPassed) + formatRunCostSuffix(cost);
@@ -965,35 +1979,88 @@ export function buildRunSummaryWithSkips(verdict, toolEval, compositeAllPassed,
965
1979
  * safe precisely because vat always writes these after the merge succeeds.
966
1980
  */
967
1981
  function assertVatWroteArtifact(path, validate, label) {
968
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
969
1982
  if (!existsSync(path)) {
970
1983
  throw new InternalHarnessError(`vat did not write ${label} at ${path} after the merge — harness bug.`);
971
1984
  }
972
1985
  let raw;
973
1986
  try {
974
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
975
1987
  raw = JSON.parse(readFileSync(path, 'utf-8'));
976
1988
  }
977
1989
  catch (err) {
978
- throw new InternalHarnessError(`vat-written ${label} at ${path} is not valid JSON (harness bug): ${err instanceof Error ? err.message : String(err)}`);
1990
+ throw new InternalHarnessError(`vat-written ${label} at ${path} is not valid JSON (harness bug)${quarantineArtifact(path)}: ` +
1991
+ `${err instanceof Error ? err.message : String(err)}`);
979
1992
  }
980
1993
  try {
981
1994
  validate(raw);
982
1995
  }
983
1996
  catch (err) {
984
- throw new InternalHarnessError(`vat-written ${label} at ${path} failed its schema (harness bug): ${err instanceof Error ? err.message : String(err)}`);
1997
+ throw new InternalHarnessError(`vat-written ${label} at ${path} failed its schema (harness bug)${quarantineArtifact(path)}: ` +
1998
+ `${err instanceof Error ? err.message : String(err)}`);
1999
+ }
2000
+ }
2001
+ /**
2002
+ * Move a gate-failing artifact aside and say where it went. See
2003
+ * {@link rejectedArtifactPath}.
2004
+ *
2005
+ * Best-effort and never throws: this runs on the way OUT of a run that is already
2006
+ * failing, and a cleanup that replaces the real diagnosis with a rename error would
2007
+ * be strictly worse than a file left in place. If the rename fails the file is
2008
+ * removed instead — an absent artifact reads as "vat wrote nothing", which is at
2009
+ * least not a false claim of authority. If THAT fails too, the file is still there
2010
+ * under its authoritative name, and the returned clause says so with both errors:
2011
+ * the operator is about to read a CI archive, and a bare '' told them nothing.
2012
+ */
2013
+ function quarantineArtifact(path) {
2014
+ const rejected = rejectedArtifactPath(path);
2015
+ let renameFailure;
2016
+ try {
2017
+ renameSync(path, rejected);
2018
+ return `; moved aside to ${rejected} so it is not read as authoritative`;
2019
+ }
2020
+ catch (err) {
2021
+ renameFailure = err instanceof Error ? err.message : String(err);
2022
+ }
2023
+ try {
2024
+ rmSync(path, { force: true });
2025
+ return '; removed so it is not read as authoritative';
2026
+ }
2027
+ catch (err) {
2028
+ return (`; it could not be moved aside (${renameFailure}) or removed ` +
2029
+ `(${err instanceof Error ? err.message : String(err)}), so it is STILL THERE under its ` +
2030
+ 'authoritative name — do not read it as authoritative');
985
2031
  }
986
2032
  }
987
2033
  /**
988
2034
  * Post-merge D2 gate over the vat-written results artifacts (vat is SOLE writer):
989
- * grading.json, friction.json, and tool-eval.json must each exist + parse + validate.
990
- * A separate, explicit check (NOT folded into any prompt-invariant linter) — fail-closed
991
- * and safe because vat wrote every one of these itself just above.
2035
+ * grading.json, friction.json, tool-eval.json — and, on a baseline run, baseline.json
2036
+ * — must each exist + parse + validate. A separate, explicit check (NOT folded into
2037
+ * any prompt-invariant linter) — fail-closed and safe because vat wrote every one of
2038
+ * these itself just above.
2039
+ *
2040
+ * `baseline.json` was EXEMPT from this gate for as long as the gate has existed, and
2041
+ * it is the artifact the exemption fits worst: on a `--baseline` run it is the only
2042
+ * durable record of the thing the command sells, and this branch has already been
2043
+ * bitten once by a default run deleting `results/` out from under it. Its validation
2044
+ * is also the strictest of the four — {@link BaselineArtifactSchema} carries the
2045
+ * `passed <= total` and `delta === with − without` refines, so a merge that produced
2046
+ * a self-contradicting number fails the run here instead of being written down.
2047
+ *
2048
+ * `baselineRun` gates it rather than "does the file exist", deliberately: an
2049
+ * existence check would pass on the one failure that matters (vat asked for a
2050
+ * baseline and wrote nothing), which is exactly what a fail-CLOSED gate must not do.
2051
+ * A non-baseline run writes no baseline.json at all and is not asked about one.
992
2052
  */
993
- function assertVatWroteArtifacts(paths) {
2053
+ function assertVatWroteArtifacts(paths, baselineRun) {
994
2054
  assertVatWroteArtifact(paths.gradingOut, (raw) => { GradingReportSchema.parse(raw); }, 'grading.json');
995
2055
  assertVatWroteArtifact(paths.frictionOut, (raw) => { FrictionReportSchema.parse(raw); }, 'friction.json');
996
2056
  assertVatWroteArtifact(paths.toolEvalOut, (raw) => { ToolEvalReportSchema.parse(raw); }, 'tool-eval.json');
2057
+ if (!baselineRun)
2058
+ return;
2059
+ assertVatWroteArtifact(paths.baselineOut, (raw) => {
2060
+ // Both halves: the grading report it IS, and the two blocks vat stamps onto it.
2061
+ GradingReportSchema.parse(raw);
2062
+ BaselineArtifactSchema.parse(raw);
2063
+ }, 'baseline.json');
997
2064
  }
998
2065
  // ---------------------------------------------------------------------------
999
2066
  // Main orchestrator
@@ -1002,17 +2069,56 @@ function assertVatWroteArtifacts(paths) {
1002
2069
  * Domain orchestrator for `vat skill test run`. Pure of process.exit — all
1003
2070
  * exit-code decisions live in the caller (run.ts).
1004
2071
  */
1005
- export async function runSkillTestHarness(opts) {
2072
+ /**
2073
+ * Resolve WHERE this run lives, and whether the harness owns that location.
2074
+ *
2075
+ * `--out` names the harness root exactly; `--workdir` names the BASE the root is
2076
+ * derived under. They are mutually exclusive: passing both used to silently
2077
+ * discard `--workdir` — the run went entirely to `--out` and the `--workdir` path
2078
+ * was never even created — so an operator using the pair to separate the
2079
+ * executor's cwd from the staged trees got no separation and no warning.
2080
+ *
2081
+ * `harnessCreated` is true only for the derived-under-OS-tmp case; an explicit
2082
+ * `--out` or `--workdir` is a location the user owns and is never auto-removed.
2083
+ */
2084
+ function resolveHarnessLocation(opts) {
2085
+ if (opts.out !== undefined && opts.workdir !== undefined) {
2086
+ throw new HarnessLocationError('--out and --workdir are mutually exclusive: --out names the harness root exactly, ' +
2087
+ '--workdir names the base it is derived under. Pass one.');
2088
+ }
1006
2089
  // §7 workdir safety: refuse a --workdir whose ancestry contains CLAUDE.md/.claude
1007
2090
  // BEFORE deriving the harness root from it (defense in depth with --setting-sources "").
1008
2091
  if (opts.workdir !== undefined) {
1009
- assertSafeWorkdir(opts.workdir);
2092
+ // Bounded at the user's home directory, which the walk stops BEFORE inspecting.
2093
+ //
2094
+ // Unbounded, this refuses every `--workdir` on Windows for every Claude Code
2095
+ // user: the OS temp dir there lives INSIDE the profile
2096
+ // (`C:\Users\<name>\AppData\Local\Temp`), so the ancestry walk climbs out of tmp
2097
+ // and finds the ambient `~/.claude` — and the resulting message, "Use an OS-tmp
2098
+ // location", names the thing the user just did. Unsatisfiable advice. It cannot
2099
+ // reproduce on POSIX, where tmp is never under `$HOME`, nor on the GitHub Windows
2100
+ // runner, whose `runneradmin` profile has no `~/.claude`; it needs a real Windows
2101
+ // developer machine, which is where it was found.
2102
+ //
2103
+ // Not a relaxation of the threat model: the gate refuses a workdir inside a
2104
+ // PROJECT, and `$HOME/.claude` is global config that every Claude Code user has on
2105
+ // every platform — it was never the target. A project at `$HOME/proj` is still
2106
+ // caught, because the walk inspects `$HOME/proj` and stops only at `$HOME` itself.
2107
+ assertSafeWorkdir(opts.workdir, homedir());
1010
2108
  }
1011
- const harnessRoot = opts.out ?? resolveHarnessRoot([opts.subject], opts.workdir);
1012
- // Only auto-remove the dir the harness itself created under OS tmp. An explicit
1013
- // --out (exact dir) or --workdir (user-chosen base) is a location the user owns;
1014
- // treat it like --keep and never delete it.
1015
- const harnessCreated = opts.out === undefined && opts.workdir === undefined;
2109
+ return {
2110
+ // `--out` is RESOLVED to an absolute path, not taken verbatim. A relative
2111
+ // `--out ./h` used to be stored as `./h` and then used as a contamination
2112
+ // needle — a two-character string that matches almost any transcript, so every
2113
+ // eval reported contaminated and the operator was told to discard a clean run.
2114
+ harnessRoot: opts.out === undefined
2115
+ ? resolveHarnessRoot([opts.subject], opts.workdir)
2116
+ : safePath.resolve(opts.out),
2117
+ harnessCreated: opts.out === undefined && opts.workdir === undefined,
2118
+ };
2119
+ }
2120
+ export async function runSkillTestHarness(opts) {
2121
+ const { harnessRoot, harnessCreated } = resolveHarnessLocation(opts);
1016
2122
  const repoRoot = opts.repoRoot ?? harnessRoot;
1017
2123
  // Two distinct questions, deliberately not one value. `evalsRef` is what the
1018
2124
  // adopter ASKED FOR — a `test.evals`/`--evals` value, or `undefined` for the
@@ -1055,6 +2161,31 @@ export async function runSkillTestHarness(opts) {
1055
2161
  // when the subject carried no suite, which is the common case) so the staging call
1056
2162
  // stays branch-free; removed by the same cleanup as the grader dir.
1057
2163
  const evalSuiteHoldDir = resolveEvalSuiteHoldDir(randomBytes(16).toString('hex'));
2164
+ // The executor's per-eval working directories. Like the grader and hold dirs
2165
+ // these live OUTSIDE harnessRoot — see resolveWorkspacesRoot for why that
2166
+ // separation is load-bearing for --baseline — so they need their own cleanup.
2167
+ const workspacesRoot = resolveWorkspacesRoot(randomBytes(16).toString('hex'));
2168
+ // Whether those workspaces OUTLIVE the run. `--keep` is the ONLY thing that
2169
+ // retains them.
2170
+ //
2171
+ // Deliberately NOT mirrored on `cleanupHarness`'s rule (keep OR user-owned
2172
+ // location). That rule exists because `--out`/`--workdir` name a directory the
2173
+ // USER owns, so vat has no business deleting it. Since these moved out of the
2174
+ // harness root they live under OS tmp, which the user did not choose and does
2175
+ // not manage — carrying the rule across would orphan a
2176
+ // `vat-skill-test-ws-<token>` dir on every `--out` run, forever. The location
2177
+ // changed, so the retention policy has to change with it.
2178
+ //
2179
+ // Decided ONCE, here, because two consumers ask the question — cleanup (delete
2180
+ // or not) and the returned `workspacesPath` (report or not) — and a second copy
2181
+ // of the predicate would drift, which is exactly how the result came to name a
2182
+ // directory cleanup had already removed.
2183
+ const retainWorkspaces = opts.keep === true;
2184
+ // The opaque per-arm segments beneath it. Minted here, with the roots, so the
2185
+ // same values reach staging, the executor's cwd, and the contamination
2186
+ // detector's sibling-arm needle — one source of truth for a value whose whole
2187
+ // job is to be unguessable and to carry no meaning.
2188
+ const armDirs = mintArmWorkspaceDirs(opts.baseline === true);
1058
2189
  const cleanup = () => {
1059
2190
  // Reap any still-in-flight executor/grader children FIRST. On the concurrent
1060
2191
  // error path `Promise.all` rejects without cancelling its siblings, and a
@@ -1064,7 +2195,13 @@ export async function runSkillTestHarness(opts) {
1064
2195
  // NOTE: the registry is a module-level singleton, so this assumes one run per
1065
2196
  // process (true for the CLI). A library embedding two concurrent runs in one
1066
2197
  // process would need per-run child scoping instead.
1067
- killAllActiveClaudeChildren();
2198
+ //
2199
+ // `cleanupHarness`, `removeVatOnlyDir` and `lock.release()` each swallow
2200
+ // explicitly at their own definition, for the stated reason that cleanup "must
2201
+ // not mask the run's real outcome". `killAllActiveClaudeChildren` lives in
2202
+ // another package, so its guard is applied here — see {@link swallowCleanupFailure}
2203
+ // for why a call-site guard is the weaker of the two forms.
2204
+ swallowCleanupFailure(killAllActiveClaudeChildren);
1068
2205
  lock.release();
1069
2206
  cleanupHarness(harnessRoot, { keep: opts.keep === true, created: harnessCreated });
1070
2207
  removeVatOnlyDir(graderOutDir);
@@ -1072,6 +2209,10 @@ export async function runSkillTestHarness(opts) {
1072
2209
  // --keep, which retains the harness dir for inspection but has no business
1073
2210
  // retaining the key), and never conditionally.
1074
2211
  removeVatOnlyDir(evalSuiteHoldDir);
2212
+ // Per-eval workspaces are the executor's cwd; `retainWorkspaces` (minted with
2213
+ // the root above) is the single author of whether they survive.
2214
+ if (!retainWorkspaces)
2215
+ removeVatOnlyDir(workspacesRoot);
1075
2216
  };
1076
2217
  const removeSignalCleanup = installSignalCleanup({ onSignal: cleanup });
1077
2218
  // Hoisted so the finally can surface packaging friction even when a throw
@@ -1081,6 +2222,23 @@ export async function runSkillTestHarness(opts) {
1081
2222
  // once merging succeeded; on a pre-merge throw the finally simply finds no
1082
2223
  // friction.json and emits nothing (a no-op).
1083
2224
  let frictionOut;
2225
+ /**
2226
+ * Emit the friction report AT MOST ONCE per run.
2227
+ *
2228
+ * There are now two call sites, and they exist for different reasons that must
2229
+ * not become two reports: the happy path calls it deliberately BEFORE the
2230
+ * baseline delta/banner (so ~1250 rows of friction cannot come between the two —
2231
+ * see the call site), and the `finally` calls it for the throw paths, where no
2232
+ * happy-path call ever happened. A flag rather than a "did we already return normally?"
2233
+ * test, because the throw can land anywhere between the two.
2234
+ */
2235
+ let frictionEmitted = false;
2236
+ const emitFrictionOnce = () => {
2237
+ if (frictionEmitted)
2238
+ return;
2239
+ frictionEmitted = true;
2240
+ emitFrictionReport(frictionOut);
2241
+ };
1084
2242
  try {
1085
2243
  // Step 3: Stage the harness FIRST — the subject's own evals/evals.json lands
1086
2244
  // inside its staged dir, so we must stage before we can locate it. The subject's
@@ -1088,6 +2246,18 @@ export async function runSkillTestHarness(opts) {
1088
2246
  // arrives via opts.subjectSource; build no longer happens in-domain here.
1089
2247
  const resolveCtx = buildResolveCtx(harnessRoot, repoRoot);
1090
2248
  const items = buildStageItems(opts, repoRoot);
2249
+ // Hold a copy of the subject's suite ONLY when this run would have to read it
2250
+ // from there. `resolveEvalSuitePath` prefers the AUTHORED source and falls back
2251
+ // to the hold dir, so when the authored copy exists the held one is never read
2252
+ // — and staging was writing it anyway, putting a second copy of the
2253
+ // `expected_output` answer key in the shared OS temp dir on the common path,
2254
+ // for nothing. The hold dir is 0700 and outside the executor's `--add-dir`
2255
+ // sandbox, but that is Claude's permission model, not an OS boundary: skill
2256
+ // code runs as the same uid and `ls $TMPDIR` finds it. The cheapest answer to
2257
+ // "can the executor reach the answer key" is not to write it there.
2258
+ // Same predicate as `resolveEvalSuitePath`'s rule 1, so the two cannot disagree
2259
+ // about which copy this run uses.
2260
+ const authoredSuiteExists = existsSync(resolveScaffoldEvalsPath(opts, repoRoot, evalsRef));
1091
2261
  const stageResult = await stageHarness({
1092
2262
  harnessRoot,
1093
2263
  items,
@@ -1095,7 +2265,7 @@ export async function runSkillTestHarness(opts) {
1095
2265
  ctx: resolveCtx,
1096
2266
  currentUid,
1097
2267
  evalsSubpath,
1098
- evalSuiteHoldDir,
2268
+ ...(authoredSuiteExists ? {} : { evalSuiteHoldDir }),
1099
2269
  });
1100
2270
  const { pluginDirs, subjectStagedDir, subjectPluginRoot, skippedOptional } = stageResult;
1101
2271
  emitSkippedOptionalWarning(skippedOptional);
@@ -1106,7 +2276,7 @@ export async function runSkillTestHarness(opts) {
1106
2276
  // staged tree — the answer key must never be reachable by the executor — so this
1107
2277
  // resolves to the AUTHORED source copy, or to the vat-only hold dir when the
1108
2278
  // suite existed nowhere but inside a fetched artifact. Absent from both →
1109
- // bootstrap a template at the source scaffold and exit 3, so an authored suite
2279
+ // bootstrap a template at the source scaffold and exit 2 with `Reason: bootstrap`, so an authored suite
1110
2280
  // is never overwritten.
1111
2281
  const evalsPath = resolveEvalSuitePath({
1112
2282
  opts,
@@ -1124,24 +2294,29 @@ export async function runSkillTestHarness(opts) {
1124
2294
  const summary = renderPreflightSummary(preflightResult.checks);
1125
2295
  return {
1126
2296
  harnessPath: harnessRoot,
1127
- exitCode: SkillTestExitCode.Preflight,
2297
+ exitCode: ExitCode.ERROR,
2298
+ reason: 'preflight',
1128
2299
  summary: `Preflight failed:\n${summary}`,
1129
2300
  };
1130
2301
  }
1131
2302
  // Step 5.5: Parse the eval suite and stage per-eval input workspaces. The
1132
2303
  // parsed suite is threaded on so the eval loop has the entries without
1133
2304
  // re-reading; declaredEvalCount is derived from it (suite.evals.length).
1134
- const workspaceStageResult = attemptStageWorkspaces(evalsPath, harnessRoot);
2305
+ const workspaceStageResult = attemptStageWorkspaces(evalsPath, harnessRoot, workspacesRoot, armDirs);
1135
2306
  if ('exitCode' in workspaceStageResult)
1136
2307
  return workspaceStageResult;
1137
- const { workspacesRoot, declaredEvalCount, suite } = workspaceStageResult;
2308
+ // `workspacesRoot` is resolved above (outside the try, so cleanup can reach it
2309
+ // on a signal); staging returns it unchanged, so only the derived values are
2310
+ // destructured here.
2311
+ const { declaredEvalCount, suite } = workspaceStageResult;
1138
2312
  emitEvalLintWarnings(suite.evals, (opts.declaredExecutables ?? []).map((e) => e.name));
1139
2313
  // Step 6: Enforce the §12 security ack (must pass --i-understand-this-runs-skill-code).
1140
2314
  // Only enforced when not a dry-run and not already acknowledged.
1141
2315
  if (!isAcknowledged(opts)) {
1142
2316
  return {
1143
2317
  harnessPath: harnessRoot,
1144
- exitCode: SkillTestExitCode.Preflight,
2318
+ exitCode: ExitCode.ERROR,
2319
+ reason: 'preflight',
1145
2320
  summary: 'Security acknowledgment required. Pass --i-understand-this-runs-skill-code to proceed.',
1146
2321
  };
1147
2322
  }
@@ -1149,11 +2324,7 @@ export async function runSkillTestHarness(opts) {
1149
2324
  // results/ — grading.json/friction.json/baseline.json come from the merged
1150
2325
  // grader fragments below, never from the (untrusted) model.
1151
2326
  const resultsDir = safePath.joinUnderRoot(harnessRoot, 'results');
1152
- // 0700, matching the harness root. `grading.json` quotes the executor
1153
- // transcript verbatim in each expectation's evidence, so whatever the skill
1154
- // read out of its input files ends up here as text — and unlike the
1155
- // workspaces, results/ SURVIVES `--keep` by design.
1156
- mkdirSyncReal(resultsDir, { recursive: true, mode: 0o700 });
2327
+ const isDryRun = opts.dryRun === true;
1157
2328
  // Record what was actually staged & tested (the subject identity + the
1158
2329
  // staged manifest fingerprint + per-entry content hashes) so a run is
1159
2330
  // auditable: which source was resolved, and whether it was rebuilt by run.ts.
@@ -1164,17 +2335,16 @@ export async function runSkillTestHarness(opts) {
1164
2335
  rebuilt: opts.rebuilt === true,
1165
2336
  };
1166
2337
  const provenancePath = safePath.join(resultsDir, 'provenance.json');
1167
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own derived results path
1168
- writeFileSync(provenancePath, JSON.stringify(provenance, null, 2) + '\n', 'utf-8');
2338
+ openResultsDir({ dryRun: isDryRun, resultsDir, provenancePath, provenance });
1169
2339
  process.stderr.write(`Provenance: ${provenance.fingerprint}\n`);
1170
- // Resolve the results/ artifact paths and WIPE any prior run's artifacts up
1171
- // front: harnessRoot is deterministic and may be reused (--keep/--out, or after
1172
- // a crash before cleanup), so a stale grading/friction/baseline.json must never
1173
- // leak into — or be echoed by the finally on — this run. `frictionOut` is
1174
- // hoisted so the finally can echo THIS run's friction (and only after the wipe).
2340
+ // Resolved here, WIPED after the dry-run short-circuit below. harnessRoot is
2341
+ // deterministic and may be reused (--keep/--out, or after a crash before
2342
+ // cleanup), so a stale grading/friction/baseline.json must never leak into — or
2343
+ // be echoed by the finally on — this run; but the wipe is a REAL run's
2344
+ // prerogative, and running it here put it ahead of the dry-run return and of
2345
+ // every remaining pre-pipeline failure point (Step 7.5 token resolution among
2346
+ // them, which is how a FAILING dry run also took the prior artifacts with it).
1175
2347
  const artifacts = resolveArtifactPaths(resultsDir);
1176
- wipeStaleArtifacts(artifacts);
1177
- frictionOut = artifacts.frictionOut;
1178
2348
  // Step 7.5: Resolve the executor's declared test env (Features A + B). Token
1179
2349
  // resolution can hard-fail (exit 2) on an unknown ${token}; do it before the
1180
2350
  // dry-run short-circuit so a dry run validates interpolation too. resolvedAuth
@@ -1200,11 +2370,12 @@ export async function runSkillTestHarness(opts) {
1200
2370
  const concurrency = opts.concurrency ?? DEFAULT_CONCURRENCY;
1201
2371
  const modelFlag = buildModelFlag(knobs.model);
1202
2372
  process.stderr.write(`Model: ${knobs.model ?? '(claude default)'} | grader: ${graderModel}\n`);
1203
- // Step 8: Dry-run short-circuit — return assembled info without spawning.
1204
- if (opts.dryRun === true) {
2373
+ // Step 8: Dry-run short-circuit — return assembled info without spawning, and
2374
+ // without having touched results/ (see Step 7).
2375
+ if (isDryRun) {
1205
2376
  return {
1206
2377
  harnessPath: harnessRoot,
1207
- exitCode: SkillTestExitCode.Ok,
2378
+ exitCode: ExitCode.OK,
1208
2379
  summary: buildDryRunSummary({
1209
2380
  wouldBuild: opts.wouldBuild === true,
1210
2381
  ...(opts.dryRunStagedExistingDist === undefined
@@ -1215,11 +2386,20 @@ export async function runSkillTestHarness(opts) {
1215
2386
  provenanceEntryCount: provenance.entries.length,
1216
2387
  modelFlag,
1217
2388
  evalCount: declaredEvalCount,
2389
+ // The one fact that doubles the spawn count, and the one the preview was
2390
+ // blind to — `opts.baseline` was in scope here the whole time.
2391
+ baseline: opts.baseline === true,
1218
2392
  concurrency,
1219
2393
  graderModel,
2394
+ maxBudgetUsd: knobs.maxBudgetUsd,
1220
2395
  }),
1221
2396
  };
1222
2397
  }
2398
+ // Past the dry-run return, so this is a REAL run: evict any prior run's
2399
+ // artifacts before writing our own. `frictionOut` is assigned WITH the wipe, so
2400
+ // the `finally` can only ever echo THIS run's friction.
2401
+ wipeStaleArtifacts(artifacts);
2402
+ frictionOut = artifacts.frictionOut;
1223
2403
  // Step 9: Run the vat-owned executor→grader pipeline. ONE secret per-run nonce
1224
2404
  // is stamped into every grader prompt (via stdin) and re-verified per fragment
1225
2405
  // on merge — this is what distinguishes a fragment produced by a grader WE
@@ -1242,8 +2422,15 @@ export async function runSkillTestHarness(opts) {
1242
2422
  const evalCtx = {
1243
2423
  subjectStagedDir,
1244
2424
  workspacesRoot,
2425
+ armDirs,
2426
+ harnessRoot,
1245
2427
  pluginDirs,
1246
2428
  graderOutDir,
2429
+ evalSuiteHoldDir,
2430
+ skillContentNeedles: resolveSkillContentNeedles(subjectStagedDir, suite.evals, {
2431
+ workspacesRoot,
2432
+ armDirs,
2433
+ }),
1247
2434
  runNonce,
1248
2435
  graderModel,
1249
2436
  costAccumulator,
@@ -1261,45 +2448,83 @@ export async function runSkillTestHarness(opts) {
1261
2448
  ...(opts.declaredExecutables === undefined ? {} : { declaredExecutables: opts.declaredExecutables }),
1262
2449
  ...(opts.spawn === undefined ? {} : { spawn: opts.spawn }),
1263
2450
  };
1264
- // Tier-ordered, cost-tiered fail-fast (issue #145 Phase G): run evals tier by
2451
+ // Tier-ordered, cost-tiered fail-fast: run evals tier by
1265
2452
  // tier (ascending / cheapest first), bounded-parallel WITHIN each tier, and
1266
2453
  // gate BETWEEN tiers — once a cheaper tier fails a gating expectation, the
1267
2454
  // higher (more expensive) tiers are SKIPPED (never graded, never passed), so a
1268
2455
  // broken foundational expectation stops the run from spending on hard tiers.
1269
2456
  // Each tier is one runPipeline: bounded-parallel, retrying a RateLimitSignal
1270
- // per item. An InternalHarnessError thrown by any executor/grader (timeout/
1271
- // stall/spawn-error, grader failure, missing fragment, nonce mismatch)
1272
- // propagates OUT unhandled → mapErrorToExitCode → exit 1; a spawn or grader
1273
- // break is never laundered into a pass/fail verdict (R1 no-laundering).
1274
- const { fragments, skipped } = await runEvalsTiered({
2457
+ // per item. An InternalHarnessError thrown by a TREATMENT-arm executor/grader
2458
+ // (timeout/stall/spawn-error, grader failure, missing fragment, nonce mismatch,
2459
+ // declared-count mismatch) propagates OUT unhandled → skillTestFailureReason →
2460
+ // ERROR/internal; a spawn or grader break is never laundered into a pass/fail verdict
2461
+ // (R1 no-laundering). The one grader outcome that IS a verdict — a fragment the
2462
+ // model could not write as valid JSON even after one re-grade — fails that eval
2463
+ // (see unparseableGradingFragment). The same break on the CONTROL arm is
2464
+ // recorded and the run continues — see runEvalWorker for why the two arms are
2465
+ // not symmetric.
2466
+ const { fragments, controlFailures, skipped } = await runEvalsTiered({
1275
2467
  evals: suite.evals,
1276
2468
  baseline: opts.baseline === true,
1277
2469
  runTier: (items) => runPipeline({
1278
2470
  items,
1279
2471
  concurrency,
1280
2472
  worker: (item) => runEvalWorker(item, evalCtx),
2473
+ // The worker rethrows every RateLimitSignal so the pipeline can retry; this
2474
+ // is where an EXHAUSTED one stops being a retry and becomes an outcome. Only
2475
+ // the pipeline knows the budget is spent (see rateLimitExhaustedOutcome).
2476
+ onRetriesExhausted: (item, _index, error) => rateLimitExhaustedOutcome(item, error),
2477
+ ...(opts.rateLimitBackoffMs === undefined ? {} : { onRateLimit: opts.rateLimitBackoffMs }),
1281
2478
  }),
1282
2479
  });
1283
2480
  // Step 10: vat merges the grader fragments and writes grading.json/friction.json/
1284
2481
  // tool-eval.json (and baseline.json for a baseline run), then reconciles the
1285
2482
  // authoritative prose-expectation verdict from the WITH-arm per-expectation
1286
2483
  // `passed` flags — NOT any self-reported summary. An executor CLEAN failure
1287
- // reaches here as a FAILing fragment → composite verdict → exit 4.
1288
- const { verdict, toolEval } = writeRunArtifactsAndReconcile(fragments, runNonce, artifacts);
2484
+ // reaches here as a FAILing fragment → composite verdict → exit FINDINGS.
2485
+ const { verdict, toolEval, baselineReport } = writeRunArtifactsAndReconcile({
2486
+ fragments,
2487
+ runNonce,
2488
+ paths: artifacts,
2489
+ signals: contaminationSignalsFor(evalCtx),
2490
+ controlFailures,
2491
+ // `skipped` was in scope here and simply not passed, which is how the delta
2492
+ // block and its printed line came to describe a truncated run as a complete one.
2493
+ skipped,
2494
+ subjectInPlace: opts.subjectInPlace,
2495
+ });
1289
2496
  // D2 fail-closed gate: vat is the SOLE writer of results/, so a missing/unparseable/
1290
- // invalid grading.json, friction.json, or tool-eval.json after the merge is a HARNESS
1291
- // bug (exit 1), never a skill fault. A SEPARATE explicit check, not a prompt invariant.
1292
- assertVatWroteArtifacts(artifacts);
2497
+ // invalid grading.json, friction.json, tool-eval.json — or, on a baseline run,
2498
+ // baseline.json — after the merge is a HARNESS bug (exit 1), never a skill fault.
2499
+ // A SEPARATE explicit check, not a prompt invariant.
2500
+ assertVatWroteArtifacts(artifacts, opts.baseline === true);
2501
+ // Friction FIRST, then the baseline report — the ORDER is the point. Friction is
2502
+ // capped at 50 lines of 2000 characters, so emitting it after the ⚠️ contaminated
2503
+ // banner (which is what the `finally` used to do) could put ~1250 terminal rows
2504
+ // between the operator and the only line saying the delta above is meaningless.
2505
+ // Emitting it here leaves nothing of vat's own between the delta and its caveat;
2506
+ // `emitFrictionOnce` keeps the `finally` call a no-op so a normal run reports it
2507
+ // exactly once. The CLI still appends `Harness:`/`Results:`/`Workspaces:`/
2508
+ // `Summary:` after the harness returns, so the banner is last-but-2-to-4, not
2509
+ // last — a bounded tail, unlike the friction report.
2510
+ emitFrictionOnce();
2511
+ emitBaselineReport(baselineReport);
1293
2512
  // Composite verdict: the run passes only when BOTH output expectations AND every
1294
2513
  // tool-expectation verdict passed — so an output all-pass where a `mustRun` never
1295
- // ran yields FAIL → exit 4. Tool verdicts stay in tool-eval.json (C2); the composite
2514
+ // ran yields FAIL → exit FINDINGS. Tool verdicts stay in tool-eval.json (C2); the composite
1296
2515
  // combines the two channels only here, at the exit-code layer. A fail-fast run
1297
2516
  // that SKIPPED higher tiers is never a pass: skipped ≠ passed forces allPassed
1298
- // false → exit 4 (never downgraded to 0 by the composite path).
2517
+ // false → exit FINDINGS (never downgraded to 0 by the composite path).
1299
2518
  const allPassed = resolveCompositeAllPassed(verdict.allPassed, toolEval, skipped);
1300
2519
  const summary = buildRunSummaryWithSkips(verdict, toolEval, allPassed, skipped, costAccumulator);
1301
2520
  return {
1302
2521
  harnessPath: harnessRoot,
2522
+ // Reported ONLY when the workspaces outlive the run. The `finally` below
2523
+ // runs cleanup BEFORE this value ever reaches a caller, so reporting it
2524
+ // unconditionally handed the operator a path to a directory that no longer
2525
+ // existed. Same predicate as cleanup's, by construction.
2526
+ ...(retainWorkspaces ? { workspacesPath: workspacesRoot } : {}),
2527
+ resultsPath: resultsDir,
1303
2528
  exitCode: verdictExitCode(allPassed, opts.tolerateEvalFailure === true),
1304
2529
  summary,
1305
2530
  };
@@ -1308,8 +2533,10 @@ export async function runSkillTestHarness(opts) {
1308
2533
  // Surface any packaging-fidelity friction VAT merged into friction.json to
1309
2534
  // STDERR BEFORE cleanup removes the harness dir. Best-effort + stderr-only
1310
2535
  // (stdout stays machine-readable); a no-op when the path is unassigned (a
1311
- // pre-merge throw) or friction.json is absent or empty.
1312
- emitFrictionReport(frictionOut);
2536
+ // pre-merge throw) or friction.json is absent or empty. On a run that reached
2537
+ // the verdict this is ALREADY a no-op — the happy path emits it earlier, ahead
2538
+ // of the baseline banner — so what is left here is the throw paths.
2539
+ emitFrictionOnce();
1313
2540
  // Remove the signal handlers first (no listener leak across runs), then run
1314
2541
  // the same cleanup: release the lock, remove the harness dir, and remove the
1315
2542
  // vat-only grader dir (which lives outside harnessRoot).
@@ -1317,4 +2544,62 @@ export async function runSkillTestHarness(opts) {
1317
2544
  cleanup();
1318
2545
  }
1319
2546
  }
2547
+ /**
2548
+ * The test-facing seam — every helper the unit tests reach that is NOT part of
2549
+ * this module's API.
2550
+ *
2551
+ * These used to be forty-four separate `export`s, indistinguishable at the export
2552
+ * keyword from the seven names the skill-test barrel actually publishes, so
2553
+ * "is this API?" was a question for a JSDoc rather than a grep. Now it is a
2554
+ * grep: a name here is reachable by a test and by nothing else. Nothing under
2555
+ * `src/` imports `__internal` and no barrel re-exports it — a dev-tools test
2556
+ * holds both lines. Types stay exported on their own: an object cannot carry
2557
+ * them, and a type is not a call site.
2558
+ */
2559
+ export const __internal = {
2560
+ assertVatWroteArtifacts,
2561
+ buildContaminationInput,
2562
+ buildContaminationSignalsInput,
2563
+ buildEvalWorkItems,
2564
+ buildFlagParseProbe,
2565
+ buildPreflightInput,
2566
+ buildResolveCtx,
2567
+ buildRunSummary,
2568
+ buildRunSummaryWithSkips,
2569
+ computeCompositeVerdict,
2570
+ detectItemPluginLayout,
2571
+ emitFrictionReport,
2572
+ FLAG_PROBE_SENTINEL,
2573
+ formatBaselineReport,
2574
+ formatRunCostSuffix,
2575
+ gradedCounts,
2576
+ helpTextDeclaresFlag,
2577
+ makeStageItem,
2578
+ mintArmWorkspaceDirs,
2579
+ partitionFragmentsByArm,
2580
+ recordSessionCost,
2581
+ rejectedArtifactPath,
2582
+ removeVatOnlyDir,
2583
+ renderPreflightSummary,
2584
+ resolveArtifactPaths,
2585
+ resolveCompositeAllPassed,
2586
+ resolveHarnessLocation,
2587
+ resolveKnobs,
2588
+ resolvePerEvalWorkspaceDir,
2589
+ resolveScaffoldEvalsPath,
2590
+ resolveSkillContentNeedles,
2591
+ resolveStallMs,
2592
+ resolveTimeoutMs,
2593
+ resolveWorkspacesRoot,
2594
+ RETAINED_RESULTS_DIRNAME,
2595
+ stageWorkspacesForRun,
2596
+ subjectSkillName,
2597
+ swallowCleanupFailure,
2598
+ wipeStaleArtifacts,
2599
+ withoutGraderContamination,
2600
+ cleanupHarness,
2601
+ buildStageItems,
2602
+ resolveGraderOutDir,
2603
+ resolveEvalSuitePath,
2604
+ };
1320
2605
  //# sourceMappingURL=run-harness.js.map