@vibe-agent-toolkit/agent-skills 0.2.0-rc.2 → 0.2.0-rc.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/dist/files-config.d.ts +34 -1
  2. package/dist/files-config.d.ts.map +1 -1
  3. package/dist/files-config.js +280 -36
  4. package/dist/files-config.js.map +1 -1
  5. package/dist/fs-attribution.d.ts +45 -0
  6. package/dist/fs-attribution.d.ts.map +1 -0
  7. package/dist/fs-attribution.js +57 -0
  8. package/dist/fs-attribution.js.map +1 -0
  9. package/dist/index.d.ts +6 -5
  10. package/dist/index.d.ts.map +1 -1
  11. package/dist/index.js +9 -3
  12. package/dist/index.js.map +1 -1
  13. package/dist/inventory/detectors/declared-but-missing.d.ts +1 -1
  14. package/dist/inventory/detectors/declared-but-missing.d.ts.map +1 -1
  15. package/dist/inventory/detectors/marketplace-source-missing.d.ts +1 -1
  16. package/dist/inventory/detectors/marketplace-source-missing.d.ts.map +1 -1
  17. package/dist/inventory/detectors/present-but-undeclared.d.ts +1 -1
  18. package/dist/inventory/detectors/present-but-undeclared.d.ts.map +1 -1
  19. package/dist/inventory/detectors/reference-target-missing.d.ts +1 -1
  20. package/dist/inventory/detectors/reference-target-missing.d.ts.map +1 -1
  21. package/dist/inventory/index.d.ts +1 -1
  22. package/dist/inventory/index.d.ts.map +1 -1
  23. package/dist/inventory/index.js +1 -1
  24. package/dist/inventory/index.js.map +1 -1
  25. package/dist/inventory/serialize.d.ts +4 -4
  26. package/dist/inventory/serialize.d.ts.map +1 -1
  27. package/dist/inventory/serialize.js +6 -6
  28. package/dist/inventory/serialize.js.map +1 -1
  29. package/dist/plugin-distribution-layout.d.ts.map +1 -1
  30. package/dist/plugin-distribution-layout.js +3 -1
  31. package/dist/plugin-distribution-layout.js.map +1 -1
  32. package/dist/post-build-checks.d.ts +1 -1
  33. package/dist/post-build-checks.d.ts.map +1 -1
  34. package/dist/post-build-checks.js.map +1 -1
  35. package/dist/projection/skill-extent.d.ts +247 -0
  36. package/dist/projection/skill-extent.d.ts.map +1 -0
  37. package/dist/projection/skill-extent.js +558 -0
  38. package/dist/projection/skill-extent.js.map +1 -0
  39. package/dist/schemas/agent-skill-frontmatter.js +1 -1
  40. package/dist/schemas/agent-skill-frontmatter.js.map +1 -1
  41. package/dist/skill-packager.d.ts +45 -7
  42. package/dist/skill-packager.d.ts.map +1 -1
  43. package/dist/skill-packager.js +123 -25
  44. package/dist/skill-packager.js.map +1 -1
  45. package/dist/skill-source/git-clone.d.ts +1 -1
  46. package/dist/skill-source/git-clone.d.ts.map +1 -1
  47. package/dist/skill-source/git-clone.js +10 -8
  48. package/dist/skill-source/git-clone.js.map +1 -1
  49. package/dist/skill-source/sources/url-source.d.ts.map +1 -1
  50. package/dist/skill-source/sources/url-source.js +2 -1
  51. package/dist/skill-source/sources/url-source.js.map +1 -1
  52. package/dist/skill-test/baseline-delta.d.ts +1575 -0
  53. package/dist/skill-test/baseline-delta.d.ts.map +1 -0
  54. package/dist/skill-test/baseline-delta.js +272 -0
  55. package/dist/skill-test/baseline-delta.js.map +1 -0
  56. package/dist/skill-test/baseline-integrity.d.ts +944 -0
  57. package/dist/skill-test/baseline-integrity.d.ts.map +1 -0
  58. package/dist/skill-test/baseline-integrity.js +3083 -0
  59. package/dist/skill-test/baseline-integrity.js.map +1 -0
  60. package/dist/skill-test/configure-writer.js +1 -1
  61. package/dist/skill-test/configure-writer.js.map +1 -1
  62. package/dist/skill-test/declared-env.d.ts.map +1 -1
  63. package/dist/skill-test/declared-env.js +2 -1
  64. package/dist/skill-test/declared-env.js.map +1 -1
  65. package/dist/skill-test/eval-executor.d.ts +20 -4
  66. package/dist/skill-test/eval-executor.d.ts.map +1 -1
  67. package/dist/skill-test/eval-executor.js +4 -4
  68. package/dist/skill-test/eval-executor.js.map +1 -1
  69. package/dist/skill-test/eval-fragment.d.ts +80 -6
  70. package/dist/skill-test/eval-fragment.d.ts.map +1 -1
  71. package/dist/skill-test/eval-fragment.js +61 -7
  72. package/dist/skill-test/eval-fragment.js.map +1 -1
  73. package/dist/skill-test/eval-grader.d.ts +1 -1
  74. package/dist/skill-test/eval-grader.d.ts.map +1 -1
  75. package/dist/skill-test/eval-grader.js +145 -8
  76. package/dist/skill-test/eval-grader.js.map +1 -1
  77. package/dist/skill-test/eval-inputs.d.ts +107 -6
  78. package/dist/skill-test/eval-inputs.d.ts.map +1 -1
  79. package/dist/skill-test/eval-inputs.js +173 -16
  80. package/dist/skill-test/eval-inputs.js.map +1 -1
  81. package/dist/skill-test/eval-lint.d.ts +13 -0
  82. package/dist/skill-test/eval-lint.d.ts.map +1 -1
  83. package/dist/skill-test/eval-lint.js +17 -4
  84. package/dist/skill-test/eval-lint.js.map +1 -1
  85. package/dist/skill-test/executor-prompt.d.ts +13 -5
  86. package/dist/skill-test/executor-prompt.d.ts.map +1 -1
  87. package/dist/skill-test/executor-prompt.js +8 -5
  88. package/dist/skill-test/executor-prompt.js.map +1 -1
  89. package/dist/skill-test/exit-codes.d.ts +53 -16
  90. package/dist/skill-test/exit-codes.d.ts.map +1 -1
  91. package/dist/skill-test/exit-codes.js +123 -40
  92. package/dist/skill-test/exit-codes.js.map +1 -1
  93. package/dist/skill-test/fragment-merge.d.ts +1 -18
  94. package/dist/skill-test/fragment-merge.d.ts.map +1 -1
  95. package/dist/skill-test/fragment-merge.js +63 -4
  96. package/dist/skill-test/fragment-merge.js.map +1 -1
  97. package/dist/skill-test/grader-prompt.d.ts +27 -7
  98. package/dist/skill-test/grader-prompt.d.ts.map +1 -1
  99. package/dist/skill-test/grader-prompt.js +253 -61
  100. package/dist/skill-test/grader-prompt.js.map +1 -1
  101. package/dist/skill-test/grader-text.d.ts +145 -0
  102. package/dist/skill-test/grader-text.d.ts.map +1 -0
  103. package/dist/skill-test/grader-text.js +410 -0
  104. package/dist/skill-test/grader-text.js.map +1 -0
  105. package/dist/skill-test/grading-adapter.d.ts +32 -0
  106. package/dist/skill-test/grading-adapter.d.ts.map +1 -1
  107. package/dist/skill-test/grading-adapter.js +50 -3
  108. package/dist/skill-test/grading-adapter.js.map +1 -1
  109. package/dist/skill-test/grading-schema.d.ts +160 -5
  110. package/dist/skill-test/grading-schema.d.ts.map +1 -1
  111. package/dist/skill-test/grading-schema.js +66 -7
  112. package/dist/skill-test/grading-schema.js.map +1 -1
  113. package/dist/skill-test/index.d.ts +1 -1
  114. package/dist/skill-test/index.d.ts.map +1 -1
  115. package/dist/skill-test/index.js +1 -1
  116. package/dist/skill-test/index.js.map +1 -1
  117. package/dist/skill-test/lock.d.ts +56 -0
  118. package/dist/skill-test/lock.d.ts.map +1 -1
  119. package/dist/skill-test/lock.js +75 -2
  120. package/dist/skill-test/lock.js.map +1 -1
  121. package/dist/skill-test/pipeline.d.ts +16 -0
  122. package/dist/skill-test/pipeline.d.ts.map +1 -1
  123. package/dist/skill-test/pipeline.js +16 -9
  124. package/dist/skill-test/pipeline.js.map +1 -1
  125. package/dist/skill-test/preflight.d.ts +1 -1
  126. package/dist/skill-test/preflight.d.ts.map +1 -1
  127. package/dist/skill-test/preflight.js +59 -15
  128. package/dist/skill-test/preflight.js.map +1 -1
  129. package/dist/skill-test/run-harness.d.ts +578 -35
  130. package/dist/skill-test/run-harness.d.ts.map +1 -1
  131. package/dist/skill-test/run-harness.js +1281 -148
  132. package/dist/skill-test/run-harness.js.map +1 -1
  133. package/dist/skill-test/staging.d.ts +8 -2
  134. package/dist/skill-test/staging.d.ts.map +1 -1
  135. package/dist/skill-test/staging.js +8 -4
  136. package/dist/skill-test/staging.js.map +1 -1
  137. package/dist/skill-test/tool-eval-schema.d.ts +6 -6
  138. package/dist/test-input.d.ts +57 -6
  139. package/dist/test-input.d.ts.map +1 -1
  140. package/dist/test-input.js +35 -12
  141. package/dist/test-input.js.map +1 -1
  142. package/dist/validators/agent-instruction-presence.d.ts +1 -1
  143. package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
  144. package/dist/validators/agent-instruction-presence.js +2 -1
  145. package/dist/validators/agent-instruction-presence.js.map +1 -1
  146. package/dist/validators/bundled-resource-link-detection.d.ts +1 -1
  147. package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
  148. package/dist/validators/bundled-resource-link-detection.js +1 -1
  149. package/dist/validators/bundled-resource-link-detection.js.map +1 -1
  150. package/dist/validators/compat-detectors.d.ts +1 -1
  151. package/dist/validators/compat-detectors.d.ts.map +1 -1
  152. package/dist/validators/compat-detectors.js +1 -1
  153. package/dist/validators/compat-detectors.js.map +1 -1
  154. package/dist/validators/cross-skill-dependency-detection.d.ts +1 -1
  155. package/dist/validators/cross-skill-dependency-detection.d.ts.map +1 -1
  156. package/dist/validators/cross-skill-dependency-detection.js +1 -1
  157. package/dist/validators/cross-skill-dependency-detection.js.map +1 -1
  158. package/dist/validators/description-style-detection.d.ts +1 -1
  159. package/dist/validators/description-style-detection.d.ts.map +1 -1
  160. package/dist/validators/description-style-detection.js +1 -1
  161. package/dist/validators/description-style-detection.js.map +1 -1
  162. package/dist/validators/frontmatter-validation.d.ts +1 -1
  163. package/dist/validators/frontmatter-validation.d.ts.map +1 -1
  164. package/dist/validators/frontmatter-validation.js +1 -1
  165. package/dist/validators/frontmatter-validation.js.map +1 -1
  166. package/dist/validators/imperative-body-detection.d.ts +1 -1
  167. package/dist/validators/imperative-body-detection.d.ts.map +1 -1
  168. package/dist/validators/imperative-body-detection.js +1 -1
  169. package/dist/validators/imperative-body-detection.js.map +1 -1
  170. package/dist/validators/index.d.ts +1 -1
  171. package/dist/validators/index.d.ts.map +1 -1
  172. package/dist/validators/kebab-case-detection.d.ts +1 -1
  173. package/dist/validators/kebab-case-detection.d.ts.map +1 -1
  174. package/dist/validators/kebab-case-detection.js +1 -1
  175. package/dist/validators/kebab-case-detection.js.map +1 -1
  176. package/dist/validators/marketplace-validator.js +1 -1
  177. package/dist/validators/marketplace-validator.js.map +1 -1
  178. package/dist/validators/packaging-validator.d.ts +96 -7
  179. package/dist/validators/packaging-validator.d.ts.map +1 -1
  180. package/dist/validators/packaging-validator.js +148 -21
  181. package/dist/validators/packaging-validator.js.map +1 -1
  182. package/dist/validators/plugin-hosted-shape.d.ts +1 -1
  183. package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
  184. package/dist/validators/plugin-hosted-shape.js +1 -1
  185. package/dist/validators/plugin-hosted-shape.js.map +1 -1
  186. package/dist/validators/plugin-recommended-fields.d.ts +1 -1
  187. package/dist/validators/plugin-recommended-fields.d.ts.map +1 -1
  188. package/dist/validators/plugin-recommended-fields.js +1 -1
  189. package/dist/validators/plugin-recommended-fields.js.map +1 -1
  190. package/dist/validators/registry-validator.js +1 -1
  191. package/dist/validators/registry-validator.js.map +1 -1
  192. package/dist/validators/rule-engine/rule-engine.d.ts +1 -1
  193. package/dist/validators/rule-engine/rule-engine.d.ts.map +1 -1
  194. package/dist/validators/rule-engine/rule-engine.js +20 -9
  195. package/dist/validators/rule-engine/rule-engine.js.map +1 -1
  196. package/dist/validators/skill-validator.d.ts.map +1 -1
  197. package/dist/validators/skill-validator.js +21 -3
  198. package/dist/validators/skill-validator.js.map +1 -1
  199. package/dist/validators/types.d.ts +1 -1
  200. package/dist/validators/types.d.ts.map +1 -1
  201. package/dist/validators/unified-validator.js +1 -1
  202. package/dist/validators/unified-validator.js.map +1 -1
  203. package/dist/validators/validation-rules.d.ts +1 -1
  204. package/dist/validators/validation-rules.d.ts.map +1 -1
  205. package/dist/validators/walker-to-issues.d.ts +7 -3
  206. package/dist/validators/walker-to-issues.d.ts.map +1 -1
  207. package/dist/validators/walker-to-issues.js +37 -8
  208. package/dist/validators/walker-to-issues.js.map +1 -1
  209. package/dist/walk-link-graph.d.ts +2 -1
  210. package/dist/walk-link-graph.d.ts.map +1 -1
  211. package/dist/walk-link-graph.js +37 -2
  212. package/dist/walk-link-graph.js.map +1 -1
  213. package/package.json +7 -7
@@ -1 +1 @@
1
- {"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wGAAwG;AACxG,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAE3C;+EAC+E;AAC/E,MAAM,sBAAsB,GAAG;IAC7B,IAAI;IACJ,QAAQ;IACR,iBAAiB;IACjB,OAAO;IACP,cAAc;IACd,MAAM;IACN,kBAAkB;CACV,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,MAAc;IAC1D,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IACtB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IACzB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe;IAC/E,IAAI,EAAE,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAC9E,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB;AACpE,CAAC;AAED,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,gFAAgF;AAChF,uEAAuE;AACvE,2EAA2E;AAC3E,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;QAChB,CAAC;aACE,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,KAAK,CACJ,eAAe,EACf,wGAAwG,CACzG;KACJ,CAAC;IACF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,2EAA2E;IAC3E,2EAA2E;IAC3E,qEAAqE;IACrE,4EAA4E;IAC5E,4EAA4E;IAC5E,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,wFAAwF;IACxF,mEAAmE;IACnE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAC/C,4EAA4E;IAC5E,kFAAkF;IAClF,qFAAqF;IACrF,+BAA+B;IAC/B,gBAAgB,EAAE,CAAC;SAChB,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC9C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QACjD,wEAAwE;QACxE,oEAAoE;QACpE,6DAA6D;QAC7D,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAClD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KAChD,CAAC;SACD,MAAM,EAAE;SACR,QAAQ,EAAE;CACd,CAAC;KACD,WAAW,EAAE;KACb,WAAW,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;IAC1B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAK,sBAA4C,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC9G,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;gBAC3B,OAAO,EAAE,uBAAuB,GAAG,qBAAqB,IAAI,kDAAkD;gBAC9G,IAAI,EAAE,CAAC,GAAG,CAAC;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,WAAW,EAAE,CAAC;AAajB,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,IAAI,cAAc,CAAC,iCAAkC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,cAAc,CAAC,wCAAwC,MAAM,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;IAC3F,CAAC;IACD,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,cAAc,CAAC,4FAA4F,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAWD;;;;;GAKG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QACpE,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACrF,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YAC9B,yEAAyE;YACzE,wEAAwE;YACxE,IAAI,GAAW,CAAC;YAChB,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAClD,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,GAAG,sCAAuC,GAAa,CAAC,OAAO,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,qJAAqJ;YACrJ,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,cAAc,CAAC,QAAQ,KAAK,CAAC,EAAE,yBAAyB,GAAG,yBAAyB,GAAG,EAAE,CAAC,CAAC;YACvG,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,CAAC;gBACH,0EAA0E;gBAC1E,kDAAkD;gBAClD,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC3E,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,gCAAgC,GAAG,yBAA0B,GAAa,CAAC,OAAO,EAAE,CACrG,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,cAAc,CAAC;AAC9B,CAAC"}
1
+ {"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,kBAAkB,CAAC;AAEnF;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,cAAe,SAAQ,KAAK;IAC9B,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,2BAA2B,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAE3C;+EAC+E;AAC/E,MAAM,sBAAsB,GAAG;IAC7B,IAAI;IACJ,QAAQ;IACR,iBAAiB;IACjB,OAAO;IACP,cAAc;IACd,MAAM;IACN,kBAAkB;CACV,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,MAAc;IAC1D,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IACtB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IACzB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe;IAC/E,IAAI,EAAE,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAC9E,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB;AACpE,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,6BAA6B,CAAC,YAAsB,EAAE,GAAoB;IACjF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACtB,SAAS;QACX,CAAC;QACD,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;YAC3B,OAAO,EACL,0BAA0B,cAAc,CAAC,WAAW,CAAC,4CAA4C;gBACjG,8FAA8F;gBAC9F,yEAAyE;YAC3E,IAAI,EAAE,CAAC,cAAc,EAAE,KAAK,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,gFAAgF;AAChF,uEAAuE;AACvE,2EAA2E;AAC3E,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;QAChB,CAAC;aACE,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,KAAK,CACJ,eAAe,EACf,wGAAwG,CACzG;KACJ,CAAC;IACF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,2EAA2E;IAC3E,2EAA2E;IAC3E,qEAAqE;IACrE,4EAA4E;IAC5E,4EAA4E;IAC5E,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,wFAAwF;IACxF,mEAAmE;IACnE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAC/C,4EAA4E;IAC5E,kFAAkF;IAClF,qFAAqF;IACrF,+BAA+B;IAC/B,gBAAgB,EAAE,CAAC;SAChB,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC9C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QACjD,wEAAwE;QACxE,oEAAoE;QACpE,6DAA6D;QAC7D,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAClD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KAChD,CAAC;SACD,MAAM,EAAE;SACR,QAAQ,EAAE;CACd,CAAC;KACD,WAAW,EAAE;KACb,WAAW,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;IAC1B,6BAA6B,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IACvD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAK,sBAA4C,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC9G,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;gBAC3B,uEAAuE;gBACvE,iEAAiE;gBACjE,OAAO,EAAE,uBAAuB,cAAc,CAAC,GAAG,CAAC,qBAAqB,IAAI,kDAAkD;gBAC9H,IAAI,EAAE,CAAC,GAAG,CAAC;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,WAAW,EAAE,CAAC;AAajB,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,wEAAwE;QACxE,4EAA4E;QAC5E,uEAAuE;QACvE,MAAM,IAAI,cAAc,CAAC,iCAAiC,cAAc,CAAE,CAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,cAAc,CAAC,wCAAwC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;IACD,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,cAAc,CAAC,4FAA4F,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAwBD,sEAAsE;AACtE,MAAM,UAAU,MAAM,CAAC,IAAsB;IAC3C,OAAO,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAE,CAAC,MAAM,CAAW,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,SAAS,CAAW,CAAC;AAC3F,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,IAAsB,EAAE,GAAY;IAChE,MAAM,OAAO,GAAG,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;IAC1D,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,cAAc,CACtB,wDAAwD,GAAG,SAAS;YAClE,kFAAkF;YAClF,mDAAmD,CACtD,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAiBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,yBAAyB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC,cAAc,CAAC;AAC9B,CAAC;AAED,SAAS,yBAAyB,CAAC,KAA+B,EAAE,GAAY;IAC9E,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAChG,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,0EAA0E;QAC1E,2BAA2B;QAC3B,aAAa,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACpC,yEAAyE;YACzE,wEAAwE;YACxE,IAAI,GAAW,CAAC;YAChB,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAClD,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,cAAc,CAAC,GAAG,CAAC,qCAAqC;oBAC/F,cAAc,CAAE,GAAa,CAAC,OAAO,CAAC,CACzC,CAAC;YACJ,CAAC;YACD,qJAAqJ;YACrJ,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,cAAc,CAAC,GAAG,CAAC,yBAAyB,cAAc,CAAC,GAAG,CAAC,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,CAAC;gBACH,0EAA0E;gBAC1E,kDAAkD;gBAClD,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC3E,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,gCAAgC,cAAc,CAAC,GAAG,CAAC,wBAAwB;oBACzF,cAAc,CAAE,GAAa,CAAC,OAAO,CAAC,CACzC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
@@ -46,6 +46,19 @@ export declare function lintEvalExpectations(evals: EvalEntry[]): EvalLintWarnin
46
46
  * not close to any declared name (a deliberate built-in/system tool reference), is
47
47
  * never flagged. When the skill declares no executables there is nothing to compare
48
48
  * against, so it returns no warnings. Pure + side-effect free.
49
+ *
50
+ * ⚠️ EVERY INTERPOLATED NAME IS NEUTRALIZED WITH {@link quoteSuiteText}, at the point
51
+ * of CONSTRUCTION rather than at the stderr write, so the message is safe wherever it
52
+ * is later carried. Both populations here are untrusted: `toolExpectations.*` is
53
+ * `z.array(z.string().min(1))` with no charset constraint on a `.passthrough()` entry,
54
+ * and `resolveEvalSuitePath` will harvest the suite out of a FETCHED artifact — i.e.
55
+ * out of the skill under test; the declared names come from that same skill's
56
+ * manifest. `run-harness.ts` writes this text with no sanitizer of its own, and does
57
+ * it at Step 5.5 — AHEAD of the `--i-understand-this-runs-skill-code` gate and ahead
58
+ * of the `--dry-run` short-circuit, so an unsanitized name repaints the terminal of a
59
+ * run that spawned nothing and that the operator believes never executed skill code.
60
+ * Matching still happens on the RAW value, so neutralization cannot change which
61
+ * names are flagged.
49
62
  */
50
63
  export declare function lintToolExpectationExecutables(evals: EvalEntry[], declaredExecutableNames: string[]): EvalLintWarning[];
51
64
  //# sourceMappingURL=eval-lint.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"eval-lint.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAoB,KAAK,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEpE,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAsBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,eAAe,EAAE,CAgB1E;AA0DD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,SAAS,EAAE,EAClB,uBAAuB,EAAE,MAAM,EAAE,GAChC,eAAe,EAAE,CAuBnB"}
1
+ {"version":3,"file":"eval-lint.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAoC,KAAK,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEpF,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAsBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,eAAe,EAAE,CAgB1E;AA0DD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,SAAS,EAAE,EAClB,uBAAuB,EAAE,MAAM,EAAE,GAChC,eAAe,EAAE,CAuBnB"}
@@ -11,7 +11,7 @@
11
11
  * than a missed one, since the grader (an LLM judging the transcript) is the real
12
12
  * backstop for eval quality.
13
13
  */
14
- import { isSingleEditAway } from './eval-inputs.js';
14
+ import { isSingleEditAway, quoteSuiteText } from './eval-inputs.js';
15
15
  // An expectation "looks like" a pure presence/mention check when it uses one of
16
16
  // these verbs — the vocabulary authors reach for when asserting "X shows up
17
17
  // somewhere in the output" without saying what a WRONG output would look like.
@@ -127,13 +127,26 @@ function probableDeclaredTypoTarget(name, declared) {
127
127
  * not close to any declared name (a deliberate built-in/system tool reference), is
128
128
  * never flagged. When the skill declares no executables there is nothing to compare
129
129
  * against, so it returns no warnings. Pure + side-effect free.
130
+ *
131
+ * ⚠️ EVERY INTERPOLATED NAME IS NEUTRALIZED WITH {@link quoteSuiteText}, at the point
132
+ * of CONSTRUCTION rather than at the stderr write, so the message is safe wherever it
133
+ * is later carried. Both populations here are untrusted: `toolExpectations.*` is
134
+ * `z.array(z.string().min(1))` with no charset constraint on a `.passthrough()` entry,
135
+ * and `resolveEvalSuitePath` will harvest the suite out of a FETCHED artifact — i.e.
136
+ * out of the skill under test; the declared names come from that same skill's
137
+ * manifest. `run-harness.ts` writes this text with no sanitizer of its own, and does
138
+ * it at Step 5.5 — AHEAD of the `--i-understand-this-runs-skill-code` gate and ahead
139
+ * of the `--dry-run` short-circuit, so an unsanitized name repaints the terminal of a
140
+ * run that spawned nothing and that the operator believes never executed skill code.
141
+ * Matching still happens on the RAW value, so neutralization cannot change which
142
+ * names are flagged.
130
143
  */
131
144
  export function lintToolExpectationExecutables(evals, declaredExecutableNames) {
132
145
  const warnings = [];
133
146
  const declared = declaredExecutableNames.map((original) => ({ original, normalized: normalizeExecutableName(original) }));
134
147
  if (declared.length === 0)
135
148
  return warnings;
136
- const declaredList = declared.map((d) => d.original).join(', ');
149
+ const declaredList = declared.map((d) => quoteSuiteText(d.original)).join(', ');
137
150
  for (const entry of evals) {
138
151
  const evalId = String(entry.id);
139
152
  const seen = new Set();
@@ -146,8 +159,8 @@ export function lintToolExpectationExecutables(evals, declaredExecutableNames) {
146
159
  continue;
147
160
  warnings.push({
148
161
  evalId,
149
- message: `eval "${evalId}": toolExpectation references executable "${name}", which no declared executable matches — ` +
150
- `did you mean "${target}"? A name that never matches a real tool makes mustRun/mustSucceed/sequence fail ` +
162
+ message: `eval "${evalId}": toolExpectation references executable "${quoteSuiteText(name)}", which no declared executable matches — ` +
163
+ `did you mean "${quoteSuiteText(target)}"? A name that never matches a real tool makes mustRun/mustSucceed/sequence fail ` +
151
164
  `for the wrong reason (or mustNotRun pass vacuously). Declared executables: ${declaredList}.`,
152
165
  });
153
166
  }
@@ -1 +1 @@
1
- {"version":3,"file":"eval-lint.js","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAkB,MAAM,kBAAkB,CAAC;AAQpE,gFAAgF;AAChF,4EAA4E;AAC5E,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,2EAA2E,CAAC;AAE1G,iFAAiF;AACjF,kFAAkF;AAClF,0EAA0E;AAC1E,4EAA4E;AAC5E,2EAA2E;AAC3E,kFAAkF;AAClF,6DAA6D;AAC7D,MAAM,sBAAsB,GAC1B,+GAA+G,CAAC;AAElH,0IAA0I;AAC1I,SAAS,yBAAyB,CAAC,WAAmB;IACpD,OAAO,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAC9F,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC9C,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS;YAAE,SAAS;QACnD,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,yBAAyB,CAAC;YAAE,SAAS;QACnE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM;YACN,OAAO,EACL,SAAS,MAAM,uFAAuF;gBACtG,yGAAyG;gBACzG,8CAA8C;SACjD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAE9E,6EAA6E;AAC7E,4FAA4F;AAC5F,oFAAoF;AACpF,gFAAgF;AAChF,iFAAiF;AACjF,SAAS;AACT,MAAM,gBAAgB,GAAG,0CAA0C,CAAC;AAEpE,yHAAyH;AACzH,SAAS,uBAAuB,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACtF,CAAC;AAED,mHAAmH;AACnH,SAAS,iCAAiC,CAAC,KAAgB;IACzD,MAAM,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;IAClC,IAAI,EAAE,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,SAAiB,EAAE,IAAY;IACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpG,OAAO,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,0BAA0B,CACjC,IAAY,EACZ,QAAoD;IAEpD,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC,CAAC,iCAAiC;IAC1G,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QAClE,IAAI,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,eAAe,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;IAChH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,8BAA8B,CAC5C,KAAkB,EAClB,uBAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1H,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC3C,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChE,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,iCAAiC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5D,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC1D,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM;gBACN,OAAO,EACL,SAAS,MAAM,6CAA6C,IAAI,4CAA4C;oBAC5G,iBAAiB,MAAM,mFAAmF;oBAC1G,8EAA8E,YAAY,GAAG;aAChG,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
1
+ {"version":3,"file":"eval-lint.js","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAQpF,gFAAgF;AAChF,4EAA4E;AAC5E,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,2EAA2E,CAAC;AAE1G,iFAAiF;AACjF,kFAAkF;AAClF,0EAA0E;AAC1E,4EAA4E;AAC5E,2EAA2E;AAC3E,kFAAkF;AAClF,6DAA6D;AAC7D,MAAM,sBAAsB,GAC1B,+GAA+G,CAAC;AAElH,0IAA0I;AAC1I,SAAS,yBAAyB,CAAC,WAAmB;IACpD,OAAO,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAC9F,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC9C,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS;YAAE,SAAS;QACnD,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,yBAAyB,CAAC;YAAE,SAAS;QACnE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM;YACN,OAAO,EACL,SAAS,MAAM,uFAAuF;gBACtG,yGAAyG;gBACzG,8CAA8C;SACjD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAE9E,6EAA6E;AAC7E,4FAA4F;AAC5F,oFAAoF;AACpF,gFAAgF;AAChF,iFAAiF;AACjF,SAAS;AACT,MAAM,gBAAgB,GAAG,0CAA0C,CAAC;AAEpE,yHAAyH;AACzH,SAAS,uBAAuB,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACtF,CAAC;AAED,mHAAmH;AACnH,SAAS,iCAAiC,CAAC,KAAgB;IACzD,MAAM,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;IAClC,IAAI,EAAE,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,SAAiB,EAAE,IAAY;IACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpG,OAAO,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,0BAA0B,CACjC,IAAY,EACZ,QAAoD;IAEpD,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC,CAAC,iCAAiC;IAC1G,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QAClE,IAAI,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,eAAe,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;IAChH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,UAAU,8BAA8B,CAC5C,KAAkB,EAClB,uBAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1H,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC3C,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChF,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,iCAAiC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5D,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC1D,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM;gBACN,OAAO,EACL,SAAS,MAAM,6CAA6C,cAAc,CAAC,IAAI,CAAC,4CAA4C;oBAC5H,iBAAiB,cAAc,CAAC,MAAM,CAAC,mFAAmF;oBAC1H,8EAA8E,YAAY,GAAG;aAChG,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
@@ -1,9 +1,17 @@
1
1
  export interface BuildExecutorPromptOptions {
2
2
  /** The eval's `prompt` field, verbatim — the task the executor must perform. */
3
3
  task: string;
4
- /** Path to the staged subject (skill/plugin under test). */
5
- subjectPath: string;
6
- /** `<workspacesRoot>/<id>` when the eval declares input `files` — the project the executor should operate on. */
4
+ /**
5
+ * Path to the staged subject (skill/plugin under test).
6
+ *
7
+ * OMITTED for the skill-absent (WITHOUT) arm of a `--baseline` run. Naming it
8
+ * there would hand the control arm the exact thing the arm exists to withhold:
9
+ * the staged dir holds both the skill's SKILL.md and any executable it ships,
10
+ * so a control arm told this path can recover the whole treatment with one
11
+ * `cat`/`ls`. The A/B is only a measurement while this stays absent.
12
+ */
13
+ subjectPath?: string;
14
+ /** The eval's working directory — the project the executor should operate on. */
7
15
  workspaceDir?: string;
8
16
  }
9
17
  /**
@@ -15,8 +23,8 @@ export interface BuildExecutorPromptOptions {
15
23
  * knowledge would change its behavior and invalidate the eval.
16
24
  *
17
25
  * `subjectPath` is where the skill/plugin under test is staged (context the
18
- * executor's environment provides); `workspaceDir`, when the eval declares
19
- * input `files`, is the project the executor is actually meant to operate on.
26
+ * executor's environment provides) and is OMITTED for the skill-absent arm;
27
+ * `workspaceDir` is the project the executor is actually meant to operate on.
20
28
  *
21
29
  * Pure function: same inputs always produce the same prompt.
22
30
  */
@@ -1 +1 @@
1
- {"version":3,"file":"executor-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/executor-prompt.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,0BAA0B;IACzC,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAC;IACpB,iHAAiH;IACjH,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,0BAA0B,GAAG,MAAM,CAM5E;AAWD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,8BAA8B,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAiBjF"}
1
+ {"version":3,"file":"executor-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/executor-prompt.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,0BAA0B;IACzC,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iFAAiF;IACjF,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,0BAA0B,GAAG,MAAM,CAS5E;AAWD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,8BAA8B,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAiBjF"}
@@ -8,17 +8,20 @@ import { PromptInvariantError } from './prompt-invariants.js';
8
8
  * knowledge would change its behavior and invalidate the eval.
9
9
  *
10
10
  * `subjectPath` is where the skill/plugin under test is staged (context the
11
- * executor's environment provides); `workspaceDir`, when the eval declares
12
- * input `files`, is the project the executor is actually meant to operate on.
11
+ * executor's environment provides) and is OMITTED for the skill-absent arm;
12
+ * `workspaceDir` is the project the executor is actually meant to operate on.
13
13
  *
14
14
  * Pure function: same inputs always produce the same prompt.
15
15
  */
16
16
  export function buildExecutorPrompt(opts) {
17
- const lines = [opts.task, '', `The relevant files are located at ${opts.subjectPath}.`];
17
+ const context = [];
18
+ if (opts.subjectPath !== undefined) {
19
+ context.push(`The relevant files are located at ${opts.subjectPath}.`);
20
+ }
18
21
  if (opts.workspaceDir !== undefined) {
19
- lines.push(`Your working directory is ${opts.workspaceDir} — the project to operate on.`);
22
+ context.push(`Your working directory is ${opts.workspaceDir} — the project to operate on.`);
20
23
  }
21
- return lines.join('\n');
24
+ return context.length === 0 ? opts.task : [opts.task, '', ...context].join('\n');
22
25
  }
23
26
  /**
24
27
  * Phrases that, if present in an executor prompt, would break the blind
@@ -1 +1 @@
1
- {"version":3,"file":"executor-prompt.js","sourceRoot":"","sources":["../../src/skill-test/executor-prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAW9D;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAgC;IAClE,MAAM,KAAK,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,qCAAqC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IACxF,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,6BAA6B,IAAI,CAAC,YAAY,+BAA+B,CAAC,CAAC;IAC5F,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,iBAAiB,GAAG,CAAC,cAAc,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,SAAS,CAAC,CAAC;AAEpG;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,8BAA8B,CAAC,MAAc,EAAE,IAAY;IACzE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACvB,MAAM,IAAI,oBAAoB,CAAC,4CAA4C,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,oBAAoB,CAAC,yDAAyD,CAAC,CAAC;IAC5F,CAAC;IACD,2EAA2E;IAC3E,4EAA4E;IAC5E,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/D,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;QACxC,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,oBAAoB,CAC5B,oFAAoF,OAAO,IAAI,CAChG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"executor-prompt.js","sourceRoot":"","sources":["../../src/skill-test/executor-prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAmB9D;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAgC;IAClE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,qCAAqC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,6BAA6B,IAAI,CAAC,YAAY,+BAA+B,CAAC,CAAC;IAC9F,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACnF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,iBAAiB,GAAG,CAAC,cAAc,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,SAAS,CAAC,CAAC;AAEpG;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,8BAA8B,CAAC,MAAc,EAAE,IAAY;IACzE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACvB,MAAM,IAAI,oBAAoB,CAAC,4CAA4C,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,oBAAoB,CAAC,yDAAyD,CAAC,CAAC;IAC5F,CAAC;IACD,2EAA2E;IAC3E,4EAA4E;IAC5E,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/D,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;QACxC,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,oBAAoB,CAC5B,oFAAoF,OAAO,IAAI,CAChG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC"}
@@ -26,9 +26,28 @@ export type SkillTestExitCodeValue = (typeof SkillTestExitCode)[keyof typeof Ski
26
26
  * annotated starter template.
27
27
  *
28
28
  * In a real run the harness has already written that template, and the message
29
- * says so. Under `--dry-run` nothing is written — a dry run must never touch the
30
- * filesystem — so the message instead describes what a real run *would* scaffold
31
- * and where, while surfacing the same exit-3 "bootstrap needed" signal.
29
+ * says so. Under `--dry-run` the TEMPLATE is not written, so the message instead
30
+ * describes what a real run *would* scaffold and where, while surfacing the same
31
+ * exit-3 "bootstrap needed" signal.
32
+ *
33
+ * That scope is exact, and it used to read "a dry run must never touch the
34
+ * filesystem", which is not true of this command. A dry run is not a read-only
35
+ * mode: it takes the harness lock, creates the harness root, stages BOTH arms'
36
+ * per-eval workspaces, and — once `--i-understand-this-runs-skill-code` is passed —
37
+ * runs the repo's `test.build` hook and builds the subject. What it never does is
38
+ * SPAWN: no executor session, no grader session, no tokens.
39
+ *
40
+ * It also never touches `results/`: it neither creates the directory, nor writes
41
+ * `provenance.json`, nor wipes the previous run's artifacts. That last one is the
42
+ * reason the ordering matters — the wipe used to sit AHEAD of the dry-run
43
+ * short-circuit, so a free `--dry-run` (or any failure after it) destroyed the
44
+ * `grading.json` / `baseline.json` / `friction.json` / `tool-eval.json` of the
45
+ * expensive real run an operator was about to read. The summary now names the
46
+ * provenance path it WOULD write rather than writing it.
47
+ *
48
+ * The scaffold is the other filesystem effect a dry run deliberately withholds,
49
+ * and for a different reason: that write lands in the AUTHOR's source
50
+ * tree rather than in vat's own scratch space.
32
51
  */
33
52
  export declare class BootstrapNeededError extends Error {
34
53
  readonly expectedPath: string;
@@ -74,19 +93,37 @@ export declare class InternalHarnessError extends Error {
74
93
  constructor(message: string);
75
94
  }
76
95
  /**
77
- * Map any thrown error to the process exit code. Errors that carry their own
78
- * `exitCode` (Bootstrap/Auth/HarnessLocation/Internal) are authoritative;
79
- * a PromptInvariantError is a defense-in-depth failure over VAT's OWN generated
80
- * executor/grader prompts (a required safety directive missing from a prompt VAT
81
- * built) — surfaced as a user-visible preflight-class problem → 2; a BuildHookError
82
- * is a pre-stage build failure → 2; a SkillBuildError is a declared-skill build
83
- * failure → 2; a SecurityAckError is a missing security ack before a build → 2;
84
- * an UnknownEnvTokenError is a bad ${token} in a
85
- * declared env value → 2; a DuplicateStagedSkillError is a staged-name collision
86
- * across subject/--with/--with-optional → 2; GradingSkewError (aggregate grading.json shape skew),
87
- * EvalFragmentError (per-eval grader fragment parse failure), and
88
- * GradingNonceError (forged/mismatched per-fragment grader nonce) are all → 1;
89
- * everything unknown → 1.
96
+ * Map any thrown error to the process exit code.
97
+ *
98
+ * ⚠️ THE `exitCode` FIELD IS THE PRIMARY MECHANISM, and until this rewrite it was
99
+ * not a mechanism at all. Ten error classes across five modules carried a
100
+ * `readonly exitCode`, this docblock called them "authoritative", and NOTHING read
101
+ * one: dispatch was a hand-maintained `instanceof` chain that happened to agree with
102
+ * every field. Deleting any of those fields changed nothing, so each new class added
103
+ * a fifth, then a sixth member to an inert pattern — and the two lists could have
104
+ * disagreed at any time with no test able to tell.
105
+ *
106
+ * So: an error that declares one of the three throwable codes ({@link
107
+ * selfDeclaredExitCode}) decides its own exit code, and what remains of the
108
+ * `instanceof` chain is exactly ONE row — `PromptInvariantError` (a
109
+ * defense-in-depth failure over VAT's OWN generated executor/grader prompts: a
110
+ * required safety directive missing from a prompt VAT built, surfaced as a
111
+ * user-visible preflight-class problem) → 2, the only class that declares nothing and
112
+ * still needs a code other than the default.
113
+ *
114
+ * `GradingSkewError` (aggregate grading.json shape skew), `EvalFragmentError`
115
+ * (per-eval grader fragment parse failure) and `GradingNonceError`
116
+ * (forged/mismatched per-fragment grader nonce) reach 1 through the FALLBACK, not
117
+ * through a row — they are indistinguishable here from `new Error('boom')`. That is
118
+ * correct (1 is what they want) but it is not dispatch, and saying otherwise implies
119
+ * a pin that does not exist: deleting any of the three changes nothing. Everything
120
+ * unknown → 1.
121
+ *
122
+ * A side benefit of reading the field: it works across a `src`/`dist` boundary, where
123
+ * `instanceof` silently cannot — a `dist` copy of a class never matches a `src`
124
+ * instance of it, and this module is imported from both. The surviving
125
+ * `PromptInvariantError` row carries exactly that risk; it is kept because the class
126
+ * declares no field to read.
90
127
  *
91
128
  * Note: SkillTestExitCode.EvalFailure (4) is NOT produced here — it is an
92
129
  * outcome of a completed run (an eval failed, the fail-closed default unless
@@ -1 +1 @@
1
- {"version":3,"file":"exit-codes.d.ts","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AASA,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;;;IAK5B;;;;;;;;;;;;OAYG;;CAEK,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAEhG;;;;;;;;;GASG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAEjB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE;CAQ9E;AAED,iHAAiH;AACjH,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;;CAKhC;AAED;;;;;;;;;;GAUG;AACH,qBAAa,yBAA0B,SAAQ,KAAK;aAEtB,SAAS,EAAE,MAAM;IAD7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,SAAS,EAAE,MAAM;CAO9C;AAED,+HAA+H;AAC/H,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAuBvD"}
1
+ {"version":3,"file":"exit-codes.d.ts","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAEA,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;;;IAK5B;;;;;;;;;;;;OAYG;;CAEK,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAEhG;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAEjB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE;CAQ9E;AAED,iHAAiH;AACjH,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;;CAKhC;AAED;;;;;;;;;;GAUG;AACH,qBAAa,yBAA0B,SAAQ,KAAK;aAEtB,SAAS,EAAE,MAAM;IAD7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,SAAS,EAAE,MAAM;CAO9C;AAED,+HAA+H;AAC/H,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAmED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAKvD"}
@@ -1,9 +1,3 @@
1
- import { AuthPreflightError } from '@vibe-agent-toolkit/utils';
2
- import { BuildHookError } from './build-hook.js';
3
- import { UnknownEnvTokenError } from './declared-env.js';
4
- import { EvalFragmentError } from './eval-fragment.js';
5
- import { GradingNonceError, GradingSkewError } from './grading-adapter.js';
6
- import { HarnessLocationError } from './harness-location.js';
7
1
  import { PromptInvariantError } from './prompt-invariants.js';
8
2
  /** Exit codes for `vat skill test` (spec §6d). */
9
3
  export const SkillTestExitCode = {
@@ -32,16 +26,35 @@ export const SkillTestExitCode = {
32
26
  * annotated starter template.
33
27
  *
34
28
  * In a real run the harness has already written that template, and the message
35
- * says so. Under `--dry-run` nothing is written — a dry run must never touch the
36
- * filesystem — so the message instead describes what a real run *would* scaffold
37
- * and where, while surfacing the same exit-3 "bootstrap needed" signal.
29
+ * says so. Under `--dry-run` the TEMPLATE is not written, so the message instead
30
+ * describes what a real run *would* scaffold and where, while surfacing the same
31
+ * exit-3 "bootstrap needed" signal.
32
+ *
33
+ * That scope is exact, and it used to read "a dry run must never touch the
34
+ * filesystem", which is not true of this command. A dry run is not a read-only
35
+ * mode: it takes the harness lock, creates the harness root, stages BOTH arms'
36
+ * per-eval workspaces, and — once `--i-understand-this-runs-skill-code` is passed —
37
+ * runs the repo's `test.build` hook and builds the subject. What it never does is
38
+ * SPAWN: no executor session, no grader session, no tokens.
39
+ *
40
+ * It also never touches `results/`: it neither creates the directory, nor writes
41
+ * `provenance.json`, nor wipes the previous run's artifacts. That last one is the
42
+ * reason the ordering matters — the wipe used to sit AHEAD of the dry-run
43
+ * short-circuit, so a free `--dry-run` (or any failure after it) destroyed the
44
+ * `grading.json` / `baseline.json` / `friction.json` / `tool-eval.json` of the
45
+ * expensive real run an operator was about to read. The summary now names the
46
+ * provenance path it WOULD write rather than writing it.
47
+ *
48
+ * The scaffold is the other filesystem effect a dry run deliberately withholds,
49
+ * and for a different reason: that write lands in the AUTHOR's source
50
+ * tree rather than in vat's own scratch space.
38
51
  */
39
52
  export class BootstrapNeededError extends Error {
40
53
  expectedPath;
41
54
  exitCode = 3;
42
55
  constructor(expectedPath, opts) {
43
56
  super(opts?.dryRun === true
44
- ? `[dry-run] No evals.json found. A real run would scaffold an annotated template at ${expectedPath} — fill it in and re-run. (dry-run: nothing was written.)`
57
+ ? `[dry-run] No evals.json found. A real run would scaffold an annotated template at ${expectedPath} — fill it in and re-run. (dry-run: nothing was written to your source tree.)`
45
58
  : `Wrote an evals.json template at ${expectedPath} — fill it in and re-run.`);
46
59
  this.expectedPath = expectedPath;
47
60
  this.name = 'BootstrapNeededError';
@@ -98,19 +111,102 @@ export class InternalHarnessError extends Error {
98
111
  }
99
112
  }
100
113
  /**
101
- * Map any thrown error to the process exit code. Errors that carry their own
102
- * `exitCode` (Bootstrap/Auth/HarnessLocation/Internal) are authoritative;
103
- * a PromptInvariantError is a defense-in-depth failure over VAT's OWN generated
104
- * executor/grader prompts (a required safety directive missing from a prompt VAT
105
- * built) — surfaced as a user-visible preflight-class problem → 2; a BuildHookError
106
- * is a pre-stage build failure → 2; a SkillBuildError is a declared-skill build
107
- * failure → 2; a SecurityAckError is a missing security ack before a build → 2;
108
- * an UnknownEnvTokenError is a bad ${token} in a
109
- * declared env value → 2; a DuplicateStagedSkillError is a staged-name collision
110
- * across subject/--with/--with-optional → 2; GradingSkewError (aggregate grading.json shape skew),
111
- * EvalFragmentError (per-eval grader fragment parse failure), and
112
- * GradingNonceError (forged/mismatched per-fragment grader nonce) are all → 1;
113
- * everything unknown → 1.
114
+ * The codes an error is allowed to declare for itself. `Ok` is not a failure and
115
+ * `EvalFailure` is an outcome rather than a throw (see below), so neither may be
116
+ * claimed by a thrown object — that also keeps a stray `exitCode: 0` from turning a
117
+ * crash into a green build.
118
+ */
119
+ const SELF_DECLARED_EXIT_CODES = new Set([
120
+ SkillTestExitCode.Internal,
121
+ SkillTestExitCode.Preflight,
122
+ SkillTestExitCode.Bootstrap,
123
+ ]);
124
+ /**
125
+ * The error's own declared exit code, if it has a usable one.
126
+ *
127
+ * Narrow ON PURPOSE. `exitCode` is a common property name on foreign errors (child
128
+ * processes, HTTP clients), so an unrestricted read would let an unrelated throw pick
129
+ * the process's exit code. Requiring an `Error` whose value is one of the three
130
+ * throwable codes above keeps the mechanism to errors that meant to opt in, and
131
+ * leaves everything else on the `instanceof` fallback below.
132
+ *
133
+ * ⚠️ THE READ IS GUARDED, because reading a property is not a total operation and the
134
+ * `instanceof` chain it replaced was. A getter or `Proxy` that throws on `exitCode`
135
+ * propagated straight out of {@link mapErrorToExitCode} — and BOTH of its call sites
136
+ * are inside `catch` blocks, so that throw escaped the handler entirely: no summary
137
+ * line, no chosen exit code, a bare stack. An error whose own `exitCode` cannot be
138
+ * read has not opted in to anything, so it takes the Internal default.
139
+ *
140
+ * ⚠️ IT IS NOT AN OWN-PROPERTY READ. An `exitCode` inherited from a PROTOTYPE is
141
+ * honoured, verified — `Base.prototype.exitCode = 3` maps to 3. That is what makes
142
+ * subclassing work (a subclass of `InternalHarnessError` keeps its code without
143
+ * redeclaring the field), and it is also what widens the residual below by one hop.
144
+ *
145
+ * ⚠️ RESIDUAL, KNOWN AND UNCLOSED. The value test rejects everything outside
146
+ * {1,2,3} but nothing INSIDE it, so a foreign error carrying `exitCode: 2` picks
147
+ * Preflight — "your environment is wrong" — for a crash. A walk of the reachable
148
+ * dependency closure found `spawnSync`, `execSync`, `fs`, `JSON.parse`, `ZodError`,
149
+ * `YAMLParseError`, `adm-zip`, `fetch`, `DOMException`, Anthropic's `APIError` and
150
+ * VAT's own `CommandExecutionError` all leaving `exitCode` undefined — but
151
+ * `commander`, a direct `@vibe-agent-toolkit/cli` dependency, defines `exitCode` on
152
+ * `CommanderError` and carries `2`. It is unreachable today (`exitOverride` and
153
+ * `CommanderError` appear nowhere in `packages/cli/src`), and adopting
154
+ * `exitOverride()` — the standard testability pattern — would silently arm it.
155
+ *
156
+ * Closing it means a VAT-OWNED BRAND rather than a name every ecosystem uses: a
157
+ * `Symbol.for('vat.exitCode')` key, which (unlike `instanceof` or a shared base
158
+ * class) is registry-global and so survives the `src`/`dist` boundary this read
159
+ * exists to cross. That is a mechanical change across all eleven declaring classes —
160
+ * five here, plus `lock.ts`, `build-hook.ts`, `declared-env.ts` (two),
161
+ * `harness-location.ts` and `utils`' `auth-resolver.ts` — and a half-migration would
162
+ * leave two mechanisms with the hole still open, so it is one change or none. The
163
+ * behaviour is pinned by a RESIDUAL row in `exit-codes.test.ts` so the migration
164
+ * has a test to flip rather than a comment to find.
165
+ */
166
+ function selfDeclaredExitCode(err) {
167
+ if (!(err instanceof Error))
168
+ return undefined;
169
+ let declared;
170
+ try {
171
+ declared = err.exitCode;
172
+ }
173
+ catch {
174
+ return undefined;
175
+ }
176
+ return typeof declared === 'number' && SELF_DECLARED_EXIT_CODES.has(declared) ? declared : undefined;
177
+ }
178
+ /**
179
+ * Map any thrown error to the process exit code.
180
+ *
181
+ * ⚠️ THE `exitCode` FIELD IS THE PRIMARY MECHANISM, and until this rewrite it was
182
+ * not a mechanism at all. Ten error classes across five modules carried a
183
+ * `readonly exitCode`, this docblock called them "authoritative", and NOTHING read
184
+ * one: dispatch was a hand-maintained `instanceof` chain that happened to agree with
185
+ * every field. Deleting any of those fields changed nothing, so each new class added
186
+ * a fifth, then a sixth member to an inert pattern — and the two lists could have
187
+ * disagreed at any time with no test able to tell.
188
+ *
189
+ * So: an error that declares one of the three throwable codes ({@link
190
+ * selfDeclaredExitCode}) decides its own exit code, and what remains of the
191
+ * `instanceof` chain is exactly ONE row — `PromptInvariantError` (a
192
+ * defense-in-depth failure over VAT's OWN generated executor/grader prompts: a
193
+ * required safety directive missing from a prompt VAT built, surfaced as a
194
+ * user-visible preflight-class problem) → 2, the only class that declares nothing and
195
+ * still needs a code other than the default.
196
+ *
197
+ * `GradingSkewError` (aggregate grading.json shape skew), `EvalFragmentError`
198
+ * (per-eval grader fragment parse failure) and `GradingNonceError`
199
+ * (forged/mismatched per-fragment grader nonce) reach 1 through the FALLBACK, not
200
+ * through a row — they are indistinguishable here from `new Error('boom')`. That is
201
+ * correct (1 is what they want) but it is not dispatch, and saying otherwise implies
202
+ * a pin that does not exist: deleting any of the three changes nothing. Everything
203
+ * unknown → 1.
204
+ *
205
+ * A side benefit of reading the field: it works across a `src`/`dist` boundary, where
206
+ * `instanceof` silently cannot — a `dist` copy of a class never matches a `src`
207
+ * instance of it, and this module is imported from both. The surviving
208
+ * `PromptInvariantError` row carries exactly that risk; it is kept because the class
209
+ * declares no field to read.
114
210
  *
115
211
  * Note: SkillTestExitCode.EvalFailure (4) is NOT produced here — it is an
116
212
  * outcome of a completed run (an eval failed, the fail-closed default unless
@@ -118,24 +214,11 @@ export class InternalHarnessError extends Error {
118
214
  * directly (see verdictExitCode).
119
215
  */
120
216
  export function mapErrorToExitCode(err) {
121
- if (err instanceof BootstrapNeededError)
122
- return SkillTestExitCode.Bootstrap;
123
- if (err instanceof AuthPreflightError ||
124
- err instanceof BuildHookError ||
125
- err instanceof HarnessLocationError ||
126
- err instanceof PromptInvariantError ||
127
- err instanceof SecurityAckError ||
128
- err instanceof SkillBuildError ||
129
- err instanceof UnknownEnvTokenError ||
130
- err instanceof DuplicateStagedSkillError) {
217
+ const declared = selfDeclaredExitCode(err);
218
+ if (declared !== undefined)
219
+ return declared;
220
+ if (err instanceof PromptInvariantError)
131
221
  return SkillTestExitCode.Preflight;
132
- }
133
- if (err instanceof EvalFragmentError ||
134
- err instanceof GradingNonceError ||
135
- err instanceof GradingSkewError ||
136
- err instanceof InternalHarnessError) {
137
- return SkillTestExitCode.Internal;
138
- }
139
222
  return SkillTestExitCode.Internal;
140
223
  }
141
224
  //# sourceMappingURL=exit-codes.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"exit-codes.js","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAC;AAE/D,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAC3E,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAE9D,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,EAAE,EAAE,CAAC;IACL,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;IACZ;;;;;;;;;;;;OAYG;IACH,WAAW,EAAE,CAAC;CACN,CAAC;AAIX;;;;;;;;;GASG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAEjB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,YAAoB,EAAE,IAA2B;QAC3E,KAAK,CACH,IAAI,EAAE,MAAM,KAAK,IAAI;YACnB,CAAC,CAAC,qFAAqF,YAAY,2DAA2D;YAC9J,CAAC,CAAC,mCAAmC,YAAY,2BAA2B,CAC/E,CAAC;QALwB,iBAAY,GAAZ,YAAY,CAAQ;QAM9C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,iHAAiH;AACjH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAC/B,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAChC,QAAQ,GAAG,CAAU,CAAC;IAC/B;QACE,KAAK,CAAC,wFAAwF,CAAC,CAAC;QAChG,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED;;;;;;;;;;GAUG;AACH,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IAEtB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,SAAiB;QAC3C,KAAK,CACH,eAAe,SAAS,mEAAmE;YACzF,4CAA4C,CAC/C,CAAC;QAJwB,cAAS,GAAT,SAAS,CAAQ;QAK3C,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED,+HAA+H;AAC/H,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,SAAS,CAAC;IAC5E,IACE,GAAG,YAAY,kBAAkB;QACjC,GAAG,YAAY,cAAc;QAC7B,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,gBAAgB;QAC/B,GAAG,YAAY,eAAe;QAC9B,GAAG,YAAY,oBAAoB;QACnC,GAAG,YAAY,yBAAyB,EACxC,CAAC;QACD,OAAO,iBAAiB,CAAC,SAAS,CAAC;IACrC,CAAC;IACD,IACE,GAAG,YAAY,iBAAiB;QAChC,GAAG,YAAY,iBAAiB;QAChC,GAAG,YAAY,gBAAgB;QAC/B,GAAG,YAAY,oBAAoB,EACnC,CAAC;QACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC;IACpC,CAAC;IACD,OAAO,iBAAiB,CAAC,QAAQ,CAAC;AACpC,CAAC"}
1
+ {"version":3,"file":"exit-codes.js","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAE9D,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,EAAE,EAAE,CAAC;IACL,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;IACZ;;;;;;;;;;;;OAYG;IACH,WAAW,EAAE,CAAC;CACN,CAAC;AAIX;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAEjB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,YAAoB,EAAE,IAA2B;QAC3E,KAAK,CACH,IAAI,EAAE,MAAM,KAAK,IAAI;YACnB,CAAC,CAAC,qFAAqF,YAAY,+EAA+E;YAClL,CAAC,CAAC,mCAAmC,YAAY,2BAA2B,CAC/E,CAAC;QALwB,iBAAY,GAAZ,YAAY,CAAQ;QAM9C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,iHAAiH;AACjH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAC/B,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAChC,QAAQ,GAAG,CAAU,CAAC;IAC/B;QACE,KAAK,CAAC,wFAAwF,CAAC,CAAC;QAChG,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED;;;;;;;;;;GAUG;AACH,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IAEtB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,SAAiB;QAC3C,KAAK,CACH,eAAe,SAAS,mEAAmE;YACzF,4CAA4C,CAC/C,CAAC;QAJwB,cAAS,GAAT,SAAS,CAAQ;QAK3C,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED,+HAA+H;AAC/H,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,wBAAwB,GAAwB,IAAI,GAAG,CAAS;IACpE,iBAAiB,CAAC,QAAQ;IAC1B,iBAAiB,CAAC,SAAS;IAC3B,iBAAiB,CAAC,SAAS;CAC5B,CAAC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,SAAS,oBAAoB,CAAC,GAAY;IACxC,IAAI,CAAC,CAAC,GAAG,YAAY,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9C,IAAI,QAAiB,CAAC;IACtB,IAAI,CAAC;QACH,QAAQ,GAAI,GAA8B,CAAC,QAAQ,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,QAAQ,KAAK,QAAQ,IAAI,wBAAwB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AACvG,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC5C,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,SAAS,CAAC;IAC5E,OAAO,iBAAiB,CAAC,QAAQ,CAAC;AACpC,CAAC"}
@@ -2,24 +2,7 @@ import type { EvalFragment } from './eval-fragment.js';
2
2
  import { type FrictionReport } from './friction-schema.js';
3
3
  import { type NormalizedGrading } from './grading-adapter.js';
4
4
  import { type ToolEvalReport } from './tool-eval-schema.js';
5
- /**
6
- * Assemble the run's aggregate grading result (skill-creator's flat
7
- * `grading.json` shape) from every eval's per-eval grader fragment (issue #145
8
- * Task 4/5). vat is the SOLE writer of the aggregate; this is that assembly
9
- * step — the result feeds {@link import('./grading-adapter.js').reconcileGrading}
10
- * unchanged.
11
- *
12
- * Every fragment must carry the SAME `runNonce` the harness stamped for this
13
- * run — a mismatch means a fragment was not produced for this run (forged,
14
- * stale, or left behind by untrusted skill code in the shared sandbox) and is
15
- * rejected via {@link GradingNonceError}, never silently dropped.
16
- *
17
- * Zero fragments in → zero expectations out. This is deliberate: the existing
18
- * "graded nothing" guard in `reconcileGrading` already throws
19
- * `GradingSkewError` on an empty `expectations[]`, so that case is NOT
20
- * special-cased here.
21
- */
22
- export declare function mergeFragmentsToGrading(fragments: EvalFragment[], runNonce: string): NormalizedGrading;
5
+ export declare function mergeFragmentsToGrading(fragments: EvalFragment[], runNonce: string, arm: 'with' | 'without'): NormalizedGrading;
23
6
  /**
24
7
  * Assemble the run's aggregate friction report from every eval's fragment
25
8
  * (vat-owned; see friction-schema.ts). Concatenates each fragment's
@@ -1 +1 @@
1
- {"version":3,"file":"fragment-merge.d.ts","sourceRoot":"","sources":["../../src/skill-test/fragment-merge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAqB,KAAK,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAElF;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,iBAAiB,CAoBtG;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,cAAc,CAYlF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,cAAc,CAMlF"}
1
+ {"version":3,"file":"fragment-merge.d.ts","sourceRoot":"","sources":["../../src/skill-test/fragment-merge.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAsC,KAAK,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAClG,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAkElF,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,YAAY,EAAE,EACzB,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,MAAM,GAAG,SAAS,GACtB,iBAAiB,CAmCnB;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,cAAc,CAYlF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,cAAc,CAMlF"}