@vibe-agent-toolkit/agent-skills 0.2.0-rc.7 → 0.2.0-rc.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/README.md +5 -5
  2. package/dist/builder.d.ts.map +1 -1
  3. package/dist/builder.js +7 -17
  4. package/dist/builder.js.map +1 -1
  5. package/dist/files-config.d.ts +2 -2
  6. package/dist/files-config.d.ts.map +1 -1
  7. package/dist/files-config.js +4 -10
  8. package/dist/files-config.js.map +1 -1
  9. package/dist/import.d.ts.map +1 -1
  10. package/dist/import.js +0 -4
  11. package/dist/import.js.map +1 -1
  12. package/dist/index.d.ts +2 -1
  13. package/dist/index.d.ts.map +1 -1
  14. package/dist/index.js +2 -1
  15. package/dist/index.js.map +1 -1
  16. package/dist/inventory/types.d.ts +8 -0
  17. package/dist/inventory/types.d.ts.map +1 -1
  18. package/dist/inventory/types.js.map +1 -1
  19. package/dist/link-graph-members.d.ts +18 -0
  20. package/dist/link-graph-members.d.ts.map +1 -0
  21. package/dist/link-graph-members.js +18 -0
  22. package/dist/link-graph-members.js.map +1 -0
  23. package/dist/plugin-distribution-layout.d.ts +70 -6
  24. package/dist/plugin-distribution-layout.d.ts.map +1 -1
  25. package/dist/plugin-distribution-layout.js +68 -10
  26. package/dist/plugin-distribution-layout.js.map +1 -1
  27. package/dist/post-build-checks.d.ts.map +1 -1
  28. package/dist/post-build-checks.js +12 -11
  29. package/dist/post-build-checks.js.map +1 -1
  30. package/dist/projection/skill-extent.d.ts +1 -1
  31. package/dist/projection/skill-extent.d.ts.map +1 -1
  32. package/dist/projection/skill-extent.js +3 -1
  33. package/dist/projection/skill-extent.js.map +1 -1
  34. package/dist/schemas/installed-plugins-registry.d.ts.map +1 -1
  35. package/dist/schemas/installed-plugins-registry.js +7 -1
  36. package/dist/schemas/installed-plugins-registry.js.map +1 -1
  37. package/dist/skill-identity.d.ts +8 -3
  38. package/dist/skill-identity.d.ts.map +1 -1
  39. package/dist/skill-identity.js +13 -6
  40. package/dist/skill-identity.js.map +1 -1
  41. package/dist/skill-packager.d.ts +2 -1
  42. package/dist/skill-packager.d.ts.map +1 -1
  43. package/dist/skill-packager.js +53 -44
  44. package/dist/skill-packager.js.map +1 -1
  45. package/dist/skill-source/content-hash.d.ts.map +1 -1
  46. package/dist/skill-source/content-hash.js +21 -10
  47. package/dist/skill-source/content-hash.js.map +1 -1
  48. package/dist/skill-source/fetch-cache.d.ts.map +1 -1
  49. package/dist/skill-source/fetch-cache.js +0 -5
  50. package/dist/skill-source/fetch-cache.js.map +1 -1
  51. package/dist/skill-source/git-clone.d.ts.map +1 -1
  52. package/dist/skill-source/git-clone.js +0 -2
  53. package/dist/skill-source/git-clone.js.map +1 -1
  54. package/dist/skill-source/sources/url-source.d.ts.map +1 -1
  55. package/dist/skill-source/sources/url-source.js +0 -1
  56. package/dist/skill-source/sources/url-source.js.map +1 -1
  57. package/dist/skill-source/stage.d.ts.map +1 -1
  58. package/dist/skill-source/stage.js +0 -6
  59. package/dist/skill-source/stage.js.map +1 -1
  60. package/dist/skill-test/baseline-integrity.d.ts.map +1 -1
  61. package/dist/skill-test/baseline-integrity.js +10 -3
  62. package/dist/skill-test/baseline-integrity.js.map +1 -1
  63. package/dist/skill-test/build-hook.d.ts +3 -2
  64. package/dist/skill-test/build-hook.d.ts.map +1 -1
  65. package/dist/skill-test/build-hook.js +4 -4
  66. package/dist/skill-test/build-hook.js.map +1 -1
  67. package/dist/skill-test/declared-env.d.ts +5 -4
  68. package/dist/skill-test/declared-env.d.ts.map +1 -1
  69. package/dist/skill-test/declared-env.js +7 -9
  70. package/dist/skill-test/declared-env.js.map +1 -1
  71. package/dist/skill-test/eval-executor.d.ts +1 -1
  72. package/dist/skill-test/eval-executor.js +2 -2
  73. package/dist/skill-test/eval-executor.js.map +1 -1
  74. package/dist/skill-test/eval-fragment.d.ts +7 -5
  75. package/dist/skill-test/eval-fragment.d.ts.map +1 -1
  76. package/dist/skill-test/eval-fragment.js +13 -7
  77. package/dist/skill-test/eval-fragment.js.map +1 -1
  78. package/dist/skill-test/eval-grader.d.ts +56 -4
  79. package/dist/skill-test/eval-grader.d.ts.map +1 -1
  80. package/dist/skill-test/eval-grader.js +138 -73
  81. package/dist/skill-test/eval-grader.js.map +1 -1
  82. package/dist/skill-test/eval-inputs.d.ts +8 -7
  83. package/dist/skill-test/eval-inputs.d.ts.map +1 -1
  84. package/dist/skill-test/eval-inputs.js +9 -11
  85. package/dist/skill-test/eval-inputs.js.map +1 -1
  86. package/dist/skill-test/eval-lint.d.ts +4 -4
  87. package/dist/skill-test/eval-lint.js +5 -5
  88. package/dist/skill-test/eval-lint.js.map +1 -1
  89. package/dist/skill-test/eval-suite-isolation.d.ts.map +1 -1
  90. package/dist/skill-test/eval-suite-isolation.js +8 -5
  91. package/dist/skill-test/eval-suite-isolation.js.map +1 -1
  92. package/dist/skill-test/evals-template.d.ts +1 -1
  93. package/dist/skill-test/evals-template.d.ts.map +1 -1
  94. package/dist/skill-test/evals-template.js +1 -3
  95. package/dist/skill-test/evals-template.js.map +1 -1
  96. package/dist/skill-test/executor-prompt.d.ts +2 -2
  97. package/dist/skill-test/executor-prompt.js +2 -2
  98. package/dist/skill-test/failure-reason.d.ts +113 -0
  99. package/dist/skill-test/failure-reason.d.ts.map +1 -0
  100. package/dist/skill-test/failure-reason.js +124 -0
  101. package/dist/skill-test/failure-reason.js.map +1 -0
  102. package/dist/skill-test/fragment-merge.d.ts +2 -2
  103. package/dist/skill-test/fragment-merge.js +4 -4
  104. package/dist/skill-test/grader-prompt.d.ts +2 -2
  105. package/dist/skill-test/grader-prompt.d.ts.map +1 -1
  106. package/dist/skill-test/grader-prompt.js +12 -8
  107. package/dist/skill-test/grader-prompt.js.map +1 -1
  108. package/dist/skill-test/grading-adapter.d.ts +7 -3
  109. package/dist/skill-test/grading-adapter.d.ts.map +1 -1
  110. package/dist/skill-test/grading-adapter.js +10 -9
  111. package/dist/skill-test/grading-adapter.js.map +1 -1
  112. package/dist/skill-test/harness-location.d.ts +3 -2
  113. package/dist/skill-test/harness-location.d.ts.map +1 -1
  114. package/dist/skill-test/harness-location.js +5 -16
  115. package/dist/skill-test/harness-location.js.map +1 -1
  116. package/dist/skill-test/index.d.ts +1 -1
  117. package/dist/skill-test/index.d.ts.map +1 -1
  118. package/dist/skill-test/index.js +1 -1
  119. package/dist/skill-test/index.js.map +1 -1
  120. package/dist/skill-test/lock.d.ts +11 -10
  121. package/dist/skill-test/lock.d.ts.map +1 -1
  122. package/dist/skill-test/lock.js +13 -14
  123. package/dist/skill-test/lock.js.map +1 -1
  124. package/dist/skill-test/pipeline.d.ts +3 -1
  125. package/dist/skill-test/pipeline.d.ts.map +1 -1
  126. package/dist/skill-test/pipeline.js +5 -1
  127. package/dist/skill-test/pipeline.js.map +1 -1
  128. package/dist/skill-test/preflight.d.ts.map +1 -1
  129. package/dist/skill-test/preflight.js +0 -1
  130. package/dist/skill-test/preflight.js.map +1 -1
  131. package/dist/skill-test/prompt-invariants.d.ts +4 -2
  132. package/dist/skill-test/prompt-invariants.d.ts.map +1 -1
  133. package/dist/skill-test/prompt-invariants.js +5 -4
  134. package/dist/skill-test/prompt-invariants.js.map +1 -1
  135. package/dist/skill-test/run-harness.d.ts +160 -123
  136. package/dist/skill-test/run-harness.d.ts.map +1 -1
  137. package/dist/skill-test/run-harness.js +310 -174
  138. package/dist/skill-test/run-harness.js.map +1 -1
  139. package/dist/skill-test/staging.d.ts +12 -6
  140. package/dist/skill-test/staging.d.ts.map +1 -1
  141. package/dist/skill-test/staging.js +17 -14
  142. package/dist/skill-test/staging.js.map +1 -1
  143. package/dist/skill-test/tier-plan.d.ts +1 -1
  144. package/dist/skill-test/tier-plan.js +1 -1
  145. package/dist/skill-test/tool-eval-schema.js +2 -2
  146. package/dist/skill-test/vendor-manifest.d.ts +2 -1
  147. package/dist/skill-test/vendor-manifest.d.ts.map +1 -1
  148. package/dist/skill-test/vendor-manifest.js +10 -10
  149. package/dist/skill-test/vendor-manifest.js.map +1 -1
  150. package/dist/test-input.d.ts +2 -2
  151. package/dist/test-input.d.ts.map +1 -1
  152. package/dist/test-input.js +6 -7
  153. package/dist/test-input.js.map +1 -1
  154. package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
  155. package/dist/validators/agent-instruction-presence.js +0 -1
  156. package/dist/validators/agent-instruction-presence.js.map +1 -1
  157. package/dist/validators/bundled-resource-link-detection.d.ts +5 -3
  158. package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
  159. package/dist/validators/bundled-resource-link-detection.js +38 -38
  160. package/dist/validators/bundled-resource-link-detection.js.map +1 -1
  161. package/dist/validators/format-detection.d.ts +3 -0
  162. package/dist/validators/format-detection.d.ts.map +1 -1
  163. package/dist/validators/format-detection.js +41 -26
  164. package/dist/validators/format-detection.js.map +1 -1
  165. package/dist/validators/frontmatter-validation.js +1 -1
  166. package/dist/validators/frontmatter-validation.js.map +1 -1
  167. package/dist/validators/marketplace-validator.d.ts.map +1 -1
  168. package/dist/validators/marketplace-validator.js +0 -1
  169. package/dist/validators/marketplace-validator.js.map +1 -1
  170. package/dist/validators/mcp-tool-qualification.d.ts +1 -1
  171. package/dist/validators/mcp-tool-qualification.js +1 -1
  172. package/dist/validators/packaged-size-limit.d.ts.map +1 -1
  173. package/dist/validators/packaged-size-limit.js +0 -2
  174. package/dist/validators/packaged-size-limit.js.map +1 -1
  175. package/dist/validators/packaging-validator.d.ts +2 -2
  176. package/dist/validators/packaging-validator.d.ts.map +1 -1
  177. package/dist/validators/packaging-validator.js +10 -13
  178. package/dist/validators/packaging-validator.js.map +1 -1
  179. package/dist/validators/plugin-hosted-shape.d.ts +3 -1
  180. package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
  181. package/dist/validators/plugin-hosted-shape.js +5 -6
  182. package/dist/validators/plugin-hosted-shape.js.map +1 -1
  183. package/dist/validators/referenced-path-missing.d.ts.map +1 -1
  184. package/dist/validators/referenced-path-missing.js +3 -4
  185. package/dist/validators/referenced-path-missing.js.map +1 -1
  186. package/dist/validators/registry-validator.d.ts.map +1 -1
  187. package/dist/validators/registry-validator.js +0 -1
  188. package/dist/validators/registry-validator.js.map +1 -1
  189. package/dist/validators/rule-engine/index.d.ts +1 -1
  190. package/dist/validators/rule-engine/rule-context.d.ts +3 -3
  191. package/dist/validators/rule-engine/rule-context.js +1 -1
  192. package/dist/validators/rule-engine/rule-engine.d.ts +1 -1
  193. package/dist/validators/rule-engine/rule-engine.js +5 -5
  194. package/dist/validators/rule-engine/rule-engine.js.map +1 -1
  195. package/dist/validators/skill-validator.d.ts.map +1 -1
  196. package/dist/validators/skill-validator.js +41 -51
  197. package/dist/validators/skill-validator.js.map +1 -1
  198. package/dist/validators/source-only-codes.js +1 -1
  199. package/dist/validators/source-only-codes.js.map +1 -1
  200. package/dist/validators/types.d.ts +7 -1
  201. package/dist/validators/types.d.ts.map +1 -1
  202. package/dist/validators/unified-validator.d.ts.map +1 -1
  203. package/dist/validators/unified-validator.js +3 -4
  204. package/dist/validators/unified-validator.js.map +1 -1
  205. package/dist/validators/validation-rules.d.ts +3 -3
  206. package/dist/validators/validation-rules.d.ts.map +1 -1
  207. package/dist/validators/validation-rules.js +9 -38
  208. package/dist/validators/validation-rules.js.map +1 -1
  209. package/dist/validators/walker-to-issues.d.ts +2 -0
  210. package/dist/validators/walker-to-issues.d.ts.map +1 -1
  211. package/dist/validators/walker-to-issues.js +17 -12
  212. package/dist/validators/walker-to-issues.js.map +1 -1
  213. package/dist/walk-link-graph.d.ts +2 -0
  214. package/dist/walk-link-graph.d.ts.map +1 -1
  215. package/dist/walk-link-graph.js +20 -8
  216. package/dist/walk-link-graph.js.map +1 -1
  217. package/package.json +12 -20
  218. package/dist/skill-test/exit-codes.d.ts +0 -134
  219. package/dist/skill-test/exit-codes.d.ts.map +0 -1
  220. package/dist/skill-test/exit-codes.js +0 -224
  221. package/dist/skill-test/exit-codes.js.map +0 -1
@@ -1 +1 @@
1
- {"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE/D,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EACL,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EAAE,iBAAiB,EAAE,iBAAiB,EAAqB,MAAM,oBAAoB,CAAC;AAE7F,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAsD1D;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,UAAwC;IAClE,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAqB;IAC1D,8EAA8E;IAC9E,iFAAiF;IACjF,8EAA8E;IAC9E,+EAA+E;IAC/E,8EAA8E;IAC9E,iFAAiF;IACjF,kFAAkF;IAClF,8EAA8E;IAC9E,+EAA+E;IAC/E,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAEpE,4EAA4E;IAC5E,2EAA2E;IAC3E,uEAAuE;IACvE,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC;IAEvF,MAAM,MAAM,GAAG,iBAAiB,CAAC;QAC/B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC;QACvF,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC7F,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC;KACvG,CAAC,CAAC;IACH,8EAA8E;IAC9E,0EAA0E;IAC1E,4EAA4E;IAC5E,8EAA8E;IAC9E,6EAA6E;IAC7E,8EAA8E;IAC9E,2EAA2E;IAC3E,0EAA0E;IAC1E,UAAU;IACV,4BAA4B,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAElD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,mBAAmB,CAAC;IAEjD,yEAAyE;IACzE,+EAA+E;IAC/E,uEAAuE;IACvE,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAQ,EAAE;QACvC,gBAAgB,IAAI,KAAK,CAAC;QAC1B,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC,CAAC;IAEF,IAAI,WAAW,CAAC;IAChB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,KAAK,CAAC;YACxB,MAAM;YACN,uEAAuE;YACvE,gEAAgE;YAChE,0CAA0C;YAC1C,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,KAAK,CAAC,YAAY;YAC9B,GAAG,EAAE,KAAK,CAAC,YAAY;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,KAAK,EAAE,KAAK,CAAC,WAAW;YACxB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,iCAAiC,KAAK,CAAC,MAAM,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAC5B,4BAA4B,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CACrF,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAC5B,8BAA8B,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,SAAS,MAAM,CAC5E,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,kCAAkC,WAAW,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,IAAI,CACpF,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,iFAAiF;IACjF,8EAA8E;IAC9E,KAAK,CAAC,QAAQ,EAAE,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEnF,MAAM,GAAG,GAAG,0BAA0B,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACtF,MAAM,QAAQ,GAAG,iBAAiB,CAAC,GAAG,EAAE,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAE9E,6EAA6E;IAC7E,6EAA6E;IAC7E,0EAA0E;IAC1E,IAAI,QAAQ,CAAC,QAAQ,KAAK,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,IAAI,iBAAiB,CACzB,SAAS,KAAK,CAAC,MAAM,iDAAiD,CACvE,CAAC;IACJ,CAAC;IAED,8BAA8B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;IACxF,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjF,iFAAiF;IACjF,+EAA+E;IAC/E,yEAAyE;IACzE,0EAA0E;IAC1E,8EAA8E;IAC9E,gFAAgF;IAChF,uEAAuE;IACvE,OAAO,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;AAC/C,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,0BAA0B,CAAC,WAAmB,EAAE,MAAc,EAAE,MAAc;IACrF,kIAAkI;IAClI,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,MAAM,mCAAmC,WAAW,cAAc,MAAM,IAAI,CACtG,CAAC;IACJ,CAAC;IACD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,kIAAkI;QAClI,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IACvD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uEAAuE;QACvE,4EAA4E;QAC5E,oEAAoE;QACpE,4EAA4E;QAC5E,6EAA6E;QAC7E,0EAA0E;QAC1E,2EAA2E;QAC3E,qDAAqD;QACrD,MAAM,IAAI,iBAAiB,CACzB,6BAA6B,MAAM,QAAQ,WAAW,sBAAsB;YAC1E,kBAAkB,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CACvE,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,kIAAkI;QAClI,UAAU,CAAC,WAAW,CAAC,CAAC;IAC1B,CAAC;IAAC,MAAM,CAAC;QACP,sDAAsD;IACxD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,SAAS,8BAA8B,CACrC,MAAc,EACd,QAA2B,EAC3B,QAAsC;IAEtC,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,OAAO;IAChD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,cAAc,QAAQ,CAAC,MAAM,oBAAoB,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG;QAC/G,6BAA6B,QAAQ,CAAC,MAAM,yDAAyD;QACrG,qGAAqG;QACrG,oCAAoC,CACvC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,2BAA2B,CAClC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,gBAAgB,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACzD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,mGAAmG,CAC9H,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,8CAA8C,IAAI,CAAC,MAAM,IAAI;YACrF,iDAAiD,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAC/E,CAAC;IACJ,CAAC;IACD,4BAA4B,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;AAC/D,CAAC;AAED,kGAAkG;AAClG,SAAS,UAAU,CAAC,KAAwB;IAC1C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAS,4BAA4B,CACnC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO;IAC/B,MAAM,QAAQ,GAAG,gBAAgB,IAAI,EAAE,CAAC;IACxC,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACjG,kBAAkB,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1G,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7G,kBAAkB,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAc,EACd,OAAe,EACf,QAAuC,EACvC,QAA2B;IAE3B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACtE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC1D,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,oBAAoB,OAAO,yCAAyC;QAC5F,8BAA8B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,gDAAgD;QACjG,qEAAqE,CACxE,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAE3D,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EACL,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EAAE,iBAAiB,EAAqB,MAAM,oBAAoB,CAAC;AAE1E,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAuD1D,oJAAoJ;AACpJ,MAAM,CAAC,MAAM,2BAA2B,GAAG,6BAA6B,CAAC;AAEzE;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,8BAA+B,SAAQ,QAAQ;IAExC;IACA;IAFlB,YACkB,MAAc,EACd,MAAc;QAE9B,KAAK,CACH,wCAAwC,EACxC,GAAG,2BAA2B,0BAA0B,MAAM,6CAA6C;YACzG,sBAAsB,MAAM,IAAI,CACnC,CAAC;QAPc,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAAQ;IAOhC,CAAC;CACF;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,UAAwC;IAClE,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAqB;IAC1D,8EAA8E;IAC9E,iFAAiF;IACjF,8EAA8E;IAC9E,+EAA+E;IAC/E,8EAA8E;IAC9E,iFAAiF;IACjF,kFAAkF;IAClF,8EAA8E;IAC9E,+EAA+E;IAC/E,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAEpE,4EAA4E;IAC5E,2EAA2E;IAC3E,uEAAuE;IACvE,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC;IAEvF,IAAI,OAAO,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACjF,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACtC,KAAK,CAAC,UAAU,CACd,0CAA0C,KAAK,CAAC,MAAM,yBAAyB,OAAO,CAAC,WAAW,uBAAuB,CAC1H,CAAC;QACF,6EAA6E;QAC7E,kCAAkC;QAClC,OAAO,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3G,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,8BAA8B,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAEzF,6EAA6E;IAC7E,0EAA0E;IAC1E,iFAAiF;IACjF,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,IAAI,iBAAiB,CACzB,SAAS,KAAK,CAAC,MAAM,iDAAiD,CACvE,CAAC;IACJ,CAAC;IAED,8BAA8B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;IACxF,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjF,iFAAiF;IACjF,+EAA+E;IAC/E,yEAAyE;IACzE,0EAA0E;IAC1E,8EAA8E;IAC9E,gFAAgF;IAChF,uEAAuE;IACvE,EAAE;IACF,4EAA4E;IAC5E,8EAA8E;IAC9E,0EAA0E;IAC1E,OAAO,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;AACtE,CAAC;AAUD;;;;;;GAMG;AACH,KAAK,UAAU,gBAAgB,CAC7B,KAAqB,EACrB,WAAmB,EACnB,KAAa,EACb,eAAmC;IAEnC,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC;QACvF,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,KAAK;QACL,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC7F,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC;KACvG,CAAC,CAAC;IACH,MAAM,MAAM,GACV,eAAe,KAAK,SAAS;QAC3B,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,GAAG,KAAK,kDAAkD,eAAe,sBAAsB,CAAC;IACtG,8EAA8E;IAC9E,0EAA0E;IAC1E,4EAA4E;IAC5E,8EAA8E;IAC9E,6EAA6E;IAC7E,8EAA8E;IAC9E,2EAA2E;IAC3E,0EAA0E;IAC1E,UAAU;IACV,4BAA4B,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAE5C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,mBAAmB,CAAC;IAEjD,yEAAyE;IACzE,+EAA+E;IAC/E,uEAAuE;IACvE,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAQ,EAAE;QACvC,gBAAgB,IAAI,KAAK,CAAC;QAC1B,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC,CAAC;IAEF,IAAI,WAAW,CAAC;IAChB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,KAAK,CAAC;YACxB,MAAM;YACN,uEAAuE;YACvE,gEAAgE;YAChE,0CAA0C;YAC1C,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,KAAK,CAAC,YAAY;YAC9B,GAAG,EAAE,KAAK,CAAC,YAAY;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,KAAK,EAAE,KAAK,CAAC,WAAW;YACxB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,iCAAiC,KAAK,CAAC,MAAM,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAC5B,4BAA4B,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CACrF,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAC5B,8BAA8B,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,SAAS,MAAM,CAC5E,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,kCAAkC,WAAW,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,IAAI,CACpF,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,iFAAiF;IACjF,8EAA8E;IAC9E,KAAK,CAAC,QAAQ,EAAE,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEnF,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,0BAA0B,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC7F,IAAI,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,CAAC,GAAG,YAAY,WAAW,CAAC;YAAE,MAAM,GAAG,CAAC;QAC7C,uEAAuE;QACvE,4EAA4E;QAC5E,oEAAoE;QACpE,4EAA4E;QAC5E,+EAA+E;QAC/E,+EAA+E;QAC/E,+EAA+E;QAC/E,UAAU;QACV,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;IACjE,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAS,0BAA0B,CACjC,WAAmB,EACnB,MAAc,EACd,MAAc,EACd,MAAiC;IAEjC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,MAAM,mCAAmC,WAAW,cAAc,MAAM,IAAI,CACtG,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAChD,IAAI,CAAC;QACH,8EAA8E;QAC9E,kEAAkE;QAClE,MAAM,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CACJ,6BAA6B,MAAM,QAAQ,WAAW,0CAA0C;YAC9F,IAAI,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,+CAA+C,CACtG,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,SAAS,8BAA8B,CACrC,MAAc,EACd,QAA2B,EAC3B,QAAsC;IAEtC,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,OAAO;IAChD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,cAAc,QAAQ,CAAC,MAAM,oBAAoB,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG;QAC/G,6BAA6B,QAAQ,CAAC,MAAM,yDAAyD;QACrG,qGAAqG;QACrG,oCAAoC,CACvC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,2BAA2B,CAClC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,gBAAgB,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACzD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,mGAAmG,CAC9H,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,8CAA8C,IAAI,CAAC,MAAM,IAAI;YACrF,iDAAiD,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAC/E,CAAC;IACJ,CAAC;IACD,4BAA4B,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;AAC/D,CAAC;AAED,kGAAkG;AAClG,SAAS,UAAU,CAAC,KAAwB;IAC1C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAS,4BAA4B,CACnC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO;IAC/B,MAAM,QAAQ,GAAG,gBAAgB,IAAI,EAAE,CAAC;IACxC,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACjG,kBAAkB,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1G,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7G,kBAAkB,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAc,EACd,OAAe,EACf,QAAuC,EACvC,QAA2B;IAE3B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACtE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC1D,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,oBAAoB,OAAO,yCAAyC;QAC5F,8BAA8B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,gDAAgD;QACjG,qEAAqE,CACxE,CAAC;AACJ,CAAC"}
@@ -1,18 +1,19 @@
1
+ import { VatError } from '@vibe-agent-toolkit/utils';
1
2
  import { z } from 'zod';
2
3
  /**
3
4
  * Raised for any eval-input problem (bad JSON, schema failure, missing input file).
4
- * Maps to exit 2.
5
+ * Reason `preflight`.
5
6
  *
6
7
  * The FIELD is what makes that sentence true, and until it was added only one of this
7
8
  * class's two routes delivered it. `attemptStageWorkspaces` catches the staging
8
9
  * instance and maps it explicitly; the instance thrown from {@link armDirSegment}
9
10
  * fires deep in the eval loop, outside that handler, and reached
10
- * `mapErrorToExitCode`'s Internal default — reporting 1 for a preflight-class
11
- * problem while the docblock above claimed 2. Exit 2 for the internal-invariant case
12
- * too is consistent with every sibling in this feature.
11
+ * `skillTestFailureReason`'s `internal` default — reporting a harness bug for a
12
+ * preflight-class problem while the docblock above claimed otherwise. `preflight`
13
+ * for the internal-invariant case too is consistent with every sibling in this feature.
13
14
  */
14
- export declare class EvalInputError extends Error {
15
- readonly exitCode: 2;
15
+ export declare class EvalInputError extends VatError {
16
+ readonly reason: "preflight";
16
17
  constructor(message: string);
17
18
  }
18
19
  /**
@@ -528,7 +529,7 @@ export declare const EvalSuiteSchema: z.ZodObject<{
528
529
  export type EvalEntry = z.infer<typeof EvalEntrySchema>;
529
530
  export type EvalSuite = z.infer<typeof EvalSuiteSchema>;
530
531
  /**
531
- * The declared tool expectations of ONE eval (issue #145 Phase T). Derived from
532
+ * The declared tool expectations of ONE eval. Derived from
532
533
  * {@link EvalEntry} so the grader input + prompt builder share the EXACT shape the
533
534
  * parser produces — no drift, and no duplicated inline `{ mustRun?; mustNotRun?;
534
535
  * sequence? }` literal across those consumers.
@@ -1 +1 @@
1
- {"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AA2CD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAUrE;AAiDD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAyDxB,CAAC;AAEL,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAMZ,CAAC;AAEjB,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAE1E,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAsB1D;AAED;;;;GAIG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;AAEzC;;;;;;;;;GASG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sEAAsE;AACtE,wBAAgB,MAAM,CAAC,IAAI,EAAE,gBAAgB,GAAG,SAAS,OAAO,EAAE,CAEjE;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,OAAO,GAAG,MAAM,CAU1E;AAED,MAAM,WAAW,wBAAwB;IACvC,6BAA6B;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,EAAE,gBAAgB,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAK3E"}
1
+ {"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAEA,OAAO,EAA2B,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,QAAQ;IAC1C,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;gBAC3B,OAAO,EAAE,MAAM;CAG5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AA2CD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAUrE;AAiDD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAyDxB,CAAC;AAEL,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAMZ,CAAC;AAEjB,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAE1E,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAsB1D;AAED;;;;GAIG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;AAEzC;;;;;;;;;GASG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sEAAsE;AACtE,wBAAgB,MAAM,CAAC,IAAI,EAAE,gBAAgB,GAAG,SAAS,OAAO,EAAE,CAEjE;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,OAAO,GAAG,MAAM,CAU1E;AAED,MAAM,WAAW,wBAAwB;IACvC,6BAA6B;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,EAAE,gBAAgB,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAK3E"}
@@ -1,24 +1,23 @@
1
1
  import { cpSync, existsSync } from 'node:fs';
2
- import { mkdirSyncReal, safePath } from '@vibe-agent-toolkit/utils';
2
+ import { mkdirSyncReal, safePath, VatError } from '@vibe-agent-toolkit/utils';
3
3
  import { z } from 'zod';
4
4
  import { sanitizeGraderText, sanitizeTextPreservingLines } from './grader-text.js';
5
5
  /**
6
6
  * Raised for any eval-input problem (bad JSON, schema failure, missing input file).
7
- * Maps to exit 2.
7
+ * Reason `preflight`.
8
8
  *
9
9
  * The FIELD is what makes that sentence true, and until it was added only one of this
10
10
  * class's two routes delivered it. `attemptStageWorkspaces` catches the staging
11
11
  * instance and maps it explicitly; the instance thrown from {@link armDirSegment}
12
12
  * fires deep in the eval loop, outside that handler, and reached
13
- * `mapErrorToExitCode`'s Internal default — reporting 1 for a preflight-class
14
- * problem while the docblock above claimed 2. Exit 2 for the internal-invariant case
15
- * too is consistent with every sibling in this feature.
13
+ * `skillTestFailureReason`'s `internal` default — reporting a harness bug for a
14
+ * preflight-class problem while the docblock above claimed otherwise. `preflight`
15
+ * for the internal-invariant case too is consistent with every sibling in this feature.
16
16
  */
17
- export class EvalInputError extends Error {
18
- exitCode = 2;
17
+ export class EvalInputError extends VatError {
18
+ reason = 'preflight';
19
19
  constructor(message) {
20
- super(message);
21
- this.name = 'EvalInputError';
20
+ super('EVAL_INPUT', message);
22
21
  }
23
22
  }
24
23
  /**
@@ -187,7 +186,7 @@ export const EvalEntrySchema = z
187
186
  mustRun: z.array(z.string().min(1)).optional(),
188
187
  mustNotRun: z.array(z.string().min(1)).optional(),
189
188
  // Each named executable must have RUN and its invoking tool_result must
190
- // NOT be an error (feature #148). Judged from the transcript by the
189
+ // NOT be an error. Judged from the transcript by the
191
190
  // grader — see tool-eval-schema.ts's ToolSucceedCheckSchema.
192
191
  mustSucceed: z.array(z.string().min(1)).optional(),
193
192
  sequence: z.array(z.string().min(1)).optional(),
@@ -323,7 +322,6 @@ function stageEvalWorkspacesForArm(input, arm) {
323
322
  throw new EvalInputError(`eval ${entry.id} declares input file "${quoteSuiteText(rel)}" that escapes the eval directory: ` +
324
323
  quoteSuiteText(err.message));
325
324
  }
326
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- src is contained under evalsDir via joinUnderRoot; suite is developer-authored
327
325
  if (!existsSync(src)) {
328
326
  throw new EvalInputError(`eval ${entry.id} declares input file "${quoteSuiteText(rel)}" but it is absent at ${quoteSuiteText(src)}`);
329
327
  }
@@ -1 +1 @@
1
- {"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,kBAAkB,CAAC;AAEnF;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,cAAe,SAAQ,KAAK;IAC9B,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,2BAA2B,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAE3C;+EAC+E;AAC/E,MAAM,sBAAsB,GAAG;IAC7B,IAAI;IACJ,QAAQ;IACR,iBAAiB;IACjB,OAAO;IACP,cAAc;IACd,MAAM;IACN,kBAAkB;CACV,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,MAAc;IAC1D,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IACtB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IACzB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe;IAC/E,IAAI,EAAE,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAC9E,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB;AACpE,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,6BAA6B,CAAC,YAAsB,EAAE,GAAoB;IACjF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACtB,SAAS;QACX,CAAC;QACD,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;YAC3B,OAAO,EACL,0BAA0B,cAAc,CAAC,WAAW,CAAC,4CAA4C;gBACjG,8FAA8F;gBAC9F,yEAAyE;YAC3E,IAAI,EAAE,CAAC,cAAc,EAAE,KAAK,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,gFAAgF;AAChF,uEAAuE;AACvE,2EAA2E;AAC3E,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;QAChB,CAAC;aACE,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,KAAK,CACJ,eAAe,EACf,wGAAwG,CACzG;KACJ,CAAC;IACF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,2EAA2E;IAC3E,2EAA2E;IAC3E,qEAAqE;IACrE,4EAA4E;IAC5E,4EAA4E;IAC5E,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,wFAAwF;IACxF,mEAAmE;IACnE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAC/C,4EAA4E;IAC5E,kFAAkF;IAClF,qFAAqF;IACrF,+BAA+B;IAC/B,gBAAgB,EAAE,CAAC;SAChB,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC9C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QACjD,wEAAwE;QACxE,oEAAoE;QACpE,6DAA6D;QAC7D,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAClD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KAChD,CAAC;SACD,MAAM,EAAE;SACR,QAAQ,EAAE;CACd,CAAC;KACD,WAAW,EAAE;KACb,WAAW,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;IAC1B,6BAA6B,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IACvD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAK,sBAA4C,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC9G,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;gBAC3B,uEAAuE;gBACvE,iEAAiE;gBACjE,OAAO,EAAE,uBAAuB,cAAc,CAAC,GAAG,CAAC,qBAAqB,IAAI,kDAAkD;gBAC9H,IAAI,EAAE,CAAC,GAAG,CAAC;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,WAAW,EAAE,CAAC;AAajB,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,wEAAwE;QACxE,4EAA4E;QAC5E,uEAAuE;QACvE,MAAM,IAAI,cAAc,CAAC,iCAAiC,cAAc,CAAE,CAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,cAAc,CAAC,wCAAwC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;IACD,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,cAAc,CAAC,4FAA4F,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAwBD,sEAAsE;AACtE,MAAM,UAAU,MAAM,CAAC,IAAsB;IAC3C,OAAO,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAE,CAAC,MAAM,CAAW,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,SAAS,CAAW,CAAC;AAC3F,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,IAAsB,EAAE,GAAY;IAChE,MAAM,OAAO,GAAG,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;IAC1D,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,cAAc,CACtB,wDAAwD,GAAG,SAAS;YAClE,kFAAkF;YAClF,mDAAmD,CACtD,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAiBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,yBAAyB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC,cAAc,CAAC;AAC9B,CAAC;AAED,SAAS,yBAAyB,CAAC,KAA+B,EAAE,GAAY;IAC9E,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAChG,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,0EAA0E;QAC1E,2BAA2B;QAC3B,aAAa,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACpC,yEAAyE;YACzE,wEAAwE;YACxE,IAAI,GAAW,CAAC;YAChB,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAClD,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,cAAc,CAAC,GAAG,CAAC,qCAAqC;oBAC/F,cAAc,CAAE,GAAa,CAAC,OAAO,CAAC,CACzC,CAAC;YACJ,CAAC;YACD,qJAAqJ;YACrJ,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,cAAc,CAAC,GAAG,CAAC,yBAAyB,cAAc,CAAC,GAAG,CAAC,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,CAAC;gBACH,0EAA0E;gBAC1E,kDAAkD;gBAClD,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC3E,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,gCAAgC,cAAc,CAAC,GAAG,CAAC,wBAAwB;oBACzF,cAAc,CAAE,GAAa,CAAC,OAAO,CAAC,CACzC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,kBAAkB,CAAC;AAEnF;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,cAAe,SAAQ,QAAQ;IACjC,MAAM,GAAG,WAAoB,CAAC;IACvC,YAAY,OAAe;QACzB,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,2BAA2B,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAE3C;+EAC+E;AAC/E,MAAM,sBAAsB,GAAG;IAC7B,IAAI;IACJ,QAAQ;IACR,iBAAiB;IACjB,OAAO;IACP,cAAc;IACd,MAAM;IACN,kBAAkB;CACV,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,MAAc;IAC1D,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IACtB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IACzB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe;IAC/E,IAAI,EAAE,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAC9E,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB;AACpE,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,6BAA6B,CAAC,YAAsB,EAAE,GAAoB;IACjF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACtB,SAAS;QACX,CAAC;QACD,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;YAC3B,OAAO,EACL,0BAA0B,cAAc,CAAC,WAAW,CAAC,4CAA4C;gBACjG,8FAA8F;gBAC9F,yEAAyE;YAC3E,IAAI,EAAE,CAAC,cAAc,EAAE,KAAK,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,gFAAgF;AAChF,uEAAuE;AACvE,2EAA2E;AAC3E,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;QAChB,CAAC;aACE,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,KAAK,CACJ,eAAe,EACf,wGAAwG,CACzG;KACJ,CAAC;IACF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,2EAA2E;IAC3E,2EAA2E;IAC3E,qEAAqE;IACrE,4EAA4E;IAC5E,4EAA4E;IAC5E,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,wFAAwF;IACxF,mEAAmE;IACnE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAC/C,4EAA4E;IAC5E,kFAAkF;IAClF,qFAAqF;IACrF,+BAA+B;IAC/B,gBAAgB,EAAE,CAAC;SAChB,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC9C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QACjD,wEAAwE;QACxE,qDAAqD;QACrD,6DAA6D;QAC7D,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAClD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KAChD,CAAC;SACD,MAAM,EAAE;SACR,QAAQ,EAAE;CACd,CAAC;KACD,WAAW,EAAE;KACb,WAAW,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;IAC1B,6BAA6B,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IACvD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAK,sBAA4C,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC9G,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;gBAC3B,uEAAuE;gBACvE,iEAAiE;gBACjE,OAAO,EAAE,uBAAuB,cAAc,CAAC,GAAG,CAAC,qBAAqB,IAAI,kDAAkD;gBAC9H,IAAI,EAAE,CAAC,GAAG,CAAC;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,WAAW,EAAE,CAAC;AAajB,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,wEAAwE;QACxE,4EAA4E;QAC5E,uEAAuE;QACvE,MAAM,IAAI,cAAc,CAAC,iCAAiC,cAAc,CAAE,CAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,cAAc,CAAC,wCAAwC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;IACD,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,cAAc,CAAC,4FAA4F,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAwBD,sEAAsE;AACtE,MAAM,UAAU,MAAM,CAAC,IAAsB;IAC3C,OAAO,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAE,CAAC,MAAM,CAAW,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,SAAS,CAAW,CAAC;AAC3F,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,IAAsB,EAAE,GAAY;IAChE,MAAM,OAAO,GAAG,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;IAC1D,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,cAAc,CACtB,wDAAwD,GAAG,SAAS;YAClE,kFAAkF;YAClF,mDAAmD,CACtD,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAiBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,yBAAyB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC,cAAc,CAAC;AAC9B,CAAC;AAED,SAAS,yBAAyB,CAAC,KAA+B,EAAE,GAAY;IAC9E,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAChG,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,0EAA0E;QAC1E,2BAA2B;QAC3B,aAAa,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACpC,yEAAyE;YACzE,wEAAwE;YACxE,IAAI,GAAW,CAAC;YAChB,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAClD,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,cAAc,CAAC,GAAG,CAAC,qCAAqC;oBAC/F,cAAc,CAAE,GAAa,CAAC,OAAO,CAAC,CACzC,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,cAAc,CAAC,GAAG,CAAC,yBAAyB,cAAc,CAAC,GAAG,CAAC,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,CAAC;gBACH,0EAA0E;gBAC1E,kDAAkD;gBAClD,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC3E,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,gCAAgC,cAAc,CAAC,GAAG,CAAC,wBAAwB;oBACzF,cAAc,CAAE,GAAa,CAAC,OAAO,CAAC,CACzC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
@@ -1,6 +1,6 @@
1
1
  /**
2
- * Advisory (never-fatal) lint over an eval suite's `expectations` prose (issue #145
3
- * follow-up). An adopter reported that WEAK expectations — pure "mentions X" /
2
+ * Advisory (never-fatal) lint over an eval suite's `expectations` prose.
3
+ * An adopter reported that WEAK expectations — pure "mentions X" /
4
4
  * "includes Y" presence checks with no discriminating or negative counterpart —
5
5
  * pass even for a hallucinated or right-for-the-wrong-reason answer, giving false
6
6
  * confidence in a green run. This module flags that pattern so authors can
@@ -34,8 +34,8 @@ export interface EvalLintWarning {
34
34
  export declare function lintEvalExpectations(evals: EvalEntry[]): EvalLintWarning[];
35
35
  /**
36
36
  * Advisory (never-fatal) lint: flag a `toolExpectations` entry that names an
37
- * executable which looks like a TYPO of one of the skill's `declaredExecutables`
38
- * (issue #145 adopter follow-up). A name that never matches a real tool is a quiet
37
+ * executable which looks like a TYPO of one of the skill's `declaredExecutables`.
38
+ * A name that never matches a real tool is a quiet
39
39
  * footgun — `mustRun`/`mustSucceed`/`sequence` then fail for the wrong reason, and
40
40
  * `mustNotRun` passes vacuously (the tool "never ran" only because the name is
41
41
  * wrong). This is zero-token static plumbing, run before any spend.
@@ -1,6 +1,6 @@
1
1
  /**
2
- * Advisory (never-fatal) lint over an eval suite's `expectations` prose (issue #145
3
- * follow-up). An adopter reported that WEAK expectations — pure "mentions X" /
2
+ * Advisory (never-fatal) lint over an eval suite's `expectations` prose.
3
+ * An adopter reported that WEAK expectations — pure "mentions X" /
4
4
  * "includes Y" presence checks with no discriminating or negative counterpart —
5
5
  * pass even for a hallucinated or right-for-the-wrong-reason answer, giving false
6
6
  * confidence in a green run. This module flags that pattern so authors can
@@ -62,7 +62,7 @@ export function lintEvalExpectations(evals) {
62
62
  return warnings;
63
63
  }
64
64
  // ---------------------------------------------------------------------------
65
- // Undeclared-executable lint (issue #145 adopter follow-up)
65
+ // Undeclared-executable lint
66
66
  // ---------------------------------------------------------------------------
67
67
  // A trailing script/binary extension we strip before comparing an executable
68
68
  // name to a declared one, so `csvsum.py`/`./csvsum`/`csvsum` all normalize to `csvsum` (the
@@ -115,8 +115,8 @@ function probableDeclaredTypoTarget(name, declared) {
115
115
  }
116
116
  /**
117
117
  * Advisory (never-fatal) lint: flag a `toolExpectations` entry that names an
118
- * executable which looks like a TYPO of one of the skill's `declaredExecutables`
119
- * (issue #145 adopter follow-up). A name that never matches a real tool is a quiet
118
+ * executable which looks like a TYPO of one of the skill's `declaredExecutables`.
119
+ * A name that never matches a real tool is a quiet
120
120
  * footgun — `mustRun`/`mustSucceed`/`sequence` then fail for the wrong reason, and
121
121
  * `mustNotRun` passes vacuously (the tool "never ran" only because the name is
122
122
  * wrong). This is zero-token static plumbing, run before any spend.
@@ -1 +1 @@
1
- {"version":3,"file":"eval-lint.js","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAQpF,gFAAgF;AAChF,4EAA4E;AAC5E,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,2EAA2E,CAAC;AAE1G,iFAAiF;AACjF,kFAAkF;AAClF,0EAA0E;AAC1E,4EAA4E;AAC5E,2EAA2E;AAC3E,kFAAkF;AAClF,6DAA6D;AAC7D,MAAM,sBAAsB,GAC1B,+GAA+G,CAAC;AAElH,0IAA0I;AAC1I,SAAS,yBAAyB,CAAC,WAAmB;IACpD,OAAO,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAC9F,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC9C,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS;YAAE,SAAS;QACnD,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,yBAAyB,CAAC;YAAE,SAAS;QACnE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM;YACN,OAAO,EACL,SAAS,MAAM,uFAAuF;gBACtG,yGAAyG;gBACzG,8CAA8C;SACjD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAE9E,6EAA6E;AAC7E,4FAA4F;AAC5F,oFAAoF;AACpF,gFAAgF;AAChF,iFAAiF;AACjF,SAAS;AACT,MAAM,gBAAgB,GAAG,0CAA0C,CAAC;AAEpE,yHAAyH;AACzH,SAAS,uBAAuB,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACtF,CAAC;AAED,mHAAmH;AACnH,SAAS,iCAAiC,CAAC,KAAgB;IACzD,MAAM,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;IAClC,IAAI,EAAE,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,SAAiB,EAAE,IAAY;IACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpG,OAAO,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,0BAA0B,CACjC,IAAY,EACZ,QAAoD;IAEpD,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC,CAAC,iCAAiC;IAC1G,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QAClE,IAAI,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,eAAe,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;IAChH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,UAAU,8BAA8B,CAC5C,KAAkB,EAClB,uBAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1H,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC3C,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChF,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,iCAAiC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5D,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC1D,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM;gBACN,OAAO,EACL,SAAS,MAAM,6CAA6C,cAAc,CAAC,IAAI,CAAC,4CAA4C;oBAC5H,iBAAiB,cAAc,CAAC,MAAM,CAAC,mFAAmF;oBAC1H,8EAA8E,YAAY,GAAG;aAChG,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
1
+ {"version":3,"file":"eval-lint.js","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAQpF,gFAAgF;AAChF,4EAA4E;AAC5E,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,2EAA2E,CAAC;AAE1G,iFAAiF;AACjF,kFAAkF;AAClF,0EAA0E;AAC1E,4EAA4E;AAC5E,2EAA2E;AAC3E,kFAAkF;AAClF,6DAA6D;AAC7D,MAAM,sBAAsB,GAC1B,+GAA+G,CAAC;AAElH,0IAA0I;AAC1I,SAAS,yBAAyB,CAAC,WAAmB;IACpD,OAAO,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAC9F,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC9C,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS;YAAE,SAAS;QACnD,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,yBAAyB,CAAC;YAAE,SAAS;QACnE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM;YACN,OAAO,EACL,SAAS,MAAM,uFAAuF;gBACtG,yGAAyG;gBACzG,8CAA8C;SACjD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,6BAA6B;AAC7B,8EAA8E;AAE9E,6EAA6E;AAC7E,4FAA4F;AAC5F,oFAAoF;AACpF,gFAAgF;AAChF,iFAAiF;AACjF,SAAS;AACT,MAAM,gBAAgB,GAAG,0CAA0C,CAAC;AAEpE,yHAAyH;AACzH,SAAS,uBAAuB,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACtF,CAAC;AAED,mHAAmH;AACnH,SAAS,iCAAiC,CAAC,KAAgB;IACzD,MAAM,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;IAClC,IAAI,EAAE,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,SAAiB,EAAE,IAAY;IACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpG,OAAO,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,0BAA0B,CACjC,IAAY,EACZ,QAAoD;IAEpD,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC,CAAC,iCAAiC;IAC1G,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QAClE,IAAI,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,eAAe,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;IAChH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,UAAU,8BAA8B,CAC5C,KAAkB,EAClB,uBAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1H,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC3C,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChF,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,iCAAiC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5D,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC1D,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM;gBACN,OAAO,EACL,SAAS,MAAM,6CAA6C,cAAc,CAAC,IAAI,CAAC,4CAA4C;oBAC5H,iBAAiB,cAAc,CAAC,MAAM,CAAC,mFAAmF;oBAC1H,8EAA8E,YAAY,GAAG;aAChG,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
@@ -1 +1 @@
1
- {"version":3,"file":"eval-suite-isolation.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-suite-isolation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAOH;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,qBAAqB,CAAC;AAExD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CASxG;AA0BD,MAAM,WAAW,qBAAqB;IACpC,kFAAkF;IAClF,SAAS,EAAE,MAAM,CAAC;IAClB,6FAA6F;IAC7F,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,qBAAqB,GAAG,OAAO,CAsBtE"}
1
+ {"version":3,"file":"eval-suite-isolation.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-suite-isolation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAOH;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,qBAAqB,CAAC;AAExD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAaxG;AA0BD,MAAM,WAAW,qBAAqB;IACpC,kFAAkF;IAClF,SAAS,EAAE,MAAM,CAAC;IAClB,6FAA6F;IAC7F,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,qBAAqB,GAAG,OAAO,CAoBtE"}
@@ -34,7 +34,7 @@
34
34
  */
35
35
  import { cpSync, existsSync, lstatSync, rmSync } from 'node:fs';
36
36
  import { basename, dirname } from 'node:path';
37
- import { mkdirSyncReal, safePath } from '@vibe-agent-toolkit/utils';
37
+ import { isVatError, mkdirSyncReal, PathEscapesRootError, safePath } from '@vibe-agent-toolkit/utils';
38
38
  /**
39
39
  * Where a skill's eval suite lives by convention. Defined here, beside the code
40
40
  * that must REMOVE it, because every lane that reasons about a suite needs the
@@ -69,8 +69,13 @@ export function evalSuiteUnitPath(skillDir, evalsSubpath) {
69
69
  try {
70
70
  return safePath.joinUnderRoot(skillDir, relative);
71
71
  }
72
- catch {
73
- return undefined;
72
+ catch (error) {
73
+ // Only the containment helper's OWN refusal means "outside the skill dir";
74
+ // anything else out of it is a bug, and `undefined` would read as
75
+ // "nothing to strip" — leaving the answer key in the staged copy.
76
+ if (isVatError(error, PathEscapesRootError.code))
77
+ return undefined;
78
+ throw error;
74
79
  }
75
80
  }
76
81
  /**
@@ -109,7 +114,6 @@ function assertInsideRoot(child, root) {
109
114
  */
110
115
  export function isolateEvalSuite(input) {
111
116
  const unit = evalSuiteUnitPath(input.stagedDir, input.evalsSubpath);
112
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- unit is proven inside our own staged dir
113
117
  if (unit === undefined || !existsSync(unit))
114
118
  return false;
115
119
  assertInsideRoot(input.stagedDir, input.stagingRoot);
@@ -120,7 +124,6 @@ export function isolateEvalSuite(input) {
120
124
  // lands at `<holdDir>/<basename(evalsSubpath)>` either way and `fixtures/` keep
121
125
  // their positions relative to it — exactly the shape the eval-input staging
122
126
  // expects from an authored evals dir.
123
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- unit is proven inside our own staged dir
124
127
  const dest = lstatSync(unit).isDirectory() ? input.holdDir : safePath.join(input.holdDir, basename(unit));
125
128
  cpSync(unit, dest, { recursive: true });
126
129
  preserved = true;
@@ -1 +1 @@
1
- {"version":3,"file":"eval-suite-isolation.js","sourceRoot":"","sources":["../../src/skill-test/eval-suite-isolation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEpE;;;;;GAKG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,kBAAkB,CAAC;AAExD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,YAAgC;IAClF,IAAI,YAAY,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACjD,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC;IACzE,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACpD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,KAAa,EAAE,IAAY;IACnD,0EAA0E;IAC1E,yEAAyE;IACzE,4EAA4E;IAC5E,+EAA+E;IAC/E,wEAAwE;IACxE,yCAAyC;IACzC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,SAAS,CACjB,wEAAwE;YACtE,mEAAmE,CACtE,CAAC;IACJ,CAAC;IACD,QAAQ,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC;AAyBD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAA4B;IAC3D,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IACpE,+GAA+G;IAC/G,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE1D,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAErD,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAChC,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,gFAAgF;QAChF,gFAAgF;QAChF,4EAA4E;QAC5E,sCAAsC;QACtC,+GAA+G;QAC/G,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1G,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,OAAO,SAAS,CAAC;AACnB,CAAC"}
1
+ {"version":3,"file":"eval-suite-isolation.js","sourceRoot":"","sources":["../../src/skill-test/eval-suite-isolation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,oBAAoB,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEtG;;;;;GAKG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,kBAAkB,CAAC;AAExD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,YAAgC;IAClF,IAAI,YAAY,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACjD,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC;IACzE,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,2EAA2E;QAC3E,kEAAkE;QAClE,kEAAkE;QAClE,IAAI,UAAU,CAAC,KAAK,EAAE,oBAAoB,CAAC,IAAI,CAAC;YAAE,OAAO,SAAS,CAAC;QACnE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,KAAa,EAAE,IAAY;IACnD,0EAA0E;IAC1E,yEAAyE;IACzE,4EAA4E;IAC5E,+EAA+E;IAC/E,wEAAwE;IACxE,yCAAyC;IACzC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,SAAS,CACjB,wEAAwE;YACtE,mEAAmE,CACtE,CAAC;IACJ,CAAC;IACD,QAAQ,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC;AAyBD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAA4B;IAC3D,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IACpE,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE1D,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAErD,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAChC,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,gFAAgF;QAChF,gFAAgF;QAChF,4EAA4E;QAC5E,sCAAsC;QACtC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1G,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,OAAO,SAAS,CAAC;AACnB,CAAC"}
@@ -1,5 +1,5 @@
1
1
  /**
2
- * evals-template.ts — scaffold a starter `evals.json` on bootstrap (exit 3).
2
+ * evals-template.ts — scaffold a starter `evals.json` on bootstrap (exit 2, `Reason: bootstrap`).
3
3
  *
4
4
  * When a subject skill has no `evals/evals.json`, the harness writes an
5
5
  * annotated template the user can fill in and re-run. The shape mirrors
@@ -1 +1 @@
1
- {"version":3,"file":"evals-template.d.ts","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,2EAA2E;AAC3E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CA6C5D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAW/E"}
1
+ {"version":3,"file":"evals-template.d.ts","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,2EAA2E;AAC3E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CA6C5D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAS/E"}
@@ -1,5 +1,5 @@
1
1
  /**
2
- * evals-template.ts — scaffold a starter `evals.json` on bootstrap (exit 3).
2
+ * evals-template.ts — scaffold a starter `evals.json` on bootstrap (exit 2, `Reason: bootstrap`).
3
3
  *
4
4
  * When a subject skill has no `evals/evals.json`, the harness writes an
5
5
  * annotated template the user can fill in and re-run. The shape mirrors
@@ -70,9 +70,7 @@ export function writeEvalsTemplate(evalsPath, skillName) {
70
70
  mkdirSyncReal(parent, { recursive: true });
71
71
  // Defensive: never clobber an authored eval suite. Callers only reach here when
72
72
  // the suite is absent, but guard so a stray call can never destroy real evals.
73
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own resolved scaffold path
74
73
  if (!existsSync(evalsPath)) {
75
- // eslint-disable-next-line security/detect-non-literal-fs-filename -- our own resolved scaffold path
76
74
  writeFileSync(evalsPath, buildEvalsTemplate(skillName), 'utf8');
77
75
  }
78
76
  return evalsPath;
@@ -1 +1 @@
1
- {"version":3,"file":"evals-template.js","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEpE,2EAA2E;AAC3E,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE;YACR,8DAA8D;YAC9D,oFAAoF;YACpF,kBAAkB;YAClB,8EAA8E;YAC9E,yFAAyF;YACzF,iFAAiF;YACjF,oFAAoF;YACpF,yFAAyF;YACzF,2FAA2F;YAC3F,6FAA6F;YAC7F,mFAAmF;YACnF,uFAAuF;YACvF,wFAAwF;YACxF,gGAAgG;YAChG,kGAAkG;YAClG,mFAAmF;YACnF,+FAA+F;YAC/F,qFAAqF;YACrF,wFAAwF;YACxF,0FAA0F;YAC1F,mFAAmF;YACnF,qFAAqF;YACrF,oFAAoF;YACpF,yFAAyF;YACzF,2GAA2G;SAC5G;QACD,UAAU,EAAE,SAAS;QACrB,KAAK,EAAE;YACL;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE,qDAAqD;gBAC/D,MAAM,EAAE,yEAAyE;gBACjF,eAAe,EAAE,kDAAkD;gBACnE,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE;oBACZ,sGAAsG;oBACtG,2GAA2G;iBAC5G;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,SAAiB;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC9C,aAAa,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,gFAAgF;IAChF,+EAA+E;IAC/E,qGAAqG;IACrG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,qGAAqG;QACrG,aAAa,CAAC,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
1
+ {"version":3,"file":"evals-template.js","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEpE,2EAA2E;AAC3E,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE;YACR,8DAA8D;YAC9D,oFAAoF;YACpF,kBAAkB;YAClB,8EAA8E;YAC9E,yFAAyF;YACzF,iFAAiF;YACjF,oFAAoF;YACpF,yFAAyF;YACzF,2FAA2F;YAC3F,6FAA6F;YAC7F,mFAAmF;YACnF,uFAAuF;YACvF,wFAAwF;YACxF,gGAAgG;YAChG,kGAAkG;YAClG,mFAAmF;YACnF,+FAA+F;YAC/F,qFAAqF;YACrF,wFAAwF;YACxF,0FAA0F;YAC1F,mFAAmF;YACnF,qFAAqF;YACrF,oFAAoF;YACpF,yFAAyF;YACzF,2GAA2G;SAC5G;QACD,UAAU,EAAE,SAAS;QACrB,KAAK,EAAE;YACL;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE,qDAAqD;gBAC/D,MAAM,EAAE,yEAAyE;gBACjF,eAAe,EAAE,kDAAkD;gBACnE,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE;oBACZ,sGAAsG;oBACtG,2GAA2G;iBAC5G;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,SAAiB;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC9C,aAAa,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,gFAAgF;IAChF,+EAA+E;IAC/E,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,aAAa,CAAC,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
@@ -15,8 +15,8 @@ export interface BuildExecutorPromptOptions {
15
15
  workspaceDir?: string;
16
16
  }
17
17
  /**
18
- * Build the prompt handed to the blind executor subagent (issue #145 —
19
- * transcript-grounded eval rewrite). The executor is a real worker, not a
18
+ * Build the prompt handed to the blind executor subagent (the
19
+ * transcript-grounded eval design). The executor is a real worker, not a
20
20
  * participant in an eval: it receives the task and its working context ONLY,
21
21
  * phrased exactly as a genuine user request. It must NEVER be told (directly
22
22
  * or by implication) that it is being tested, evaluated, or graded — that
@@ -1,7 +1,7 @@
1
1
  import { PromptInvariantError } from './prompt-invariants.js';
2
2
  /**
3
- * Build the prompt handed to the blind executor subagent (issue #145 —
4
- * transcript-grounded eval rewrite). The executor is a real worker, not a
3
+ * Build the prompt handed to the blind executor subagent (the
4
+ * transcript-grounded eval design). The executor is a real worker, not a
5
5
  * participant in an eval: it receives the task and its working context ONLY,
6
6
  * phrased exactly as a genuine user request. It must NEVER be told (directly
7
7
  * or by implication) that it is being tested, evaluated, or graded — that
@@ -0,0 +1,113 @@
1
+ import { VatError } from '@vibe-agent-toolkit/utils';
2
+ /**
3
+ * WHY a `vat skill test run` ended on `ExitCode.ERROR`.
4
+ *
5
+ * This used to be three exit codes — 1 internal, 2 preflight, 3 bootstrap —
6
+ * beside 4 for "an eval failed". Under the one exit-code contract every verb
7
+ * shares (`ExitCode` in `@vibe-agent-toolkit/schema`) a failed eval is
8
+ * `FINDINGS` (1) like every other gate that ran and did not pass, and every way
9
+ * the harness could NOT run is `ERROR` (2). The distinction a CI author still
10
+ * needs — "did the harness break, or is my environment wrong?" — is said on
11
+ * stderr as `Reason: <reason>`, never in a fourth number a wrapper would have
12
+ * to look up per command.
13
+ *
14
+ * - `internal` — the harness broke: a spawn failure, a watchdog stall, a grader
15
+ * fragment that did not parse, a forged nonce. VAT's fault until proven
16
+ * otherwise.
17
+ * - `preflight` — the operator can fix it: a bad config, a missing binary, an
18
+ * absent security ack, a held lock, a companion that would not build.
19
+ * - `bootstrap` — nothing was wrong; the run scaffolded an `evals.json`
20
+ * template and stopped so the author can fill it in.
21
+ */
22
+ export type SkillTestFailureReason = 'internal' | 'preflight' | 'bootstrap';
23
+ /**
24
+ * The reason an error, thrown out of the harness, gives for the run's `ERROR`.
25
+ *
26
+ * An error DECLARES its reason with a `readonly reason` field — every VAT error
27
+ * class in this feature carries one — and this reads it. Only a VAT error is
28
+ * consulted (the `Symbol.for('vat.error')` brand, which survives the
29
+ * `src`/`dist` boundary that `instanceof` does not): a foreign error carrying
30
+ * `reason` is not opting in to anything, and a foreign error carrying nothing
31
+ * is the harness's own fault, `internal`. That closes the residual the old
32
+ * `exitCode` read left open, where `commander`'s `CommanderError` carried a 2
33
+ * and would have reported "your environment is wrong" for a crash.
34
+ *
35
+ * The read is guarded: a getter or `Proxy` that throws on `reason` is a foreign
36
+ * object misbehaving, and this runs inside the CLI's own `catch`, where a
37
+ * rethrow escapes the handler entirely — no summary, no exit code, a bare
38
+ * stack.
39
+ */
40
+ export declare function skillTestFailureReason(err: unknown): SkillTestFailureReason;
41
+ /**
42
+ * A required input is absent in a way vat can scaffold (missing evals.json).
43
+ * Reason `bootstrap`, NOT a failure. `expectedPath` is the persistent location
44
+ * of the annotated starter template.
45
+ *
46
+ * In a real run the harness has already written that template, and the message
47
+ * says so. Under `--dry-run` the TEMPLATE is not written, so the message instead
48
+ * describes what a real run *would* scaffold and where, while surfacing the same
49
+ * "bootstrap needed" signal.
50
+ *
51
+ * That scope is exact, and it used to read "a dry run must never touch the
52
+ * filesystem", which is not true of this command. A dry run is not a read-only
53
+ * mode: it takes the harness lock, creates the harness root, stages BOTH arms'
54
+ * per-eval workspaces, and — once `--i-understand-this-runs-skill-code` is passed —
55
+ * runs the repo's `test.build` hook and builds the subject. What it never does is
56
+ * SPAWN: no executor session, no grader session, no tokens.
57
+ *
58
+ * It also never touches `results/`: it neither creates the directory, nor writes
59
+ * `provenance.json`, nor wipes the previous run's artifacts. That last one is the
60
+ * reason the ordering matters — the wipe used to sit AHEAD of the dry-run
61
+ * short-circuit, so a free `--dry-run` (or any failure after it) destroyed the
62
+ * `grading.json` / `baseline.json` / `friction.json` / `tool-eval.json` of the
63
+ * expensive real run an operator was about to read. The summary now names the
64
+ * provenance path it WOULD write rather than writing it.
65
+ *
66
+ * The scaffold is the other filesystem effect a dry run deliberately withholds,
67
+ * and for a different reason: that write lands in the AUTHOR's source
68
+ * tree rather than in vat's own scratch space.
69
+ */
70
+ export declare class BootstrapNeededError extends VatError {
71
+ readonly expectedPath: string;
72
+ readonly reason: "bootstrap";
73
+ constructor(expectedPath: string, opts?: {
74
+ dryRun?: boolean;
75
+ });
76
+ }
77
+ /** Thrown when building a declared skill (pool packageSkill or plugin build) fails. Reason `preflight`. */
78
+ export declare class SkillBuildError extends VatError {
79
+ readonly reason: "preflight";
80
+ constructor(message: string);
81
+ }
82
+ /**
83
+ * The §12 security acknowledgment is required but absent, thrown BEFORE any
84
+ * build/pre-stage command (which executes untrusted repo code) runs for a
85
+ * buildable subject. Reason `preflight`. Mirrors the harness Step-6 ack
86
+ * message wording so the two enforcement points read identically.
87
+ */
88
+ export declare class SecurityAckError extends VatError {
89
+ readonly reason: "preflight";
90
+ constructor();
91
+ }
92
+ /**
93
+ * A skill name is staged more than once in a single run — the subject, a
94
+ * `--with` companion, and a `--with-optional` companion must all have distinct
95
+ * names. Both `--with` and `--with-optional` STAGE the named companion and make
96
+ * it invocable (they differ only in required-vs-optional resolution, not in
97
+ * whether they are staged), so a colliding name would silently overwrite an
98
+ * earlier staged copy under the same slot — never an error, never a manifest
99
+ * trace — which makes a routing/deferral eval look correct while testing
100
+ * something else. Reason `preflight`: user-correctable input, like a bad env
101
+ * token or missing security ack.
102
+ */
103
+ export declare class DuplicateStagedSkillError extends VatError {
104
+ readonly skillName: string;
105
+ readonly reason: "preflight";
106
+ constructor(skillName: string);
107
+ }
108
+ /** Internal harness failure (an executor/grader spawn error, watchdog timeout/stall, or a missing grader fragment). Reason `internal`. */
109
+ export declare class InternalHarnessError extends VatError {
110
+ readonly reason: "internal";
111
+ constructor(message: string);
112
+ }
113
+ //# sourceMappingURL=failure-reason.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"failure-reason.d.ts","sourceRoot":"","sources":["../../src/skill-test/failure-reason.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEjE;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,MAAM,sBAAsB,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC;AAQ5E;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,OAAO,GAAG,sBAAsB,CAY3E;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,oBAAqB,SAAQ,QAAQ;aAEpB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;gBACX,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE;CAQ9E;AAED,2GAA2G;AAC3G,qBAAa,eAAgB,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;gBAC3B,OAAO,EAAE,MAAM;CAG5B;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,QAAQ;IAC5C,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;;CAOxC;AAED;;;;;;;;;;GAUG;AACH,qBAAa,yBAA0B,SAAQ,QAAQ;aAEzB,SAAS,EAAE,MAAM;IAD7C,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;gBACX,SAAS,EAAE,MAAM;CAO9C;AAED,0IAA0I;AAC1I,qBAAa,oBAAqB,SAAQ,QAAQ;IAChD,QAAQ,CAAC,MAAM,EAAG,UAAU,CAAU;gBAC1B,OAAO,EAAE,MAAM;CAG5B"}
@@ -0,0 +1,124 @@
1
+ import { isVatError, VatError } from '@vibe-agent-toolkit/utils';
2
+ const FAILURE_REASONS = new Set([
3
+ 'internal',
4
+ 'preflight',
5
+ 'bootstrap',
6
+ ]);
7
+ /**
8
+ * The reason an error, thrown out of the harness, gives for the run's `ERROR`.
9
+ *
10
+ * An error DECLARES its reason with a `readonly reason` field — every VAT error
11
+ * class in this feature carries one — and this reads it. Only a VAT error is
12
+ * consulted (the `Symbol.for('vat.error')` brand, which survives the
13
+ * `src`/`dist` boundary that `instanceof` does not): a foreign error carrying
14
+ * `reason` is not opting in to anything, and a foreign error carrying nothing
15
+ * is the harness's own fault, `internal`. That closes the residual the old
16
+ * `exitCode` read left open, where `commander`'s `CommanderError` carried a 2
17
+ * and would have reported "your environment is wrong" for a crash.
18
+ *
19
+ * The read is guarded: a getter or `Proxy` that throws on `reason` is a foreign
20
+ * object misbehaving, and this runs inside the CLI's own `catch`, where a
21
+ * rethrow escapes the handler entirely — no summary, no exit code, a bare
22
+ * stack.
23
+ */
24
+ export function skillTestFailureReason(err) {
25
+ if (!isVatError(err))
26
+ return 'internal';
27
+ let declared;
28
+ try {
29
+ declared = err.reason;
30
+ // eslint-disable-next-line local/no-blind-catch -- a property read throws only from a getter or Proxy trap on a misbehaving object, and "the read itself threw" is exactly "declared nothing": there is no second case to tell apart, and this runs inside the CLI's own catch, where a rethrow escapes the handler entirely (pinned in failure-reason.test.ts).
31
+ }
32
+ catch {
33
+ return 'internal';
34
+ }
35
+ return typeof declared === 'string' && FAILURE_REASONS.has(declared)
36
+ ? declared
37
+ : 'internal';
38
+ }
39
+ /**
40
+ * A required input is absent in a way vat can scaffold (missing evals.json).
41
+ * Reason `bootstrap`, NOT a failure. `expectedPath` is the persistent location
42
+ * of the annotated starter template.
43
+ *
44
+ * In a real run the harness has already written that template, and the message
45
+ * says so. Under `--dry-run` the TEMPLATE is not written, so the message instead
46
+ * describes what a real run *would* scaffold and where, while surfacing the same
47
+ * "bootstrap needed" signal.
48
+ *
49
+ * That scope is exact, and it used to read "a dry run must never touch the
50
+ * filesystem", which is not true of this command. A dry run is not a read-only
51
+ * mode: it takes the harness lock, creates the harness root, stages BOTH arms'
52
+ * per-eval workspaces, and — once `--i-understand-this-runs-skill-code` is passed —
53
+ * runs the repo's `test.build` hook and builds the subject. What it never does is
54
+ * SPAWN: no executor session, no grader session, no tokens.
55
+ *
56
+ * It also never touches `results/`: it neither creates the directory, nor writes
57
+ * `provenance.json`, nor wipes the previous run's artifacts. That last one is the
58
+ * reason the ordering matters — the wipe used to sit AHEAD of the dry-run
59
+ * short-circuit, so a free `--dry-run` (or any failure after it) destroyed the
60
+ * `grading.json` / `baseline.json` / `friction.json` / `tool-eval.json` of the
61
+ * expensive real run an operator was about to read. The summary now names the
62
+ * provenance path it WOULD write rather than writing it.
63
+ *
64
+ * The scaffold is the other filesystem effect a dry run deliberately withholds,
65
+ * and for a different reason: that write lands in the AUTHOR's source
66
+ * tree rather than in vat's own scratch space.
67
+ */
68
+ export class BootstrapNeededError extends VatError {
69
+ expectedPath;
70
+ reason = 'bootstrap';
71
+ constructor(expectedPath, opts) {
72
+ super('SKILL_TEST_BOOTSTRAP_NEEDED', opts?.dryRun === true
73
+ ? `[dry-run] No evals.json found. A real run would scaffold an annotated template at ${expectedPath} — fill it in and re-run. (dry-run: nothing was written to your source tree.)`
74
+ : `Wrote an evals.json template at ${expectedPath} — fill it in and re-run.`);
75
+ this.expectedPath = expectedPath;
76
+ }
77
+ }
78
+ /** Thrown when building a declared skill (pool packageSkill or plugin build) fails. Reason `preflight`. */
79
+ export class SkillBuildError extends VatError {
80
+ reason = 'preflight';
81
+ constructor(message) {
82
+ super('SKILL_TEST_BUILD_FAILED', message);
83
+ }
84
+ }
85
+ /**
86
+ * The §12 security acknowledgment is required but absent, thrown BEFORE any
87
+ * build/pre-stage command (which executes untrusted repo code) runs for a
88
+ * buildable subject. Reason `preflight`. Mirrors the harness Step-6 ack
89
+ * message wording so the two enforcement points read identically.
90
+ */
91
+ export class SecurityAckError extends VatError {
92
+ reason = 'preflight';
93
+ constructor() {
94
+ super('SKILL_TEST_SECURITY_ACK_MISSING', 'Security acknowledgment required. Pass --i-understand-this-runs-skill-code to proceed.');
95
+ }
96
+ }
97
+ /**
98
+ * A skill name is staged more than once in a single run — the subject, a
99
+ * `--with` companion, and a `--with-optional` companion must all have distinct
100
+ * names. Both `--with` and `--with-optional` STAGE the named companion and make
101
+ * it invocable (they differ only in required-vs-optional resolution, not in
102
+ * whether they are staged), so a colliding name would silently overwrite an
103
+ * earlier staged copy under the same slot — never an error, never a manifest
104
+ * trace — which makes a routing/deferral eval look correct while testing
105
+ * something else. Reason `preflight`: user-correctable input, like a bad env
106
+ * token or missing security ack.
107
+ */
108
+ export class DuplicateStagedSkillError extends VatError {
109
+ skillName;
110
+ reason = 'preflight';
111
+ constructor(skillName) {
112
+ super('SKILL_TEST_DUPLICATE_STAGED_SKILL', `Skill name "${skillName}" is staged more than once (subject / --with / --with-optional). ` +
113
+ `Each staged skill must have a unique name.`);
114
+ this.skillName = skillName;
115
+ }
116
+ }
117
+ /** Internal harness failure (an executor/grader spawn error, watchdog timeout/stall, or a missing grader fragment). Reason `internal`. */
118
+ export class InternalHarnessError extends VatError {
119
+ reason = 'internal';
120
+ constructor(message) {
121
+ super('SKILL_TEST_INTERNAL', message);
122
+ }
123
+ }
124
+ //# sourceMappingURL=failure-reason.js.map