@vibe-agent-toolkit/agent-skills 0.2.0-rc.2 → 0.2.0-rc.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/files-config.d.ts +34 -1
- package/dist/files-config.d.ts.map +1 -1
- package/dist/files-config.js +280 -36
- package/dist/files-config.js.map +1 -1
- package/dist/fs-attribution.d.ts +45 -0
- package/dist/fs-attribution.d.ts.map +1 -0
- package/dist/fs-attribution.js +57 -0
- package/dist/fs-attribution.js.map +1 -0
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -3
- package/dist/index.js.map +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts +1 -1
- package/dist/inventory/detectors/declared-but-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts +1 -1
- package/dist/inventory/detectors/marketplace-source-missing.d.ts.map +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts +1 -1
- package/dist/inventory/detectors/present-but-undeclared.d.ts.map +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts +1 -1
- package/dist/inventory/detectors/reference-target-missing.d.ts.map +1 -1
- package/dist/inventory/index.d.ts +1 -1
- package/dist/inventory/index.d.ts.map +1 -1
- package/dist/inventory/index.js +1 -1
- package/dist/inventory/index.js.map +1 -1
- package/dist/inventory/serialize.d.ts +4 -4
- package/dist/inventory/serialize.d.ts.map +1 -1
- package/dist/inventory/serialize.js +6 -6
- package/dist/inventory/serialize.js.map +1 -1
- package/dist/plugin-distribution-layout.d.ts.map +1 -1
- package/dist/plugin-distribution-layout.js +3 -1
- package/dist/plugin-distribution-layout.js.map +1 -1
- package/dist/post-build-checks.d.ts +1 -1
- package/dist/post-build-checks.d.ts.map +1 -1
- package/dist/post-build-checks.js.map +1 -1
- package/dist/projection/skill-extent.d.ts +247 -0
- package/dist/projection/skill-extent.d.ts.map +1 -0
- package/dist/projection/skill-extent.js +558 -0
- package/dist/projection/skill-extent.js.map +1 -0
- package/dist/schemas/agent-skill-frontmatter.js +1 -1
- package/dist/schemas/agent-skill-frontmatter.js.map +1 -1
- package/dist/skill-packager.d.ts +45 -7
- package/dist/skill-packager.d.ts.map +1 -1
- package/dist/skill-packager.js +123 -25
- package/dist/skill-packager.js.map +1 -1
- package/dist/skill-source/git-clone.d.ts +1 -1
- package/dist/skill-source/git-clone.d.ts.map +1 -1
- package/dist/skill-source/git-clone.js +10 -8
- package/dist/skill-source/git-clone.js.map +1 -1
- package/dist/skill-source/sources/url-source.d.ts.map +1 -1
- package/dist/skill-source/sources/url-source.js +2 -1
- package/dist/skill-source/sources/url-source.js.map +1 -1
- package/dist/skill-test/baseline-delta.d.ts +1575 -0
- package/dist/skill-test/baseline-delta.d.ts.map +1 -0
- package/dist/skill-test/baseline-delta.js +272 -0
- package/dist/skill-test/baseline-delta.js.map +1 -0
- package/dist/skill-test/baseline-integrity.d.ts +944 -0
- package/dist/skill-test/baseline-integrity.d.ts.map +1 -0
- package/dist/skill-test/baseline-integrity.js +3083 -0
- package/dist/skill-test/baseline-integrity.js.map +1 -0
- package/dist/skill-test/configure-writer.js +1 -1
- package/dist/skill-test/configure-writer.js.map +1 -1
- package/dist/skill-test/declared-env.d.ts.map +1 -1
- package/dist/skill-test/declared-env.js +2 -1
- package/dist/skill-test/declared-env.js.map +1 -1
- package/dist/skill-test/eval-executor.d.ts +20 -4
- package/dist/skill-test/eval-executor.d.ts.map +1 -1
- package/dist/skill-test/eval-executor.js +4 -4
- package/dist/skill-test/eval-executor.js.map +1 -1
- package/dist/skill-test/eval-fragment.d.ts +80 -6
- package/dist/skill-test/eval-fragment.d.ts.map +1 -1
- package/dist/skill-test/eval-fragment.js +61 -7
- package/dist/skill-test/eval-fragment.js.map +1 -1
- package/dist/skill-test/eval-grader.d.ts +1 -1
- package/dist/skill-test/eval-grader.d.ts.map +1 -1
- package/dist/skill-test/eval-grader.js +145 -8
- package/dist/skill-test/eval-grader.js.map +1 -1
- package/dist/skill-test/eval-inputs.d.ts +107 -6
- package/dist/skill-test/eval-inputs.d.ts.map +1 -1
- package/dist/skill-test/eval-inputs.js +173 -16
- package/dist/skill-test/eval-inputs.js.map +1 -1
- package/dist/skill-test/eval-lint.d.ts +13 -0
- package/dist/skill-test/eval-lint.d.ts.map +1 -1
- package/dist/skill-test/eval-lint.js +17 -4
- package/dist/skill-test/eval-lint.js.map +1 -1
- package/dist/skill-test/executor-prompt.d.ts +13 -5
- package/dist/skill-test/executor-prompt.d.ts.map +1 -1
- package/dist/skill-test/executor-prompt.js +8 -5
- package/dist/skill-test/executor-prompt.js.map +1 -1
- package/dist/skill-test/exit-codes.d.ts +53 -16
- package/dist/skill-test/exit-codes.d.ts.map +1 -1
- package/dist/skill-test/exit-codes.js +123 -40
- package/dist/skill-test/exit-codes.js.map +1 -1
- package/dist/skill-test/fragment-merge.d.ts +1 -18
- package/dist/skill-test/fragment-merge.d.ts.map +1 -1
- package/dist/skill-test/fragment-merge.js +63 -4
- package/dist/skill-test/fragment-merge.js.map +1 -1
- package/dist/skill-test/grader-prompt.d.ts +27 -7
- package/dist/skill-test/grader-prompt.d.ts.map +1 -1
- package/dist/skill-test/grader-prompt.js +253 -61
- package/dist/skill-test/grader-prompt.js.map +1 -1
- package/dist/skill-test/grader-text.d.ts +145 -0
- package/dist/skill-test/grader-text.d.ts.map +1 -0
- package/dist/skill-test/grader-text.js +410 -0
- package/dist/skill-test/grader-text.js.map +1 -0
- package/dist/skill-test/grading-adapter.d.ts +32 -0
- package/dist/skill-test/grading-adapter.d.ts.map +1 -1
- package/dist/skill-test/grading-adapter.js +50 -3
- package/dist/skill-test/grading-adapter.js.map +1 -1
- package/dist/skill-test/grading-schema.d.ts +160 -5
- package/dist/skill-test/grading-schema.d.ts.map +1 -1
- package/dist/skill-test/grading-schema.js +66 -7
- package/dist/skill-test/grading-schema.js.map +1 -1
- package/dist/skill-test/index.d.ts +1 -1
- package/dist/skill-test/index.d.ts.map +1 -1
- package/dist/skill-test/index.js +1 -1
- package/dist/skill-test/index.js.map +1 -1
- package/dist/skill-test/lock.d.ts +56 -0
- package/dist/skill-test/lock.d.ts.map +1 -1
- package/dist/skill-test/lock.js +75 -2
- package/dist/skill-test/lock.js.map +1 -1
- package/dist/skill-test/pipeline.d.ts +16 -0
- package/dist/skill-test/pipeline.d.ts.map +1 -1
- package/dist/skill-test/pipeline.js +16 -9
- package/dist/skill-test/pipeline.js.map +1 -1
- package/dist/skill-test/preflight.d.ts +1 -1
- package/dist/skill-test/preflight.d.ts.map +1 -1
- package/dist/skill-test/preflight.js +59 -15
- package/dist/skill-test/preflight.js.map +1 -1
- package/dist/skill-test/run-harness.d.ts +578 -35
- package/dist/skill-test/run-harness.d.ts.map +1 -1
- package/dist/skill-test/run-harness.js +1281 -148
- package/dist/skill-test/run-harness.js.map +1 -1
- package/dist/skill-test/staging.d.ts +8 -2
- package/dist/skill-test/staging.d.ts.map +1 -1
- package/dist/skill-test/staging.js +8 -4
- package/dist/skill-test/staging.js.map +1 -1
- package/dist/skill-test/tool-eval-schema.d.ts +6 -6
- package/dist/test-input.d.ts +57 -6
- package/dist/test-input.d.ts.map +1 -1
- package/dist/test-input.js +35 -12
- package/dist/test-input.js.map +1 -1
- package/dist/validators/agent-instruction-presence.d.ts +1 -1
- package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
- package/dist/validators/agent-instruction-presence.js +2 -1
- package/dist/validators/agent-instruction-presence.js.map +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
- package/dist/validators/bundled-resource-link-detection.js +1 -1
- package/dist/validators/bundled-resource-link-detection.js.map +1 -1
- package/dist/validators/compat-detectors.d.ts +1 -1
- package/dist/validators/compat-detectors.d.ts.map +1 -1
- package/dist/validators/compat-detectors.js +1 -1
- package/dist/validators/compat-detectors.js.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts +1 -1
- package/dist/validators/cross-skill-dependency-detection.d.ts.map +1 -1
- package/dist/validators/cross-skill-dependency-detection.js +1 -1
- package/dist/validators/cross-skill-dependency-detection.js.map +1 -1
- package/dist/validators/description-style-detection.d.ts +1 -1
- package/dist/validators/description-style-detection.d.ts.map +1 -1
- package/dist/validators/description-style-detection.js +1 -1
- package/dist/validators/description-style-detection.js.map +1 -1
- package/dist/validators/frontmatter-validation.d.ts +1 -1
- package/dist/validators/frontmatter-validation.d.ts.map +1 -1
- package/dist/validators/frontmatter-validation.js +1 -1
- package/dist/validators/frontmatter-validation.js.map +1 -1
- package/dist/validators/imperative-body-detection.d.ts +1 -1
- package/dist/validators/imperative-body-detection.d.ts.map +1 -1
- package/dist/validators/imperative-body-detection.js +1 -1
- package/dist/validators/imperative-body-detection.js.map +1 -1
- package/dist/validators/index.d.ts +1 -1
- package/dist/validators/index.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.d.ts +1 -1
- package/dist/validators/kebab-case-detection.d.ts.map +1 -1
- package/dist/validators/kebab-case-detection.js +1 -1
- package/dist/validators/kebab-case-detection.js.map +1 -1
- package/dist/validators/marketplace-validator.js +1 -1
- package/dist/validators/marketplace-validator.js.map +1 -1
- package/dist/validators/packaging-validator.d.ts +96 -7
- package/dist/validators/packaging-validator.d.ts.map +1 -1
- package/dist/validators/packaging-validator.js +148 -21
- package/dist/validators/packaging-validator.js.map +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
- package/dist/validators/plugin-hosted-shape.js +1 -1
- package/dist/validators/plugin-hosted-shape.js.map +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts +1 -1
- package/dist/validators/plugin-recommended-fields.d.ts.map +1 -1
- package/dist/validators/plugin-recommended-fields.js +1 -1
- package/dist/validators/plugin-recommended-fields.js.map +1 -1
- package/dist/validators/registry-validator.js +1 -1
- package/dist/validators/registry-validator.js.map +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts.map +1 -1
- package/dist/validators/rule-engine/rule-engine.js +20 -9
- package/dist/validators/rule-engine/rule-engine.js.map +1 -1
- package/dist/validators/skill-validator.d.ts.map +1 -1
- package/dist/validators/skill-validator.js +21 -3
- package/dist/validators/skill-validator.js.map +1 -1
- package/dist/validators/types.d.ts +1 -1
- package/dist/validators/types.d.ts.map +1 -1
- package/dist/validators/unified-validator.js +1 -1
- package/dist/validators/unified-validator.js.map +1 -1
- package/dist/validators/validation-rules.d.ts +1 -1
- package/dist/validators/validation-rules.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.d.ts +7 -3
- package/dist/validators/walker-to-issues.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.js +37 -8
- package/dist/validators/walker-to-issues.js.map +1 -1
- package/dist/walk-link-graph.d.ts +2 -1
- package/dist/walk-link-graph.d.ts.map +1 -1
- package/dist/walk-link-graph.js +37 -2
- package/dist/walk-link-graph.js.map +1 -1
- package/package.json +7 -7
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,
|
|
1
|
+
{"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,kBAAkB,CAAC;AAEnF;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,cAAe,SAAQ,KAAK;IAC9B,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;IAC/B,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,2BAA2B,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAE3C;+EAC+E;AAC/E,MAAM,sBAAsB,GAAG;IAC7B,IAAI;IACJ,QAAQ;IACR,iBAAiB;IACjB,OAAO;IACP,cAAc;IACd,MAAM;IACN,kBAAkB;CACV,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,MAAc;IAC1D,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IACtB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IACzB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe;IAC/E,IAAI,EAAE,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAC9E,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB;AACpE,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,6BAA6B,CAAC,YAAsB,EAAE,GAAoB;IACjF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACtB,SAAS;QACX,CAAC;QACD,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;YAC3B,OAAO,EACL,0BAA0B,cAAc,CAAC,WAAW,CAAC,4CAA4C;gBACjG,8FAA8F;gBAC9F,yEAAyE;YAC3E,IAAI,EAAE,CAAC,cAAc,EAAE,KAAK,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,gFAAgF;AAChF,uEAAuE;AACvE,2EAA2E;AAC3E,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;QAChB,CAAC;aACE,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,KAAK,CACJ,eAAe,EACf,wGAAwG,CACzG;KACJ,CAAC;IACF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,2EAA2E;IAC3E,2EAA2E;IAC3E,qEAAqE;IACrE,4EAA4E;IAC5E,4EAA4E;IAC5E,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,wFAAwF;IACxF,mEAAmE;IACnE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAC/C,4EAA4E;IAC5E,kFAAkF;IAClF,qFAAqF;IACrF,+BAA+B;IAC/B,gBAAgB,EAAE,CAAC;SAChB,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC9C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QACjD,wEAAwE;QACxE,oEAAoE;QACpE,6DAA6D;QAC7D,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAClD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KAChD,CAAC;SACD,MAAM,EAAE;SACR,QAAQ,EAAE;CACd,CAAC;KACD,WAAW,EAAE;KACb,WAAW,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;IAC1B,6BAA6B,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IACvD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAK,sBAA4C,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC9G,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;gBAC3B,uEAAuE;gBACvE,iEAAiE;gBACjE,OAAO,EAAE,uBAAuB,cAAc,CAAC,GAAG,CAAC,qBAAqB,IAAI,kDAAkD;gBAC9H,IAAI,EAAE,CAAC,GAAG,CAAC;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,WAAW,EAAE,CAAC;AAajB,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,wEAAwE;QACxE,4EAA4E;QAC5E,uEAAuE;QACvE,MAAM,IAAI,cAAc,CAAC,iCAAiC,cAAc,CAAE,CAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,cAAc,CAAC,wCAAwC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;IACD,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,cAAc,CAAC,4FAA4F,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAwBD,sEAAsE;AACtE,MAAM,UAAU,MAAM,CAAC,IAAsB;IAC3C,OAAO,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAE,CAAC,MAAM,CAAW,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,SAAS,CAAW,CAAC;AAC3F,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,IAAsB,EAAE,GAAY;IAChE,MAAM,OAAO,GAAG,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;IAC1D,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,cAAc,CACtB,wDAAwD,GAAG,SAAS;YAClE,kFAAkF;YAClF,mDAAmD,CACtD,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAiBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,yBAAyB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC,cAAc,CAAC;AAC9B,CAAC;AAED,SAAS,yBAAyB,CAAC,KAA+B,EAAE,GAAY;IAC9E,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAChG,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,0EAA0E;QAC1E,2BAA2B;QAC3B,aAAa,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACpC,yEAAyE;YACzE,wEAAwE;YACxE,IAAI,GAAW,CAAC;YAChB,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAClD,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,cAAc,CAAC,GAAG,CAAC,qCAAqC;oBAC/F,cAAc,CAAE,GAAa,CAAC,OAAO,CAAC,CACzC,CAAC;YACJ,CAAC;YACD,qJAAqJ;YACrJ,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,cAAc,CAAC,GAAG,CAAC,yBAAyB,cAAc,CAAC,GAAG,CAAC,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,CAAC;gBACH,0EAA0E;gBAC1E,kDAAkD;gBAClD,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC3E,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,gCAAgC,cAAc,CAAC,GAAG,CAAC,wBAAwB;oBACzF,cAAc,CAAE,GAAa,CAAC,OAAO,CAAC,CACzC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -46,6 +46,19 @@ export declare function lintEvalExpectations(evals: EvalEntry[]): EvalLintWarnin
|
|
|
46
46
|
* not close to any declared name (a deliberate built-in/system tool reference), is
|
|
47
47
|
* never flagged. When the skill declares no executables there is nothing to compare
|
|
48
48
|
* against, so it returns no warnings. Pure + side-effect free.
|
|
49
|
+
*
|
|
50
|
+
* ⚠️ EVERY INTERPOLATED NAME IS NEUTRALIZED WITH {@link quoteSuiteText}, at the point
|
|
51
|
+
* of CONSTRUCTION rather than at the stderr write, so the message is safe wherever it
|
|
52
|
+
* is later carried. Both populations here are untrusted: `toolExpectations.*` is
|
|
53
|
+
* `z.array(z.string().min(1))` with no charset constraint on a `.passthrough()` entry,
|
|
54
|
+
* and `resolveEvalSuitePath` will harvest the suite out of a FETCHED artifact — i.e.
|
|
55
|
+
* out of the skill under test; the declared names come from that same skill's
|
|
56
|
+
* manifest. `run-harness.ts` writes this text with no sanitizer of its own, and does
|
|
57
|
+
* it at Step 5.5 — AHEAD of the `--i-understand-this-runs-skill-code` gate and ahead
|
|
58
|
+
* of the `--dry-run` short-circuit, so an unsanitized name repaints the terminal of a
|
|
59
|
+
* run that spawned nothing and that the operator believes never executed skill code.
|
|
60
|
+
* Matching still happens on the RAW value, so neutralization cannot change which
|
|
61
|
+
* names are flagged.
|
|
49
62
|
*/
|
|
50
63
|
export declare function lintToolExpectationExecutables(evals: EvalEntry[], declaredExecutableNames: string[]): EvalLintWarning[];
|
|
51
64
|
//# sourceMappingURL=eval-lint.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-lint.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,
|
|
1
|
+
{"version":3,"file":"eval-lint.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAoC,KAAK,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAEpF,oGAAoG;AACpG,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAsBD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,eAAe,EAAE,CAgB1E;AA0DD;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,8BAA8B,CAC5C,KAAK,EAAE,SAAS,EAAE,EAClB,uBAAuB,EAAE,MAAM,EAAE,GAChC,eAAe,EAAE,CAuBnB"}
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* than a missed one, since the grader (an LLM judging the transcript) is the real
|
|
12
12
|
* backstop for eval quality.
|
|
13
13
|
*/
|
|
14
|
-
import { isSingleEditAway } from './eval-inputs.js';
|
|
14
|
+
import { isSingleEditAway, quoteSuiteText } from './eval-inputs.js';
|
|
15
15
|
// An expectation "looks like" a pure presence/mention check when it uses one of
|
|
16
16
|
// these verbs — the vocabulary authors reach for when asserting "X shows up
|
|
17
17
|
// somewhere in the output" without saying what a WRONG output would look like.
|
|
@@ -127,13 +127,26 @@ function probableDeclaredTypoTarget(name, declared) {
|
|
|
127
127
|
* not close to any declared name (a deliberate built-in/system tool reference), is
|
|
128
128
|
* never flagged. When the skill declares no executables there is nothing to compare
|
|
129
129
|
* against, so it returns no warnings. Pure + side-effect free.
|
|
130
|
+
*
|
|
131
|
+
* ⚠️ EVERY INTERPOLATED NAME IS NEUTRALIZED WITH {@link quoteSuiteText}, at the point
|
|
132
|
+
* of CONSTRUCTION rather than at the stderr write, so the message is safe wherever it
|
|
133
|
+
* is later carried. Both populations here are untrusted: `toolExpectations.*` is
|
|
134
|
+
* `z.array(z.string().min(1))` with no charset constraint on a `.passthrough()` entry,
|
|
135
|
+
* and `resolveEvalSuitePath` will harvest the suite out of a FETCHED artifact — i.e.
|
|
136
|
+
* out of the skill under test; the declared names come from that same skill's
|
|
137
|
+
* manifest. `run-harness.ts` writes this text with no sanitizer of its own, and does
|
|
138
|
+
* it at Step 5.5 — AHEAD of the `--i-understand-this-runs-skill-code` gate and ahead
|
|
139
|
+
* of the `--dry-run` short-circuit, so an unsanitized name repaints the terminal of a
|
|
140
|
+
* run that spawned nothing and that the operator believes never executed skill code.
|
|
141
|
+
* Matching still happens on the RAW value, so neutralization cannot change which
|
|
142
|
+
* names are flagged.
|
|
130
143
|
*/
|
|
131
144
|
export function lintToolExpectationExecutables(evals, declaredExecutableNames) {
|
|
132
145
|
const warnings = [];
|
|
133
146
|
const declared = declaredExecutableNames.map((original) => ({ original, normalized: normalizeExecutableName(original) }));
|
|
134
147
|
if (declared.length === 0)
|
|
135
148
|
return warnings;
|
|
136
|
-
const declaredList = declared.map((d) => d.original).join(', ');
|
|
149
|
+
const declaredList = declared.map((d) => quoteSuiteText(d.original)).join(', ');
|
|
137
150
|
for (const entry of evals) {
|
|
138
151
|
const evalId = String(entry.id);
|
|
139
152
|
const seen = new Set();
|
|
@@ -146,8 +159,8 @@ export function lintToolExpectationExecutables(evals, declaredExecutableNames) {
|
|
|
146
159
|
continue;
|
|
147
160
|
warnings.push({
|
|
148
161
|
evalId,
|
|
149
|
-
message: `eval "${evalId}": toolExpectation references executable "${name}", which no declared executable matches — ` +
|
|
150
|
-
`did you mean "${target}"? A name that never matches a real tool makes mustRun/mustSucceed/sequence fail ` +
|
|
162
|
+
message: `eval "${evalId}": toolExpectation references executable "${quoteSuiteText(name)}", which no declared executable matches — ` +
|
|
163
|
+
`did you mean "${quoteSuiteText(target)}"? A name that never matches a real tool makes mustRun/mustSucceed/sequence fail ` +
|
|
151
164
|
`for the wrong reason (or mustNotRun pass vacuously). Declared executables: ${declaredList}.`,
|
|
152
165
|
});
|
|
153
166
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-lint.js","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAkB,MAAM,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-lint.js","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAQpF,gFAAgF;AAChF,4EAA4E;AAC5E,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,2EAA2E,CAAC;AAE1G,iFAAiF;AACjF,kFAAkF;AAClF,0EAA0E;AAC1E,4EAA4E;AAC5E,2EAA2E;AAC3E,kFAAkF;AAClF,6DAA6D;AAC7D,MAAM,sBAAsB,GAC1B,+GAA+G,CAAC;AAElH,0IAA0I;AAC1I,SAAS,yBAAyB,CAAC,WAAmB;IACpD,OAAO,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAC9F,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC9C,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS;YAAE,SAAS;QACnD,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,yBAAyB,CAAC;YAAE,SAAS;QACnE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM;YACN,OAAO,EACL,SAAS,MAAM,uFAAuF;gBACtG,yGAAyG;gBACzG,8CAA8C;SACjD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,4DAA4D;AAC5D,8EAA8E;AAE9E,6EAA6E;AAC7E,4FAA4F;AAC5F,oFAAoF;AACpF,gFAAgF;AAChF,iFAAiF;AACjF,SAAS;AACT,MAAM,gBAAgB,GAAG,0CAA0C,CAAC;AAEpE,yHAAyH;AACzH,SAAS,uBAAuB,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACtF,CAAC;AAED,mHAAmH;AACnH,SAAS,iCAAiC,CAAC,KAAgB;IACzD,MAAM,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;IAClC,IAAI,EAAE,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,SAAiB,EAAE,IAAY;IACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpG,OAAO,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,0BAA0B,CACjC,IAAY,EACZ,QAAoD;IAEpD,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC,CAAC,iCAAiC;IAC1G,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QAClE,IAAI,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,eAAe,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;IAChH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,UAAU,8BAA8B,CAC5C,KAAkB,EAClB,uBAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1H,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC3C,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChF,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,iCAAiC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5D,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC1D,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM;gBACN,OAAO,EACL,SAAS,MAAM,6CAA6C,cAAc,CAAC,IAAI,CAAC,4CAA4C;oBAC5H,iBAAiB,cAAc,CAAC,MAAM,CAAC,mFAAmF;oBAC1H,8EAA8E,YAAY,GAAG;aAChG,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -1,9 +1,17 @@
|
|
|
1
1
|
export interface BuildExecutorPromptOptions {
|
|
2
2
|
/** The eval's `prompt` field, verbatim — the task the executor must perform. */
|
|
3
3
|
task: string;
|
|
4
|
-
/**
|
|
5
|
-
|
|
6
|
-
|
|
4
|
+
/**
|
|
5
|
+
* Path to the staged subject (skill/plugin under test).
|
|
6
|
+
*
|
|
7
|
+
* OMITTED for the skill-absent (WITHOUT) arm of a `--baseline` run. Naming it
|
|
8
|
+
* there would hand the control arm the exact thing the arm exists to withhold:
|
|
9
|
+
* the staged dir holds both the skill's SKILL.md and any executable it ships,
|
|
10
|
+
* so a control arm told this path can recover the whole treatment with one
|
|
11
|
+
* `cat`/`ls`. The A/B is only a measurement while this stays absent.
|
|
12
|
+
*/
|
|
13
|
+
subjectPath?: string;
|
|
14
|
+
/** The eval's working directory — the project the executor should operate on. */
|
|
7
15
|
workspaceDir?: string;
|
|
8
16
|
}
|
|
9
17
|
/**
|
|
@@ -15,8 +23,8 @@ export interface BuildExecutorPromptOptions {
|
|
|
15
23
|
* knowledge would change its behavior and invalidate the eval.
|
|
16
24
|
*
|
|
17
25
|
* `subjectPath` is where the skill/plugin under test is staged (context the
|
|
18
|
-
* executor's environment provides)
|
|
19
|
-
*
|
|
26
|
+
* executor's environment provides) and is OMITTED for the skill-absent arm;
|
|
27
|
+
* `workspaceDir` is the project the executor is actually meant to operate on.
|
|
20
28
|
*
|
|
21
29
|
* Pure function: same inputs always produce the same prompt.
|
|
22
30
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/executor-prompt.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,0BAA0B;IACzC,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAC;IACb
|
|
1
|
+
{"version":3,"file":"executor-prompt.d.ts","sourceRoot":"","sources":["../../src/skill-test/executor-prompt.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,0BAA0B;IACzC,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAC;IACb;;;;;;;;OAQG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iFAAiF;IACjF,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,0BAA0B,GAAG,MAAM,CAS5E;AAWD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,8BAA8B,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAiBjF"}
|
|
@@ -8,17 +8,20 @@ import { PromptInvariantError } from './prompt-invariants.js';
|
|
|
8
8
|
* knowledge would change its behavior and invalidate the eval.
|
|
9
9
|
*
|
|
10
10
|
* `subjectPath` is where the skill/plugin under test is staged (context the
|
|
11
|
-
* executor's environment provides)
|
|
12
|
-
*
|
|
11
|
+
* executor's environment provides) and is OMITTED for the skill-absent arm;
|
|
12
|
+
* `workspaceDir` is the project the executor is actually meant to operate on.
|
|
13
13
|
*
|
|
14
14
|
* Pure function: same inputs always produce the same prompt.
|
|
15
15
|
*/
|
|
16
16
|
export function buildExecutorPrompt(opts) {
|
|
17
|
-
const
|
|
17
|
+
const context = [];
|
|
18
|
+
if (opts.subjectPath !== undefined) {
|
|
19
|
+
context.push(`The relevant files are located at ${opts.subjectPath}.`);
|
|
20
|
+
}
|
|
18
21
|
if (opts.workspaceDir !== undefined) {
|
|
19
|
-
|
|
22
|
+
context.push(`Your working directory is ${opts.workspaceDir} — the project to operate on.`);
|
|
20
23
|
}
|
|
21
|
-
return
|
|
24
|
+
return context.length === 0 ? opts.task : [opts.task, '', ...context].join('\n');
|
|
22
25
|
}
|
|
23
26
|
/**
|
|
24
27
|
* Phrases that, if present in an executor prompt, would break the blind
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"executor-prompt.js","sourceRoot":"","sources":["../../src/skill-test/executor-prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"executor-prompt.js","sourceRoot":"","sources":["../../src/skill-test/executor-prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAmB9D;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAgC;IAClE,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,IAAI,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,CAAC,IAAI,CAAC,qCAAqC,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACpC,OAAO,CAAC,IAAI,CAAC,6BAA6B,IAAI,CAAC,YAAY,+BAA+B,CAAC,CAAC;IAC9F,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACnF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,iBAAiB,GAAG,CAAC,cAAc,EAAE,iBAAiB,EAAE,yBAAyB,EAAE,SAAS,CAAC,CAAC;AAEpG;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,8BAA8B,CAAC,MAAc,EAAE,IAAY;IACzE,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACvB,MAAM,IAAI,oBAAoB,CAAC,4CAA4C,CAAC,CAAC;IAC/E,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,oBAAoB,CAAC,yDAAyD,CAAC,CAAC;IAC5F,CAAC;IACD,2EAA2E;IAC3E,4EAA4E;IAC5E,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC/D,KAAK,MAAM,OAAO,IAAI,iBAAiB,EAAE,CAAC;QACxC,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,oBAAoB,CAC5B,oFAAoF,OAAO,IAAI,CAChG,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -26,9 +26,28 @@ export type SkillTestExitCodeValue = (typeof SkillTestExitCode)[keyof typeof Ski
|
|
|
26
26
|
* annotated starter template.
|
|
27
27
|
*
|
|
28
28
|
* In a real run the harness has already written that template, and the message
|
|
29
|
-
* says so. Under `--dry-run`
|
|
30
|
-
*
|
|
31
|
-
*
|
|
29
|
+
* says so. Under `--dry-run` the TEMPLATE is not written, so the message instead
|
|
30
|
+
* describes what a real run *would* scaffold and where, while surfacing the same
|
|
31
|
+
* exit-3 "bootstrap needed" signal.
|
|
32
|
+
*
|
|
33
|
+
* That scope is exact, and it used to read "a dry run must never touch the
|
|
34
|
+
* filesystem", which is not true of this command. A dry run is not a read-only
|
|
35
|
+
* mode: it takes the harness lock, creates the harness root, stages BOTH arms'
|
|
36
|
+
* per-eval workspaces, and — once `--i-understand-this-runs-skill-code` is passed —
|
|
37
|
+
* runs the repo's `test.build` hook and builds the subject. What it never does is
|
|
38
|
+
* SPAWN: no executor session, no grader session, no tokens.
|
|
39
|
+
*
|
|
40
|
+
* It also never touches `results/`: it neither creates the directory, nor writes
|
|
41
|
+
* `provenance.json`, nor wipes the previous run's artifacts. That last one is the
|
|
42
|
+
* reason the ordering matters — the wipe used to sit AHEAD of the dry-run
|
|
43
|
+
* short-circuit, so a free `--dry-run` (or any failure after it) destroyed the
|
|
44
|
+
* `grading.json` / `baseline.json` / `friction.json` / `tool-eval.json` of the
|
|
45
|
+
* expensive real run an operator was about to read. The summary now names the
|
|
46
|
+
* provenance path it WOULD write rather than writing it.
|
|
47
|
+
*
|
|
48
|
+
* The scaffold is the other filesystem effect a dry run deliberately withholds,
|
|
49
|
+
* and for a different reason: that write lands in the AUTHOR's source
|
|
50
|
+
* tree rather than in vat's own scratch space.
|
|
32
51
|
*/
|
|
33
52
|
export declare class BootstrapNeededError extends Error {
|
|
34
53
|
readonly expectedPath: string;
|
|
@@ -74,19 +93,37 @@ export declare class InternalHarnessError extends Error {
|
|
|
74
93
|
constructor(message: string);
|
|
75
94
|
}
|
|
76
95
|
/**
|
|
77
|
-
* Map any thrown error to the process exit code.
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
96
|
+
* Map any thrown error to the process exit code.
|
|
97
|
+
*
|
|
98
|
+
* ⚠️ THE `exitCode` FIELD IS THE PRIMARY MECHANISM, and until this rewrite it was
|
|
99
|
+
* not a mechanism at all. Ten error classes across five modules carried a
|
|
100
|
+
* `readonly exitCode`, this docblock called them "authoritative", and NOTHING read
|
|
101
|
+
* one: dispatch was a hand-maintained `instanceof` chain that happened to agree with
|
|
102
|
+
* every field. Deleting any of those fields changed nothing, so each new class added
|
|
103
|
+
* a fifth, then a sixth member to an inert pattern — and the two lists could have
|
|
104
|
+
* disagreed at any time with no test able to tell.
|
|
105
|
+
*
|
|
106
|
+
* So: an error that declares one of the three throwable codes ({@link
|
|
107
|
+
* selfDeclaredExitCode}) decides its own exit code, and what remains of the
|
|
108
|
+
* `instanceof` chain is exactly ONE row — `PromptInvariantError` (a
|
|
109
|
+
* defense-in-depth failure over VAT's OWN generated executor/grader prompts: a
|
|
110
|
+
* required safety directive missing from a prompt VAT built, surfaced as a
|
|
111
|
+
* user-visible preflight-class problem) → 2, the only class that declares nothing and
|
|
112
|
+
* still needs a code other than the default.
|
|
113
|
+
*
|
|
114
|
+
* `GradingSkewError` (aggregate grading.json shape skew), `EvalFragmentError`
|
|
115
|
+
* (per-eval grader fragment parse failure) and `GradingNonceError`
|
|
116
|
+
* (forged/mismatched per-fragment grader nonce) reach 1 through the FALLBACK, not
|
|
117
|
+
* through a row — they are indistinguishable here from `new Error('boom')`. That is
|
|
118
|
+
* correct (1 is what they want) but it is not dispatch, and saying otherwise implies
|
|
119
|
+
* a pin that does not exist: deleting any of the three changes nothing. Everything
|
|
120
|
+
* unknown → 1.
|
|
121
|
+
*
|
|
122
|
+
* A side benefit of reading the field: it works across a `src`/`dist` boundary, where
|
|
123
|
+
* `instanceof` silently cannot — a `dist` copy of a class never matches a `src`
|
|
124
|
+
* instance of it, and this module is imported from both. The surviving
|
|
125
|
+
* `PromptInvariantError` row carries exactly that risk; it is kept because the class
|
|
126
|
+
* declares no field to read.
|
|
90
127
|
*
|
|
91
128
|
* Note: SkillTestExitCode.EvalFailure (4) is NOT produced here — it is an
|
|
92
129
|
* outcome of a completed run (an eval failed, the fail-closed default unless
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"exit-codes.d.ts","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"exit-codes.d.ts","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAEA,kDAAkD;AAClD,eAAO,MAAM,iBAAiB;;;;;IAK5B;;;;;;;;;;;;OAYG;;CAEK,CAAC;AAEX,MAAM,MAAM,sBAAsB,GAAG,CAAC,OAAO,iBAAiB,CAAC,CAAC,MAAM,OAAO,iBAAiB,CAAC,CAAC;AAEhG;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;aAEjB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE;CAQ9E;AAED,iHAAiH;AACjH,qBAAa,eAAgB,SAAQ,KAAK;IACxC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,KAAK;IACzC,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;;CAKhC;AAED;;;;;;;;;;GAUG;AACH,qBAAa,yBAA0B,SAAQ,KAAK;aAEtB,SAAS,EAAE,MAAM;IAD7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACH,SAAS,EAAE,MAAM;CAO9C;AAED,+HAA+H;AAC/H,qBAAa,oBAAqB,SAAQ,KAAK;IAC7C,QAAQ,CAAC,QAAQ,EAAG,CAAC,CAAU;gBACnB,OAAO,EAAE,MAAM;CAI5B;AAmED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CAKvD"}
|
|
@@ -1,9 +1,3 @@
|
|
|
1
|
-
import { AuthPreflightError } from '@vibe-agent-toolkit/utils';
|
|
2
|
-
import { BuildHookError } from './build-hook.js';
|
|
3
|
-
import { UnknownEnvTokenError } from './declared-env.js';
|
|
4
|
-
import { EvalFragmentError } from './eval-fragment.js';
|
|
5
|
-
import { GradingNonceError, GradingSkewError } from './grading-adapter.js';
|
|
6
|
-
import { HarnessLocationError } from './harness-location.js';
|
|
7
1
|
import { PromptInvariantError } from './prompt-invariants.js';
|
|
8
2
|
/** Exit codes for `vat skill test` (spec §6d). */
|
|
9
3
|
export const SkillTestExitCode = {
|
|
@@ -32,16 +26,35 @@ export const SkillTestExitCode = {
|
|
|
32
26
|
* annotated starter template.
|
|
33
27
|
*
|
|
34
28
|
* In a real run the harness has already written that template, and the message
|
|
35
|
-
* says so. Under `--dry-run`
|
|
36
|
-
*
|
|
37
|
-
*
|
|
29
|
+
* says so. Under `--dry-run` the TEMPLATE is not written, so the message instead
|
|
30
|
+
* describes what a real run *would* scaffold and where, while surfacing the same
|
|
31
|
+
* exit-3 "bootstrap needed" signal.
|
|
32
|
+
*
|
|
33
|
+
* That scope is exact, and it used to read "a dry run must never touch the
|
|
34
|
+
* filesystem", which is not true of this command. A dry run is not a read-only
|
|
35
|
+
* mode: it takes the harness lock, creates the harness root, stages BOTH arms'
|
|
36
|
+
* per-eval workspaces, and — once `--i-understand-this-runs-skill-code` is passed —
|
|
37
|
+
* runs the repo's `test.build` hook and builds the subject. What it never does is
|
|
38
|
+
* SPAWN: no executor session, no grader session, no tokens.
|
|
39
|
+
*
|
|
40
|
+
* It also never touches `results/`: it neither creates the directory, nor writes
|
|
41
|
+
* `provenance.json`, nor wipes the previous run's artifacts. That last one is the
|
|
42
|
+
* reason the ordering matters — the wipe used to sit AHEAD of the dry-run
|
|
43
|
+
* short-circuit, so a free `--dry-run` (or any failure after it) destroyed the
|
|
44
|
+
* `grading.json` / `baseline.json` / `friction.json` / `tool-eval.json` of the
|
|
45
|
+
* expensive real run an operator was about to read. The summary now names the
|
|
46
|
+
* provenance path it WOULD write rather than writing it.
|
|
47
|
+
*
|
|
48
|
+
* The scaffold is the other filesystem effect a dry run deliberately withholds,
|
|
49
|
+
* and for a different reason: that write lands in the AUTHOR's source
|
|
50
|
+
* tree rather than in vat's own scratch space.
|
|
38
51
|
*/
|
|
39
52
|
export class BootstrapNeededError extends Error {
|
|
40
53
|
expectedPath;
|
|
41
54
|
exitCode = 3;
|
|
42
55
|
constructor(expectedPath, opts) {
|
|
43
56
|
super(opts?.dryRun === true
|
|
44
|
-
? `[dry-run] No evals.json found. A real run would scaffold an annotated template at ${expectedPath} — fill it in and re-run. (dry-run: nothing was written.)`
|
|
57
|
+
? `[dry-run] No evals.json found. A real run would scaffold an annotated template at ${expectedPath} — fill it in and re-run. (dry-run: nothing was written to your source tree.)`
|
|
45
58
|
: `Wrote an evals.json template at ${expectedPath} — fill it in and re-run.`);
|
|
46
59
|
this.expectedPath = expectedPath;
|
|
47
60
|
this.name = 'BootstrapNeededError';
|
|
@@ -98,19 +111,102 @@ export class InternalHarnessError extends Error {
|
|
|
98
111
|
}
|
|
99
112
|
}
|
|
100
113
|
/**
|
|
101
|
-
*
|
|
102
|
-
* `
|
|
103
|
-
* a
|
|
104
|
-
*
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
*
|
|
113
|
-
*
|
|
114
|
+
* The codes an error is allowed to declare for itself. `Ok` is not a failure and
|
|
115
|
+
* `EvalFailure` is an outcome rather than a throw (see below), so neither may be
|
|
116
|
+
* claimed by a thrown object — that also keeps a stray `exitCode: 0` from turning a
|
|
117
|
+
* crash into a green build.
|
|
118
|
+
*/
|
|
119
|
+
const SELF_DECLARED_EXIT_CODES = new Set([
|
|
120
|
+
SkillTestExitCode.Internal,
|
|
121
|
+
SkillTestExitCode.Preflight,
|
|
122
|
+
SkillTestExitCode.Bootstrap,
|
|
123
|
+
]);
|
|
124
|
+
/**
|
|
125
|
+
* The error's own declared exit code, if it has a usable one.
|
|
126
|
+
*
|
|
127
|
+
* Narrow ON PURPOSE. `exitCode` is a common property name on foreign errors (child
|
|
128
|
+
* processes, HTTP clients), so an unrestricted read would let an unrelated throw pick
|
|
129
|
+
* the process's exit code. Requiring an `Error` whose value is one of the three
|
|
130
|
+
* throwable codes above keeps the mechanism to errors that meant to opt in, and
|
|
131
|
+
* leaves everything else on the `instanceof` fallback below.
|
|
132
|
+
*
|
|
133
|
+
* ⚠️ THE READ IS GUARDED, because reading a property is not a total operation and the
|
|
134
|
+
* `instanceof` chain it replaced was. A getter or `Proxy` that throws on `exitCode`
|
|
135
|
+
* propagated straight out of {@link mapErrorToExitCode} — and BOTH of its call sites
|
|
136
|
+
* are inside `catch` blocks, so that throw escaped the handler entirely: no summary
|
|
137
|
+
* line, no chosen exit code, a bare stack. An error whose own `exitCode` cannot be
|
|
138
|
+
* read has not opted in to anything, so it takes the Internal default.
|
|
139
|
+
*
|
|
140
|
+
* ⚠️ IT IS NOT AN OWN-PROPERTY READ. An `exitCode` inherited from a PROTOTYPE is
|
|
141
|
+
* honoured, verified — `Base.prototype.exitCode = 3` maps to 3. That is what makes
|
|
142
|
+
* subclassing work (a subclass of `InternalHarnessError` keeps its code without
|
|
143
|
+
* redeclaring the field), and it is also what widens the residual below by one hop.
|
|
144
|
+
*
|
|
145
|
+
* ⚠️ RESIDUAL, KNOWN AND UNCLOSED. The value test rejects everything outside
|
|
146
|
+
* {1,2,3} but nothing INSIDE it, so a foreign error carrying `exitCode: 2` picks
|
|
147
|
+
* Preflight — "your environment is wrong" — for a crash. A walk of the reachable
|
|
148
|
+
* dependency closure found `spawnSync`, `execSync`, `fs`, `JSON.parse`, `ZodError`,
|
|
149
|
+
* `YAMLParseError`, `adm-zip`, `fetch`, `DOMException`, Anthropic's `APIError` and
|
|
150
|
+
* VAT's own `CommandExecutionError` all leaving `exitCode` undefined — but
|
|
151
|
+
* `commander`, a direct `@vibe-agent-toolkit/cli` dependency, defines `exitCode` on
|
|
152
|
+
* `CommanderError` and carries `2`. It is unreachable today (`exitOverride` and
|
|
153
|
+
* `CommanderError` appear nowhere in `packages/cli/src`), and adopting
|
|
154
|
+
* `exitOverride()` — the standard testability pattern — would silently arm it.
|
|
155
|
+
*
|
|
156
|
+
* Closing it means a VAT-OWNED BRAND rather than a name every ecosystem uses: a
|
|
157
|
+
* `Symbol.for('vat.exitCode')` key, which (unlike `instanceof` or a shared base
|
|
158
|
+
* class) is registry-global and so survives the `src`/`dist` boundary this read
|
|
159
|
+
* exists to cross. That is a mechanical change across all eleven declaring classes —
|
|
160
|
+
* five here, plus `lock.ts`, `build-hook.ts`, `declared-env.ts` (two),
|
|
161
|
+
* `harness-location.ts` and `utils`' `auth-resolver.ts` — and a half-migration would
|
|
162
|
+
* leave two mechanisms with the hole still open, so it is one change or none. The
|
|
163
|
+
* behaviour is pinned by a RESIDUAL row in `exit-codes.test.ts` so the migration
|
|
164
|
+
* has a test to flip rather than a comment to find.
|
|
165
|
+
*/
|
|
166
|
+
function selfDeclaredExitCode(err) {
|
|
167
|
+
if (!(err instanceof Error))
|
|
168
|
+
return undefined;
|
|
169
|
+
let declared;
|
|
170
|
+
try {
|
|
171
|
+
declared = err.exitCode;
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
return undefined;
|
|
175
|
+
}
|
|
176
|
+
return typeof declared === 'number' && SELF_DECLARED_EXIT_CODES.has(declared) ? declared : undefined;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Map any thrown error to the process exit code.
|
|
180
|
+
*
|
|
181
|
+
* ⚠️ THE `exitCode` FIELD IS THE PRIMARY MECHANISM, and until this rewrite it was
|
|
182
|
+
* not a mechanism at all. Ten error classes across five modules carried a
|
|
183
|
+
* `readonly exitCode`, this docblock called them "authoritative", and NOTHING read
|
|
184
|
+
* one: dispatch was a hand-maintained `instanceof` chain that happened to agree with
|
|
185
|
+
* every field. Deleting any of those fields changed nothing, so each new class added
|
|
186
|
+
* a fifth, then a sixth member to an inert pattern — and the two lists could have
|
|
187
|
+
* disagreed at any time with no test able to tell.
|
|
188
|
+
*
|
|
189
|
+
* So: an error that declares one of the three throwable codes ({@link
|
|
190
|
+
* selfDeclaredExitCode}) decides its own exit code, and what remains of the
|
|
191
|
+
* `instanceof` chain is exactly ONE row — `PromptInvariantError` (a
|
|
192
|
+
* defense-in-depth failure over VAT's OWN generated executor/grader prompts: a
|
|
193
|
+
* required safety directive missing from a prompt VAT built, surfaced as a
|
|
194
|
+
* user-visible preflight-class problem) → 2, the only class that declares nothing and
|
|
195
|
+
* still needs a code other than the default.
|
|
196
|
+
*
|
|
197
|
+
* `GradingSkewError` (aggregate grading.json shape skew), `EvalFragmentError`
|
|
198
|
+
* (per-eval grader fragment parse failure) and `GradingNonceError`
|
|
199
|
+
* (forged/mismatched per-fragment grader nonce) reach 1 through the FALLBACK, not
|
|
200
|
+
* through a row — they are indistinguishable here from `new Error('boom')`. That is
|
|
201
|
+
* correct (1 is what they want) but it is not dispatch, and saying otherwise implies
|
|
202
|
+
* a pin that does not exist: deleting any of the three changes nothing. Everything
|
|
203
|
+
* unknown → 1.
|
|
204
|
+
*
|
|
205
|
+
* A side benefit of reading the field: it works across a `src`/`dist` boundary, where
|
|
206
|
+
* `instanceof` silently cannot — a `dist` copy of a class never matches a `src`
|
|
207
|
+
* instance of it, and this module is imported from both. The surviving
|
|
208
|
+
* `PromptInvariantError` row carries exactly that risk; it is kept because the class
|
|
209
|
+
* declares no field to read.
|
|
114
210
|
*
|
|
115
211
|
* Note: SkillTestExitCode.EvalFailure (4) is NOT produced here — it is an
|
|
116
212
|
* outcome of a completed run (an eval failed, the fail-closed default unless
|
|
@@ -118,24 +214,11 @@ export class InternalHarnessError extends Error {
|
|
|
118
214
|
* directly (see verdictExitCode).
|
|
119
215
|
*/
|
|
120
216
|
export function mapErrorToExitCode(err) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
err instanceof HarnessLocationError ||
|
|
126
|
-
err instanceof PromptInvariantError ||
|
|
127
|
-
err instanceof SecurityAckError ||
|
|
128
|
-
err instanceof SkillBuildError ||
|
|
129
|
-
err instanceof UnknownEnvTokenError ||
|
|
130
|
-
err instanceof DuplicateStagedSkillError) {
|
|
217
|
+
const declared = selfDeclaredExitCode(err);
|
|
218
|
+
if (declared !== undefined)
|
|
219
|
+
return declared;
|
|
220
|
+
if (err instanceof PromptInvariantError)
|
|
131
221
|
return SkillTestExitCode.Preflight;
|
|
132
|
-
}
|
|
133
|
-
if (err instanceof EvalFragmentError ||
|
|
134
|
-
err instanceof GradingNonceError ||
|
|
135
|
-
err instanceof GradingSkewError ||
|
|
136
|
-
err instanceof InternalHarnessError) {
|
|
137
|
-
return SkillTestExitCode.Internal;
|
|
138
|
-
}
|
|
139
222
|
return SkillTestExitCode.Internal;
|
|
140
223
|
}
|
|
141
224
|
//# sourceMappingURL=exit-codes.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"exit-codes.js","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"exit-codes.js","sourceRoot":"","sources":["../../src/skill-test/exit-codes.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,wBAAwB,CAAC;AAE9D,kDAAkD;AAClD,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,EAAE,EAAE,CAAC;IACL,QAAQ,EAAE,CAAC;IACX,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,CAAC;IACZ;;;;;;;;;;;;OAYG;IACH,WAAW,EAAE,CAAC;CACN,CAAC;AAIX;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAEjB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,YAAoB,EAAE,IAA2B;QAC3E,KAAK,CACH,IAAI,EAAE,MAAM,KAAK,IAAI;YACnB,CAAC,CAAC,qFAAqF,YAAY,+EAA+E;YAClL,CAAC,CAAC,mCAAmC,YAAY,2BAA2B,CAC/E,CAAC;QALwB,iBAAY,GAAZ,YAAY,CAAQ;QAM9C,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,iHAAiH;AACjH,MAAM,OAAO,eAAgB,SAAQ,KAAK;IAC/B,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,iBAAiB,CAAC;IAChC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,gBAAiB,SAAQ,KAAK;IAChC,QAAQ,GAAG,CAAU,CAAC;IAC/B;QACE,KAAK,CAAC,wFAAwF,CAAC,CAAC;QAChG,IAAI,CAAC,IAAI,GAAG,kBAAkB,CAAC;IACjC,CAAC;CACF;AAED;;;;;;;;;;GAUG;AACH,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IAEtB;IADnB,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAA4B,SAAiB;QAC3C,KAAK,CACH,eAAe,SAAS,mEAAmE;YACzF,4CAA4C,CAC/C,CAAC;QAJwB,cAAS,GAAT,SAAS,CAAQ;QAK3C,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED,+HAA+H;AAC/H,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IACpC,QAAQ,GAAG,CAAU,CAAC;IAC/B,YAAY,OAAe;QACzB,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,wBAAwB,GAAwB,IAAI,GAAG,CAAS;IACpE,iBAAiB,CAAC,QAAQ;IAC1B,iBAAiB,CAAC,SAAS;IAC3B,iBAAiB,CAAC,SAAS;CAC5B,CAAC,CAAC;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,SAAS,oBAAoB,CAAC,GAAY;IACxC,IAAI,CAAC,CAAC,GAAG,YAAY,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAC9C,IAAI,QAAiB,CAAC;IACtB,IAAI,CAAC;QACH,QAAQ,GAAI,GAA8B,CAAC,QAAQ,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,QAAQ,KAAK,QAAQ,IAAI,wBAAwB,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;AACvG,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,MAAM,QAAQ,GAAG,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAC3C,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC5C,IAAI,GAAG,YAAY,oBAAoB;QAAE,OAAO,iBAAiB,CAAC,SAAS,CAAC;IAC5E,OAAO,iBAAiB,CAAC,QAAQ,CAAC;AACpC,CAAC"}
|
|
@@ -2,24 +2,7 @@ import type { EvalFragment } from './eval-fragment.js';
|
|
|
2
2
|
import { type FrictionReport } from './friction-schema.js';
|
|
3
3
|
import { type NormalizedGrading } from './grading-adapter.js';
|
|
4
4
|
import { type ToolEvalReport } from './tool-eval-schema.js';
|
|
5
|
-
|
|
6
|
-
* Assemble the run's aggregate grading result (skill-creator's flat
|
|
7
|
-
* `grading.json` shape) from every eval's per-eval grader fragment (issue #145
|
|
8
|
-
* Task 4/5). vat is the SOLE writer of the aggregate; this is that assembly
|
|
9
|
-
* step — the result feeds {@link import('./grading-adapter.js').reconcileGrading}
|
|
10
|
-
* unchanged.
|
|
11
|
-
*
|
|
12
|
-
* Every fragment must carry the SAME `runNonce` the harness stamped for this
|
|
13
|
-
* run — a mismatch means a fragment was not produced for this run (forged,
|
|
14
|
-
* stale, or left behind by untrusted skill code in the shared sandbox) and is
|
|
15
|
-
* rejected via {@link GradingNonceError}, never silently dropped.
|
|
16
|
-
*
|
|
17
|
-
* Zero fragments in → zero expectations out. This is deliberate: the existing
|
|
18
|
-
* "graded nothing" guard in `reconcileGrading` already throws
|
|
19
|
-
* `GradingSkewError` on an empty `expectations[]`, so that case is NOT
|
|
20
|
-
* special-cased here.
|
|
21
|
-
*/
|
|
22
|
-
export declare function mergeFragmentsToGrading(fragments: EvalFragment[], runNonce: string): NormalizedGrading;
|
|
5
|
+
export declare function mergeFragmentsToGrading(fragments: EvalFragment[], runNonce: string, arm: 'with' | 'without'): NormalizedGrading;
|
|
23
6
|
/**
|
|
24
7
|
* Assemble the run's aggregate friction report from every eval's fragment
|
|
25
8
|
* (vat-owned; see friction-schema.ts). Concatenates each fragment's
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fragment-merge.d.ts","sourceRoot":"","sources":["../../src/skill-test/fragment-merge.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"fragment-merge.d.ts","sourceRoot":"","sources":["../../src/skill-test/fragment-merge.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACjF,OAAO,EAAsC,KAAK,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAClG,OAAO,EAAwB,KAAK,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAkElF,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,YAAY,EAAE,EACzB,QAAQ,EAAE,MAAM,EAChB,GAAG,EAAE,MAAM,GAAG,SAAS,GACtB,iBAAiB,CAmCnB;AAED;;;;;;;GAOG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,cAAc,CAYlF;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,wBAAwB,CAAC,SAAS,EAAE,YAAY,EAAE,GAAG,cAAc,CAMlF"}
|