@vibe-agent-toolkit/agent-skills 0.2.0-rc.7 → 0.2.0-rc.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/dist/builder.d.ts.map +1 -1
- package/dist/builder.js +7 -17
- package/dist/builder.js.map +1 -1
- package/dist/files-config.d.ts +2 -2
- package/dist/files-config.d.ts.map +1 -1
- package/dist/files-config.js +4 -10
- package/dist/files-config.js.map +1 -1
- package/dist/import.d.ts.map +1 -1
- package/dist/import.js +0 -4
- package/dist/import.js.map +1 -1
- package/dist/index.d.ts +2 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/inventory/types.d.ts +8 -0
- package/dist/inventory/types.d.ts.map +1 -1
- package/dist/inventory/types.js.map +1 -1
- package/dist/link-graph-members.d.ts +18 -0
- package/dist/link-graph-members.d.ts.map +1 -0
- package/dist/link-graph-members.js +18 -0
- package/dist/link-graph-members.js.map +1 -0
- package/dist/plugin-distribution-layout.d.ts +70 -6
- package/dist/plugin-distribution-layout.d.ts.map +1 -1
- package/dist/plugin-distribution-layout.js +68 -10
- package/dist/plugin-distribution-layout.js.map +1 -1
- package/dist/post-build-checks.d.ts.map +1 -1
- package/dist/post-build-checks.js +12 -11
- package/dist/post-build-checks.js.map +1 -1
- package/dist/projection/skill-extent.d.ts +1 -1
- package/dist/projection/skill-extent.d.ts.map +1 -1
- package/dist/projection/skill-extent.js +3 -1
- package/dist/projection/skill-extent.js.map +1 -1
- package/dist/schemas/installed-plugins-registry.d.ts.map +1 -1
- package/dist/schemas/installed-plugins-registry.js +7 -1
- package/dist/schemas/installed-plugins-registry.js.map +1 -1
- package/dist/skill-identity.d.ts +8 -3
- package/dist/skill-identity.d.ts.map +1 -1
- package/dist/skill-identity.js +13 -6
- package/dist/skill-identity.js.map +1 -1
- package/dist/skill-packager.d.ts +2 -1
- package/dist/skill-packager.d.ts.map +1 -1
- package/dist/skill-packager.js +53 -44
- package/dist/skill-packager.js.map +1 -1
- package/dist/skill-source/content-hash.d.ts.map +1 -1
- package/dist/skill-source/content-hash.js +21 -10
- package/dist/skill-source/content-hash.js.map +1 -1
- package/dist/skill-source/fetch-cache.d.ts.map +1 -1
- package/dist/skill-source/fetch-cache.js +0 -5
- package/dist/skill-source/fetch-cache.js.map +1 -1
- package/dist/skill-source/git-clone.d.ts.map +1 -1
- package/dist/skill-source/git-clone.js +0 -2
- package/dist/skill-source/git-clone.js.map +1 -1
- package/dist/skill-source/sources/url-source.d.ts.map +1 -1
- package/dist/skill-source/sources/url-source.js +0 -1
- package/dist/skill-source/sources/url-source.js.map +1 -1
- package/dist/skill-source/stage.d.ts.map +1 -1
- package/dist/skill-source/stage.js +0 -6
- package/dist/skill-source/stage.js.map +1 -1
- package/dist/skill-test/baseline-integrity.d.ts.map +1 -1
- package/dist/skill-test/baseline-integrity.js +10 -3
- package/dist/skill-test/baseline-integrity.js.map +1 -1
- package/dist/skill-test/build-hook.d.ts +3 -2
- package/dist/skill-test/build-hook.d.ts.map +1 -1
- package/dist/skill-test/build-hook.js +4 -4
- package/dist/skill-test/build-hook.js.map +1 -1
- package/dist/skill-test/declared-env.d.ts +5 -4
- package/dist/skill-test/declared-env.d.ts.map +1 -1
- package/dist/skill-test/declared-env.js +7 -9
- package/dist/skill-test/declared-env.js.map +1 -1
- package/dist/skill-test/eval-executor.d.ts +1 -1
- package/dist/skill-test/eval-executor.js +2 -2
- package/dist/skill-test/eval-executor.js.map +1 -1
- package/dist/skill-test/eval-fragment.d.ts +7 -5
- package/dist/skill-test/eval-fragment.d.ts.map +1 -1
- package/dist/skill-test/eval-fragment.js +13 -7
- package/dist/skill-test/eval-fragment.js.map +1 -1
- package/dist/skill-test/eval-grader.d.ts +56 -4
- package/dist/skill-test/eval-grader.d.ts.map +1 -1
- package/dist/skill-test/eval-grader.js +138 -73
- package/dist/skill-test/eval-grader.js.map +1 -1
- package/dist/skill-test/eval-inputs.d.ts +8 -7
- package/dist/skill-test/eval-inputs.d.ts.map +1 -1
- package/dist/skill-test/eval-inputs.js +9 -11
- package/dist/skill-test/eval-inputs.js.map +1 -1
- package/dist/skill-test/eval-lint.d.ts +4 -4
- package/dist/skill-test/eval-lint.js +5 -5
- package/dist/skill-test/eval-lint.js.map +1 -1
- package/dist/skill-test/eval-suite-isolation.d.ts.map +1 -1
- package/dist/skill-test/eval-suite-isolation.js +8 -5
- package/dist/skill-test/eval-suite-isolation.js.map +1 -1
- package/dist/skill-test/evals-template.d.ts +1 -1
- package/dist/skill-test/evals-template.d.ts.map +1 -1
- package/dist/skill-test/evals-template.js +1 -3
- package/dist/skill-test/evals-template.js.map +1 -1
- package/dist/skill-test/executor-prompt.d.ts +2 -2
- package/dist/skill-test/executor-prompt.js +2 -2
- package/dist/skill-test/failure-reason.d.ts +113 -0
- package/dist/skill-test/failure-reason.d.ts.map +1 -0
- package/dist/skill-test/failure-reason.js +124 -0
- package/dist/skill-test/failure-reason.js.map +1 -0
- package/dist/skill-test/fragment-merge.d.ts +2 -2
- package/dist/skill-test/fragment-merge.js +4 -4
- package/dist/skill-test/grader-prompt.d.ts +2 -2
- package/dist/skill-test/grader-prompt.d.ts.map +1 -1
- package/dist/skill-test/grader-prompt.js +12 -8
- package/dist/skill-test/grader-prompt.js.map +1 -1
- package/dist/skill-test/grading-adapter.d.ts +7 -3
- package/dist/skill-test/grading-adapter.d.ts.map +1 -1
- package/dist/skill-test/grading-adapter.js +10 -9
- package/dist/skill-test/grading-adapter.js.map +1 -1
- package/dist/skill-test/harness-location.d.ts +3 -2
- package/dist/skill-test/harness-location.d.ts.map +1 -1
- package/dist/skill-test/harness-location.js +5 -16
- package/dist/skill-test/harness-location.js.map +1 -1
- package/dist/skill-test/index.d.ts +1 -1
- package/dist/skill-test/index.d.ts.map +1 -1
- package/dist/skill-test/index.js +1 -1
- package/dist/skill-test/index.js.map +1 -1
- package/dist/skill-test/lock.d.ts +11 -10
- package/dist/skill-test/lock.d.ts.map +1 -1
- package/dist/skill-test/lock.js +13 -14
- package/dist/skill-test/lock.js.map +1 -1
- package/dist/skill-test/pipeline.d.ts +3 -1
- package/dist/skill-test/pipeline.d.ts.map +1 -1
- package/dist/skill-test/pipeline.js +5 -1
- package/dist/skill-test/pipeline.js.map +1 -1
- package/dist/skill-test/preflight.d.ts.map +1 -1
- package/dist/skill-test/preflight.js +0 -1
- package/dist/skill-test/preflight.js.map +1 -1
- package/dist/skill-test/prompt-invariants.d.ts +4 -2
- package/dist/skill-test/prompt-invariants.d.ts.map +1 -1
- package/dist/skill-test/prompt-invariants.js +5 -4
- package/dist/skill-test/prompt-invariants.js.map +1 -1
- package/dist/skill-test/run-harness.d.ts +160 -123
- package/dist/skill-test/run-harness.d.ts.map +1 -1
- package/dist/skill-test/run-harness.js +310 -174
- package/dist/skill-test/run-harness.js.map +1 -1
- package/dist/skill-test/staging.d.ts +12 -6
- package/dist/skill-test/staging.d.ts.map +1 -1
- package/dist/skill-test/staging.js +17 -14
- package/dist/skill-test/staging.js.map +1 -1
- package/dist/skill-test/tier-plan.d.ts +1 -1
- package/dist/skill-test/tier-plan.js +1 -1
- package/dist/skill-test/tool-eval-schema.js +2 -2
- package/dist/skill-test/vendor-manifest.d.ts +2 -1
- package/dist/skill-test/vendor-manifest.d.ts.map +1 -1
- package/dist/skill-test/vendor-manifest.js +10 -10
- package/dist/skill-test/vendor-manifest.js.map +1 -1
- package/dist/test-input.d.ts +2 -2
- package/dist/test-input.d.ts.map +1 -1
- package/dist/test-input.js +6 -7
- package/dist/test-input.js.map +1 -1
- package/dist/validators/agent-instruction-presence.d.ts.map +1 -1
- package/dist/validators/agent-instruction-presence.js +0 -1
- package/dist/validators/agent-instruction-presence.js.map +1 -1
- package/dist/validators/bundled-resource-link-detection.d.ts +5 -3
- package/dist/validators/bundled-resource-link-detection.d.ts.map +1 -1
- package/dist/validators/bundled-resource-link-detection.js +38 -38
- package/dist/validators/bundled-resource-link-detection.js.map +1 -1
- package/dist/validators/format-detection.d.ts +3 -0
- package/dist/validators/format-detection.d.ts.map +1 -1
- package/dist/validators/format-detection.js +41 -26
- package/dist/validators/format-detection.js.map +1 -1
- package/dist/validators/frontmatter-validation.js +1 -1
- package/dist/validators/frontmatter-validation.js.map +1 -1
- package/dist/validators/marketplace-validator.d.ts.map +1 -1
- package/dist/validators/marketplace-validator.js +0 -1
- package/dist/validators/marketplace-validator.js.map +1 -1
- package/dist/validators/mcp-tool-qualification.d.ts +1 -1
- package/dist/validators/mcp-tool-qualification.js +1 -1
- package/dist/validators/packaged-size-limit.d.ts.map +1 -1
- package/dist/validators/packaged-size-limit.js +0 -2
- package/dist/validators/packaged-size-limit.js.map +1 -1
- package/dist/validators/packaging-validator.d.ts +2 -2
- package/dist/validators/packaging-validator.d.ts.map +1 -1
- package/dist/validators/packaging-validator.js +10 -13
- package/dist/validators/packaging-validator.js.map +1 -1
- package/dist/validators/plugin-hosted-shape.d.ts +3 -1
- package/dist/validators/plugin-hosted-shape.d.ts.map +1 -1
- package/dist/validators/plugin-hosted-shape.js +5 -6
- package/dist/validators/plugin-hosted-shape.js.map +1 -1
- package/dist/validators/referenced-path-missing.d.ts.map +1 -1
- package/dist/validators/referenced-path-missing.js +3 -4
- package/dist/validators/referenced-path-missing.js.map +1 -1
- package/dist/validators/registry-validator.d.ts.map +1 -1
- package/dist/validators/registry-validator.js +0 -1
- package/dist/validators/registry-validator.js.map +1 -1
- package/dist/validators/rule-engine/index.d.ts +1 -1
- package/dist/validators/rule-engine/rule-context.d.ts +3 -3
- package/dist/validators/rule-engine/rule-context.js +1 -1
- package/dist/validators/rule-engine/rule-engine.d.ts +1 -1
- package/dist/validators/rule-engine/rule-engine.js +5 -5
- package/dist/validators/rule-engine/rule-engine.js.map +1 -1
- package/dist/validators/skill-validator.d.ts.map +1 -1
- package/dist/validators/skill-validator.js +41 -51
- package/dist/validators/skill-validator.js.map +1 -1
- package/dist/validators/source-only-codes.js +1 -1
- package/dist/validators/source-only-codes.js.map +1 -1
- package/dist/validators/types.d.ts +7 -1
- package/dist/validators/types.d.ts.map +1 -1
- package/dist/validators/unified-validator.d.ts.map +1 -1
- package/dist/validators/unified-validator.js +3 -4
- package/dist/validators/unified-validator.js.map +1 -1
- package/dist/validators/validation-rules.d.ts +3 -3
- package/dist/validators/validation-rules.d.ts.map +1 -1
- package/dist/validators/validation-rules.js +9 -38
- package/dist/validators/validation-rules.js.map +1 -1
- package/dist/validators/walker-to-issues.d.ts +2 -0
- package/dist/validators/walker-to-issues.d.ts.map +1 -1
- package/dist/validators/walker-to-issues.js +17 -12
- package/dist/validators/walker-to-issues.js.map +1 -1
- package/dist/walk-link-graph.d.ts +2 -0
- package/dist/walk-link-graph.d.ts.map +1 -1
- package/dist/walk-link-graph.js +20 -8
- package/dist/walk-link-graph.js.map +1 -1
- package/package.json +12 -20
- package/dist/skill-test/exit-codes.d.ts +0 -134
- package/dist/skill-test/exit-codes.d.ts.map +0 -1
- package/dist/skill-test/exit-codes.js +0 -224
- package/dist/skill-test/exit-codes.js.map +0 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,
|
|
1
|
+
{"version":3,"file":"eval-grader.js","sourceRoot":"","sources":["../../src/skill-test/eval-grader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAE3D,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EACL,yBAAyB,EACzB,mBAAmB,GACpB,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EAAE,iBAAiB,EAAqB,MAAM,oBAAoB,CAAC;AAE1E,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACrF,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAuD1D,oJAAoJ;AACpJ,MAAM,CAAC,MAAM,2BAA2B,GAAG,6BAA6B,CAAC;AAEzE;;;;;;;;;;;;;GAaG;AACH,MAAM,OAAO,8BAA+B,SAAQ,QAAQ;IAExC;IACA;IAFlB,YACkB,MAAc,EACd,MAAc;QAE9B,KAAK,CACH,wCAAwC,EACxC,GAAG,2BAA2B,0BAA0B,MAAM,6CAA6C;YACzG,sBAAsB,MAAM,IAAI,CACnC,CAAC;QAPc,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAAQ;IAOhC,CAAC;CACF;AAED;;;;GAIG;AACH,SAAS,kBAAkB,CAAC,UAAwC;IAClE,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,gBAAgB,OAAO,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,KAAqB;IAC1D,8EAA8E;IAC9E,iFAAiF;IACjF,8EAA8E;IAC9E,+EAA+E;IAC/E,8EAA8E;IAC9E,iFAAiF;IACjF,kFAAkF;IAClF,8EAA8E;IAC9E,+EAA+E;IAC/E,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAEpE,4EAA4E;IAC5E,2EAA2E;IAC3E,uEAAuE;IACvE,MAAM,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC;IAEvF,IAAI,OAAO,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACjF,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACtC,KAAK,CAAC,UAAU,CACd,0CAA0C,KAAK,CAAC,MAAM,yBAAyB,OAAO,CAAC,WAAW,uBAAuB,CAC1H,CAAC;QACF,6EAA6E;QAC7E,kCAAkC;QAClC,OAAO,GAAG,MAAM,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3G,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;YACtC,MAAM,IAAI,8BAA8B,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC9E,CAAC;IACH,CAAC;IAED,MAAM,QAAQ,GAAG,iBAAiB,CAAC,OAAO,CAAC,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IAEzF,6EAA6E;IAC7E,0EAA0E;IAC1E,iFAAiF;IACjF,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,IAAI,iBAAiB,CACzB,SAAS,KAAK,CAAC,MAAM,iDAAiD,CACvE,CAAC;IACJ,CAAC;IAED,8BAA8B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,YAAY,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;IACxF,2BAA2B,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,gBAAgB,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjF,iFAAiF;IACjF,+EAA+E;IAC/E,yEAAyE;IACzE,0EAA0E;IAC1E,8EAA8E;IAC9E,gFAAgF;IAChF,uEAAuE;IACvE,EAAE;IACF,4EAA4E;IAC5E,8EAA8E;IAC9E,0EAA0E;IAC1E,OAAO,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;AACtE,CAAC;AAUD;;;;;;GAMG;AACH,KAAK,UAAU,gBAAgB,CAC7B,KAAqB,EACrB,WAAmB,EACnB,KAAa,EACb,eAAmC;IAEnC,MAAM,KAAK,GAAG,iBAAiB,CAAC;QAC9B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,YAAY,EAAE,KAAK,CAAC,YAAY;QAChC,GAAG,CAAC,KAAK,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,KAAK,CAAC,cAAc,EAAE,CAAC;QACvF,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,KAAK;QACL,GAAG,CAAC,KAAK,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAC7F,GAAG,CAAC,KAAK,CAAC,mBAAmB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,mBAAmB,EAAE,KAAK,CAAC,mBAAmB,EAAE,CAAC;KACvG,CAAC,CAAC;IACH,MAAM,MAAM,GACV,eAAe,KAAK,SAAS;QAC3B,CAAC,CAAC,KAAK;QACP,CAAC,CAAC,GAAG,KAAK,kDAAkD,eAAe,sBAAsB,CAAC;IACtG,8EAA8E;IAC9E,0EAA0E;IAC1E,4EAA4E;IAC5E,8EAA8E;IAC9E,6EAA6E;IAC7E,8EAA8E;IAC9E,2EAA2E;IAC3E,0EAA0E;IAC1E,UAAU;IACV,4BAA4B,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAE5C,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,mBAAmB,CAAC;IAEjD,yEAAyE;IACzE,+EAA+E;IAC/E,uEAAuE;IACvE,IAAI,gBAAgB,GAAG,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAG,CAAC,KAAa,EAAQ,EAAE;QACvC,gBAAgB,IAAI,KAAK,CAAC;QAC1B,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAC1B,CAAC,CAAC;IAEF,IAAI,WAAW,CAAC;IAChB,IAAI,CAAC;QACH,WAAW,GAAG,MAAM,KAAK,CAAC;YACxB,MAAM;YACN,uEAAuE;YACvE,gEAAgE;YAChE,0CAA0C;YAC1C,UAAU,EAAE,EAAE;YACd,UAAU,EAAE,KAAK,CAAC,YAAY;YAC9B,GAAG,EAAE,KAAK,CAAC,YAAY;YACvB,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ;YACR,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,YAAY,EAAE,KAAK,CAAC,YAAY;YAChC,KAAK,EAAE,KAAK,CAAC,WAAW;YACxB,GAAG,CAAC,KAAK,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,CAAC;SACnE,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,oBAAoB,CAC5B,iCAAiC,KAAK,CAAC,MAAM,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACtG,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,2EAA2E;IAC3E,wDAAwD;IACxD,IAAI,WAAW,CAAC,OAAO,EAAE,CAAC;QACxB,MAAM,IAAI,oBAAoB,CAC5B,4BAA4B,KAAK,CAAC,MAAM,oBAAoB,KAAK,CAAC,OAAO,IAAI,CAAC,MAAM,CACrF,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,QAAQ,EAAE,CAAC;QACzB,MAAM,IAAI,oBAAoB,CAC5B,8BAA8B,KAAK,CAAC,MAAM,YAAY,KAAK,CAAC,SAAS,MAAM,CAC5E,CAAC;IACJ,CAAC;IACD,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,kCAAkC,WAAW,CAAC,MAAM,eAAe,KAAK,CAAC,MAAM,IAAI,CACpF,CAAC;IACJ,CAAC;IAED,+EAA+E;IAC/E,iFAAiF;IACjF,8EAA8E;IAC9E,KAAK,CAAC,QAAQ,EAAE,CAAC,yBAAyB,CAAC,gBAAgB,CAAC,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IAEnF,MAAM,IAAI,GAAG,kBAAkB,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,0BAA0B,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAC7F,IAAI,CAAC;QACH,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;IAC7C,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,CAAC,GAAG,YAAY,WAAW,CAAC;YAAE,MAAM,GAAG,CAAC;QAC7C,uEAAuE;QACvE,4EAA4E;QAC5E,oEAAoE;QACpE,4EAA4E;QAC5E,+EAA+E;QAC/E,+EAA+E;QAC/E,+EAA+E;QAC/E,UAAU;QACV,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,kBAAkB,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;IACjE,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,SAAS,0BAA0B,CACjC,WAAmB,EACnB,MAAc,EACd,MAAc,EACd,MAAiC;IAEjC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,oBAAoB,CAC5B,yBAAyB,MAAM,mCAAmC,WAAW,cAAc,MAAM,IAAI,CACtG,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAAG,YAAY,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IAChD,IAAI,CAAC;QACH,8EAA8E;QAC9E,kEAAkE;QAClE,MAAM,CAAC,WAAW,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CACJ,6BAA6B,MAAM,QAAQ,WAAW,0CAA0C;YAC9F,IAAI,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,+CAA+C,CACtG,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AACH,SAAS,8BAA8B,CACrC,MAAc,EACd,QAA2B,EAC3B,QAAsC;IAEtC,IAAI,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM;QAAE,OAAO;IAChD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,cAAc,QAAQ,CAAC,MAAM,oBAAoB,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG;QAC/G,6BAA6B,QAAQ,CAAC,MAAM,yDAAyD;QACrG,qGAAqG;QACrG,oCAAoC,CACvC,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,SAAS,2BAA2B,CAClC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,gBAAgB,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACzD,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,mGAAmG,CAC9H,CAAC;IACJ,CAAC;IACD,IAAI,IAAI,KAAK,SAAS,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC;QAClE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,8CAA8C,IAAI,CAAC,MAAM,IAAI;YACrF,iDAAiD,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAC/E,CAAC;IACJ,CAAC;IACD,4BAA4B,CAAC,MAAM,EAAE,gBAAgB,EAAE,IAAI,CAAC,CAAC;AAC/D,CAAC;AAED,kGAAkG;AAClG,SAAS,UAAU,CAAC,KAAwB;IAC1C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,SAAS,4BAA4B,CACnC,MAAc,EACd,gBAAoD,EACpD,IAA0B;IAE1B,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO;IAC/B,MAAM,QAAQ,GAAG,gBAAgB,IAAI,EAAE,CAAC;IACxC,kBAAkB,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACjG,kBAAkB,CAAC,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,UAAU,EAAE,CAAC,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1G,kBAAkB,CAAC,MAAM,EAAE,aAAa,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7G,kBAAkB,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,kBAAkB,CACzB,MAAc,EACd,OAAe,EACf,QAAuC,EACvC,QAA2B;IAE3B,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;IACtE,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAC1D,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAC3E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,QAAQ,CAAC,IAAI,CAAC,WAAW,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACxE,MAAM,IAAI,oBAAoB,CAC5B,oBAAoB,MAAM,oBAAoB,OAAO,yCAAyC;QAC5F,8BAA8B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,gDAAgD;QACjG,qEAAqE,CACxE,CAAC;AACJ,CAAC"}
|
|
@@ -1,18 +1,19 @@
|
|
|
1
|
+
import { VatError } from '@vibe-agent-toolkit/utils';
|
|
1
2
|
import { z } from 'zod';
|
|
2
3
|
/**
|
|
3
4
|
* Raised for any eval-input problem (bad JSON, schema failure, missing input file).
|
|
4
|
-
*
|
|
5
|
+
* Reason `preflight`.
|
|
5
6
|
*
|
|
6
7
|
* The FIELD is what makes that sentence true, and until it was added only one of this
|
|
7
8
|
* class's two routes delivered it. `attemptStageWorkspaces` catches the staging
|
|
8
9
|
* instance and maps it explicitly; the instance thrown from {@link armDirSegment}
|
|
9
10
|
* fires deep in the eval loop, outside that handler, and reached
|
|
10
|
-
* `
|
|
11
|
-
* problem while the docblock above claimed
|
|
12
|
-
* too is consistent with every sibling in this feature.
|
|
11
|
+
* `skillTestFailureReason`'s `internal` default — reporting a harness bug for a
|
|
12
|
+
* preflight-class problem while the docblock above claimed otherwise. `preflight`
|
|
13
|
+
* for the internal-invariant case too is consistent with every sibling in this feature.
|
|
13
14
|
*/
|
|
14
|
-
export declare class EvalInputError extends
|
|
15
|
-
readonly
|
|
15
|
+
export declare class EvalInputError extends VatError {
|
|
16
|
+
readonly reason: "preflight";
|
|
16
17
|
constructor(message: string);
|
|
17
18
|
}
|
|
18
19
|
/**
|
|
@@ -528,7 +529,7 @@ export declare const EvalSuiteSchema: z.ZodObject<{
|
|
|
528
529
|
export type EvalEntry = z.infer<typeof EvalEntrySchema>;
|
|
529
530
|
export type EvalSuite = z.infer<typeof EvalSuiteSchema>;
|
|
530
531
|
/**
|
|
531
|
-
* The declared tool expectations of ONE eval
|
|
532
|
+
* The declared tool expectations of ONE eval. Derived from
|
|
532
533
|
* {@link EvalEntry} so the grader input + prompt builder share the EXACT shape the
|
|
533
534
|
* parser produces — no drift, and no duplicated inline `{ mustRun?; mustNotRun?;
|
|
534
535
|
* sequence? }` literal across those consumers.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"eval-inputs.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAEA,OAAO,EAA2B,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB;;;;;;;;;;;GAWG;AACH,qBAAa,cAAe,SAAQ,QAAQ;IAC1C,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;gBAC3B,OAAO,EAAE,MAAM;CAG5B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEpD;AA2CD;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,OAAO,CAUrE;AAiDD,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAyDxB,CAAC;AAEL,eAAO,MAAM,eAAe;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;gCAMZ,CAAC;AAEjB,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AACxD,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,eAAe,CAAC,CAAC;AAExD;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC;AAE1E,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAsB1D;AAED;;;;GAIG;AACH,MAAM,MAAM,OAAO,GAAG,MAAM,GAAG,SAAS,CAAC;AAEzC;;;;;;;;;GASG;AACH,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,sEAAsE;AACtE,wBAAgB,MAAM,CAAC,IAAI,EAAE,gBAAgB,GAAG,SAAS,OAAO,EAAE,CAEjE;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,gBAAgB,EAAE,GAAG,EAAE,OAAO,GAAG,MAAM,CAU1E;AAED,MAAM,WAAW,wBAAwB;IACvC,6BAA6B;IAC7B,KAAK,EAAE,SAAS,CAAC;IACjB,mFAAmF;IACnF,QAAQ,EAAE,MAAM,CAAC;IACjB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,EAAE,gBAAgB,CAAC;CAC3B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,wBAAwB,GAAG,MAAM,CAK3E"}
|
|
@@ -1,24 +1,23 @@
|
|
|
1
1
|
import { cpSync, existsSync } from 'node:fs';
|
|
2
|
-
import { mkdirSyncReal, safePath } from '@vibe-agent-toolkit/utils';
|
|
2
|
+
import { mkdirSyncReal, safePath, VatError } from '@vibe-agent-toolkit/utils';
|
|
3
3
|
import { z } from 'zod';
|
|
4
4
|
import { sanitizeGraderText, sanitizeTextPreservingLines } from './grader-text.js';
|
|
5
5
|
/**
|
|
6
6
|
* Raised for any eval-input problem (bad JSON, schema failure, missing input file).
|
|
7
|
-
*
|
|
7
|
+
* Reason `preflight`.
|
|
8
8
|
*
|
|
9
9
|
* The FIELD is what makes that sentence true, and until it was added only one of this
|
|
10
10
|
* class's two routes delivered it. `attemptStageWorkspaces` catches the staging
|
|
11
11
|
* instance and maps it explicitly; the instance thrown from {@link armDirSegment}
|
|
12
12
|
* fires deep in the eval loop, outside that handler, and reached
|
|
13
|
-
* `
|
|
14
|
-
* problem while the docblock above claimed
|
|
15
|
-
* too is consistent with every sibling in this feature.
|
|
13
|
+
* `skillTestFailureReason`'s `internal` default — reporting a harness bug for a
|
|
14
|
+
* preflight-class problem while the docblock above claimed otherwise. `preflight`
|
|
15
|
+
* for the internal-invariant case too is consistent with every sibling in this feature.
|
|
16
16
|
*/
|
|
17
|
-
export class EvalInputError extends
|
|
18
|
-
|
|
17
|
+
export class EvalInputError extends VatError {
|
|
18
|
+
reason = 'preflight';
|
|
19
19
|
constructor(message) {
|
|
20
|
-
super(message);
|
|
21
|
-
this.name = 'EvalInputError';
|
|
20
|
+
super('EVAL_INPUT', message);
|
|
22
21
|
}
|
|
23
22
|
}
|
|
24
23
|
/**
|
|
@@ -187,7 +186,7 @@ export const EvalEntrySchema = z
|
|
|
187
186
|
mustRun: z.array(z.string().min(1)).optional(),
|
|
188
187
|
mustNotRun: z.array(z.string().min(1)).optional(),
|
|
189
188
|
// Each named executable must have RUN and its invoking tool_result must
|
|
190
|
-
// NOT be an error
|
|
189
|
+
// NOT be an error. Judged from the transcript by the
|
|
191
190
|
// grader — see tool-eval-schema.ts's ToolSucceedCheckSchema.
|
|
192
191
|
mustSucceed: z.array(z.string().min(1)).optional(),
|
|
193
192
|
sequence: z.array(z.string().min(1)).optional(),
|
|
@@ -323,7 +322,6 @@ function stageEvalWorkspacesForArm(input, arm) {
|
|
|
323
322
|
throw new EvalInputError(`eval ${entry.id} declares input file "${quoteSuiteText(rel)}" that escapes the eval directory: ` +
|
|
324
323
|
quoteSuiteText(err.message));
|
|
325
324
|
}
|
|
326
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- src is contained under evalsDir via joinUnderRoot; suite is developer-authored
|
|
327
325
|
if (!existsSync(src)) {
|
|
328
326
|
throw new EvalInputError(`eval ${entry.id} declares input file "${quoteSuiteText(rel)}" but it is absent at ${quoteSuiteText(src)}`);
|
|
329
327
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-inputs.js","sourceRoot":"","sources":["../../src/skill-test/eval-inputs.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAE7C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAC9E,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,2BAA2B,EAAE,MAAM,kBAAkB,CAAC;AAEnF;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,cAAe,SAAQ,QAAQ;IACjC,MAAM,GAAG,WAAoB,CAAC;IACvC,YAAY,OAAe;QACzB,KAAK,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IAC/B,CAAC;CACF;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,2BAA2B,CAAC,KAAK,CAAC,CAAC;AAC5C,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,eAAe,GAAG,kBAAkB,CAAC;AAE3C;+EAC+E;AAC/E,MAAM,sBAAsB,GAAG;IAC7B,IAAI;IACJ,QAAQ;IACR,iBAAiB;IACjB,OAAO;IACP,cAAc;IACd,MAAM;IACN,kBAAkB;CACV,CAAC;AAEX;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAW,EAAE,MAAc;IAC1D,IAAI,GAAG,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IACjC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IACtB,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IACzB,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,CAAC;QAAE,CAAC,EAAE,CAAC;IACrD,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,eAAe;IAC/E,IAAI,EAAE,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,oBAAoB;IAC9E,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,qBAAqB;AACpE,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,6BAA6B,CAAC,YAAsB,EAAE,GAAoB;IACjF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC,IAAI,YAAY,CAAC,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAC3B,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YACtB,SAAS;QACX,CAAC;QACD,GAAG,CAAC,QAAQ,CAAC;YACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;YAC3B,OAAO,EACL,0BAA0B,cAAc,CAAC,WAAW,CAAC,4CAA4C;gBACjG,8FAA8F;gBAC9F,yEAAyE;YAC3E,IAAI,EAAE,CAAC,cAAc,EAAE,KAAK,CAAC;SAC9B,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,+EAA+E;AAC/E,iFAAiF;AACjF,gFAAgF;AAChF,uEAAuE;AACvE,2EAA2E;AAC3E,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC;QACV,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE;QAChB,CAAC;aACE,MAAM,EAAE;aACR,GAAG,CAAC,CAAC,CAAC;aACN,KAAK,CACJ,eAAe,EACf,wGAAwG,CACzG;KACJ,CAAC;IACF,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,2EAA2E;IAC3E,2EAA2E;IAC3E,qEAAqE;IACrE,4EAA4E;IAC5E,4EAA4E;IAC5E,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC7C,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5C,YAAY,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,wFAAwF;IACxF,mEAAmE;IACnE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC,QAAQ,EAAE;IAC/C,4EAA4E;IAC5E,kFAAkF;IAClF,qFAAqF;IACrF,+BAA+B;IAC/B,gBAAgB,EAAE,CAAC;SAChB,MAAM,CAAC;QACN,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAC9C,UAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QACjD,wEAAwE;QACxE,qDAAqD;QACrD,6DAA6D;QAC7D,WAAW,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;QAClD,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;KAChD,CAAC;SACD,MAAM,EAAE;SACR,QAAQ,EAAE;CACd,CAAC;KACD,WAAW,EAAE;KACb,WAAW,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;IAC1B,6BAA6B,CAAC,KAAK,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IACvD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAK,sBAA4C,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,SAAS;QAC1E,MAAM,IAAI,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,WAAW,EAAE,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QAC9G,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,GAAG,CAAC,QAAQ,CAAC;gBACX,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,MAAM;gBAC3B,uEAAuE;gBACvE,iEAAiE;gBACjE,OAAO,EAAE,uBAAuB,cAAc,CAAC,GAAG,CAAC,qBAAqB,IAAI,kDAAkD;gBAC9H,IAAI,EAAE,CAAC,GAAG,CAAC;aACZ,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC;KAC7B,MAAM,CAAC;IACN,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CACvC,CAAC;KACD,WAAW,EAAE,CAAC;AAajB,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,QAAgB;IAC7C,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC7B,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,wEAAwE;QACxE,4EAA4E;QAC5E,uEAAuE;QACvE,MAAM,IAAI,cAAc,CAAC,iCAAiC,cAAc,CAAE,CAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IACpG,CAAC;IACD,MAAM,MAAM,GAAG,eAAe,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,cAAc,CAAC,wCAAwC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC5G,CAAC;IACD,4EAA4E;IAC5E,6EAA6E;IAC7E,uEAAuE;IACvE,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,cAAc,CAAC,4FAA4F,CAAC,CAAC;IACzH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC;AACrB,CAAC;AAwBD,sEAAsE;AACtE,MAAM,UAAU,MAAM,CAAC,IAAsB;IAC3C,OAAO,IAAI,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAE,CAAC,MAAM,CAAW,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,SAAS,CAAW,CAAC;AAC3F,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,IAAsB,EAAE,GAAY;IAChE,MAAM,OAAO,GAAG,GAAG,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;IAC1D,IAAI,OAAO,KAAK,SAAS,IAAI,OAAO,KAAK,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,cAAc,CACtB,wDAAwD,GAAG,SAAS;YAClE,kFAAkF;YAClF,mDAAmD,CACtD,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAiBD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA+B;IACjE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,yBAAyB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC,cAAc,CAAC;AAC9B,CAAC;AAED,SAAS,yBAAyB,CAAC,KAA+B,EAAE,GAAY;IAC9E,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,cAAc,EAAE,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IAChG,aAAa,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,KAAK,MAAM,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACtC,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,0EAA0E;QAC1E,2BAA2B;QAC3B,aAAa,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC;YACpC,yEAAyE;YACzE,wEAAwE;YACxE,IAAI,GAAW,CAAC;YAChB,IAAI,IAAY,CAAC;YACjB,IAAI,CAAC;gBACH,GAAG,GAAG,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBAClD,IAAI,GAAG,QAAQ,CAAC,aAAa,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,cAAc,CAAC,GAAG,CAAC,qCAAqC;oBAC/F,cAAc,CAAE,GAAa,CAAC,OAAO,CAAC,CACzC,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,yBAAyB,cAAc,CAAC,GAAG,CAAC,yBAAyB,cAAc,CAAC,GAAG,CAAC,EAAE,CAC3G,CAAC;YACJ,CAAC;YACD,sEAAsE;YACtE,sEAAsE;YACtE,IAAI,CAAC;gBACH,0EAA0E;gBAC1E,kDAAkD;gBAClD,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;gBAC3E,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACzC,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,IAAI,cAAc,CACtB,QAAQ,KAAK,CAAC,EAAE,gCAAgC,cAAc,CAAC,GAAG,CAAC,wBAAwB;oBACzF,cAAc,CAAE,GAAa,CAAC,OAAO,CAAC,CACzC,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Advisory (never-fatal) lint over an eval suite's `expectations` prose
|
|
3
|
-
*
|
|
2
|
+
* Advisory (never-fatal) lint over an eval suite's `expectations` prose.
|
|
3
|
+
* An adopter reported that WEAK expectations — pure "mentions X" /
|
|
4
4
|
* "includes Y" presence checks with no discriminating or negative counterpart —
|
|
5
5
|
* pass even for a hallucinated or right-for-the-wrong-reason answer, giving false
|
|
6
6
|
* confidence in a green run. This module flags that pattern so authors can
|
|
@@ -34,8 +34,8 @@ export interface EvalLintWarning {
|
|
|
34
34
|
export declare function lintEvalExpectations(evals: EvalEntry[]): EvalLintWarning[];
|
|
35
35
|
/**
|
|
36
36
|
* Advisory (never-fatal) lint: flag a `toolExpectations` entry that names an
|
|
37
|
-
* executable which looks like a TYPO of one of the skill's `declaredExecutables
|
|
38
|
-
*
|
|
37
|
+
* executable which looks like a TYPO of one of the skill's `declaredExecutables`.
|
|
38
|
+
* A name that never matches a real tool is a quiet
|
|
39
39
|
* footgun — `mustRun`/`mustSucceed`/`sequence` then fail for the wrong reason, and
|
|
40
40
|
* `mustNotRun` passes vacuously (the tool "never ran" only because the name is
|
|
41
41
|
* wrong). This is zero-token static plumbing, run before any spend.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Advisory (never-fatal) lint over an eval suite's `expectations` prose
|
|
3
|
-
*
|
|
2
|
+
* Advisory (never-fatal) lint over an eval suite's `expectations` prose.
|
|
3
|
+
* An adopter reported that WEAK expectations — pure "mentions X" /
|
|
4
4
|
* "includes Y" presence checks with no discriminating or negative counterpart —
|
|
5
5
|
* pass even for a hallucinated or right-for-the-wrong-reason answer, giving false
|
|
6
6
|
* confidence in a green run. This module flags that pattern so authors can
|
|
@@ -62,7 +62,7 @@ export function lintEvalExpectations(evals) {
|
|
|
62
62
|
return warnings;
|
|
63
63
|
}
|
|
64
64
|
// ---------------------------------------------------------------------------
|
|
65
|
-
// Undeclared-executable lint
|
|
65
|
+
// Undeclared-executable lint
|
|
66
66
|
// ---------------------------------------------------------------------------
|
|
67
67
|
// A trailing script/binary extension we strip before comparing an executable
|
|
68
68
|
// name to a declared one, so `csvsum.py`/`./csvsum`/`csvsum` all normalize to `csvsum` (the
|
|
@@ -115,8 +115,8 @@ function probableDeclaredTypoTarget(name, declared) {
|
|
|
115
115
|
}
|
|
116
116
|
/**
|
|
117
117
|
* Advisory (never-fatal) lint: flag a `toolExpectations` entry that names an
|
|
118
|
-
* executable which looks like a TYPO of one of the skill's `declaredExecutables
|
|
119
|
-
*
|
|
118
|
+
* executable which looks like a TYPO of one of the skill's `declaredExecutables`.
|
|
119
|
+
* A name that never matches a real tool is a quiet
|
|
120
120
|
* footgun — `mustRun`/`mustSucceed`/`sequence` then fail for the wrong reason, and
|
|
121
121
|
* `mustNotRun` passes vacuously (the tool "never ran" only because the name is
|
|
122
122
|
* wrong). This is zero-token static plumbing, run before any spend.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-lint.js","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAQpF,gFAAgF;AAChF,4EAA4E;AAC5E,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,2EAA2E,CAAC;AAE1G,iFAAiF;AACjF,kFAAkF;AAClF,0EAA0E;AAC1E,4EAA4E;AAC5E,2EAA2E;AAC3E,kFAAkF;AAClF,6DAA6D;AAC7D,MAAM,sBAAsB,GAC1B,+GAA+G,CAAC;AAElH,0IAA0I;AAC1I,SAAS,yBAAyB,CAAC,WAAmB;IACpD,OAAO,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAC9F,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC9C,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS;YAAE,SAAS;QACnD,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,yBAAyB,CAAC;YAAE,SAAS;QACnE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM;YACN,OAAO,EACL,SAAS,MAAM,uFAAuF;gBACtG,yGAAyG;gBACzG,8CAA8C;SACjD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,
|
|
1
|
+
{"version":3,"file":"eval-lint.js","sourceRoot":"","sources":["../../src/skill-test/eval-lint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAkB,MAAM,kBAAkB,CAAC;AAQpF,gFAAgF;AAChF,4EAA4E;AAC5E,+EAA+E;AAC/E,MAAM,qBAAqB,GAAG,2EAA2E,CAAC;AAE1G,iFAAiF;AACjF,kFAAkF;AAClF,0EAA0E;AAC1E,4EAA4E;AAC5E,2EAA2E;AAC3E,kFAAkF;AAClF,6DAA6D;AAC7D,MAAM,sBAAsB,GAC1B,+GAA+G,CAAC;AAElH,0IAA0I;AAC1I,SAAS,yBAAyB,CAAC,WAAmB;IACpD,OAAO,qBAAqB,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;AAC9F,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAkB;IACrD,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAC9C,IAAI,KAAK,CAAC,gBAAgB,KAAK,SAAS;YAAE,SAAS;QACnD,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,yBAAyB,CAAC;YAAE,SAAS;QACnE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,QAAQ,CAAC,IAAI,CAAC;YACZ,MAAM;YACN,OAAO,EACL,SAAS,MAAM,uFAAuF;gBACtG,yGAAyG;gBACzG,8CAA8C;SACjD,CAAC,CAAC;IACL,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,8EAA8E;AAC9E,6BAA6B;AAC7B,8EAA8E;AAE9E,6EAA6E;AAC7E,4FAA4F;AAC5F,oFAAoF;AACpF,gFAAgF;AAChF,iFAAiF;AACjF,SAAS;AACT,MAAM,gBAAgB,GAAG,0CAA0C,CAAC;AAEpE,yHAAyH;AACzH,SAAS,uBAAuB,CAAC,IAAY;IAC3C,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;AACtF,CAAC;AAED,mHAAmH;AACnH,SAAS,iCAAiC,CAAC,KAAgB;IACzD,MAAM,EAAE,GAAG,KAAK,CAAC,gBAAgB,CAAC;IAClC,IAAI,EAAE,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAChC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;AAC9G,CAAC;AAED;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,SAAiB,EAAE,IAAY;IACtD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IACpG,OAAO,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,0BAA0B,CACjC,IAAY,EACZ,QAAoD;IAEpD,MAAM,UAAU,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,KAAK,UAAU,CAAC;QAAE,OAAO,SAAS,CAAC,CAAC,iCAAiC;IAC1G,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,IAAI,gBAAgB,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QAClE,IAAI,eAAe,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,IAAI,eAAe,CAAC,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC;YAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;IAChH,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,UAAU,8BAA8B,CAC5C,KAAkB,EAClB,uBAAiC;IAEjC,MAAM,QAAQ,GAAsB,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAG,uBAAuB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1H,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC3C,MAAM,YAAY,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChF,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,iCAAiC,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5D,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC7B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACf,MAAM,MAAM,GAAG,0BAA0B,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YAC1D,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YACnC,QAAQ,CAAC,IAAI,CAAC;gBACZ,MAAM;gBACN,OAAO,EACL,SAAS,MAAM,6CAA6C,cAAc,CAAC,IAAI,CAAC,4CAA4C;oBAC5H,iBAAiB,cAAc,CAAC,MAAM,CAAC,mFAAmF;oBAC1H,8EAA8E,YAAY,GAAG;aAChG,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite-isolation.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-suite-isolation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAOH;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,qBAAqB,CAAC;AAExD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,
|
|
1
|
+
{"version":3,"file":"eval-suite-isolation.d.ts","sourceRoot":"","sources":["../../src/skill-test/eval-suite-isolation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAOH;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,qBAAqB,CAAC;AAExD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,SAAS,CAaxG;AA0BD,MAAM,WAAW,qBAAqB;IACpC,kFAAkF;IAClF,SAAS,EAAE,MAAM,CAAC;IAClB,6FAA6F;IAC7F,WAAW,EAAE,MAAM,CAAC;IACpB;;;;;OAKG;IACH,YAAY,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;;;;;OAOG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,qBAAqB,GAAG,OAAO,CAoBtE"}
|
|
@@ -34,7 +34,7 @@
|
|
|
34
34
|
*/
|
|
35
35
|
import { cpSync, existsSync, lstatSync, rmSync } from 'node:fs';
|
|
36
36
|
import { basename, dirname } from 'node:path';
|
|
37
|
-
import { mkdirSyncReal, safePath } from '@vibe-agent-toolkit/utils';
|
|
37
|
+
import { isVatError, mkdirSyncReal, PathEscapesRootError, safePath } from '@vibe-agent-toolkit/utils';
|
|
38
38
|
/**
|
|
39
39
|
* Where a skill's eval suite lives by convention. Defined here, beside the code
|
|
40
40
|
* that must REMOVE it, because every lane that reasons about a suite needs the
|
|
@@ -69,8 +69,13 @@ export function evalSuiteUnitPath(skillDir, evalsSubpath) {
|
|
|
69
69
|
try {
|
|
70
70
|
return safePath.joinUnderRoot(skillDir, relative);
|
|
71
71
|
}
|
|
72
|
-
catch {
|
|
73
|
-
|
|
72
|
+
catch (error) {
|
|
73
|
+
// Only the containment helper's OWN refusal means "outside the skill dir";
|
|
74
|
+
// anything else out of it is a bug, and `undefined` would read as
|
|
75
|
+
// "nothing to strip" — leaving the answer key in the staged copy.
|
|
76
|
+
if (isVatError(error, PathEscapesRootError.code))
|
|
77
|
+
return undefined;
|
|
78
|
+
throw error;
|
|
74
79
|
}
|
|
75
80
|
}
|
|
76
81
|
/**
|
|
@@ -109,7 +114,6 @@ function assertInsideRoot(child, root) {
|
|
|
109
114
|
*/
|
|
110
115
|
export function isolateEvalSuite(input) {
|
|
111
116
|
const unit = evalSuiteUnitPath(input.stagedDir, input.evalsSubpath);
|
|
112
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- unit is proven inside our own staged dir
|
|
113
117
|
if (unit === undefined || !existsSync(unit))
|
|
114
118
|
return false;
|
|
115
119
|
assertInsideRoot(input.stagedDir, input.stagingRoot);
|
|
@@ -120,7 +124,6 @@ export function isolateEvalSuite(input) {
|
|
|
120
124
|
// lands at `<holdDir>/<basename(evalsSubpath)>` either way and `fixtures/` keep
|
|
121
125
|
// their positions relative to it — exactly the shape the eval-input staging
|
|
122
126
|
// expects from an authored evals dir.
|
|
123
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- unit is proven inside our own staged dir
|
|
124
127
|
const dest = lstatSync(unit).isDirectory() ? input.holdDir : safePath.join(input.holdDir, basename(unit));
|
|
125
128
|
cpSync(unit, dest, { recursive: true });
|
|
126
129
|
preserved = true;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"eval-suite-isolation.js","sourceRoot":"","sources":["../../src/skill-test/eval-suite-isolation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"eval-suite-isolation.js","sourceRoot":"","sources":["../../src/skill-test/eval-suite-isolation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,oBAAoB,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEtG;;;;;GAKG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAG,kBAAkB,CAAC;AAExD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB,EAAE,YAAgC;IAClF,IAAI,YAAY,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACjD,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC;IACzE,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACpD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,2EAA2E;QAC3E,kEAAkE;QAClE,kEAAkE;QAClE,IAAI,UAAU,CAAC,KAAK,EAAE,oBAAoB,CAAC,IAAI,CAAC;YAAE,OAAO,SAAS,CAAC;QACnE,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,gBAAgB,CAAC,KAAa,EAAE,IAAY;IACnD,0EAA0E;IAC1E,yEAAyE;IACzE,4EAA4E;IAC5E,+EAA+E;IAC/E,wEAAwE;IACxE,yCAAyC;IACzC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,SAAS,CACjB,wEAAwE;YACtE,mEAAmE,CACtE,CAAC;IACJ,CAAC;IACD,QAAQ,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAC/D,CAAC;AAyBD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAA4B;IAC3D,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IACpE,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC;IAE1D,gBAAgB,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAErD,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,KAAK,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAChC,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC/D,gFAAgF;QAChF,gFAAgF;QAChF,4EAA4E;QAC5E,sCAAsC;QACtC,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;QAC1G,MAAM,CAAC,IAAI,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* evals-template.ts — scaffold a starter `evals.json` on bootstrap (exit
|
|
2
|
+
* evals-template.ts — scaffold a starter `evals.json` on bootstrap (exit 2, `Reason: bootstrap`).
|
|
3
3
|
*
|
|
4
4
|
* When a subject skill has no `evals/evals.json`, the harness writes an
|
|
5
5
|
* annotated template the user can fill in and re-run. The shape mirrors
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evals-template.d.ts","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,2EAA2E;AAC3E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CA6C5D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"evals-template.d.ts","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAMH,2EAA2E;AAC3E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CA6C5D;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAS/E"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* evals-template.ts — scaffold a starter `evals.json` on bootstrap (exit
|
|
2
|
+
* evals-template.ts — scaffold a starter `evals.json` on bootstrap (exit 2, `Reason: bootstrap`).
|
|
3
3
|
*
|
|
4
4
|
* When a subject skill has no `evals/evals.json`, the harness writes an
|
|
5
5
|
* annotated template the user can fill in and re-run. The shape mirrors
|
|
@@ -70,9 +70,7 @@ export function writeEvalsTemplate(evalsPath, skillName) {
|
|
|
70
70
|
mkdirSyncReal(parent, { recursive: true });
|
|
71
71
|
// Defensive: never clobber an authored eval suite. Callers only reach here when
|
|
72
72
|
// the suite is absent, but guard so a stray call can never destroy real evals.
|
|
73
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own resolved scaffold path
|
|
74
73
|
if (!existsSync(evalsPath)) {
|
|
75
|
-
// eslint-disable-next-line security/detect-non-literal-fs-filename -- our own resolved scaffold path
|
|
76
74
|
writeFileSync(evalsPath, buildEvalsTemplate(skillName), 'utf8');
|
|
77
75
|
}
|
|
78
76
|
return evalsPath;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evals-template.js","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEpE,2EAA2E;AAC3E,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE;YACR,8DAA8D;YAC9D,oFAAoF;YACpF,kBAAkB;YAClB,8EAA8E;YAC9E,yFAAyF;YACzF,iFAAiF;YACjF,oFAAoF;YACpF,yFAAyF;YACzF,2FAA2F;YAC3F,6FAA6F;YAC7F,mFAAmF;YACnF,uFAAuF;YACvF,wFAAwF;YACxF,gGAAgG;YAChG,kGAAkG;YAClG,mFAAmF;YACnF,+FAA+F;YAC/F,qFAAqF;YACrF,wFAAwF;YACxF,0FAA0F;YAC1F,mFAAmF;YACnF,qFAAqF;YACrF,oFAAoF;YACpF,yFAAyF;YACzF,2GAA2G;SAC5G;QACD,UAAU,EAAE,SAAS;QACrB,KAAK,EAAE;YACL;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE,qDAAqD;gBAC/D,MAAM,EAAE,yEAAyE;gBACjF,eAAe,EAAE,kDAAkD;gBACnE,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE;oBACZ,sGAAsG;oBACtG,2GAA2G;iBAC5G;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,SAAiB;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC9C,aAAa,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,gFAAgF;IAChF,+EAA+E;IAC/E,
|
|
1
|
+
{"version":3,"file":"evals-template.js","sourceRoot":"","sources":["../../src/skill-test/evals-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEpD,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEpE,2EAA2E;AAC3E,MAAM,UAAU,kBAAkB,CAAC,SAAiB;IAClD,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE;YACR,8DAA8D;YAC9D,oFAAoF;YACpF,kBAAkB;YAClB,8EAA8E;YAC9E,yFAAyF;YACzF,iFAAiF;YACjF,oFAAoF;YACpF,yFAAyF;YACzF,2FAA2F;YAC3F,6FAA6F;YAC7F,mFAAmF;YACnF,uFAAuF;YACvF,wFAAwF;YACxF,gGAAgG;YAChG,kGAAkG;YAClG,mFAAmF;YACnF,+FAA+F;YAC/F,qFAAqF;YACrF,wFAAwF;YACxF,0FAA0F;YAC1F,mFAAmF;YACnF,qFAAqF;YACrF,oFAAoF;YACpF,yFAAyF;YACzF,2GAA2G;SAC5G;QACD,UAAU,EAAE,SAAS;QACrB,KAAK,EAAE;YACL;gBACE,EAAE,EAAE,CAAC;gBACL,QAAQ,EAAE,qDAAqD;gBAC/D,MAAM,EAAE,yEAAyE;gBACjF,eAAe,EAAE,kDAAkD;gBACnE,KAAK,EAAE,EAAE;gBACT,YAAY,EAAE;oBACZ,sGAAsG;oBACtG,2GAA2G;iBAC5G;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAClD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,SAAiB,EAAE,SAAiB;IACrE,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IAC9C,aAAa,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,gFAAgF;IAChF,+EAA+E;IAC/E,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,aAAa,CAAC,SAAS,EAAE,kBAAkB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC"}
|
|
@@ -15,8 +15,8 @@ export interface BuildExecutorPromptOptions {
|
|
|
15
15
|
workspaceDir?: string;
|
|
16
16
|
}
|
|
17
17
|
/**
|
|
18
|
-
* Build the prompt handed to the blind executor subagent (
|
|
19
|
-
* transcript-grounded eval
|
|
18
|
+
* Build the prompt handed to the blind executor subagent (the
|
|
19
|
+
* transcript-grounded eval design). The executor is a real worker, not a
|
|
20
20
|
* participant in an eval: it receives the task and its working context ONLY,
|
|
21
21
|
* phrased exactly as a genuine user request. It must NEVER be told (directly
|
|
22
22
|
* or by implication) that it is being tested, evaluated, or graded — that
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { PromptInvariantError } from './prompt-invariants.js';
|
|
2
2
|
/**
|
|
3
|
-
* Build the prompt handed to the blind executor subagent (
|
|
4
|
-
* transcript-grounded eval
|
|
3
|
+
* Build the prompt handed to the blind executor subagent (the
|
|
4
|
+
* transcript-grounded eval design). The executor is a real worker, not a
|
|
5
5
|
* participant in an eval: it receives the task and its working context ONLY,
|
|
6
6
|
* phrased exactly as a genuine user request. It must NEVER be told (directly
|
|
7
7
|
* or by implication) that it is being tested, evaluated, or graded — that
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { VatError } from '@vibe-agent-toolkit/utils';
|
|
2
|
+
/**
|
|
3
|
+
* WHY a `vat skill test run` ended on `ExitCode.ERROR`.
|
|
4
|
+
*
|
|
5
|
+
* This used to be three exit codes — 1 internal, 2 preflight, 3 bootstrap —
|
|
6
|
+
* beside 4 for "an eval failed". Under the one exit-code contract every verb
|
|
7
|
+
* shares (`ExitCode` in `@vibe-agent-toolkit/schema`) a failed eval is
|
|
8
|
+
* `FINDINGS` (1) like every other gate that ran and did not pass, and every way
|
|
9
|
+
* the harness could NOT run is `ERROR` (2). The distinction a CI author still
|
|
10
|
+
* needs — "did the harness break, or is my environment wrong?" — is said on
|
|
11
|
+
* stderr as `Reason: <reason>`, never in a fourth number a wrapper would have
|
|
12
|
+
* to look up per command.
|
|
13
|
+
*
|
|
14
|
+
* - `internal` — the harness broke: a spawn failure, a watchdog stall, a grader
|
|
15
|
+
* fragment that did not parse, a forged nonce. VAT's fault until proven
|
|
16
|
+
* otherwise.
|
|
17
|
+
* - `preflight` — the operator can fix it: a bad config, a missing binary, an
|
|
18
|
+
* absent security ack, a held lock, a companion that would not build.
|
|
19
|
+
* - `bootstrap` — nothing was wrong; the run scaffolded an `evals.json`
|
|
20
|
+
* template and stopped so the author can fill it in.
|
|
21
|
+
*/
|
|
22
|
+
export type SkillTestFailureReason = 'internal' | 'preflight' | 'bootstrap';
|
|
23
|
+
/**
|
|
24
|
+
* The reason an error, thrown out of the harness, gives for the run's `ERROR`.
|
|
25
|
+
*
|
|
26
|
+
* An error DECLARES its reason with a `readonly reason` field — every VAT error
|
|
27
|
+
* class in this feature carries one — and this reads it. Only a VAT error is
|
|
28
|
+
* consulted (the `Symbol.for('vat.error')` brand, which survives the
|
|
29
|
+
* `src`/`dist` boundary that `instanceof` does not): a foreign error carrying
|
|
30
|
+
* `reason` is not opting in to anything, and a foreign error carrying nothing
|
|
31
|
+
* is the harness's own fault, `internal`. That closes the residual the old
|
|
32
|
+
* `exitCode` read left open, where `commander`'s `CommanderError` carried a 2
|
|
33
|
+
* and would have reported "your environment is wrong" for a crash.
|
|
34
|
+
*
|
|
35
|
+
* The read is guarded: a getter or `Proxy` that throws on `reason` is a foreign
|
|
36
|
+
* object misbehaving, and this runs inside the CLI's own `catch`, where a
|
|
37
|
+
* rethrow escapes the handler entirely — no summary, no exit code, a bare
|
|
38
|
+
* stack.
|
|
39
|
+
*/
|
|
40
|
+
export declare function skillTestFailureReason(err: unknown): SkillTestFailureReason;
|
|
41
|
+
/**
|
|
42
|
+
* A required input is absent in a way vat can scaffold (missing evals.json).
|
|
43
|
+
* Reason `bootstrap`, NOT a failure. `expectedPath` is the persistent location
|
|
44
|
+
* of the annotated starter template.
|
|
45
|
+
*
|
|
46
|
+
* In a real run the harness has already written that template, and the message
|
|
47
|
+
* says so. Under `--dry-run` the TEMPLATE is not written, so the message instead
|
|
48
|
+
* describes what a real run *would* scaffold and where, while surfacing the same
|
|
49
|
+
* "bootstrap needed" signal.
|
|
50
|
+
*
|
|
51
|
+
* That scope is exact, and it used to read "a dry run must never touch the
|
|
52
|
+
* filesystem", which is not true of this command. A dry run is not a read-only
|
|
53
|
+
* mode: it takes the harness lock, creates the harness root, stages BOTH arms'
|
|
54
|
+
* per-eval workspaces, and — once `--i-understand-this-runs-skill-code` is passed —
|
|
55
|
+
* runs the repo's `test.build` hook and builds the subject. What it never does is
|
|
56
|
+
* SPAWN: no executor session, no grader session, no tokens.
|
|
57
|
+
*
|
|
58
|
+
* It also never touches `results/`: it neither creates the directory, nor writes
|
|
59
|
+
* `provenance.json`, nor wipes the previous run's artifacts. That last one is the
|
|
60
|
+
* reason the ordering matters — the wipe used to sit AHEAD of the dry-run
|
|
61
|
+
* short-circuit, so a free `--dry-run` (or any failure after it) destroyed the
|
|
62
|
+
* `grading.json` / `baseline.json` / `friction.json` / `tool-eval.json` of the
|
|
63
|
+
* expensive real run an operator was about to read. The summary now names the
|
|
64
|
+
* provenance path it WOULD write rather than writing it.
|
|
65
|
+
*
|
|
66
|
+
* The scaffold is the other filesystem effect a dry run deliberately withholds,
|
|
67
|
+
* and for a different reason: that write lands in the AUTHOR's source
|
|
68
|
+
* tree rather than in vat's own scratch space.
|
|
69
|
+
*/
|
|
70
|
+
export declare class BootstrapNeededError extends VatError {
|
|
71
|
+
readonly expectedPath: string;
|
|
72
|
+
readonly reason: "bootstrap";
|
|
73
|
+
constructor(expectedPath: string, opts?: {
|
|
74
|
+
dryRun?: boolean;
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
/** Thrown when building a declared skill (pool packageSkill or plugin build) fails. Reason `preflight`. */
|
|
78
|
+
export declare class SkillBuildError extends VatError {
|
|
79
|
+
readonly reason: "preflight";
|
|
80
|
+
constructor(message: string);
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* The §12 security acknowledgment is required but absent, thrown BEFORE any
|
|
84
|
+
* build/pre-stage command (which executes untrusted repo code) runs for a
|
|
85
|
+
* buildable subject. Reason `preflight`. Mirrors the harness Step-6 ack
|
|
86
|
+
* message wording so the two enforcement points read identically.
|
|
87
|
+
*/
|
|
88
|
+
export declare class SecurityAckError extends VatError {
|
|
89
|
+
readonly reason: "preflight";
|
|
90
|
+
constructor();
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* A skill name is staged more than once in a single run — the subject, a
|
|
94
|
+
* `--with` companion, and a `--with-optional` companion must all have distinct
|
|
95
|
+
* names. Both `--with` and `--with-optional` STAGE the named companion and make
|
|
96
|
+
* it invocable (they differ only in required-vs-optional resolution, not in
|
|
97
|
+
* whether they are staged), so a colliding name would silently overwrite an
|
|
98
|
+
* earlier staged copy under the same slot — never an error, never a manifest
|
|
99
|
+
* trace — which makes a routing/deferral eval look correct while testing
|
|
100
|
+
* something else. Reason `preflight`: user-correctable input, like a bad env
|
|
101
|
+
* token or missing security ack.
|
|
102
|
+
*/
|
|
103
|
+
export declare class DuplicateStagedSkillError extends VatError {
|
|
104
|
+
readonly skillName: string;
|
|
105
|
+
readonly reason: "preflight";
|
|
106
|
+
constructor(skillName: string);
|
|
107
|
+
}
|
|
108
|
+
/** Internal harness failure (an executor/grader spawn error, watchdog timeout/stall, or a missing grader fragment). Reason `internal`. */
|
|
109
|
+
export declare class InternalHarnessError extends VatError {
|
|
110
|
+
readonly reason: "internal";
|
|
111
|
+
constructor(message: string);
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=failure-reason.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"failure-reason.d.ts","sourceRoot":"","sources":["../../src/skill-test/failure-reason.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AAEjE;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,MAAM,sBAAsB,GAAG,UAAU,GAAG,WAAW,GAAG,WAAW,CAAC;AAQ5E;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,OAAO,GAAG,sBAAsB,CAY3E;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,oBAAqB,SAAQ,QAAQ;aAEpB,YAAY,EAAE,MAAM;IADhD,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;gBACX,YAAY,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAE;CAQ9E;AAED,2GAA2G;AAC3G,qBAAa,eAAgB,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;gBAC3B,OAAO,EAAE,MAAM;CAG5B;AAED;;;;;GAKG;AACH,qBAAa,gBAAiB,SAAQ,QAAQ;IAC5C,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;;CAOxC;AAED;;;;;;;;;;GAUG;AACH,qBAAa,yBAA0B,SAAQ,QAAQ;aAEzB,SAAS,EAAE,MAAM;IAD7C,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;gBACX,SAAS,EAAE,MAAM;CAO9C;AAED,0IAA0I;AAC1I,qBAAa,oBAAqB,SAAQ,QAAQ;IAChD,QAAQ,CAAC,MAAM,EAAG,UAAU,CAAU;gBAC1B,OAAO,EAAE,MAAM;CAG5B"}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { isVatError, VatError } from '@vibe-agent-toolkit/utils';
|
|
2
|
+
const FAILURE_REASONS = new Set([
|
|
3
|
+
'internal',
|
|
4
|
+
'preflight',
|
|
5
|
+
'bootstrap',
|
|
6
|
+
]);
|
|
7
|
+
/**
|
|
8
|
+
* The reason an error, thrown out of the harness, gives for the run's `ERROR`.
|
|
9
|
+
*
|
|
10
|
+
* An error DECLARES its reason with a `readonly reason` field — every VAT error
|
|
11
|
+
* class in this feature carries one — and this reads it. Only a VAT error is
|
|
12
|
+
* consulted (the `Symbol.for('vat.error')` brand, which survives the
|
|
13
|
+
* `src`/`dist` boundary that `instanceof` does not): a foreign error carrying
|
|
14
|
+
* `reason` is not opting in to anything, and a foreign error carrying nothing
|
|
15
|
+
* is the harness's own fault, `internal`. That closes the residual the old
|
|
16
|
+
* `exitCode` read left open, where `commander`'s `CommanderError` carried a 2
|
|
17
|
+
* and would have reported "your environment is wrong" for a crash.
|
|
18
|
+
*
|
|
19
|
+
* The read is guarded: a getter or `Proxy` that throws on `reason` is a foreign
|
|
20
|
+
* object misbehaving, and this runs inside the CLI's own `catch`, where a
|
|
21
|
+
* rethrow escapes the handler entirely — no summary, no exit code, a bare
|
|
22
|
+
* stack.
|
|
23
|
+
*/
|
|
24
|
+
export function skillTestFailureReason(err) {
|
|
25
|
+
if (!isVatError(err))
|
|
26
|
+
return 'internal';
|
|
27
|
+
let declared;
|
|
28
|
+
try {
|
|
29
|
+
declared = err.reason;
|
|
30
|
+
// eslint-disable-next-line local/no-blind-catch -- a property read throws only from a getter or Proxy trap on a misbehaving object, and "the read itself threw" is exactly "declared nothing": there is no second case to tell apart, and this runs inside the CLI's own catch, where a rethrow escapes the handler entirely (pinned in failure-reason.test.ts).
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return 'internal';
|
|
34
|
+
}
|
|
35
|
+
return typeof declared === 'string' && FAILURE_REASONS.has(declared)
|
|
36
|
+
? declared
|
|
37
|
+
: 'internal';
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* A required input is absent in a way vat can scaffold (missing evals.json).
|
|
41
|
+
* Reason `bootstrap`, NOT a failure. `expectedPath` is the persistent location
|
|
42
|
+
* of the annotated starter template.
|
|
43
|
+
*
|
|
44
|
+
* In a real run the harness has already written that template, and the message
|
|
45
|
+
* says so. Under `--dry-run` the TEMPLATE is not written, so the message instead
|
|
46
|
+
* describes what a real run *would* scaffold and where, while surfacing the same
|
|
47
|
+
* "bootstrap needed" signal.
|
|
48
|
+
*
|
|
49
|
+
* That scope is exact, and it used to read "a dry run must never touch the
|
|
50
|
+
* filesystem", which is not true of this command. A dry run is not a read-only
|
|
51
|
+
* mode: it takes the harness lock, creates the harness root, stages BOTH arms'
|
|
52
|
+
* per-eval workspaces, and — once `--i-understand-this-runs-skill-code` is passed —
|
|
53
|
+
* runs the repo's `test.build` hook and builds the subject. What it never does is
|
|
54
|
+
* SPAWN: no executor session, no grader session, no tokens.
|
|
55
|
+
*
|
|
56
|
+
* It also never touches `results/`: it neither creates the directory, nor writes
|
|
57
|
+
* `provenance.json`, nor wipes the previous run's artifacts. That last one is the
|
|
58
|
+
* reason the ordering matters — the wipe used to sit AHEAD of the dry-run
|
|
59
|
+
* short-circuit, so a free `--dry-run` (or any failure after it) destroyed the
|
|
60
|
+
* `grading.json` / `baseline.json` / `friction.json` / `tool-eval.json` of the
|
|
61
|
+
* expensive real run an operator was about to read. The summary now names the
|
|
62
|
+
* provenance path it WOULD write rather than writing it.
|
|
63
|
+
*
|
|
64
|
+
* The scaffold is the other filesystem effect a dry run deliberately withholds,
|
|
65
|
+
* and for a different reason: that write lands in the AUTHOR's source
|
|
66
|
+
* tree rather than in vat's own scratch space.
|
|
67
|
+
*/
|
|
68
|
+
export class BootstrapNeededError extends VatError {
|
|
69
|
+
expectedPath;
|
|
70
|
+
reason = 'bootstrap';
|
|
71
|
+
constructor(expectedPath, opts) {
|
|
72
|
+
super('SKILL_TEST_BOOTSTRAP_NEEDED', opts?.dryRun === true
|
|
73
|
+
? `[dry-run] No evals.json found. A real run would scaffold an annotated template at ${expectedPath} — fill it in and re-run. (dry-run: nothing was written to your source tree.)`
|
|
74
|
+
: `Wrote an evals.json template at ${expectedPath} — fill it in and re-run.`);
|
|
75
|
+
this.expectedPath = expectedPath;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
/** Thrown when building a declared skill (pool packageSkill or plugin build) fails. Reason `preflight`. */
|
|
79
|
+
export class SkillBuildError extends VatError {
|
|
80
|
+
reason = 'preflight';
|
|
81
|
+
constructor(message) {
|
|
82
|
+
super('SKILL_TEST_BUILD_FAILED', message);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* The §12 security acknowledgment is required but absent, thrown BEFORE any
|
|
87
|
+
* build/pre-stage command (which executes untrusted repo code) runs for a
|
|
88
|
+
* buildable subject. Reason `preflight`. Mirrors the harness Step-6 ack
|
|
89
|
+
* message wording so the two enforcement points read identically.
|
|
90
|
+
*/
|
|
91
|
+
export class SecurityAckError extends VatError {
|
|
92
|
+
reason = 'preflight';
|
|
93
|
+
constructor() {
|
|
94
|
+
super('SKILL_TEST_SECURITY_ACK_MISSING', 'Security acknowledgment required. Pass --i-understand-this-runs-skill-code to proceed.');
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* A skill name is staged more than once in a single run — the subject, a
|
|
99
|
+
* `--with` companion, and a `--with-optional` companion must all have distinct
|
|
100
|
+
* names. Both `--with` and `--with-optional` STAGE the named companion and make
|
|
101
|
+
* it invocable (they differ only in required-vs-optional resolution, not in
|
|
102
|
+
* whether they are staged), so a colliding name would silently overwrite an
|
|
103
|
+
* earlier staged copy under the same slot — never an error, never a manifest
|
|
104
|
+
* trace — which makes a routing/deferral eval look correct while testing
|
|
105
|
+
* something else. Reason `preflight`: user-correctable input, like a bad env
|
|
106
|
+
* token or missing security ack.
|
|
107
|
+
*/
|
|
108
|
+
export class DuplicateStagedSkillError extends VatError {
|
|
109
|
+
skillName;
|
|
110
|
+
reason = 'preflight';
|
|
111
|
+
constructor(skillName) {
|
|
112
|
+
super('SKILL_TEST_DUPLICATE_STAGED_SKILL', `Skill name "${skillName}" is staged more than once (subject / --with / --with-optional). ` +
|
|
113
|
+
`Each staged skill must have a unique name.`);
|
|
114
|
+
this.skillName = skillName;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/** Internal harness failure (an executor/grader spawn error, watchdog timeout/stall, or a missing grader fragment). Reason `internal`. */
|
|
118
|
+
export class InternalHarnessError extends VatError {
|
|
119
|
+
reason = 'internal';
|
|
120
|
+
constructor(message) {
|
|
121
|
+
super('SKILL_TEST_INTERNAL', message);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
//# sourceMappingURL=failure-reason.js.map
|