@ztothez/design-engineering 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +15 -0
- package/README.md +448 -0
- package/ROADMAP.md +127 -0
- package/SKILL.md +698 -0
- package/V2-ROADMAP.md +431 -0
- package/V3-ROADMAP.md +605 -0
- package/V4-ROADMAP.md +283 -0
- package/clean-room-ux-design-spec.md +1431 -0
- package/dist/cli/aggregate-gates.d.ts +2 -0
- package/dist/cli/aggregate-gates.js +86 -0
- package/dist/cli/aggregate-gates.js.map +1 -0
- package/dist/cli/assess-maintainer-review.d.ts +2 -0
- package/dist/cli/assess-maintainer-review.js +72 -0
- package/dist/cli/assess-maintainer-review.js.map +1 -0
- package/dist/cli/audit.d.ts +2 -0
- package/dist/cli/audit.js +71 -0
- package/dist/cli/audit.js.map +1 -0
- package/dist/cli/compile-comparison-review.d.ts +2 -0
- package/dist/cli/compile-comparison-review.js +154 -0
- package/dist/cli/compile-comparison-review.js.map +1 -0
- package/dist/cli/compile-plan.d.ts +2 -0
- package/dist/cli/compile-plan.js +66 -0
- package/dist/cli/compile-plan.js.map +1 -0
- package/dist/cli/evaluate-corpus.d.ts +2 -0
- package/dist/cli/evaluate-corpus.js +65 -0
- package/dist/cli/evaluate-corpus.js.map +1 -0
- package/dist/cli/generate-react.d.ts +2 -0
- package/dist/cli/generate-react.js +83 -0
- package/dist/cli/generate-react.js.map +1 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +58 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/portfolio.d.ts +2 -0
- package/dist/cli/portfolio.js +316 -0
- package/dist/cli/portfolio.js.map +1 -0
- package/dist/cli/quality-gate.d.ts +2 -0
- package/dist/cli/quality-gate.js +131 -0
- package/dist/cli/quality-gate.js.map +1 -0
- package/dist/cli/repair-react.d.ts +2 -0
- package/dist/cli/repair-react.js +163 -0
- package/dist/cli/repair-react.js.map +1 -0
- package/dist/cli/review-heuristics.d.ts +2 -0
- package/dist/cli/review-heuristics.js +68 -0
- package/dist/cli/review-heuristics.js.map +1 -0
- package/dist/cli/validate-brief.d.ts +2 -0
- package/dist/cli/validate-brief.js +58 -0
- package/dist/cli/validate-brief.js.map +1 -0
- package/dist/cli/validate-comparison.d.ts +2 -0
- package/dist/cli/validate-comparison.js +68 -0
- package/dist/cli/validate-comparison.js.map +1 -0
- package/dist/cli/validate-contract.d.ts +2 -0
- package/dist/cli/validate-contract.js +54 -0
- package/dist/cli/validate-contract.js.map +1 -0
- package/dist/cli/validate-design.d.ts +2 -0
- package/dist/cli/validate-design.js +53 -0
- package/dist/cli/validate-design.js.map +1 -0
- package/dist/cli/validate-information.d.ts +2 -0
- package/dist/cli/validate-information.js +53 -0
- package/dist/cli/validate-information.js.map +1 -0
- package/dist/cli/validate-trust.d.ts +2 -0
- package/dist/cli/validate-trust.js +53 -0
- package/dist/cli/validate-trust.js.map +1 -0
- package/dist/cli/verify-ui.d.ts +2 -0
- package/dist/cli/verify-ui.js +181 -0
- package/dist/cli/verify-ui.js.map +1 -0
- package/dist/src/acceptance/evaluator.d.ts +13 -0
- package/dist/src/acceptance/evaluator.js +201 -0
- package/dist/src/acceptance/evaluator.js.map +1 -0
- package/dist/src/acceptance/loader.d.ts +2 -0
- package/dist/src/acceptance/loader.js +18 -0
- package/dist/src/acceptance/loader.js.map +1 -0
- package/dist/src/acceptance/report.d.ts +2 -0
- package/dist/src/acceptance/report.js +18 -0
- package/dist/src/acceptance/report.js.map +1 -0
- package/dist/src/acceptance/schema.d.ts +64 -0
- package/dist/src/acceptance/schema.js +57 -0
- package/dist/src/acceptance/schema.js.map +1 -0
- package/dist/src/acceptance/types.d.ts +47 -0
- package/dist/src/acceptance/types.js +2 -0
- package/dist/src/acceptance/types.js.map +1 -0
- package/dist/src/aggregate/report.d.ts +2 -0
- package/dist/src/aggregate/report.js +33 -0
- package/dist/src/aggregate/report.js.map +1 -0
- package/dist/src/aggregate/runner.d.ts +2 -0
- package/dist/src/aggregate/runner.js +165 -0
- package/dist/src/aggregate/runner.js.map +1 -0
- package/dist/src/aggregate/schema.d.ts +182 -0
- package/dist/src/aggregate/schema.js +54 -0
- package/dist/src/aggregate/schema.js.map +1 -0
- package/dist/src/aggregate/types.d.ts +57 -0
- package/dist/src/aggregate/types.js +2 -0
- package/dist/src/aggregate/types.js.map +1 -0
- package/dist/src/audit/policy.d.ts +3 -0
- package/dist/src/audit/policy.js +45 -0
- package/dist/src/audit/policy.js.map +1 -0
- package/dist/src/audit/report.d.ts +2 -0
- package/dist/src/audit/report.js +32 -0
- package/dist/src/audit/report.js.map +1 -0
- package/dist/src/audit/rules/accessibility-name.d.ts +2 -0
- package/dist/src/audit/rules/accessibility-name.js +64 -0
- package/dist/src/audit/rules/accessibility-name.js.map +1 -0
- package/dist/src/audit/rules/component-size.d.ts +2 -0
- package/dist/src/audit/rules/component-size.js +27 -0
- package/dist/src/audit/rules/component-size.js.map +1 -0
- package/dist/src/audit/rules/credential-placeholder.d.ts +2 -0
- package/dist/src/audit/rules/credential-placeholder.js +83 -0
- package/dist/src/audit/rules/credential-placeholder.js.map +1 -0
- package/dist/src/audit/rules/helpers.d.ts +16 -0
- package/dist/src/audit/rules/helpers.js +64 -0
- package/dist/src/audit/rules/helpers.js.map +1 -0
- package/dist/src/audit/rules/index.d.ts +2 -0
- package/dist/src/audit/rules/index.js +23 -0
- package/dist/src/audit/rules/index.js.map +1 -0
- package/dist/src/audit/rules/interactive-integrity.d.ts +2 -0
- package/dist/src/audit/rules/interactive-integrity.js +111 -0
- package/dist/src/audit/rules/interactive-integrity.js.map +1 -0
- package/dist/src/audit/rules/mixed-responsibilities.d.ts +2 -0
- package/dist/src/audit/rules/mixed-responsibilities.js +39 -0
- package/dist/src/audit/rules/mixed-responsibilities.js.map +1 -0
- package/dist/src/audit/rules/mock-production-path.d.ts +2 -0
- package/dist/src/audit/rules/mock-production-path.js +39 -0
- package/dist/src/audit/rules/mock-production-path.js.map +1 -0
- package/dist/src/audit/rules/network-state.d.ts +2 -0
- package/dist/src/audit/rules/network-state.js +39 -0
- package/dist/src/audit/rules/network-state.js.map +1 -0
- package/dist/src/audit/rules/operational-claim.d.ts +2 -0
- package/dist/src/audit/rules/operational-claim.js +94 -0
- package/dist/src/audit/rules/operational-claim.js.map +1 -0
- package/dist/src/audit/rules/placeholder-interaction.d.ts +2 -0
- package/dist/src/audit/rules/placeholder-interaction.js +46 -0
- package/dist/src/audit/rules/placeholder-interaction.js.map +1 -0
- package/dist/src/audit/rules/raw-design-values.d.ts +2 -0
- package/dist/src/audit/rules/raw-design-values.js +97 -0
- package/dist/src/audit/rules/raw-design-values.js.map +1 -0
- package/dist/src/audit/scanner.d.ts +2 -0
- package/dist/src/audit/scanner.js +215 -0
- package/dist/src/audit/scanner.js.map +1 -0
- package/dist/src/audit/types.d.ts +62 -0
- package/dist/src/audit/types.js +2 -0
- package/dist/src/audit/types.js.map +1 -0
- package/dist/src/comparison/evaluator.d.ts +2 -0
- package/dist/src/comparison/evaluator.js +478 -0
- package/dist/src/comparison/evaluator.js.map +1 -0
- package/dist/src/comparison/loader.d.ts +3 -0
- package/dist/src/comparison/loader.js +26 -0
- package/dist/src/comparison/loader.js.map +1 -0
- package/dist/src/comparison/maintainer.d.ts +47 -0
- package/dist/src/comparison/maintainer.js +218 -0
- package/dist/src/comparison/maintainer.js.map +1 -0
- package/dist/src/comparison/report.d.ts +2 -0
- package/dist/src/comparison/report.js +48 -0
- package/dist/src/comparison/report.js.map +1 -0
- package/dist/src/comparison/schema.d.ts +464 -0
- package/dist/src/comparison/schema.js +502 -0
- package/dist/src/comparison/schema.js.map +1 -0
- package/dist/src/contracts/archetypes.d.ts +21 -0
- package/dist/src/contracts/archetypes.js +102 -0
- package/dist/src/contracts/archetypes.js.map +1 -0
- package/dist/src/contracts/journeys.d.ts +7 -0
- package/dist/src/contracts/journeys.js +32 -0
- package/dist/src/contracts/journeys.js.map +1 -0
- package/dist/src/contracts/report.d.ts +2 -0
- package/dist/src/contracts/report.js +33 -0
- package/dist/src/contracts/report.js.map +1 -0
- package/dist/src/contracts/schema.d.ts +1038 -0
- package/dist/src/contracts/schema.js +385 -0
- package/dist/src/contracts/schema.js.map +1 -0
- package/dist/src/contracts/validator.d.ts +12 -0
- package/dist/src/contracts/validator.js +303 -0
- package/dist/src/contracts/validator.js.map +1 -0
- package/dist/src/corpus/evaluator.d.ts +2 -0
- package/dist/src/corpus/evaluator.js +203 -0
- package/dist/src/corpus/evaluator.js.map +1 -0
- package/dist/src/corpus/loader.d.ts +2 -0
- package/dist/src/corpus/loader.js +20 -0
- package/dist/src/corpus/loader.js.map +1 -0
- package/dist/src/corpus/report.d.ts +2 -0
- package/dist/src/corpus/report.js +33 -0
- package/dist/src/corpus/report.js.map +1 -0
- package/dist/src/corpus/schema.d.ts +316 -0
- package/dist/src/corpus/schema.js +269 -0
- package/dist/src/corpus/schema.js.map +1 -0
- package/dist/src/design-intelligence/loader.d.ts +2 -0
- package/dist/src/design-intelligence/loader.js +21 -0
- package/dist/src/design-intelligence/loader.js.map +1 -0
- package/dist/src/design-intelligence/report.d.ts +2 -0
- package/dist/src/design-intelligence/report.js +36 -0
- package/dist/src/design-intelligence/report.js.map +1 -0
- package/dist/src/design-intelligence/schema.d.ts +838 -0
- package/dist/src/design-intelligence/schema.js +854 -0
- package/dist/src/design-intelligence/schema.js.map +1 -0
- package/dist/src/design-intelligence/validator.d.ts +2 -0
- package/dist/src/design-intelligence/validator.js +1371 -0
- package/dist/src/design-intelligence/validator.js.map +1 -0
- package/dist/src/design-plan/compiler.d.ts +8 -0
- package/dist/src/design-plan/compiler.js +288 -0
- package/dist/src/design-plan/compiler.js.map +1 -0
- package/dist/src/design-plan/report.d.ts +2 -0
- package/dist/src/design-plan/report.js +28 -0
- package/dist/src/design-plan/report.js.map +1 -0
- package/dist/src/design-plan/schema.d.ts +202 -0
- package/dist/src/design-plan/schema.js +76 -0
- package/dist/src/design-plan/schema.js.map +1 -0
- package/dist/src/generation/loader.d.ts +2 -0
- package/dist/src/generation/loader.js +21 -0
- package/dist/src/generation/loader.js.map +1 -0
- package/dist/src/generation/policy.d.ts +12 -0
- package/dist/src/generation/policy.js +62 -0
- package/dist/src/generation/policy.js.map +1 -0
- package/dist/src/generation/react-typescript.d.ts +6 -0
- package/dist/src/generation/react-typescript.js +538 -0
- package/dist/src/generation/react-typescript.js.map +1 -0
- package/dist/src/generation/report.d.ts +2 -0
- package/dist/src/generation/report.js +21 -0
- package/dist/src/generation/report.js.map +1 -0
- package/dist/src/generation/schema.d.ts +44 -0
- package/dist/src/generation/schema.js +50 -0
- package/dist/src/generation/schema.js.map +1 -0
- package/dist/src/generation/templates/tokens.d.ts +1 -0
- package/dist/src/generation/templates/tokens.js +59 -0
- package/dist/src/generation/templates/tokens.js.map +1 -0
- package/dist/src/heuristics/evaluator.d.ts +2 -0
- package/dist/src/heuristics/evaluator.js +65 -0
- package/dist/src/heuristics/evaluator.js.map +1 -0
- package/dist/src/heuristics/loader.d.ts +2 -0
- package/dist/src/heuristics/loader.js +21 -0
- package/dist/src/heuristics/loader.js.map +1 -0
- package/dist/src/heuristics/report.d.ts +3 -0
- package/dist/src/heuristics/report.js +31 -0
- package/dist/src/heuristics/report.js.map +1 -0
- package/dist/src/heuristics/schema.d.ts +198 -0
- package/dist/src/heuristics/schema.js +197 -0
- package/dist/src/heuristics/schema.js.map +1 -0
- package/dist/src/information-design/loader.d.ts +2 -0
- package/dist/src/information-design/loader.js +21 -0
- package/dist/src/information-design/loader.js.map +1 -0
- package/dist/src/information-design/report.d.ts +2 -0
- package/dist/src/information-design/report.js +27 -0
- package/dist/src/information-design/report.js.map +1 -0
- package/dist/src/information-design/schema.d.ts +370 -0
- package/dist/src/information-design/schema.js +329 -0
- package/dist/src/information-design/schema.js.map +1 -0
- package/dist/src/information-design/validator.d.ts +2 -0
- package/dist/src/information-design/validator.js +242 -0
- package/dist/src/information-design/validator.js.map +1 -0
- package/dist/src/interface-trust/loader.d.ts +2 -0
- package/dist/src/interface-trust/loader.js +21 -0
- package/dist/src/interface-trust/loader.js.map +1 -0
- package/dist/src/interface-trust/report.d.ts +2 -0
- package/dist/src/interface-trust/report.js +26 -0
- package/dist/src/interface-trust/report.js.map +1 -0
- package/dist/src/interface-trust/schema.d.ts +299 -0
- package/dist/src/interface-trust/schema.js +238 -0
- package/dist/src/interface-trust/schema.js.map +1 -0
- package/dist/src/interface-trust/validator.d.ts +2 -0
- package/dist/src/interface-trust/validator.js +283 -0
- package/dist/src/interface-trust/validator.js.map +1 -0
- package/dist/src/portfolio/adapter-report.d.ts +3 -0
- package/dist/src/portfolio/adapter-report.js +38 -0
- package/dist/src/portfolio/adapter-report.js.map +1 -0
- package/dist/src/portfolio/adapters.d.ts +37 -0
- package/dist/src/portfolio/adapters.js +325 -0
- package/dist/src/portfolio/adapters.js.map +1 -0
- package/dist/src/portfolio/evidence.d.ts +9 -0
- package/dist/src/portfolio/evidence.js +233 -0
- package/dist/src/portfolio/evidence.js.map +1 -0
- package/dist/src/portfolio/files.d.ts +8 -0
- package/dist/src/portfolio/files.js +77 -0
- package/dist/src/portfolio/files.js.map +1 -0
- package/dist/src/portfolio/fixture-server.d.ts +11 -0
- package/dist/src/portfolio/fixture-server.js +131 -0
- package/dist/src/portfolio/fixture-server.js.map +1 -0
- package/dist/src/portfolio/loader.d.ts +2 -0
- package/dist/src/portfolio/loader.js +21 -0
- package/dist/src/portfolio/loader.js.map +1 -0
- package/dist/src/portfolio/mcp.d.ts +186 -0
- package/dist/src/portfolio/mcp.js +117 -0
- package/dist/src/portfolio/mcp.js.map +1 -0
- package/dist/src/portfolio/promotion-schema.d.ts +220 -0
- package/dist/src/portfolio/promotion-schema.js +148 -0
- package/dist/src/portfolio/promotion-schema.js.map +1 -0
- package/dist/src/portfolio/promotion.d.ts +4 -0
- package/dist/src/portfolio/promotion.js +160 -0
- package/dist/src/portfolio/promotion.js.map +1 -0
- package/dist/src/portfolio/qualification-schema.d.ts +268 -0
- package/dist/src/portfolio/qualification-schema.js +112 -0
- package/dist/src/portfolio/qualification-schema.js.map +1 -0
- package/dist/src/portfolio/qualification.d.ts +4 -0
- package/dist/src/portfolio/qualification.js +157 -0
- package/dist/src/portfolio/qualification.js.map +1 -0
- package/dist/src/portfolio/registry.d.ts +34 -0
- package/dist/src/portfolio/registry.js +240 -0
- package/dist/src/portfolio/registry.js.map +1 -0
- package/dist/src/portfolio/report.d.ts +4 -0
- package/dist/src/portfolio/report.js +47 -0
- package/dist/src/portfolio/report.js.map +1 -0
- package/dist/src/portfolio/run-schema.d.ts +135 -0
- package/dist/src/portfolio/run-schema.js +81 -0
- package/dist/src/portfolio/run-schema.js.map +1 -0
- package/dist/src/portfolio/runner.d.ts +118 -0
- package/dist/src/portfolio/runner.js +423 -0
- package/dist/src/portfolio/runner.js.map +1 -0
- package/dist/src/portfolio/schema.d.ts +251 -0
- package/dist/src/portfolio/schema.js +317 -0
- package/dist/src/portfolio/schema.js.map +1 -0
- package/dist/src/portfolio/snapshot.d.ts +89 -0
- package/dist/src/portfolio/snapshot.js +548 -0
- package/dist/src/portfolio/snapshot.js.map +1 -0
- package/dist/src/portfolio/taxonomy-schema.d.ts +215 -0
- package/dist/src/portfolio/taxonomy-schema.js +74 -0
- package/dist/src/portfolio/taxonomy-schema.js.map +1 -0
- package/dist/src/portfolio/taxonomy.d.ts +5 -0
- package/dist/src/portfolio/taxonomy.js +236 -0
- package/dist/src/portfolio/taxonomy.js.map +1 -0
- package/dist/src/portfolio/vault.d.ts +53 -0
- package/dist/src/portfolio/vault.js +176 -0
- package/dist/src/portfolio/vault.js.map +1 -0
- package/dist/src/product-brief/loader.d.ts +2 -0
- package/dist/src/product-brief/loader.js +21 -0
- package/dist/src/product-brief/loader.js.map +1 -0
- package/dist/src/product-brief/report.d.ts +2 -0
- package/dist/src/product-brief/report.js +22 -0
- package/dist/src/product-brief/report.js.map +1 -0
- package/dist/src/product-brief/schema.d.ts +367 -0
- package/dist/src/product-brief/schema.js +272 -0
- package/dist/src/product-brief/schema.js.map +1 -0
- package/dist/src/product-brief/validator.d.ts +2 -0
- package/dist/src/product-brief/validator.js +251 -0
- package/dist/src/product-brief/validator.js.map +1 -0
- package/dist/src/product.d.ts +7 -0
- package/dist/src/product.js +28 -0
- package/dist/src/product.js.map +1 -0
- package/dist/src/quality-gate/report.d.ts +2 -0
- package/dist/src/quality-gate/report.js +46 -0
- package/dist/src/quality-gate/report.js.map +1 -0
- package/dist/src/quality-gate/runner.d.ts +2 -0
- package/dist/src/quality-gate/runner.js +175 -0
- package/dist/src/quality-gate/runner.js.map +1 -0
- package/dist/src/quality-gate/schema.d.ts +75 -0
- package/dist/src/quality-gate/schema.js +37 -0
- package/dist/src/quality-gate/schema.js.map +1 -0
- package/dist/src/quality-gate/types.d.ts +49 -0
- package/dist/src/quality-gate/types.js +2 -0
- package/dist/src/quality-gate/types.js.map +1 -0
- package/dist/src/repair/loader.d.ts +2 -0
- package/dist/src/repair/loader.js +11 -0
- package/dist/src/repair/loader.js.map +1 -0
- package/dist/src/repair/policy.d.ts +16 -0
- package/dist/src/repair/policy.js +65 -0
- package/dist/src/repair/policy.js.map +1 -0
- package/dist/src/repair/report.d.ts +2 -0
- package/dist/src/repair/report.js +32 -0
- package/dist/src/repair/report.js.map +1 -0
- package/dist/src/repair/runner.d.ts +17 -0
- package/dist/src/repair/runner.js +511 -0
- package/dist/src/repair/runner.js.map +1 -0
- package/dist/src/repair/schema.d.ts +264 -0
- package/dist/src/repair/schema.js +202 -0
- package/dist/src/repair/schema.js.map +1 -0
- package/dist/src/retrieval/report.d.ts +2 -0
- package/dist/src/retrieval/report.js +21 -0
- package/dist/src/retrieval/report.js.map +1 -0
- package/dist/src/retrieval/schema.d.ts +123 -0
- package/dist/src/retrieval/schema.js +77 -0
- package/dist/src/retrieval/schema.js.map +1 -0
- package/dist/src/retrieval/search.d.ts +26 -0
- package/dist/src/retrieval/search.js +415 -0
- package/dist/src/retrieval/search.js.map +1 -0
- package/dist/src/runtime/advanced-checks.d.ts +6 -0
- package/dist/src/runtime/advanced-checks.js +487 -0
- package/dist/src/runtime/advanced-checks.js.map +1 -0
- package/dist/src/runtime/composition-checks.d.ts +5 -0
- package/dist/src/runtime/composition-checks.js +307 -0
- package/dist/src/runtime/composition-checks.js.map +1 -0
- package/dist/src/runtime/interface-checks.d.ts +12 -0
- package/dist/src/runtime/interface-checks.js +224 -0
- package/dist/src/runtime/interface-checks.js.map +1 -0
- package/dist/src/runtime/policy.d.ts +13 -0
- package/dist/src/runtime/policy.js +97 -0
- package/dist/src/runtime/policy.js.map +1 -0
- package/dist/src/runtime/report.d.ts +2 -0
- package/dist/src/runtime/report.js +48 -0
- package/dist/src/runtime/report.js.map +1 -0
- package/dist/src/runtime/schema.d.ts +406 -0
- package/dist/src/runtime/schema.js +308 -0
- package/dist/src/runtime/schema.js.map +1 -0
- package/dist/src/runtime/types.d.ts +177 -0
- package/dist/src/runtime/types.js +2 -0
- package/dist/src/runtime/types.js.map +1 -0
- package/dist/src/runtime/verifier.d.ts +2 -0
- package/dist/src/runtime/verifier.js +1201 -0
- package/dist/src/runtime/verifier.js.map +1 -0
- package/dist/src/server.d.ts +2 -0
- package/dist/src/server.js +1352 -0
- package/dist/src/server.js.map +1 -0
- package/docs/installation.md +337 -0
- package/knowledge-base/INDEX.md +285 -0
- package/knowledge-base/benchmarks/aegisops/CALIBRATION.md +52 -0
- package/knowledge-base/benchmarks/aegisops/MANIFEST.md +56 -0
- package/knowledge-base/benchmarks/aegisops/SOURCE-EVIDENCE.md +45 -0
- package/knowledge-base/benchmarks/aegisops/acceptance-criteria.md +42 -0
- package/knowledge-base/benchmarks/aegisops/anti-patterns.md +19 -0
- package/knowledge-base/benchmarks/aegisops/attestations.template.yaml +3 -0
- package/knowledge-base/benchmarks/aegisops/attestations.yaml +115 -0
- package/knowledge-base/benchmarks/aegisops/journeys.json +91 -0
- package/knowledge-base/benchmarks/aegisops/product-contract.yaml +328 -0
- package/knowledge-base/benchmarks/azure-optimizer/SOURCE-EVIDENCE.md +26 -0
- package/knowledge-base/benchmarks/azure-optimizer/V2-IMPLEMENTATION.md +39 -0
- package/knowledge-base/benchmarks/azure-optimizer/acceptance-criteria.md +23 -0
- package/knowledge-base/benchmarks/azure-optimizer/anti-patterns.md +13 -0
- package/knowledge-base/benchmarks/azure-optimizer/journeys.json +88 -0
- package/knowledge-base/benchmarks/azure-optimizer/product-contract.yaml +216 -0
- package/knowledge-base/benchmarks/azure-optimizer/v2-human-review-methodology.yaml +171 -0
- package/knowledge-base/benchmarks/azure-optimizer/v2-journeys.json +139 -0
- package/knowledge-base/benchmarks/corpus/PROVENANCE.md +21 -0
- package/knowledge-base/benchmarks/corpus/README.md +30 -0
- package/knowledge-base/benchmarks/corpus/cases/anti-slop-negative/package.json +9 -0
- package/knowledge-base/benchmarks/corpus/cases/anti-slop-negative/src/FakeDashboard.tsx +10 -0
- package/knowledge-base/benchmarks/corpus/cases/anti-slop-positive/package.json +9 -0
- package/knowledge-base/benchmarks/corpus/cases/anti-slop-positive/src/Progress.tsx +15 -0
- package/knowledge-base/benchmarks/corpus/cases/architecture-negative/package.json +9 -0
- package/knowledge-base/benchmarks/corpus/cases/architecture-negative/src/OperationsPanel.tsx +40 -0
- package/knowledge-base/benchmarks/corpus/cases/architecture-positive/package.json +9 -0
- package/knowledge-base/benchmarks/corpus/cases/architecture-positive/src/Dashboard.tsx +17 -0
- package/knowledge-base/benchmarks/corpus/cases/architecture-positive/src/theme.css +15 -0
- package/knowledge-base/benchmarks/corpus/cases/task-incomplete/SOURCE-EVIDENCE.md +3 -0
- package/knowledge-base/benchmarks/corpus/cases/task-incomplete/journeys.json +20 -0
- package/knowledge-base/benchmarks/corpus/cases/task-incomplete/product-contract.yaml +78 -0
- package/knowledge-base/benchmarks/corpus/corpus.schema.yaml +159 -0
- package/knowledge-base/benchmarks/corpus/corpus.yaml +209 -0
- package/knowledge-base/benchmarks/interface-quality/README.md +26 -0
- package/knowledge-base/benchmarks/interface-quality/SOLO-MAINTAINER-TRACK.md +35 -0
- package/knowledge-base/benchmarks/interface-quality/comparison-methodology-v1.1.schema.yaml +288 -0
- package/knowledge-base/benchmarks/interface-quality/comparison-methodology.schema.yaml +235 -0
- package/knowledge-base/benchmarks/interface-quality/comparison-methodology.template.yaml +181 -0
- package/knowledge-base/benchmarks/interface-quality/evidence/example-runtime-report.json +11 -0
- package/knowledge-base/benchmarks/interface-quality/review-session.schema.yaml +14 -0
- package/knowledge-base/benchmarks/interface-quality/review.schema.yaml +296 -0
- package/knowledge-base/benchmarks/interface-quality/review.template.yaml +43 -0
- package/knowledge-base/benchmarks/portfolio-corpus/ADAPTERS.md +68 -0
- package/knowledge-base/benchmarks/portfolio-corpus/PORTFOLIO-RUNNER.md +131 -0
- package/knowledge-base/benchmarks/portfolio-corpus/PRODUCT-TASK-CONTRACTS.md +29 -0
- package/knowledge-base/benchmarks/portfolio-corpus/PROMOTED-RULES.md +31 -0
- package/knowledge-base/benchmarks/portfolio-corpus/archetype-profiles.yaml +26 -0
- package/knowledge-base/benchmarks/portfolio-corpus/product-task-profile.schema.yaml +105 -0
- package/knowledge-base/benchmarks/portfolio-corpus/registry.schema.yaml +188 -0
- package/knowledge-base/benchmarks/portfolio-corpus/registry.template.yaml +63 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-fixtures/component-size-abstention.json +10 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-fixtures/component-size-negative.json +10 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-fixtures/component-size-positive.json +10 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-fixtures/interactive-integrity-abstention.json +10 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-fixtures/interactive-integrity-negative.json +10 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-fixtures/interactive-integrity-positive.json +10 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-fixtures/raw-design-values-abstention.json +10 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-fixtures/raw-design-values-negative.json +10 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-fixtures/raw-design-values-positive.json +10 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-holdout-evaluation.template.json +14 -0
- package/knowledge-base/benchmarks/portfolio-corpus/rule-promotion-evidence.template.json +24 -0
- package/knowledge-base/benchmarks/portfolio-corpus/v3-command-evidence.template.json +17 -0
- package/knowledge-base/benchmarks/portfolio-corpus/v3-qualification-evidence.template.json +29 -0
- package/knowledge-base/benchmarks/scenestart/CALIBRATION.md +95 -0
- package/knowledge-base/benchmarks/scenestart/MANIFEST.md +53 -0
- package/knowledge-base/benchmarks/scenestart/SOURCE-EVIDENCE.md +43 -0
- package/knowledge-base/benchmarks/scenestart/acceptance-criteria.md +59 -0
- package/knowledge-base/benchmarks/scenestart/anti-patterns.md +26 -0
- package/knowledge-base/benchmarks/scenestart/attestations.template.yaml +3 -0
- package/knowledge-base/benchmarks/scenestart/attestations.yaml +21 -0
- package/knowledge-base/benchmarks/scenestart/human-review.md +99 -0
- package/knowledge-base/benchmarks/scenestart/human-review.template.md +105 -0
- package/knowledge-base/benchmarks/scenestart/journeys.json +141 -0
- package/knowledge-base/benchmarks/scenestart/product-contract.yaml +427 -0
- package/knowledge-base/dependencies.yaml +69 -0
- package/knowledge-base/design-intelligence/MASTER.md +87 -0
- package/knowledge-base/design-intelligence/asset-generation.md +66 -0
- package/knowledge-base/design-intelligence/brand-systems.md +69 -0
- package/knowledge-base/design-intelligence/closed-loop-repair.md +95 -0
- package/knowledge-base/design-intelligence/design-deliverable.schema.yaml +1194 -0
- package/knowledge-base/design-intelligence/design-deliverable.template.yaml +591 -0
- package/knowledge-base/design-intelligence/design-plan.md +78 -0
- package/knowledge-base/design-intelligence/design-plan.schema.yaml +621 -0
- package/knowledge-base/design-intelligence/figma-production.md +63 -0
- package/knowledge-base/design-intelligence/generation-adapter.md +72 -0
- package/knowledge-base/design-intelligence/generation-adapter.schema.yaml +47 -0
- package/knowledge-base/design-intelligence/iconography.md +46 -0
- package/knowledge-base/design-intelligence/information-design.md +104 -0
- package/knowledge-base/design-intelligence/information-design.schema.yaml +390 -0
- package/knowledge-base/design-intelligence/information-design.template.yaml +281 -0
- package/knowledge-base/design-intelligence/interaction-recovery-verification.md +72 -0
- package/knowledge-base/design-intelligence/interface-trust.md +103 -0
- package/knowledge-base/design-intelligence/interface-trust.schema.yaml +263 -0
- package/knowledge-base/design-intelligence/interface-trust.template.yaml +173 -0
- package/knowledge-base/design-intelligence/licensing-and-provenance.md +59 -0
- package/knowledge-base/design-intelligence/presentation-design.md +56 -0
- package/knowledge-base/design-intelligence/product-design-brief.md +67 -0
- package/knowledge-base/design-intelligence/product-design-brief.schema.yaml +279 -0
- package/knowledge-base/design-intelligence/product-design-brief.template.yaml +225 -0
- package/knowledge-base/design-intelligence/repair-request.schema.yaml +95 -0
- package/knowledge-base/design-intelligence/repair-request.template.yaml +39 -0
- package/knowledge-base/design-intelligence/visual-accessibility.md +56 -0
- package/knowledge-base/design-intelligence/visual-polish.md +172 -0
- package/knowledge-base/maintained/architecture/MASTER.md +22 -0
- package/knowledge-base/maintained/architecture/component-boundaries.md +30 -0
- package/knowledge-base/maintained/architecture/product-platforms.md +25 -0
- package/knowledge-base/maintained/architecture/quality-attributes-and-evaluation.md +35 -0
- package/knowledge-base/maintained/product-patterns/MASTER.md +10 -0
- package/knowledge-base/maintained/product-patterns/ai-workspaces.md +26 -0
- package/knowledge-base/maintained/product-patterns/operational-dashboards.md +22 -0
- package/knowledge-base/provenance.yaml +196 -0
- package/knowledge-base/retrieval-scope.yaml +43 -0
- package/knowledge-base/usability-evaluation/HEURISTIC_EVALUATION.md +115 -0
- package/knowledge-base/usability-evaluation/heuristic-review.schema.yaml +210 -0
- package/knowledge-base/usability-evaluation/heuristic-review.template.yaml +37 -0
- package/package.json +133 -0
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
import { readFile, stat } from "node:fs/promises";
|
|
2
|
+
import { isAbsolute, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { parse } from "yaml";
|
|
4
|
+
import { journeySuiteSchema, productContractSchema, } from "./schema.js";
|
|
5
|
+
import { validateArchetypeActivation } from "./archetypes.js";
|
|
6
|
+
const REPORT_VERSION = "1.0.0";
|
|
7
|
+
const MAX_CONTRACT_BYTES = 1024 * 1024;
|
|
8
|
+
function isContained(root, candidate) {
|
|
9
|
+
const relation = relative(root, candidate);
|
|
10
|
+
return relation === "" || (!relation.startsWith(`..${sep}`) && relation !== ".." && !isAbsolute(relation));
|
|
11
|
+
}
|
|
12
|
+
function issue(code, path, message) {
|
|
13
|
+
return { code, path, message };
|
|
14
|
+
}
|
|
15
|
+
function zodIssues(prefix, errors) {
|
|
16
|
+
return errors.map((entry) => issue("CONTRACT-SCHEMA", [prefix, ...entry.path.map(String)].filter(Boolean).join("."), entry.message));
|
|
17
|
+
}
|
|
18
|
+
function duplicateIssues(label, values) {
|
|
19
|
+
const seen = new Set();
|
|
20
|
+
const duplicates = new Set();
|
|
21
|
+
for (const value of values) {
|
|
22
|
+
if (seen.has(value))
|
|
23
|
+
duplicates.add(value);
|
|
24
|
+
seen.add(value);
|
|
25
|
+
}
|
|
26
|
+
return [...duplicates].map((value) => issue("CONTRACT-DUPLICATE-ID", label, `Duplicate identifier: ${value}`));
|
|
27
|
+
}
|
|
28
|
+
async function readStructuredFile(path) {
|
|
29
|
+
const fileStats = await stat(path);
|
|
30
|
+
if (!fileStats.isFile())
|
|
31
|
+
throw new Error("Path is not a regular file");
|
|
32
|
+
if (fileStats.size > MAX_CONTRACT_BYTES) {
|
|
33
|
+
throw new Error(`File exceeds the ${MAX_CONTRACT_BYTES}-byte contract limit`);
|
|
34
|
+
}
|
|
35
|
+
const text = await readFile(path, "utf8");
|
|
36
|
+
return path.endsWith(".json") ? JSON.parse(text) : parse(text);
|
|
37
|
+
}
|
|
38
|
+
async function validateDeclaredFile(declaredPath, baseDirectory, projectRoot, fieldPath, issues) {
|
|
39
|
+
if (declaredPath.includes("\0") || isAbsolute(declaredPath)) {
|
|
40
|
+
issues.push(issue("CONTRACT-PATH", fieldPath, "Declared files must use repository-relative paths"));
|
|
41
|
+
return undefined;
|
|
42
|
+
}
|
|
43
|
+
const candidate = resolve(baseDirectory, declaredPath);
|
|
44
|
+
if (projectRoot && !isContained(projectRoot, candidate)) {
|
|
45
|
+
issues.push(issue("CONTRACT-PATH", fieldPath, "Declared file resolves outside the project root"));
|
|
46
|
+
return undefined;
|
|
47
|
+
}
|
|
48
|
+
try {
|
|
49
|
+
const fileStats = await stat(candidate);
|
|
50
|
+
if (!fileStats.isFile())
|
|
51
|
+
throw new Error("not a regular file");
|
|
52
|
+
return candidate;
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
issues.push(issue("CONTRACT-MISSING-SOURCE", fieldPath, `Declared file does not exist: ${declaredPath}`));
|
|
56
|
+
return undefined;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function validateContractReferences(contract, suite) {
|
|
60
|
+
const issues = [];
|
|
61
|
+
const actorIds = new Set(contract.actors.map((actor) => actor.id));
|
|
62
|
+
const modeIds = new Set(contract.modes.map((mode) => mode.id));
|
|
63
|
+
const outputIds = new Set(contract.outputs.map((output) => output.id));
|
|
64
|
+
const criterionIds = new Set(contract.acceptanceCriteria.map((criterion) => criterion.id));
|
|
65
|
+
const profiles = new Map(suite.profiles.map((profile) => [profile.id, profile]));
|
|
66
|
+
issues.push(...duplicateIssues("actors", contract.actors.map((actor) => actor.id)));
|
|
67
|
+
issues.push(...duplicateIssues("modes", contract.modes.map((mode) => mode.id)));
|
|
68
|
+
issues.push(...duplicateIssues("outputs", contract.outputs.map((output) => output.id)));
|
|
69
|
+
issues.push(...duplicateIssues("metrics", contract.metrics.map((metric) => metric.id)));
|
|
70
|
+
issues.push(...duplicateIssues("acceptanceCriteria", contract.acceptanceCriteria.map((criterion) => criterion.id)));
|
|
71
|
+
issues.push(...duplicateIssues("journeyProfiles", suite.profiles.map((profile) => profile.id)));
|
|
72
|
+
for (const [modeIndex, mode] of contract.modes.entries()) {
|
|
73
|
+
for (const output of mode.outputs) {
|
|
74
|
+
if (!outputIds.has(output)) {
|
|
75
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `modes.${modeIndex}.outputs`, `Unknown output: ${output}`));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
for (const [machineIndex, machine] of contract.stateMachines.entries()) {
|
|
80
|
+
const states = new Set(machine.states.map((state) => state.id));
|
|
81
|
+
issues.push(...duplicateIssues(`stateMachines.${machineIndex}.states`, machine.states.map((state) => state.id)));
|
|
82
|
+
if (!states.has(machine.initial)) {
|
|
83
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `stateMachines.${machineIndex}.initial`, `Unknown initial state: ${machine.initial}`));
|
|
84
|
+
}
|
|
85
|
+
for (const [transitionIndex, transition] of machine.transitions.entries()) {
|
|
86
|
+
if (!states.has(transition.from)) {
|
|
87
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `stateMachines.${machineIndex}.transitions.${transitionIndex}.from`, `Unknown state: ${transition.from}`));
|
|
88
|
+
}
|
|
89
|
+
if (!states.has(transition.to)) {
|
|
90
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `stateMachines.${machineIndex}.transitions.${transitionIndex}.to`, `Unknown state: ${transition.to}`));
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
for (const [criterionIndex, criterion] of contract.acceptanceCriteria.entries()) {
|
|
95
|
+
for (const mode of criterion.appliesToModes) {
|
|
96
|
+
if (!modeIds.has(mode)) {
|
|
97
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `acceptanceCriteria.${criterionIndex}.appliesToModes`, `Unknown mode: ${mode}`));
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
if (suite.contract !== contract.id) {
|
|
102
|
+
issues.push(issue("CONTRACT-MISMATCH", "journeys.contract", `Journey suite targets ${suite.contract}, expected ${contract.id}`));
|
|
103
|
+
}
|
|
104
|
+
for (const [profileIndex, profile] of suite.profiles.entries()) {
|
|
105
|
+
issues.push(...duplicateIssues(`journeyProfiles.${profileIndex}.journeys`, profile.journeys.map((journey) => journey.id)));
|
|
106
|
+
}
|
|
107
|
+
for (const [bindingIndex, binding] of contract.verification.bindings.entries()) {
|
|
108
|
+
const path = `verification.bindings.${bindingIndex}`;
|
|
109
|
+
const profile = profiles.get(binding.profile);
|
|
110
|
+
if (!profile) {
|
|
111
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.profile`, `Unknown journey profile: ${binding.profile}`));
|
|
112
|
+
}
|
|
113
|
+
else if (!profile.journeys.some((journey) => journey.id === binding.journey)) {
|
|
114
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.journey`, `Unknown journey ${binding.journey} in profile ${binding.profile}`));
|
|
115
|
+
}
|
|
116
|
+
if (!actorIds.has(binding.actor)) {
|
|
117
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.actor`, `Unknown actor: ${binding.actor}`));
|
|
118
|
+
}
|
|
119
|
+
if (!modeIds.has(binding.mode)) {
|
|
120
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.mode`, `Unknown mode: ${binding.mode}`));
|
|
121
|
+
}
|
|
122
|
+
for (const criterion of binding.acceptanceCriteria) {
|
|
123
|
+
if (!criterionIds.has(criterion)) {
|
|
124
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.acceptanceCriteria`, `Unknown acceptance criterion: ${criterion}`));
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
if (contract.version !== "1.0") {
|
|
129
|
+
const benchmark = contract.benchmark;
|
|
130
|
+
issues.push(...validateArchetypeActivation(benchmark));
|
|
131
|
+
issues.push(...duplicateIssues("benchmark.tasks", benchmark.tasks.map((task) => task.id)));
|
|
132
|
+
const machines = new Map(contract.stateMachines.map((machine) => [machine.id, machine]));
|
|
133
|
+
const viewportByName = new Map(contract.verification.viewports.map((viewport) => [viewport.name, viewport]));
|
|
134
|
+
const validateState = (stateRef, path, allowedCategories) => {
|
|
135
|
+
const machine = machines.get(stateRef.stateMachine);
|
|
136
|
+
if (!machine) {
|
|
137
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.stateMachine`, `Unknown state machine: ${stateRef.stateMachine}`));
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
const state = machine.states.find((entry) => entry.id === stateRef.state);
|
|
141
|
+
if (!state) {
|
|
142
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.state`, `Unknown state: ${stateRef.state}`));
|
|
143
|
+
}
|
|
144
|
+
else if (allowedCategories && !allowedCategories.has(state.category)) {
|
|
145
|
+
issues.push(issue("CONTRACT-TASK-STATE", `${path}.state`, `State ${state.id} has category ${state.category}; expected ${[...allowedCategories].join(" or ")}.`));
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
for (const [taskIndex, task] of benchmark.tasks.entries()) {
|
|
149
|
+
const path = `benchmark.tasks.${taskIndex}`;
|
|
150
|
+
if (!actorIds.has(task.actor)) {
|
|
151
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.actor`, `Unknown actor: ${task.actor}`));
|
|
152
|
+
}
|
|
153
|
+
if (!modeIds.has(task.mode)) {
|
|
154
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.mode`, `Unknown mode: ${task.mode}`));
|
|
155
|
+
}
|
|
156
|
+
validateState(task.start, `${path}.start`);
|
|
157
|
+
validateState(task.success, `${path}.success`, new Set(["success", "terminal"]));
|
|
158
|
+
validateState(task.recovery.failure, `${path}.recovery.failure`, new Set(["error", "fallback", "terminal"]));
|
|
159
|
+
if (task.journey) {
|
|
160
|
+
const journeyProfile = profiles.get(task.journey.profile);
|
|
161
|
+
const journey = journeyProfile?.journeys.find((entry) => entry.id === task.journey.journey);
|
|
162
|
+
if (!journeyProfile) {
|
|
163
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.journey.profile`, `Unknown journey profile: ${task.journey.profile}`));
|
|
164
|
+
}
|
|
165
|
+
else if (!journey) {
|
|
166
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.journey.journey`, `Unknown journey: ${task.journey.journey}`));
|
|
167
|
+
}
|
|
168
|
+
else if (contract.version === "1.2") {
|
|
169
|
+
if (suite.version !== "1.1") {
|
|
170
|
+
issues.push(issue("CONTRACT-INTERACTION-SUITE", `${path}.journey`, "V1.2 product contracts require a V1.1 interaction journey suite."));
|
|
171
|
+
}
|
|
172
|
+
else if (!journey.interaction) {
|
|
173
|
+
issues.push(issue("CONTRACT-INTERACTION-COVERAGE", `${path}.journey`, "The bound journey must declare interaction coverage."));
|
|
174
|
+
}
|
|
175
|
+
else {
|
|
176
|
+
if (journey.interaction.task !== task.id) {
|
|
177
|
+
issues.push(issue("CONTRACT-INTERACTION-TASK", `${path}.journey`, `Journey interaction task ${journey.interaction.task} must match product task ${task.id}.`));
|
|
178
|
+
}
|
|
179
|
+
if (task.primary && !journey.interaction.phases.includes("primary")) {
|
|
180
|
+
issues.push(issue("CONTRACT-INTERACTION-COVERAGE", `${path}.journey`, "Primary product tasks require primary journey checkpoints."));
|
|
181
|
+
}
|
|
182
|
+
if (task.recovery.required && !journey.interaction.phases.includes("recovery")) {
|
|
183
|
+
issues.push(issue("CONTRACT-INTERACTION-COVERAGE", `${path}.journey`, "Tasks with required recovery require failure and preserved-state checkpoints."));
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
const binding = contract.verification.bindings.find((entry) => entry.profile === task.journey.profile &&
|
|
188
|
+
entry.journey === task.journey.journey &&
|
|
189
|
+
entry.actor === task.actor &&
|
|
190
|
+
entry.mode === task.mode);
|
|
191
|
+
if (!binding) {
|
|
192
|
+
issues.push(issue("CONTRACT-TASK-BINDING", `${path}.journey`, "Task journey must have a verification binding with the same actor and mode."));
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
if (task.browser) {
|
|
196
|
+
const viewport = viewportByName.get(task.browser.narrowViewport);
|
|
197
|
+
if (!viewport) {
|
|
198
|
+
issues.push(issue("CONTRACT-UNKNOWN-REFERENCE", `${path}.browser.narrowViewport`, `Unknown viewport: ${task.browser.narrowViewport}`));
|
|
199
|
+
}
|
|
200
|
+
else if (viewport.width > 768) {
|
|
201
|
+
issues.push(issue("CONTRACT-NARROW-VIEWPORT", `${path}.browser.narrowViewport`, `Viewport ${viewport.name} is ${viewport.width}px wide; task paths require 768px or narrower.`));
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
return issues;
|
|
207
|
+
}
|
|
208
|
+
export async function inspectProductContract(requestedContractPath, options = {}) {
|
|
209
|
+
const contractPath = resolve(requestedContractPath);
|
|
210
|
+
const projectRoot = options.projectRoot ? resolve(options.projectRoot) : undefined;
|
|
211
|
+
const baseDirectory = resolve(contractPath, "..");
|
|
212
|
+
const issues = [];
|
|
213
|
+
let contract;
|
|
214
|
+
let suite;
|
|
215
|
+
let journeyPath;
|
|
216
|
+
if (projectRoot && !isContained(projectRoot, contractPath)) {
|
|
217
|
+
issues.push(issue("CONTRACT-PATH", "contract", "Contract path is outside the project root"));
|
|
218
|
+
}
|
|
219
|
+
else {
|
|
220
|
+
try {
|
|
221
|
+
const parsed = productContractSchema.safeParse(await readStructuredFile(contractPath));
|
|
222
|
+
if (parsed.success)
|
|
223
|
+
contract = parsed.data;
|
|
224
|
+
else
|
|
225
|
+
issues.push(...zodIssues("contract", parsed.error.issues));
|
|
226
|
+
}
|
|
227
|
+
catch (error) {
|
|
228
|
+
issues.push(issue("CONTRACT-PARSE", "contract", error instanceof Error ? error.message : String(error)));
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
if (contract) {
|
|
232
|
+
for (const [index, source] of contract.authority.precedence.entries()) {
|
|
233
|
+
await validateDeclaredFile(source.path, baseDirectory, projectRoot, `authority.precedence.${index}.path`, issues);
|
|
234
|
+
}
|
|
235
|
+
journeyPath = await validateDeclaredFile(contract.verification.journeyFile, baseDirectory, projectRoot, "verification.journeyFile", issues);
|
|
236
|
+
if (journeyPath) {
|
|
237
|
+
try {
|
|
238
|
+
const parsed = journeySuiteSchema.safeParse(await readStructuredFile(journeyPath));
|
|
239
|
+
if (parsed.success)
|
|
240
|
+
suite = parsed.data;
|
|
241
|
+
else
|
|
242
|
+
issues.push(...zodIssues("journeys", parsed.error.issues));
|
|
243
|
+
}
|
|
244
|
+
catch (error) {
|
|
245
|
+
issues.push(issue("CONTRACT-PARSE", "journeys", error instanceof Error ? error.message : String(error)));
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
if (contract && suite)
|
|
250
|
+
issues.push(...validateContractReferences(contract, suite));
|
|
251
|
+
const counts = {
|
|
252
|
+
actors: contract?.actors.length ?? 0,
|
|
253
|
+
modes: contract?.modes.length ?? 0,
|
|
254
|
+
acceptanceCriteria: contract?.acceptanceCriteria.length ?? 0,
|
|
255
|
+
journeyProfiles: suite?.profiles.length ?? 0,
|
|
256
|
+
journeys: suite?.profiles.reduce((sum, profile) => sum + profile.journeys.length, 0) ?? 0,
|
|
257
|
+
};
|
|
258
|
+
const taskModel = contract?.version === "1.1"
|
|
259
|
+
? {
|
|
260
|
+
status: issues.some((entry) => entry.path.startsWith("benchmark.")) ? "invalid" : "ready",
|
|
261
|
+
archetype: contract.benchmark.archetype,
|
|
262
|
+
primaryTasks: contract.benchmark.tasks.filter((task) => task.primary).length,
|
|
263
|
+
recoveryTasks: contract.benchmark.tasks.filter((task) => task.recovery.required).length,
|
|
264
|
+
narrowViewportTasks: contract.benchmark.tasks.filter((task) => Boolean(task.browser)).length,
|
|
265
|
+
evidencePolicy: contract.benchmark.evidencePolicy,
|
|
266
|
+
}
|
|
267
|
+
: {
|
|
268
|
+
status: contract ? "legacy" : "invalid",
|
|
269
|
+
primaryTasks: 0,
|
|
270
|
+
recoveryTasks: 0,
|
|
271
|
+
narrowViewportTasks: 0,
|
|
272
|
+
evidencePolicy: {
|
|
273
|
+
missingEvidence: "unverified",
|
|
274
|
+
failedBehavior: "failed",
|
|
275
|
+
unsupportedCapability: "limitation",
|
|
276
|
+
},
|
|
277
|
+
};
|
|
278
|
+
const limitations = contract?.version === "1.0"
|
|
279
|
+
? ["Legacy version 1.0 contract has no archetype-aware product task model."]
|
|
280
|
+
: [];
|
|
281
|
+
const report = {
|
|
282
|
+
version: REPORT_VERSION,
|
|
283
|
+
contractPath,
|
|
284
|
+
...(journeyPath ? { journeyPath } : {}),
|
|
285
|
+
generatedAt: new Date().toISOString(),
|
|
286
|
+
...(contract ? { contractId: contract.id } : {}),
|
|
287
|
+
counts,
|
|
288
|
+
taskModel,
|
|
289
|
+
limitations,
|
|
290
|
+
issues,
|
|
291
|
+
passed: issues.length === 0,
|
|
292
|
+
};
|
|
293
|
+
return {
|
|
294
|
+
report,
|
|
295
|
+
...(contract ? { contract } : {}),
|
|
296
|
+
...(suite ? { suite } : {}),
|
|
297
|
+
...(journeyPath ? { journeyPath } : {}),
|
|
298
|
+
};
|
|
299
|
+
}
|
|
300
|
+
export async function validateProductContract(requestedContractPath, options = {}) {
|
|
301
|
+
return (await inspectProductContract(requestedContractPath, options)).report;
|
|
302
|
+
}
|
|
303
|
+
//# sourceMappingURL=validator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"validator.js","sourceRoot":"","sources":["../../../src/contracts/validator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAE/D,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE7B,OAAO,EACL,kBAAkB,EAClB,qBAAqB,GAKtB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,2BAA2B,EAAE,MAAM,iBAAiB,CAAC;AAE9D,MAAM,cAAc,GAAG,OAAO,CAAC;AAC/B,MAAM,kBAAkB,GAAG,IAAI,GAAG,IAAI,CAAC;AAavC,SAAS,WAAW,CAAC,IAAY,EAAE,SAAiB;IAClD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC3C,OAAO,QAAQ,KAAK,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,SAAS,KAAK,CAAC,IAAY,EAAE,IAAY,EAAE,OAAe;IACxD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,SAAS,CAAC,MAAc,EAAE,MAA2D;IAC5F,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAC1B,KAAK,CACH,iBAAiB,EACjB,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAC7D,KAAK,CAAC,OAAO,CACd,CACF,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,MAAgB;IACtD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3C,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAClB,CAAC;IACD,OAAO,CAAC,GAAG,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CACnC,KAAK,CAAC,uBAAuB,EAAE,KAAK,EAAE,yBAAyB,KAAK,EAAE,CAAC,CACxE,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,IAAY;IAC5C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IACvE,IAAI,SAAS,CAAC,IAAI,GAAG,kBAAkB,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,oBAAoB,kBAAkB,sBAAsB,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC1C,OAAO,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AACjE,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,YAAoB,EACpB,aAAqB,EACrB,WAA+B,EAC/B,SAAiB,EACjB,MAAuB;IAEvB,IAAI,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC5D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,mDAAmD,CAAC,CAAC,CAAC;QACpG,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACvD,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,SAAS,CAAC,EAAE,CAAC;QACxD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,SAAS,EAAE,iDAAiD,CAAC,CAAC,CAAC;QAClG,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QAC/D,OAAO,SAAS,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,yBAAyB,EAAE,SAAS,EAAE,iCAAiC,YAAY,EAAE,CAAC,CAAC,CAAC;QAC1G,OAAO,SAAS,CAAC;IACnB,CAAC;AACH,CAAC;AAED,SAAS,0BAA0B,CAAC,QAAyB,EAAE,KAAmB;IAChF,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IACvE,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3F,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAEjF,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACpF,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACxF,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,SAAS,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACxF,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,oBAAoB,EAAE,QAAQ,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACpH,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAEhG,KAAK,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;QACzD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,SAAS,SAAS,UAAU,EAAE,mBAAmB,MAAM,EAAE,CAAC,CAAC,CAAC;YAC9G,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,OAAO,EAAE,EAAE,CAAC;QACvE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QAChE,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,iBAAiB,YAAY,SAAS,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjH,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,iBAAiB,YAAY,UAAU,EAAE,0BAA0B,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACzI,CAAC;QACD,KAAK,MAAM,CAAC,eAAe,EAAE,UAAU,CAAC,IAAI,OAAO,CAAC,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1E,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,iBAAiB,YAAY,gBAAgB,eAAe,OAAO,EAAE,kBAAkB,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YAC7J,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC/B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,iBAAiB,YAAY,gBAAgB,eAAe,KAAK,EAAE,kBAAkB,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;YACzJ,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,MAAM,CAAC,cAAc,EAAE,SAAS,CAAC,IAAI,QAAQ,CAAC,kBAAkB,CAAC,OAAO,EAAE,EAAE,CAAC;QAChF,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,cAAc,EAAE,CAAC;YAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,sBAAsB,cAAc,iBAAiB,EAAE,iBAAiB,IAAI,EAAE,CAAC,CAAC,CAAC;YACnI,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,mBAAmB,EAAE,mBAAmB,EAAE,yBAAyB,KAAK,CAAC,QAAQ,cAAc,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;IACnI,CAAC;IAED,KAAK,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/D,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,mBAAmB,YAAY,WAAW,EAAE,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC7H,CAAC;IAED,KAAK,MAAM,CAAC,YAAY,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QAC/E,MAAM,IAAI,GAAG,yBAAyB,YAAY,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,UAAU,EAAE,4BAA4B,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACrH,CAAC;aAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,KAAK,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/E,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,UAAU,EAAE,mBAAmB,OAAO,CAAC,OAAO,eAAe,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC1I,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,QAAQ,EAAE,kBAAkB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACvG,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,OAAO,EAAE,iBAAiB,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACpG,CAAC;QACD,KAAK,MAAM,SAAS,IAAI,OAAO,CAAC,kBAAkB,EAAE,CAAC;YACnD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;gBACjC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,qBAAqB,EAAE,iCAAiC,SAAS,EAAE,CAAC,CAAC,CAAC;YAC/H,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,CAAC;QACrC,MAAM,CAAC,IAAI,CAAC,GAAG,2BAA2B,CAAC,SAAS,CAAC,CAAC,CAAC;QAEvD,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,iBAAiB,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3F,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;QACzF,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC7G,MAAM,aAAa,GAAG,CACpB,QAAiD,EACjD,IAAY,EACZ,iBAA+B,EAC/B,EAAE;YACF,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;YACpD,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,eAAe,EAAE,0BAA0B,QAAQ,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;gBAC5H,OAAO;YACT,CAAC;YACD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC1E,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,QAAQ,EAAE,kBAAkB,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACxG,CAAC;iBAAM,IAAI,iBAAiB,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACvE,MAAM,CAAC,IAAI,CACT,KAAK,CACH,qBAAqB,EACrB,GAAG,IAAI,QAAQ,EACf,SAAS,KAAK,CAAC,EAAE,iBAAiB,KAAK,CAAC,QAAQ,cAAc,CAAC,GAAG,iBAAiB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CACrG,CACF,CAAC;YACJ,CAAC;QACH,CAAC,CAAC;QAEF,KAAK,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;YAC1D,MAAM,IAAI,GAAG,mBAAmB,SAAS,EAAE,CAAC;YAC5C,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,QAAQ,EAAE,kBAAkB,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;YACpG,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,OAAO,EAAE,iBAAiB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACjG,CAAC;YACD,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,QAAQ,CAAC,CAAC;YAC3C,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,UAAU,EAAE,IAAI,GAAG,CAAC,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;YACjF,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,GAAG,IAAI,mBAAmB,EAAE,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;YAE7G,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,MAAM,cAAc,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC1D,MAAM,OAAO,GAAG,cAAc,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,OAAQ,CAAC,OAAO,CAAC,CAAC;gBAC7F,IAAI,CAAC,cAAc,EAAE,CAAC;oBACpB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,kBAAkB,EAAE,4BAA4B,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAClI,CAAC;qBAAM,IAAI,CAAC,OAAO,EAAE,CAAC;oBACpB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,kBAAkB,EAAE,oBAAoB,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBAC1H,CAAC;qBAAM,IAAI,QAAQ,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;oBACtC,IAAI,KAAK,CAAC,OAAO,KAAK,KAAK,EAAE,CAAC;wBAC5B,MAAM,CAAC,IAAI,CACT,KAAK,CACH,4BAA4B,EAC5B,GAAG,IAAI,UAAU,EACjB,kEAAkE,CACnE,CACF,CAAC;oBACJ,CAAC;yBAAM,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;wBAChC,MAAM,CAAC,IAAI,CACT,KAAK,CACH,+BAA+B,EAC/B,GAAG,IAAI,UAAU,EACjB,sDAAsD,CACvD,CACF,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACN,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,EAAE,CAAC;4BACzC,MAAM,CAAC,IAAI,CACT,KAAK,CACH,2BAA2B,EAC3B,GAAG,IAAI,UAAU,EACjB,4BAA4B,OAAO,CAAC,WAAW,CAAC,IAAI,4BAA4B,IAAI,CAAC,EAAE,GAAG,CAC3F,CACF,CAAC;wBACJ,CAAC;wBACD,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;4BACpE,MAAM,CAAC,IAAI,CACT,KAAK,CACH,+BAA+B,EAC/B,GAAG,IAAI,UAAU,EACjB,4DAA4D,CAC7D,CACF,CAAC;wBACJ,CAAC;wBACD,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;4BAC/E,MAAM,CAAC,IAAI,CACT,KAAK,CACH,+BAA+B,EAC/B,GAAG,IAAI,UAAU,EACjB,+EAA+E,CAChF,CACF,CAAC;wBACJ,CAAC;oBACH,CAAC;gBACH,CAAC;gBACD,MAAM,OAAO,GAAG,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,IAAI,CACjD,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAQ,CAAC,OAAO;oBACvC,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,OAAQ,CAAC,OAAO;oBACvC,KAAK,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK;oBAC1B,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CAC3B,CAAC;gBACF,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,MAAM,CAAC,IAAI,CACT,KAAK,CACH,uBAAuB,EACvB,GAAG,IAAI,UAAU,EACjB,6EAA6E,CAC9E,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;gBACjE,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B,EAAE,GAAG,IAAI,yBAAyB,EAAE,qBAAqB,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;gBACzI,CAAC;qBAAM,IAAI,QAAQ,CAAC,KAAK,GAAG,GAAG,EAAE,CAAC;oBAChC,MAAM,CAAC,IAAI,CACT,KAAK,CACH,0BAA0B,EAC1B,GAAG,IAAI,yBAAyB,EAChC,YAAY,QAAQ,CAAC,IAAI,OAAO,QAAQ,CAAC,KAAK,gDAAgD,CAC/F,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,qBAA6B,EAC7B,UAA6B,EAAE;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACpD,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACnF,MAAM,aAAa,GAAG,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAClD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,IAAI,QAAqC,CAAC;IAC1C,IAAI,KAA+B,CAAC;IACpC,IAAI,WAA+B,CAAC;IAEpC,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,YAAY,CAAC,EAAE,CAAC;QAC3D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,eAAe,EAAE,UAAU,EAAE,2CAA2C,CAAC,CAAC,CAAC;IAC/F,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,qBAAqB,CAAC,SAAS,CAAC,MAAM,kBAAkB,CAAC,YAAY,CAAC,CAAC,CAAC;YACvF,IAAI,MAAM,CAAC,OAAO;gBAAE,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC;;gBACtC,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;QAClE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,UAAU,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC3G,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,EAAE,CAAC;QACb,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,QAAQ,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;YACtE,MAAM,oBAAoB,CAAC,MAAM,CAAC,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,wBAAwB,KAAK,OAAO,EAAE,MAAM,CAAC,CAAC;QACpH,CAAC;QACD,WAAW,GAAG,MAAM,oBAAoB,CAAC,QAAQ,CAAC,YAAY,CAAC,WAAW,EAAE,aAAa,EAAE,WAAW,EAAE,0BAA0B,EAAE,MAAM,CAAC,CAAC;QAC5I,IAAI,WAAW,EAAE,CAAC;YAChB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,MAAM,kBAAkB,CAAC,WAAW,CAAC,CAAC,CAAC;gBACnF,IAAI,MAAM,CAAC,OAAO;oBAAE,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC;;oBACnC,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;YAClE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,UAAU,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC3G,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,IAAI,KAAK;QAAE,MAAM,CAAC,IAAI,CAAC,GAAG,0BAA0B,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;IAEnF,MAAM,MAAM,GAAG;QACb,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,MAAM,IAAI,CAAC;QACpC,KAAK,EAAE,QAAQ,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC;QAClC,kBAAkB,EAAE,QAAQ,EAAE,kBAAkB,CAAC,MAAM,IAAI,CAAC;QAC5D,eAAe,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,IAAI,CAAC;QAC5C,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC;KAC1F,CAAC;IACF,MAAM,SAAS,GAAG,QAAQ,EAAE,OAAO,KAAK,KAAK;QAC3C,CAAC,CAAC;YACE,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,SAAkB,CAAC,CAAC,CAAC,OAAgB;YAC3G,SAAS,EAAE,QAAQ,CAAC,SAAS,CAAC,SAAS;YACvC,YAAY,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM;YAC5E,aAAa,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,MAAM;YACvF,mBAAmB,EAAE,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;YAC5F,cAAc,EAAE,QAAQ,CAAC,SAAS,CAAC,cAAc;SAClD;QACH,CAAC,CAAC;YACE,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAiB,CAAC,CAAC,CAAC,SAAkB;YACzD,YAAY,EAAE,CAAC;YACf,aAAa,EAAE,CAAC;YAChB,mBAAmB,EAAE,CAAC;YACtB,cAAc,EAAE;gBACd,eAAe,EAAE,YAAqB;gBACtC,cAAc,EAAE,QAAiB;gBACjC,qBAAqB,EAAE,YAAqB;aAC7C;SACF,CAAC;IACN,MAAM,WAAW,GAAG,QAAQ,EAAE,OAAO,KAAK,KAAK;QAC7C,CAAC,CAAC,CAAC,wEAAwE,CAAC;QAC5E,CAAC,CAAC,EAAE,CAAC;IAEP,MAAM,MAAM,GAA6B;QACvC,OAAO,EAAE,cAAc;QACvB,YAAY;QACZ,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAChD,MAAM;QACN,SAAS;QACT,WAAW;QACX,MAAM;QACN,MAAM,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC;KAC5B,CAAC;IACF,OAAO;QACL,MAAM;QACN,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3B,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,qBAA6B,EAC7B,UAA6B,EAAE;IAE/B,OAAO,CAAC,MAAM,sBAAsB,CAAC,qBAAqB,EAAE,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/E,CAAC"}
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import { realpath, stat } from "node:fs/promises";
|
|
2
|
+
import { isAbsolute, relative, resolve, sep } from "node:path";
|
|
3
|
+
import { auditRepository } from "../audit/scanner.js";
|
|
4
|
+
import { validateProductContract } from "../contracts/validator.js";
|
|
5
|
+
import { buildKnowledgeIndex, searchKnowledge } from "../retrieval/search.js";
|
|
6
|
+
import { loadCorpusManifest } from "./loader.js";
|
|
7
|
+
const REPORT_VERSION = "1.0.0";
|
|
8
|
+
function isContained(root, candidate) {
|
|
9
|
+
const relation = relative(root, candidate);
|
|
10
|
+
return relation === "" || (!relation.startsWith(`..${sep}`) && relation !== ".." && !isAbsolute(relation));
|
|
11
|
+
}
|
|
12
|
+
async function resolveCorpusPath(projectRoot, requestedPath, kind) {
|
|
13
|
+
if (requestedPath.includes("\0") || isAbsolute(requestedPath)) {
|
|
14
|
+
throw new Error(`Corpus ${kind} paths must be repository-relative`);
|
|
15
|
+
}
|
|
16
|
+
const candidate = await realpath(resolve(projectRoot, requestedPath));
|
|
17
|
+
if (!isContained(projectRoot, candidate)) {
|
|
18
|
+
throw new Error(`Corpus ${kind} path resolves outside the project root: ${requestedPath}`);
|
|
19
|
+
}
|
|
20
|
+
const fileStats = await stat(candidate);
|
|
21
|
+
if ((kind === "file" && !fileStats.isFile()) || (kind === "directory" && !fileStats.isDirectory())) {
|
|
22
|
+
throw new Error(`Corpus ${kind} path has the wrong type: ${requestedPath}`);
|
|
23
|
+
}
|
|
24
|
+
return candidate;
|
|
25
|
+
}
|
|
26
|
+
function sortedUnique(values) {
|
|
27
|
+
return [...new Set(values)].sort();
|
|
28
|
+
}
|
|
29
|
+
function containsAll(observed, expected) {
|
|
30
|
+
return expected.every((entry) => observed.has(entry));
|
|
31
|
+
}
|
|
32
|
+
function containsNone(observed, forbidden) {
|
|
33
|
+
return forbidden.every((entry) => !observed.has(entry));
|
|
34
|
+
}
|
|
35
|
+
async function evaluateRetrievalCase(benchmarkCase, index) {
|
|
36
|
+
const report = searchKnowledge(index, {
|
|
37
|
+
query: benchmarkCase.query,
|
|
38
|
+
...(benchmarkCase.categories ? { categories: benchmarkCase.categories } : {}),
|
|
39
|
+
limit: Math.max(benchmarkCase.expected.maxRank ?? 5, 5),
|
|
40
|
+
});
|
|
41
|
+
const expectedPath = benchmarkCase.expected.path;
|
|
42
|
+
const matched = expectedPath
|
|
43
|
+
? report.results.find((entry) => entry.path === expectedPath)
|
|
44
|
+
: undefined;
|
|
45
|
+
const rank = matched?.rank;
|
|
46
|
+
const passed = report.status === benchmarkCase.expected.status &&
|
|
47
|
+
(benchmarkCase.dimension === "abstention" ||
|
|
48
|
+
(rank !== undefined && rank <= (benchmarkCase.expected.maxRank ?? 5)));
|
|
49
|
+
const reciprocalRank = benchmarkCase.dimension === "recommendation-relevance"
|
|
50
|
+
? rank === undefined ? 0 : 1 / rank
|
|
51
|
+
: undefined;
|
|
52
|
+
return {
|
|
53
|
+
id: benchmarkCase.id,
|
|
54
|
+
kind: benchmarkCase.kind,
|
|
55
|
+
dimension: benchmarkCase.dimension,
|
|
56
|
+
polarity: benchmarkCase.polarity,
|
|
57
|
+
source: benchmarkCase.source,
|
|
58
|
+
passed,
|
|
59
|
+
score: passed ? 1 : 0,
|
|
60
|
+
expected: expectedPath
|
|
61
|
+
? `${benchmarkCase.expected.status}; ${expectedPath} at rank <= ${benchmarkCase.expected.maxRank ?? 5}`
|
|
62
|
+
: benchmarkCase.expected.status,
|
|
63
|
+
observed: report.status === "no-match"
|
|
64
|
+
? "no-match"
|
|
65
|
+
: `matches; top path ${report.results[0]?.path ?? "none"}`,
|
|
66
|
+
...(rank === undefined ? {} : { rank }),
|
|
67
|
+
...(reciprocalRank === undefined ? {} : { reciprocalRank }),
|
|
68
|
+
...(matched ? { matchedPath: matched.path } : {}),
|
|
69
|
+
ruleIds: [],
|
|
70
|
+
issueCodes: [],
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
async function evaluateAuditCase(benchmarkCase, projectRoot) {
|
|
74
|
+
const target = await resolveCorpusPath(projectRoot, benchmarkCase.target, "directory");
|
|
75
|
+
const report = await auditRepository(target, benchmarkCase.policy);
|
|
76
|
+
const ruleIds = sortedUnique(report.findings.map((finding) => finding.ruleId));
|
|
77
|
+
const observed = new Set(ruleIds);
|
|
78
|
+
const required = benchmarkCase.expected.requiredRuleIds;
|
|
79
|
+
const forbidden = benchmarkCase.expected.forbiddenRuleIds;
|
|
80
|
+
const expectedAccept = benchmarkCase.expected.verdict === "accept";
|
|
81
|
+
const verdictMatches = expectedAccept
|
|
82
|
+
? report.findings.length <= (benchmarkCase.expected.maximumFindings ?? 0)
|
|
83
|
+
: report.findings.length > 0;
|
|
84
|
+
const passed = verdictMatches && containsAll(observed, required) && containsNone(observed, forbidden);
|
|
85
|
+
return {
|
|
86
|
+
id: benchmarkCase.id,
|
|
87
|
+
kind: benchmarkCase.kind,
|
|
88
|
+
dimension: benchmarkCase.dimension,
|
|
89
|
+
polarity: benchmarkCase.polarity,
|
|
90
|
+
source: benchmarkCase.source,
|
|
91
|
+
passed,
|
|
92
|
+
score: passed ? 1 : 0,
|
|
93
|
+
expected: `${benchmarkCase.expected.verdict}; required ${required.join(", ") || "none"}; forbidden ${forbidden.join(", ") || "none"}`,
|
|
94
|
+
observed: `${report.findings.length} findings; ${ruleIds.join(", ") || "no rules"}`,
|
|
95
|
+
ruleIds,
|
|
96
|
+
issueCodes: [],
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
async function evaluateContractCase(benchmarkCase, projectRoot) {
|
|
100
|
+
const contract = await resolveCorpusPath(projectRoot, benchmarkCase.contract, "file");
|
|
101
|
+
const report = await validateProductContract(contract, { projectRoot });
|
|
102
|
+
const issueCodes = sortedUnique(report.issues.map((entry) => entry.code));
|
|
103
|
+
const observed = new Set(issueCodes);
|
|
104
|
+
const expectedPass = benchmarkCase.expected.verdict === "accept";
|
|
105
|
+
const passed = report.passed === expectedPass &&
|
|
106
|
+
containsAll(observed, benchmarkCase.expected.requiredIssueCodes) &&
|
|
107
|
+
containsNone(observed, benchmarkCase.expected.forbiddenIssueCodes);
|
|
108
|
+
return {
|
|
109
|
+
id: benchmarkCase.id,
|
|
110
|
+
kind: benchmarkCase.kind,
|
|
111
|
+
dimension: benchmarkCase.dimension,
|
|
112
|
+
polarity: benchmarkCase.polarity,
|
|
113
|
+
source: benchmarkCase.source,
|
|
114
|
+
passed,
|
|
115
|
+
score: passed ? 1 : 0,
|
|
116
|
+
expected: `${benchmarkCase.expected.verdict}; required ${benchmarkCase.expected.requiredIssueCodes.join(", ") || "none"}`,
|
|
117
|
+
observed: `${report.passed ? "accept" : "reject"}; ${issueCodes.join(", ") || "no issues"}`,
|
|
118
|
+
ruleIds: [],
|
|
119
|
+
issueCodes,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
function dimensionResult(dimension, cases, threshold) {
|
|
123
|
+
const relevant = cases.filter((entry) => entry.dimension === dimension);
|
|
124
|
+
const passedCases = relevant.filter((entry) => entry.passed).length;
|
|
125
|
+
const score = passedCases / relevant.length;
|
|
126
|
+
const reciprocalRanks = relevant
|
|
127
|
+
.map((entry) => entry.reciprocalRank)
|
|
128
|
+
.filter((value) => value !== undefined);
|
|
129
|
+
const meanReciprocalRank = reciprocalRanks.length
|
|
130
|
+
? reciprocalRanks.reduce((sum, value) => sum + value, 0) / reciprocalRanks.length
|
|
131
|
+
: undefined;
|
|
132
|
+
const rankPassed = threshold.minimumMeanReciprocalRank === undefined ||
|
|
133
|
+
(meanReciprocalRank !== undefined && meanReciprocalRank >= threshold.minimumMeanReciprocalRank);
|
|
134
|
+
return {
|
|
135
|
+
dimension,
|
|
136
|
+
cases: relevant.length,
|
|
137
|
+
passedCases,
|
|
138
|
+
failedCases: relevant.length - passedCases,
|
|
139
|
+
score: Number(score.toFixed(4)),
|
|
140
|
+
minimumScore: threshold.minimumScore,
|
|
141
|
+
...(meanReciprocalRank === undefined
|
|
142
|
+
? {}
|
|
143
|
+
: { meanReciprocalRank: Number(meanReciprocalRank.toFixed(4)) }),
|
|
144
|
+
...(threshold.minimumMeanReciprocalRank === undefined
|
|
145
|
+
? {}
|
|
146
|
+
: { minimumMeanReciprocalRank: threshold.minimumMeanReciprocalRank }),
|
|
147
|
+
passed: score >= threshold.minimumScore && rankPassed,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
export async function evaluateCorpusBenchmark(manifestPath, projectRoot = process.cwd()) {
|
|
151
|
+
const resolvedProjectRoot = await realpath(resolve(projectRoot));
|
|
152
|
+
const resolvedManifest = await realpath(resolve(manifestPath));
|
|
153
|
+
if (!isContained(resolvedProjectRoot, resolvedManifest)) {
|
|
154
|
+
throw new Error("Corpus manifest is outside the project root");
|
|
155
|
+
}
|
|
156
|
+
const manifest = await loadCorpusManifest(resolvedManifest);
|
|
157
|
+
for (const source of manifest.sources) {
|
|
158
|
+
await resolveCorpusPath(resolvedProjectRoot, source.evidence, "file");
|
|
159
|
+
}
|
|
160
|
+
const retrievalCases = manifest.cases.some((entry) => entry.kind === "retrieval");
|
|
161
|
+
const index = retrievalCases ? await buildKnowledgeIndex(resolvedProjectRoot) : undefined;
|
|
162
|
+
const caseResults = [];
|
|
163
|
+
for (const benchmarkCase of manifest.cases) {
|
|
164
|
+
if (benchmarkCase.kind === "retrieval") {
|
|
165
|
+
caseResults.push(await evaluateRetrievalCase(benchmarkCase, index));
|
|
166
|
+
}
|
|
167
|
+
else if (benchmarkCase.kind === "audit") {
|
|
168
|
+
caseResults.push(await evaluateAuditCase(benchmarkCase, resolvedProjectRoot));
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
caseResults.push(await evaluateContractCase(benchmarkCase, resolvedProjectRoot));
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
const dimensions = Object.keys(manifest.thresholds.dimensions)
|
|
175
|
+
.sort()
|
|
176
|
+
.map((dimension) => dimensionResult(dimension, caseResults, manifest.thresholds.dimensions[dimension]));
|
|
177
|
+
const overallScore = caseResults.reduce((sum, entry) => sum + entry.score, 0) / caseResults.length;
|
|
178
|
+
return {
|
|
179
|
+
version: REPORT_VERSION,
|
|
180
|
+
generatedAt: new Date().toISOString(),
|
|
181
|
+
manifestPath: resolvedManifest,
|
|
182
|
+
corpusId: manifest.id,
|
|
183
|
+
corpusVersion: manifest.version,
|
|
184
|
+
sources: manifest.sources.length,
|
|
185
|
+
sourceRecords: manifest.sources.map((source) => ({
|
|
186
|
+
id: source.id,
|
|
187
|
+
title: source.title,
|
|
188
|
+
origin: source.origin,
|
|
189
|
+
owner: source.owner,
|
|
190
|
+
license: source.license,
|
|
191
|
+
...(source.sourceUrl ? { sourceUrl: source.sourceUrl } : {}),
|
|
192
|
+
evidence: source.evidence,
|
|
193
|
+
})),
|
|
194
|
+
positiveCases: manifest.cases.filter((entry) => entry.polarity === "positive").length,
|
|
195
|
+
negativeCases: manifest.cases.filter((entry) => entry.polarity === "negative").length,
|
|
196
|
+
caseResults,
|
|
197
|
+
dimensions,
|
|
198
|
+
overallScore: Number(overallScore.toFixed(4)),
|
|
199
|
+
overallMinimumScore: manifest.thresholds.overallMinimumScore,
|
|
200
|
+
passed: overallScore >= manifest.thresholds.overallMinimumScore && dimensions.every((entry) => entry.passed),
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
//# sourceMappingURL=evaluator.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"evaluator.js","sourceRoot":"","sources":["../../../src/corpus/evaluator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAE/D,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEtD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAC9E,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AASjD,MAAM,cAAc,GAAG,OAAO,CAAC;AAE/B,SAAS,WAAW,CAAC,IAAY,EAAE,SAAiB;IAClD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;IAC3C,OAAO,QAAQ,KAAK,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC;AAC7G,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,WAAmB,EAAE,aAAqB,EAAE,IAA0B;IACrG,IAAI,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,oCAAoC,CAAC,CAAC;IACtE,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,aAAa,CAAC,CAAC,CAAC;IACtE,IAAI,CAAC,WAAW,CAAC,WAAW,EAAE,SAAS,CAAC,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,4CAA4C,aAAa,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAC;IACxC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;QACnG,MAAM,IAAI,KAAK,CAAC,UAAU,IAAI,6BAA6B,aAAa,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,YAAY,CAAC,MAAgB;IACpC,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACrC,CAAC;AAED,SAAS,WAAW,CAAC,QAAqB,EAAE,QAAkB;IAC5D,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,SAAS,YAAY,CAAC,QAAqB,EAAE,SAAmB;IAC9D,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,KAAK,UAAU,qBAAqB,CAClC,aAAyD,EACzD,KAAsD;IAEtD,MAAM,MAAM,GAAG,eAAe,CAAC,KAAK,EAAE;QACpC,KAAK,EAAE,aAAa,CAAC,KAAK;QAC1B,GAAG,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7E,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC;KACxD,CAAC,CAAC;IACH,MAAM,YAAY,GAAG,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC;IACjD,MAAM,OAAO,GAAG,YAAY;QAC1B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,YAAY,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,CAAC;IAC3B,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,KAAK,aAAa,CAAC,QAAQ,CAAC,MAAM;QAC/C,CAAC,aAAa,CAAC,SAAS,KAAK,YAAY;YACvC,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,MAAM,cAAc,GAAG,aAAa,CAAC,SAAS,KAAK,0BAA0B;QAC3E,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI;QACnC,CAAC,CAAC,SAAS,CAAC;IAEd,OAAO;QACL,EAAE,EAAE,aAAa,CAAC,EAAE;QACpB,IAAI,EAAE,aAAa,CAAC,IAAI;QACxB,SAAS,EAAE,aAAa,CAAC,SAAS;QAClC,QAAQ,EAAE,aAAa,CAAC,QAAQ;QAChC,MAAM,EAAE,aAAa,CAAC,MAAM;QAC5B,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,QAAQ,EAAE,YAAY;YACpB,CAAC,CAAC,GAAG,aAAa,CAAC,QAAQ,CAAC,MAAM,KAAK,YAAY,eAAe,aAAa,CAAC,QAAQ,CAAC,OAAO,IAAI,CAAC,EAAE;YACvG,CAAC,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM;QACjC,QAAQ,EAAE,MAAM,CAAC,MAAM,KAAK,UAAU;YACpC,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,qBAAqB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,IAAI,MAAM,EAAE;QAC5D,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;QACvC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;QAC3D,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,EAAE,EAAE;QACX,UAAU,EAAE,EAAE;KACf,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,aAAqD,EACrD,WAAmB;IAEnB,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACvF,MAAM,MAAM,GAAG,MAAM,eAAe,CAAC,MAAM,EAAE,aAAa,CAAC,MAA0C,CAAC,CAAC;IACvG,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,eAAe,CAAC;IACxD,MAAM,SAAS,GAAG,aAAa,CAAC,QAAQ,CAAC,gBAAgB,CAAC;IAC1D,MAAM,cAAc,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,KAAK,QAAQ,CAAC;IACnE,MAAM,cAAc,GAAG,cAAc;QACnC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,eAAe,IAAI,CAAC,CAAC;QACzE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/B,MAAM,MAAM,GAAG,cAAc,IAAI,WAAW,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,YAAY,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAEtG,OAAO;QACL,EAAE,EAAE,aAAa,CAAC,EAAE;QACpB,IAAI,EAAE,aAAa,CAAC,IAAI;QACxB,SAAS,EAAE,aAAa,CAAC,SAAS;QAClC,QAAQ,EAAE,aAAa,CAAC,QAAQ;QAChC,MAAM,EAAE,aAAa,CAAC,MAAM;QAC5B,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,QAAQ,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,cAAc,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,eAAe,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE;QACrI,QAAQ,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,cAAc,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,UAAU,EAAE;QACnF,OAAO;QACP,UAAU,EAAE,EAAE;KACf,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,aAAwD,EACxD,WAAmB;IAEnB,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,WAAW,EAAE,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACtF,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAAC,QAAQ,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC;IACxE,MAAM,UAAU,GAAG,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,YAAY,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,KAAK,QAAQ,CAAC;IACjE,MAAM,MAAM,GACV,MAAM,CAAC,MAAM,KAAK,YAAY;QAC9B,WAAW,CAAC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,kBAAkB,CAAC;QAChE,YAAY,CAAC,QAAQ,EAAE,aAAa,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAC;IAErE,OAAO;QACL,EAAE,EAAE,aAAa,CAAC,EAAE;QACpB,IAAI,EAAE,aAAa,CAAC,IAAI;QACxB,SAAS,EAAE,aAAa,CAAC,SAAS;QAClC,QAAQ,EAAE,aAAa,CAAC,QAAQ;QAChC,MAAM,EAAE,aAAa,CAAC,MAAM;QAC5B,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,QAAQ,EAAE,GAAG,aAAa,CAAC,QAAQ,CAAC,OAAO,cAAc,aAAa,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,MAAM,EAAE;QACzH,QAAQ,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,KAAK,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,WAAW,EAAE;QAC3F,OAAO,EAAE,EAAE;QACX,UAAU;KACX,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,SAA0B,EAC1B,KAAyB,EACzB,SAAuE;IAEvE,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC;IACxE,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACpE,MAAM,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC5C,MAAM,eAAe,GAAG,QAAQ;SAC7B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC;SACpC,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAC3D,MAAM,kBAAkB,GAAG,eAAe,CAAC,MAAM;QAC/C,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,eAAe,CAAC,MAAM;QACjF,CAAC,CAAC,SAAS,CAAC;IACd,MAAM,UAAU,GAAG,SAAS,CAAC,yBAAyB,KAAK,SAAS;QAClE,CAAC,kBAAkB,KAAK,SAAS,IAAI,kBAAkB,IAAI,SAAS,CAAC,yBAAyB,CAAC,CAAC;IAElG,OAAO;QACL,SAAS;QACT,KAAK,EAAE,QAAQ,CAAC,MAAM;QACtB,WAAW;QACX,WAAW,EAAE,QAAQ,CAAC,MAAM,GAAG,WAAW;QAC1C,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/B,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,GAAG,CAAC,kBAAkB,KAAK,SAAS;YAClC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,GAAG,CAAC,SAAS,CAAC,yBAAyB,KAAK,SAAS;YACnD,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,yBAAyB,EAAE,SAAS,CAAC,yBAAyB,EAAE,CAAC;QACvE,MAAM,EAAE,KAAK,IAAI,SAAS,CAAC,YAAY,IAAI,UAAU;KACtD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,YAAoB,EAAE,WAAW,GAAG,OAAO,CAAC,GAAG,EAAE;IAC7F,MAAM,mBAAmB,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC;IACjE,MAAM,gBAAgB,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;IAC/D,IAAI,CAAC,WAAW,CAAC,mBAAmB,EAAE,gBAAgB,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;IAC5D,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,iBAAiB,CAAC,mBAAmB,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;IAClF,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,MAAM,mBAAmB,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1F,MAAM,WAAW,GAAuB,EAAE,CAAC;IAE3C,KAAK,MAAM,aAAa,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAC3C,IAAI,aAAa,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;YACvC,WAAW,CAAC,IAAI,CAAC,MAAM,qBAAqB,CAAC,aAAa,EAAE,KAAM,CAAC,CAAC,CAAC;QACvE,CAAC;aAAM,IAAI,aAAa,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1C,WAAW,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,aAAa,EAAE,mBAAmB,CAAC,CAAC,CAAC;QAChF,CAAC;aAAM,CAAC;YACN,WAAW,CAAC,IAAI,CAAC,MAAM,oBAAoB,CAAC,aAAa,EAAE,mBAAmB,CAAC,CAAC,CAAC;QACnF,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAuB;SAClF,IAAI,EAAE;SACN,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CACjB,eAAe,CAAC,SAAS,EAAE,WAAW,EAAE,QAAQ,CAAC,UAAU,CAAC,UAAU,CAAC,SAAS,CAAE,CAAC,CACpF,CAAC;IACJ,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC;IAEnG,OAAO;QACL,OAAO,EAAE,cAAc;QACvB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,YAAY,EAAE,gBAAgB;QAC9B,QAAQ,EAAE,QAAQ,CAAC,EAAE;QACrB,aAAa,EAAE,QAAQ,CAAC,OAAO;QAC/B,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM;QAChC,aAAa,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC/C,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5D,QAAQ,EAAE,MAAM,CAAC,QAAQ;SAC1B,CAAC,CAAC;QACH,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,MAAM;QACrF,aAAa,EAAE,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,UAAU,CAAC,CAAC,MAAM;QACrF,WAAW;QACX,UAAU;QACV,YAAY,EAAE,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7C,mBAAmB,EAAE,QAAQ,CAAC,UAAU,CAAC,mBAAmB;QAC5D,MAAM,EAAE,YAAY,IAAI,QAAQ,CAAC,UAAU,CAAC,mBAAmB,IAAI,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;KAC7G,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { readFile, stat } from "node:fs/promises";
|
|
2
|
+
import { extname, resolve } from "node:path";
|
|
3
|
+
import { parse } from "yaml";
|
|
4
|
+
import { corpusManifestSchema } from "./schema.js";
|
|
5
|
+
const MAX_CORPUS_BYTES = 2 * 1024 * 1024;
|
|
6
|
+
export async function loadCorpusManifest(path) {
|
|
7
|
+
const resolvedPath = resolve(path);
|
|
8
|
+
if (!/^\.(?:json|ya?ml)$/i.test(extname(resolvedPath))) {
|
|
9
|
+
throw new Error("Corpus manifests must end in .json, .yaml, or .yml");
|
|
10
|
+
}
|
|
11
|
+
const fileStats = await stat(resolvedPath);
|
|
12
|
+
if (!fileStats.isFile())
|
|
13
|
+
throw new Error("Corpus manifest path is not a regular file");
|
|
14
|
+
if (fileStats.size > MAX_CORPUS_BYTES) {
|
|
15
|
+
throw new Error(`Corpus manifest exceeds ${MAX_CORPUS_BYTES} bytes`);
|
|
16
|
+
}
|
|
17
|
+
const content = await readFile(resolvedPath, "utf8");
|
|
18
|
+
return corpusManifestSchema.parse(extname(resolvedPath).toLowerCase() === ".json" ? JSON.parse(content) : parse(content));
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=loader.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loader.js","sourceRoot":"","sources":["../../../src/corpus/loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAE7C,OAAO,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,oBAAoB,EAAuB,MAAM,aAAa,CAAC;AAExE,MAAM,gBAAgB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AAEzC,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY;IACnD,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACnC,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC;IAC3C,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IACvF,IAAI,SAAS,CAAC,IAAI,GAAG,gBAAgB,EAAE,CAAC;QACtC,MAAM,IAAI,KAAK,CAAC,2BAA2B,gBAAgB,QAAQ,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IACrD,OAAO,oBAAoB,CAAC,KAAK,CAC/B,OAAO,CAAC,YAAY,CAAC,CAAC,WAAW,EAAE,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CACvF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
function percent(value) {
|
|
2
|
+
return `${(value * 100).toFixed(1)}%`;
|
|
3
|
+
}
|
|
4
|
+
export function formatCorpusReport(report) {
|
|
5
|
+
const lines = [
|
|
6
|
+
"# ZtotheZ Corpus Benchmark",
|
|
7
|
+
"",
|
|
8
|
+
`- Corpus: \`${report.corpusId}\` version ${report.corpusVersion}`,
|
|
9
|
+
`- Sources: ${report.sources}`,
|
|
10
|
+
`- Cases: ${report.caseResults.length} total, ${report.positiveCases} positive, ${report.negativeCases} negative`,
|
|
11
|
+
`- Overall: ${percent(report.overallScore)} required ${percent(report.overallMinimumScore)}`,
|
|
12
|
+
`- Result: ${report.passed ? "PASS" : "FAIL"}`,
|
|
13
|
+
"",
|
|
14
|
+
"## Dimensions",
|
|
15
|
+
"",
|
|
16
|
+
];
|
|
17
|
+
for (const dimension of report.dimensions) {
|
|
18
|
+
const rank = dimension.meanReciprocalRank === undefined
|
|
19
|
+
? ""
|
|
20
|
+
: `; MRR ${dimension.meanReciprocalRank.toFixed(4)} required ${(dimension.minimumMeanReciprocalRank ?? 0).toFixed(4)}`;
|
|
21
|
+
lines.push(`- ${dimension.passed ? "PASS" : "FAIL"} \`${dimension.dimension}\`: ${percent(dimension.score)} required ${percent(dimension.minimumScore)}; ${dimension.passedCases}/${dimension.cases} cases${rank}`);
|
|
22
|
+
}
|
|
23
|
+
lines.push("", "## Provenance", "");
|
|
24
|
+
for (const source of report.sourceRecords) {
|
|
25
|
+
lines.push(`- \`${source.id}\`: ${source.title}; ${source.origin}; owner ${source.owner}; license ${source.license}; evidence \`${source.evidence}\`${source.sourceUrl ? `; source ${source.sourceUrl}` : ""}`);
|
|
26
|
+
}
|
|
27
|
+
lines.push("", "## Cases", "");
|
|
28
|
+
for (const benchmarkCase of report.caseResults) {
|
|
29
|
+
lines.push(`### ${benchmarkCase.passed ? "PASS" : "FAIL"}: ${benchmarkCase.id}`, "", `- Dimension: ${benchmarkCase.dimension}`, `- Polarity: ${benchmarkCase.polarity}`, `- Source: ${benchmarkCase.source}`, `- Expected: ${benchmarkCase.expected}`, `- Observed: ${benchmarkCase.observed}`, "");
|
|
30
|
+
}
|
|
31
|
+
return lines.join("\n").trimEnd();
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=report.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/corpus/report.ts"],"names":[],"mappings":"AAEA,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,MAAoB;IACrD,MAAM,KAAK,GAAG;QACZ,4BAA4B;QAC5B,EAAE;QACF,eAAe,MAAM,CAAC,QAAQ,cAAc,MAAM,CAAC,aAAa,EAAE;QAClE,cAAc,MAAM,CAAC,OAAO,EAAE;QAC9B,YAAY,MAAM,CAAC,WAAW,CAAC,MAAM,WAAW,MAAM,CAAC,aAAa,cAAc,MAAM,CAAC,aAAa,WAAW;QACjH,cAAc,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,aAAa,OAAO,CAAC,MAAM,CAAC,mBAAmB,CAAC,EAAE;QAC5F,aAAa,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE;QAC9C,EAAE;QACF,eAAe;QACf,EAAE;KACH,CAAC;IAEF,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,SAAS,CAAC,kBAAkB,KAAK,SAAS;YACrD,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,SAAS,SAAS,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,SAAS,CAAC,yBAAyB,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzH,KAAK,CAAC,IAAI,CACR,KAAK,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,SAAS,CAAC,SAAS,OAAO,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,aAAa,OAAO,CAAC,SAAS,CAAC,YAAY,CAAC,KAAK,SAAS,CAAC,WAAW,IAAI,SAAS,CAAC,KAAK,SAAS,IAAI,EAAE,CACxM,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,eAAe,EAAE,EAAE,CAAC,CAAC;IACpC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;QAC1C,KAAK,CAAC,IAAI,CACR,OAAO,MAAM,CAAC,EAAE,OAAO,MAAM,CAAC,KAAK,KAAK,MAAM,CAAC,MAAM,WAAW,MAAM,CAAC,KAAK,aAAa,MAAM,CAAC,OAAO,gBAAgB,MAAM,CAAC,QAAQ,KAAK,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACpM,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,CAAC,CAAC;IAC/B,KAAK,MAAM,aAAa,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;QAC/C,KAAK,CAAC,IAAI,CACR,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,aAAa,CAAC,EAAE,EAAE,EACpE,EAAE,EACF,gBAAgB,aAAa,CAAC,SAAS,EAAE,EACzC,eAAe,aAAa,CAAC,QAAQ,EAAE,EACvC,aAAa,aAAa,CAAC,MAAM,EAAE,EACnC,eAAe,aAAa,CAAC,QAAQ,EAAE,EACvC,eAAe,aAAa,CAAC,QAAQ,EAAE,EACvC,EAAE,CACH,CAAC;IACJ,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;AACpC,CAAC"}
|