@mutagent/helix 0.1.0-alpha.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/mutagent-helix/SKILL.md +113 -0
- package/.claude/skills/mutagent-helix/scripts/cli/codex-transcode.ts +143 -0
- package/.claude/skills/mutagent-helix/scripts/cli/doctor.ts +111 -0
- package/.claude/skills/mutagent-helix/scripts/cli/init.ts +382 -0
- package/.claude/skills/mutagent-helix/scripts/cli/run.sh +61 -0
- package/CLAUDE.md +193 -0
- package/LICENSE +201 -0
- package/NOTICE +22 -0
- package/README.md +209 -0
- package/assets/adl-loop.svg +162 -0
- package/assets/system-status.svg +422 -0
- package/bin/mutagent-cli.mjs +9950 -0
- package/bin/mutagent-helix.mjs +78 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/CLAUDE.md +28 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/SKILL.md +138 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/assets/templates/agentspec.yaml.tpl +247 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/frameworks/doc-pins.md +53 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/principles.md +91 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/requirements.yaml +295 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/workflows/orchestrator-protocol.md +302 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/doctor.ts +116 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/init.ts +230 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/run.sh +60 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/config/resolve-spec-dir.ts +199 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/contract/agentspec.schema.ts +455 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/setup/detect.ts +67 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/validate/validate-spec.ts +93 -0
- package/mutagent-agentspec/LICENSE +201 -0
- package/mutagent-agentspec/NOTICE +19 -0
- package/mutagent-agentspec/README.md +44 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/SKILL.md +100 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/assets/agents/ai-architect.md +312 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/assets/agents/ai-engineer.md +357 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/assets/templates/build-report.md.tpl +39 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/references/principles.md +31 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/references/workflows/build-protocol.md +61 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/references/workflows/improve-handoff.md +23 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/cli/run.sh +60 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/handoff/validate-build-input.ts +72 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/sync-spec/check-sync-spec.ts +499 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/verify/spec-impl-coverage.ts +182 -0
- package/mutagent-builder/CLAUDE.md +25 -0
- package/mutagent-builder/README.md +26 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/README.md +192 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/SKILL.md +352 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/ai-architect.md +256 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/ai-engineer.md +344 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/diagnostics-analyzer.md +576 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/discovery.md +425 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/audit.json.tpl +24 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/audit.md.tpl +49 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/config.yaml.tpl +100 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/pr-body.md.tpl +56 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/report.html.tpl +327 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/README.md +27 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/iter-N-handover.md.tpl +125 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/self-diagnosis-contract.v0.1.0.yaml.tpl +234 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/spec.yaml.tpl +152 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/team.yaml.tpl +127 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/wave-N-dashboard.html.tpl +1712 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.1-nl-query.md +59 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.2-tier0-progress.md +58 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.3-analysis-progress.md +44 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.4-html-report.md +61 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.5-rca-detail.md +69 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.6-stale-warning.md +53 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.1-welcome.md +60 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.2-source-platform-picker.md +41 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.3-source-auth.md +75 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.4-target-picker.md +48 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.5-ask-tool.md +43 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.6-config-review.md +57 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.7-filter-defaults.md +52 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.8-first-run.md +59 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/examples/sample-findings.json +92 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/examples/sample-meta.json +47 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/adapter-strategy.md +75 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/config-migration.md +84 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/config.md +159 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/filter-search-matrix.md +66 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/harness-knowledge.md +22 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/memory-format.md +78 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/onboarding-decisions.yaml +187 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/operation-inventory.md +59 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/overview.md +167 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/reference.md +119 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/script-index.md +101 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-adapter-contract.md +93 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/claude-code-transcripts.md +66 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/codex-transcripts.md +78 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/codex.md +145 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/install-paths.md +57 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/langfuse.md +178 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/local-jsonl.md +55 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/openobserve.md +102 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/otel.md +63 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/cloud-rest.md +84 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-claude.md +65 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-cloud-agent-sdk.md +92 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-codex.md +72 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-cursor.md +108 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-mastra.md +77 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-opencode.md +70 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/apply-pr-comment-format.md +82 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/diagnostics.md +35 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/handover-contract.md +247 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/onboarding.md +300 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/orchestrator-protocol.md +1861 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/rca.md +331 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/rendering-anatomy.md +314 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/schedule-prep.md +100 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/scope-model.md +128 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/verification-methodology.md +174 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/aggregate/sort-findings.ts +0 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/awareness/blind-spots.ts +167 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/awareness/llm-sample.ts +681 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/doctor.ts +215 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/init.ts +1013 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/install-agents.ts +350 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/run.sh +61 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/shared-templates.ts +79 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/load.ts +392 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/resolve-credential.ts +120 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/schema.ts +641 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/validate.ts +134 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/context/build-diagnosis-context.ts +417 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/context/load-context.ts +105 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/contract/types.ts +166 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/build-render-input.ts +1964 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/corpus-severity.ts +172 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/dismissal-match.ts +270 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/rank-remedies.ts +75 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/hooks/install-hooks.sh +46 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/hooks/pre-push.sh +78 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/invocation/parse-brief.ts +188 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/match.ts +114 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/paths.ts +95 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/store.ts +535 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/types.ts +181 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/lint/template-inline-js.ts +267 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/memory/append.ts +188 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/memory/read.ts +218 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/platforms/entity-context.ts +686 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/read-unitf.ts +267 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/trace.ts +1078 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/unitf-adapter.ts +0 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/unitf-types.ts +213 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/report/persist-selections.ts +174 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/report/render.ts +3254 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/run/diagnose.ts +64 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/run/session.ts +167 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/caps.ts +345 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/deep-read-gate.ts +424 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/representative.ts +584 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/scan/objection.ts +224 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/scan/trajectory.ts +489 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/self-diagnostics/dispatch.ts +135 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/self-diagnostics/probe.ts +170 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/detect.ts +307 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/ensure-cli.ts +315 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/reconfigure.ts +77 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/verify-agents.ts +207 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/slicer.ts +166 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/claude-code.ts +17 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/codex.ts +11 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/langfuse.ts +17 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/local-jsonl.ts +12 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/otel.ts +12 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/index.ts +54 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/registry.ts +45 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/types.ts +86 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/stale-detector.ts +74 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/test/render-playwright.cli.ts +477 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0/claude-code.ts +133 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0/langfuse.ts +143 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0-scan.ts +423 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/completeness-check.ts +436 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/doc-refs.ts +190 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/finalize-gate.ts +417 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/findings-contract.ts +628 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/render-js-syntax.ts +188 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/report-checklist.yaml +217 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/trace-shape.ts +262 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/wave6-checklist.ts +326 -0
- package/mutagent-diagnostics/CLAUDE.md +41 -0
- package/mutagent-diagnostics/LICENSE +201 -0
- package/mutagent-diagnostics/NOTICE +18 -0
- package/mutagent-diagnostics/README.md +343 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/SKILL.md +548 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/audit-executor.md +169 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/dataset-builder.md +160 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/discovery.md +425 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/evaluator.md +1108 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/brand/theme.css +213 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/brand/wordmark.html +9 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/context-flow-lens.md +85 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/data-lens.md +47 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/decision-lens.md +48 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/methodology-critic-lens.md +53 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/trajectory-lens.md +44 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/build-review-interface.md +83 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/edd-loop.md +134 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/error-analysis.md +113 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/eval-audit.md +154 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/eval-stage.md +168 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/generate-synthetic-data.md +81 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/grounded-adjudication.md +221 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/memory-format.md +65 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/methodology.md +201 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/operation-inventory.md +197 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/validate-evaluator.md +125 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/workflows/orchestrator-protocol.md +300 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/write-judge-prompt.md +123 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/behavior-tree.schema.yaml +73 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/dataset.schema.yaml +66 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/edd-change-request.schema.yaml +114 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/eval-matrix.schema.yaml +74 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/flow-graph.schema.yaml +69 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/flow-profile.schema.yaml +49 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/methodology-review.schema.yaml +40 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/scorecard.schema.yaml +85 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/agent-dispatch.ts +0 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/aggregate-discover.ts +543 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/artifact-paths.ts +115 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/assemble-scorecard.ts +172 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-dataset.ts +186 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-evals.ts +93 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-review-ui.ts +393 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/check-method-router.ts +170 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/aggregate.ts +112 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/audit-run.ts +182 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/doctor.ts +211 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/dogfood.ts +133 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/init.ts +601 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/methodology-review.ts +122 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/prep.ts +295 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/profile-subject.ts +165 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/run.sh +56 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/variance-check.ts +105 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/code-eval.ts +248 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/code-quality-verdict.ts +285 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/codegen-evals.ts +173 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cold-start-project.ts +151 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cold-start-sampler.ts +267 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/config/load.ts +343 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/config/schema.ts +380 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts +85 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/dataset.ts +149 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-engine.ts +118 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-matrix.ts +577 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-types.ts +1074 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/flow-graph.ts +194 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/types.ts +303 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/validation.ts +193 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/derive-dataset.ts +152 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/determine-outcome.ts +219 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/diff-discriminate.ts +160 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/discover-criteria.ts +348 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/change-request.ts +232 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/edd-types.ts +210 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/variance-gate.ts +186 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/emit-completeness.ts +111 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/eval-engine.ts +160 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/evaluate.ts +333 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/flow-graph.ts +230 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/judge-prompt-template.ts +253 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/judge-provider.ts +135 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/lint-grounding.ts +229 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/lint-uniformity.ts +168 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/living-suite.ts +109 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-bundle.ts +203 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-profile-vocab.ts +64 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-profile.ts +106 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/mask.ts +138 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/materialize-dataset.ts +113 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/matrix-judge.ts +750 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/memory/append.ts +215 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/memory/read.ts +168 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/persist-eval-criteria.ts +232 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts +125 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/profile-subject.ts +310 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/publish-report.ts +131 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/read-unitf-traces.ts +81 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-build-cards.ts +195 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-discover-report.ts +1640 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-eval-report.ts +3838 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-report.ts +212 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/resolve-credential.ts +110 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/resolve-ref.ts +98 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/result-verify.ts +129 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/route-failures.ts +330 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-deterministic.ts +367 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-evaluate.ts +719 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-judge.ts +155 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-pipeline.ts +175 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/sample-traces.ts +210 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/self-audit.ts +387 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/source-map.ts +106 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/subject-profile.ts +134 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/substrate.ts +162 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/sync-eval-criteria.ts +244 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/ui-slots.ts +119 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/unitf-to-evaltrace.ts +284 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/validate-judge.ts +358 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/variance-compare.ts +177 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/behavior-tree.yaml +140 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/eval-matrix.yaml +1270 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/methodology-review.yaml +105 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/audit.workflow.js +82 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/data-leak.workflow.js +236 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/variance.workflow.js +163 -0
- package/mutagent-evaluator/CLAUDE.md +45 -0
- package/mutagent-evaluator/LICENSE +201 -0
- package/mutagent-evaluator/NOTICE +23 -0
- package/mutagent-evaluator/README.md +88 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/SKILL.md +287 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/contracts/amend-request.ts +503 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/improve-loop-run.ts +304 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/loop-state-cli.ts +300 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/loop-state.ts +440 -0
- package/mutagent-optimize/CLAUDE.md +49 -0
- package/mutagent-optimize/README.md +26 -0
- package/mutagent-orchestrator/CLAUDE.md +102 -0
- package/mutagent-orchestrator/README.md +20 -0
- package/mutagent-orchestrator/assets/agents/discovery.md +425 -0
- package/mutagent-orchestrator/assets/agents/dogfood-monitor.md +266 -0
- package/mutagent-orchestrator/assets/agents/monitor.md +258 -0
- package/mutagent-orchestrator/orchestrator.md +782 -0
- package/mutagent-orchestrator/references/MONITORING.md +209 -0
- package/mutagent-orchestrator/references/config-fields.md +132 -0
- package/mutagent-orchestrator/references/config-migration.md +132 -0
- package/mutagent-orchestrator/references/index-registry.md +92 -0
- package/mutagent-orchestrator/routing.yaml +391 -0
- package/mutagent-orchestrator/scripts/config-schema.ts +1010 -0
- package/mutagent-orchestrator/scripts/dispatch.ts +439 -0
- package/mutagent-orchestrator/scripts/dogfood/extract-feedback.ts +203 -0
- package/mutagent-orchestrator/scripts/dogfood/reconstruct-trajectory.ts +327 -0
- package/mutagent-orchestrator/scripts/dogfood/render-dogfood-report.ts +347 -0
- package/mutagent-orchestrator/scripts/dogfood/resolve-subagents.ts +160 -0
- package/mutagent-orchestrator/scripts/dogfood/run-report.ts +264 -0
- package/mutagent-orchestrator/scripts/dogfood/slack-thread.ts +106 -0
- package/mutagent-orchestrator/scripts/dogfood/types.ts +257 -0
- package/mutagent-orchestrator/scripts/eval-routing.ts +753 -0
- package/mutagent-orchestrator/scripts/gate.ts +219 -0
- package/mutagent-orchestrator/scripts/handover-contract.ts +362 -0
- package/mutagent-orchestrator/scripts/index/build-index.ts +351 -0
- package/mutagent-orchestrator/scripts/index/render-index.ts +110 -0
- package/mutagent-orchestrator/scripts/monitor/slack-notify.ts +111 -0
- package/mutagent-orchestrator/scripts/monitor/triggers.ts +162 -0
- package/mutagent-orchestrator/scripts/onboarding-check.ts +369 -0
- package/mutagent-orchestrator/scripts/render-roster.ts +308 -0
- package/mutagent-orchestrator/scripts/resolve-credential.ts +185 -0
- package/mutagent-orchestrator/scripts/resolve-paths.ts +114 -0
- package/mutagent-orchestrator/scripts/slack/format.ts +63 -0
- package/mutagent-orchestrator/scripts/slack/post.ts +115 -0
- package/mutagent-orchestrator/scripts/sync-index.ts +311 -0
- package/package.json +64 -0
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* scripts/substrate.ts — EV-050 judge-runtime selection.
|
|
3
|
+
* ---------------------------------------------------------------------------
|
|
4
|
+
* NOTE (v0.2.0 rename): `substrate` → `judge_runtime` in config + the public API
|
|
5
|
+
* (`resolveSubstrate`→`resolveJudgeRuntime`, `judgeForSubstrate`→`judgeForRuntime`).
|
|
6
|
+
* The enum VALUES are UNCHANGED (`agent-dispatch`/`in-house`/`code-based`/
|
|
7
|
+
* `user-framework`) and the EV-050 `Substrate` enum symbol (contracts/eval-types.ts)
|
|
8
|
+
* is kept — only the config KEY + the two selector functions were renamed.
|
|
9
|
+
*
|
|
10
|
+
* The W1 DEFAULT judge-runtime is **agent-dispatch** (scripts/agent-dispatch.ts):
|
|
11
|
+
* verdicts are produced by parent-session-dispatched eval-judge / error-analyst
|
|
12
|
+
* leaf subagents reasoning on the HOST runtime (Claude Code, diagnostics-style
|
|
13
|
+
* mass-dispatch) and read back from verdict FILES — the default path calls NO
|
|
14
|
+
* provider SDK. The onboarding choice (PRD §7.9) stays a real fork; this picks
|
|
15
|
+
* the DEFAULT the spine + dogfood build against, and STRUCTURES the rest as
|
|
16
|
+
* seams rather than hardcoding any single transport:
|
|
17
|
+
*
|
|
18
|
+
* - agent-dispatch (DEFAULT) → a verdict-file-backed JudgeInvoke; the LLM
|
|
19
|
+
* reasoning happens in dispatched subagents on the host,
|
|
20
|
+
* NOT in a script. Requires `verdictDir` (where the
|
|
21
|
+
* dispatched subagents wrote their verdict files).
|
|
22
|
+
* - in-house (OPTIONAL) → a live LLM JudgeInvoke (google-genai shape);
|
|
23
|
+
* kept, not deleted — DEMOTED from default. For the CI /
|
|
24
|
+
* code-based export path that wants a provider call.
|
|
25
|
+
* - code-based → objective checks over tool outputs; runs WITHOUT an LLM
|
|
26
|
+
* judge → judgeForRuntime THROWS (deterministic code).
|
|
27
|
+
* - user-framework → an EXPORT target (Vitest/promptfoo/Braintrust/…); the
|
|
28
|
+
* evaluator emits to it at onboarding → not a live judge.
|
|
29
|
+
*
|
|
30
|
+
* Why agent-dispatch is the default (operator correction 2026-06-19): the
|
|
31
|
+
* Evaluator's functions must be performed BY an Evaluator agent on Claude Code
|
|
32
|
+
* itself, mass-dispatched for throughput (mirror diagnostics), NOT by a script
|
|
33
|
+
* calling Gemini. In-house Gemini remains a real onboarding option (EV-050) but
|
|
34
|
+
* is no longer the spine default. Model intent stays sacred either way — the
|
|
35
|
+
* host's pinned model for agent-dispatch, the asserted provider model in-house.
|
|
36
|
+
*/
|
|
37
|
+
import { Substrate, type SubstrateValue } from "./contracts/eval-types.ts";
|
|
38
|
+
import { createInHouseJudge } from "./judge-provider.ts";
|
|
39
|
+
import { createAgentDispatchJudge } from "./agent-dispatch.ts";
|
|
40
|
+
import type { JudgeInvoke } from "./determine-outcome.ts";
|
|
41
|
+
|
|
42
|
+
const KNOWN_SUBSTRATES: ReadonlySet<string> = new Set([
|
|
43
|
+
Substrate.AgentDispatch,
|
|
44
|
+
Substrate.InHouse,
|
|
45
|
+
Substrate.UserFramework,
|
|
46
|
+
Substrate.CodeBased,
|
|
47
|
+
]);
|
|
48
|
+
|
|
49
|
+
/** Resolve the judge-runtime choice; default = agent-dispatch. Unknown → THROW (no silent default). */
|
|
50
|
+
export function resolveJudgeRuntime(choice?: string): SubstrateValue {
|
|
51
|
+
if (choice === undefined || choice === "") return Substrate.AgentDispatch;
|
|
52
|
+
if (!KNOWN_SUBSTRATES.has(choice)) {
|
|
53
|
+
throw new Error(
|
|
54
|
+
`resolveJudgeRuntime: unknown judge_runtime '${choice}'. Choose one of ` +
|
|
55
|
+
`{${[...KNOWN_SUBSTRATES].join(", ")}}.`,
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
return choice as SubstrateValue;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export interface SubstrateDescription {
|
|
62
|
+
kind: SubstrateValue;
|
|
63
|
+
/** true for transports that yield a live JudgeInvoke (agent-dispatch · in-house);
|
|
64
|
+
* false for the seams (code-based · user-framework). */
|
|
65
|
+
liveJudge: boolean;
|
|
66
|
+
/** the verdict transport: where the judge text comes from. */
|
|
67
|
+
transport: "agent-dispatch" | "provider-sdk" | "code" | "export";
|
|
68
|
+
/** true ONLY for the in-house provider-SDK path (the others never call a provider). */
|
|
69
|
+
callsProvider: boolean;
|
|
70
|
+
note: string;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Describe a substrate (which one provides a live judge, and via what transport). */
|
|
74
|
+
export function describeSubstrate(s: SubstrateValue): SubstrateDescription {
|
|
75
|
+
switch (s) {
|
|
76
|
+
case Substrate.AgentDispatch:
|
|
77
|
+
return {
|
|
78
|
+
kind: s,
|
|
79
|
+
liveJudge: true,
|
|
80
|
+
transport: "agent-dispatch",
|
|
81
|
+
callsProvider: false,
|
|
82
|
+
note: "Claude Code agent-dispatch (DEFAULT) — verdicts from host-runtime subagents (verdict files), no provider",
|
|
83
|
+
};
|
|
84
|
+
case Substrate.InHouse:
|
|
85
|
+
return {
|
|
86
|
+
kind: s,
|
|
87
|
+
liveJudge: true,
|
|
88
|
+
transport: "provider-sdk",
|
|
89
|
+
callsProvider: true,
|
|
90
|
+
note: "in-house AI-SDK judge (OPTIONAL — demoted from default; provider-SDK call)",
|
|
91
|
+
};
|
|
92
|
+
case Substrate.CodeBased:
|
|
93
|
+
return {
|
|
94
|
+
kind: s,
|
|
95
|
+
liveJudge: false,
|
|
96
|
+
transport: "code",
|
|
97
|
+
callsProvider: false,
|
|
98
|
+
note: "objective code-checks over tool outputs — no LLM judge",
|
|
99
|
+
};
|
|
100
|
+
case Substrate.UserFramework:
|
|
101
|
+
default:
|
|
102
|
+
return {
|
|
103
|
+
kind: s,
|
|
104
|
+
liveJudge: false,
|
|
105
|
+
transport: "export",
|
|
106
|
+
callsProvider: false,
|
|
107
|
+
note: "export target (user's eval framework) — emit, not a live judge",
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export interface SubstrateJudgeConfig {
|
|
113
|
+
substrate: SubstrateValue;
|
|
114
|
+
/** agent-dispatch: the dir the dispatched subagents wrote their verdict files into. */
|
|
115
|
+
verdictDir?: string;
|
|
116
|
+
/** in-house: the resolved (pinned) provider model. */
|
|
117
|
+
model?: string;
|
|
118
|
+
apiKey?: string;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Return a live `JudgeInvoke` for the judge-runtime.
|
|
123
|
+
* - agent-dispatch (DEFAULT) → a verdict-file-backed judge (requires verdictDir);
|
|
124
|
+
* NO provider is loaded.
|
|
125
|
+
* - in-house (OPTIONAL) → the google-genai judge (requires a resolved model —
|
|
126
|
+
* model intent sacred; createInHouseJudge asserts it).
|
|
127
|
+
* - code-based / user-framework → SEAMS; THROW (no live judge to return).
|
|
128
|
+
*/
|
|
129
|
+
export function judgeForRuntime(cfg: SubstrateJudgeConfig): JudgeInvoke {
|
|
130
|
+
if (cfg.substrate === Substrate.AgentDispatch) {
|
|
131
|
+
if (cfg.verdictDir === undefined || cfg.verdictDir === "") {
|
|
132
|
+
throw new Error(
|
|
133
|
+
"judgeForRuntime(agent-dispatch): a verdictDir is required — it is the " +
|
|
134
|
+
"directory the parent session's dispatched eval-judge / error-analyst " +
|
|
135
|
+
"subagents wrote their verdict files into. PREP → dispatch → collect, " +
|
|
136
|
+
"then aggregate (references/workflows/orchestrator-protocol.md).",
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
return createAgentDispatchJudge({ verdictDir: cfg.verdictDir });
|
|
140
|
+
}
|
|
141
|
+
if (cfg.substrate === Substrate.InHouse) {
|
|
142
|
+
if (cfg.model === undefined || cfg.model === "") {
|
|
143
|
+
throw new Error(
|
|
144
|
+
"judgeForRuntime(in-house): a model is required (model intent sacred). " +
|
|
145
|
+
"Resolve it via resolveJudgeModel() and pass it through.",
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
return createInHouseJudge({ model: cfg.model, ...(cfg.apiKey !== undefined ? { apiKey: cfg.apiKey } : {}) });
|
|
149
|
+
}
|
|
150
|
+
if (cfg.substrate === Substrate.CodeBased) {
|
|
151
|
+
throw new Error(
|
|
152
|
+
"judgeForRuntime(code-based): code-based evals run WITHOUT an LLM judge — " +
|
|
153
|
+
"there is no live judge to return. Run the deterministic code-checks " +
|
|
154
|
+
"instead (this is a seam, not a live-judge path).",
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
throw new Error(
|
|
158
|
+
"judgeForRuntime(user-framework): the user's framework is an EXPORT seam, " +
|
|
159
|
+
"not a live judge. Emit the suite to the framework at onboarding; do not " +
|
|
160
|
+
"expect a JudgeInvoke here.",
|
|
161
|
+
);
|
|
162
|
+
}
|
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* scripts/sync-eval-criteria.ts — the EVAL-LEG reconcile CONTRACT (Wave-2 W2I5 · KP-003).
|
|
3
|
+
* ---------------------------------------------------------------------------
|
|
4
|
+
* The THIRD leg of the def → impl → eval triad. Today `#sync-spec` reconciles two
|
|
5
|
+
* legs — spec ↔ impl. When an implementation amends (the ⑤ IMPROVE loop's ai-engineer,
|
|
6
|
+
* or a brownfield drift), the eval criteria that GROUND the subject's evaluation can go
|
|
7
|
+
* stale w.r.t. the changed impl. This module is the EVALUATOR-SIDE CONTRACT that the
|
|
8
|
+
* reconcile (ai-architect REASONING, session-dispatched — Model-B) READS and WRITES:
|
|
9
|
+
*
|
|
10
|
+
* 1. WHICH eval leg applies — the subject-kind resolver. The eval leg's shape differs
|
|
11
|
+
* by subject kind (this is why W2I5 was gated behind W2I1's code-quality leg):
|
|
12
|
+
* - agent / skill / composite subject → the EVAL-SUITE criteria (the evaluator's
|
|
13
|
+
* discovered / maintained criteria for that subject).
|
|
14
|
+
* - code subject → the CODE-QUALITY criteria (W2I1's `DEFAULT_CODE_QUALITY_CRITERIA`
|
|
15
|
+
* / `#mode-judge-code-quality`).
|
|
16
|
+
* 2. A deterministic STALENESS FLAG predicate — the eval-leg half of the drift trigger
|
|
17
|
+
* (mirrors the builder freshness probe's spec/code compare; same 3-value semantics).
|
|
18
|
+
* 3. The monotonic APPLY primitive — the ai-architect reasons a criteria delta; this
|
|
19
|
+
* applies it as criteria MAINTENANCE: upsert-by-id, append the novel, and NEVER drop
|
|
20
|
+
* an existing criterion (EV-053, guarded by `assertMonotonicGrowth`).
|
|
21
|
+
*
|
|
22
|
+
* JUDGE-ONLY PRESERVED (EV-051): this file NEVER scores a subject and NEVER decides a
|
|
23
|
+
* subject pass/fail. It FLAGS freshness (pure epoch compare) and GROWS a criteria set
|
|
24
|
+
* (pure set algebra). The reconcile REASONING — which criteria a changed impl makes stale,
|
|
25
|
+
* what new criterion it needs — is the ai-architect's, not this code's (Model-B: code =
|
|
26
|
+
* drift predicates + deterministic maintenance only, never agent dispatch).
|
|
27
|
+
*
|
|
28
|
+
* PURE + deterministic (C-PIN): no clock / random / network; same inputs → byte-identical.
|
|
29
|
+
*/
|
|
30
|
+
import { type Static, Type } from "@sinclair/typebox";
|
|
31
|
+
import { Value } from "@sinclair/typebox/value";
|
|
32
|
+
|
|
33
|
+
import { assertMonotonicGrowth } from "./living-suite.ts";
|
|
34
|
+
|
|
35
|
+
// ── subject kind → eval leg ──────────────────────────────────────────────────
|
|
36
|
+
|
|
37
|
+
/** The subject kinds the triad reconciles. `agent|skill|composite` come from the
|
|
38
|
+
* agentspec `definition.identity.kind`; `code` is the ⑤ IMPROVE code-target subject. */
|
|
39
|
+
export const EvalSubjectKind = {
|
|
40
|
+
Agent: "agent",
|
|
41
|
+
Skill: "skill",
|
|
42
|
+
Composite: "composite",
|
|
43
|
+
Code: "code",
|
|
44
|
+
} as const;
|
|
45
|
+
export type EvalSubjectKindValue = (typeof EvalSubjectKind)[keyof typeof EvalSubjectKind];
|
|
46
|
+
|
|
47
|
+
/** Which eval leg grounds a subject's evaluation — the W2I5 core rule. */
|
|
48
|
+
export const EvalLegKind = {
|
|
49
|
+
/** agent / skill / composite → the evaluator's discovered/maintained eval-suite criteria. */
|
|
50
|
+
EvalSuite: "eval-suite",
|
|
51
|
+
/** code → W2I1's code-quality criteria (`DEFAULT_CODE_QUALITY_CRITERIA`). */
|
|
52
|
+
CodeQuality: "code-quality",
|
|
53
|
+
} as const;
|
|
54
|
+
export type EvalLegKindValue = (typeof EvalLegKind)[keyof typeof EvalLegKind];
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Resolve WHICH eval leg a subject kind reconciles against. Code subjects use the
|
|
58
|
+
* code-quality criteria (a deterministic-test-gate PLUS a quality judge); every other
|
|
59
|
+
* kind uses the discovered eval-suite criteria. This single mapping is the shape
|
|
60
|
+
* differentiator the whole triad turns on.
|
|
61
|
+
*/
|
|
62
|
+
export function evalLegForSubjectKind(kind: EvalSubjectKindValue): EvalLegKindValue {
|
|
63
|
+
return kind === EvalSubjectKind.Code ? EvalLegKind.CodeQuality : EvalLegKind.EvalSuite;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// ── the staleness flag predicate (eval-leg half of the drift trigger) ────────
|
|
67
|
+
|
|
68
|
+
/** The eval-leg freshness verdict — mirrors the builder freshness probe's 3-value spec/code semantics. */
|
|
69
|
+
export const EvalLegDrift = {
|
|
70
|
+
/** no eval-criteria artifact yet — the eval leg must be CONSTRUCTED (cold). */
|
|
71
|
+
MissingEval: "missing-eval",
|
|
72
|
+
/** the eval criteria are as fresh as the impl — no reconcile write needed. */
|
|
73
|
+
InSync: "in-sync",
|
|
74
|
+
/** the impl is newer than the eval criteria — the eval leg is flagged for RECONCILE. */
|
|
75
|
+
NeedsSync: "needs-sync",
|
|
76
|
+
} as const;
|
|
77
|
+
export type EvalLegDriftValue = (typeof EvalLegDrift)[keyof typeof EvalLegDrift];
|
|
78
|
+
|
|
79
|
+
/** Effective freshness epochs (seconds) the flag predicate compares. `null` = unknown/absent. */
|
|
80
|
+
export interface EvalLegFreshnessInput {
|
|
81
|
+
/** effective freshness epoch of the eval-criteria artifact; `null` = artifact absent. */
|
|
82
|
+
evalCriteriaEpoch: number | null;
|
|
83
|
+
/** effective freshness epoch of the implementation; `null` = unknown. */
|
|
84
|
+
implEpoch: number | null;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* FLAG whether the eval leg is stale w.r.t. the impl. Deterministic — a pure epoch
|
|
89
|
+
* compare, identical in spirit to the builder `check-sync-spec` spec/code compare so the
|
|
90
|
+
* two legs agree on "impl is newer → reconcile". This is a FLAG only: it decides nothing
|
|
91
|
+
* about the subject and proposes no criteria — the reconcile reasoning is the agent's.
|
|
92
|
+
*/
|
|
93
|
+
export function flagEvalLegDrift(input: EvalLegFreshnessInput): EvalLegDriftValue {
|
|
94
|
+
if (input.evalCriteriaEpoch == null) return EvalLegDrift.MissingEval;
|
|
95
|
+
if (input.implEpoch != null && input.implEpoch > input.evalCriteriaEpoch) {
|
|
96
|
+
return EvalLegDrift.NeedsSync;
|
|
97
|
+
}
|
|
98
|
+
return EvalLegDrift.InSync;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** The resolved eval leg for a subject: which criteria kind + the current staleness flag. */
|
|
102
|
+
export interface EvalLegAssessment {
|
|
103
|
+
leg: EvalLegKindValue;
|
|
104
|
+
drift: EvalLegDriftValue;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Convenience: resolve the eval leg for a subject kind AND flag its staleness in one
|
|
109
|
+
* call — the exact pair the reconcile reads ("which leg, is it stale?").
|
|
110
|
+
*/
|
|
111
|
+
export function assessEvalLeg(
|
|
112
|
+
subjectKind: EvalSubjectKindValue,
|
|
113
|
+
freshness: EvalLegFreshnessInput,
|
|
114
|
+
): EvalLegAssessment {
|
|
115
|
+
return { leg: evalLegForSubjectKind(subjectKind), drift: flagEvalLegDrift(freshness) };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
// ── the criteria representation + the monotonic reconcile-apply ──────────────
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* One eval criterion — subject-kind-agnostic (a stable `id` key + the binary `statement`,
|
|
122
|
+
* plus the `leg` it belongs to and an optional severity). Serves BOTH legs: a code-quality
|
|
123
|
+
* `Qn` criterion (`severity` ∈ critical|high|medium) and an eval-suite discovered criterion
|
|
124
|
+
* flow through the same shape. `additionalProperties: true` keeps a richer criterion (e.g.
|
|
125
|
+
* a code-quality `passDefinition`/`failDefinition`, an eval-suite `checkMethod`) forward-parsing.
|
|
126
|
+
*/
|
|
127
|
+
export const EvalCriterionSchema = Type.Object(
|
|
128
|
+
{
|
|
129
|
+
id: Type.String({ minLength: 1 }),
|
|
130
|
+
statement: Type.String({ minLength: 1 }),
|
|
131
|
+
leg: Type.Union([Type.Literal(EvalLegKind.EvalSuite), Type.Literal(EvalLegKind.CodeQuality)]),
|
|
132
|
+
severity: Type.Optional(Type.String({ minLength: 1 })),
|
|
133
|
+
},
|
|
134
|
+
{ additionalProperties: true },
|
|
135
|
+
);
|
|
136
|
+
export type EvalCriterion = Static<typeof EvalCriterionSchema>;
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* The reconcile REQUEST the ai-architect emits: the persisted criteria `existing`, plus the
|
|
140
|
+
* agent-reasoned `proposed` additions/updates. `proposed` is an UPSERT set (append the novel,
|
|
141
|
+
* revise a same-id criterion's wording for the changed impl) — it is NEVER a delete list; the
|
|
142
|
+
* monotonic-growth invariant (EV-053) forbids dropping a criterion.
|
|
143
|
+
*/
|
|
144
|
+
export const EvalCriteriaReconcileRequestSchema = Type.Object(
|
|
145
|
+
{
|
|
146
|
+
subjectId: Type.String({ minLength: 1 }),
|
|
147
|
+
subjectKind: Type.Union([
|
|
148
|
+
Type.Literal(EvalSubjectKind.Agent),
|
|
149
|
+
Type.Literal(EvalSubjectKind.Skill),
|
|
150
|
+
Type.Literal(EvalSubjectKind.Composite),
|
|
151
|
+
Type.Literal(EvalSubjectKind.Code),
|
|
152
|
+
]),
|
|
153
|
+
leg: Type.Union([Type.Literal(EvalLegKind.EvalSuite), Type.Literal(EvalLegKind.CodeQuality)]),
|
|
154
|
+
existing: Type.Array(EvalCriterionSchema),
|
|
155
|
+
proposed: Type.Array(EvalCriterionSchema),
|
|
156
|
+
},
|
|
157
|
+
{ additionalProperties: false },
|
|
158
|
+
);
|
|
159
|
+
export type EvalCriteriaReconcileRequest = Static<typeof EvalCriteriaReconcileRequestSchema>;
|
|
160
|
+
|
|
161
|
+
/** Provenance for a reconcile — pure counters (NO clock, for byte-identity / C-PIN). */
|
|
162
|
+
export interface EvalCriteriaReconcileProvenance {
|
|
163
|
+
/** how many genuinely-novel criteria were appended. */
|
|
164
|
+
added: number;
|
|
165
|
+
/** how many existing criteria (same id) had their content revised. */
|
|
166
|
+
updated: number;
|
|
167
|
+
/** total criteria after the reconcile. */
|
|
168
|
+
total: number;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** The reconcile RESULT — the grown/maintained criteria set + provenance. */
|
|
172
|
+
export interface EvalCriteriaReconcileResult {
|
|
173
|
+
subjectId: string;
|
|
174
|
+
leg: EvalLegKindValue;
|
|
175
|
+
criteria: EvalCriterion[];
|
|
176
|
+
provenance: EvalCriteriaReconcileProvenance;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/** Parse + narrow a reconcile request (guarded). THROWS on schema violation. PURE. */
|
|
180
|
+
export function parseEvalCriteriaReconcileRequest(value: unknown): EvalCriteriaReconcileRequest {
|
|
181
|
+
if (!Value.Check(EvalCriteriaReconcileRequestSchema, value)) {
|
|
182
|
+
const first = [...Value.Errors(EvalCriteriaReconcileRequestSchema, value)][0];
|
|
183
|
+
throw new Error(
|
|
184
|
+
`parseEvalCriteriaReconcileRequest: schema violation at '${first?.path ?? "(root)"}': ` +
|
|
185
|
+
`${first?.message ?? "invalid eval-criteria reconcile request"}`,
|
|
186
|
+
);
|
|
187
|
+
}
|
|
188
|
+
return value;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const criterionId = (c: EvalCriterion): string => c.id;
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* APPLY the reconcile: maintain the eval-criteria set for a changed impl. Criteria
|
|
195
|
+
* MAINTENANCE, not judging (EV-051): upsert each `proposed` criterion by `id` (revise a
|
|
196
|
+
* same-id criterion's content in place, keeping the id), append the genuinely-novel, and
|
|
197
|
+
* assert the id-set NEVER shrank (EV-053, via the shared `assertMonotonicGrowth`). The
|
|
198
|
+
* `leg` on every proposed criterion must match the request's `leg` — mixing a code-quality
|
|
199
|
+
* criterion into an eval-suite reconcile (or vice versa) is a hard error, not a silent
|
|
200
|
+
* accept. DETERMINISTIC + PURE: existing order preserved, novel appended in proposal order;
|
|
201
|
+
* no clock / random. Same (existing, proposed) → byte-identical result.
|
|
202
|
+
*/
|
|
203
|
+
export function reconcileEvalCriteria(
|
|
204
|
+
request: EvalCriteriaReconcileRequest,
|
|
205
|
+
): EvalCriteriaReconcileResult {
|
|
206
|
+
const req = parseEvalCriteriaReconcileRequest(request);
|
|
207
|
+
|
|
208
|
+
for (const p of req.proposed) {
|
|
209
|
+
if (p.leg !== req.leg) {
|
|
210
|
+
throw new Error(
|
|
211
|
+
`reconcileEvalCriteria: proposed criterion '${p.id}' has leg '${p.leg}' but the ` +
|
|
212
|
+
`reconcile targets leg '${req.leg}'. An eval-suite and a code-quality criterion ` +
|
|
213
|
+
"are different legs — refusing to mix them (subject-kind integrity).",
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
// Insertion-ordered upsert: existing order first (content possibly revised), novel appended.
|
|
219
|
+
const byId = new Map<string, EvalCriterion>();
|
|
220
|
+
for (const c of req.existing) byId.set(c.id, c);
|
|
221
|
+
|
|
222
|
+
let added = 0;
|
|
223
|
+
let updated = 0;
|
|
224
|
+
for (const p of req.proposed) {
|
|
225
|
+
if (byId.has(p.id)) {
|
|
226
|
+
byId.set(p.id, p); // revise content, id (and thus the key/order) retained
|
|
227
|
+
updated += 1;
|
|
228
|
+
} else {
|
|
229
|
+
byId.set(p.id, p); // novel → appended after the existing keys
|
|
230
|
+
added += 1;
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
const criteria = [...byId.values()];
|
|
235
|
+
// EV-053: a maintained criteria set may GROW or revise, but never DROP a criterion.
|
|
236
|
+
assertMonotonicGrowth(req.existing, criteria, criterionId);
|
|
237
|
+
|
|
238
|
+
return {
|
|
239
|
+
subjectId: req.subjectId,
|
|
240
|
+
leg: req.leg,
|
|
241
|
+
criteria,
|
|
242
|
+
provenance: { added, updated, total: criteria.length },
|
|
243
|
+
};
|
|
244
|
+
}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* scripts/ui-slots.ts — EV-039/040 HTML-artifact missing-data audit (Code-only).
|
|
3
|
+
* ---------------------------------------------------------------------------
|
|
4
|
+
* The operator's first-class case: *"if a skill/agent produces an HTML artifact,
|
|
5
|
+
* data-leak is concerned with MISSING DATA in the UI representation."* A value
|
|
6
|
+
* the agent COMPUTED but did NOT render is a missing-data leak.
|
|
7
|
+
*
|
|
8
|
+
* This is the deterministic, SUBJECT-AGNOSTIC half of the EV-039/040 Hybrid: it
|
|
9
|
+
* cross-references a PROFILE-SUPPLIED expected-UI-slot list (EV-037
|
|
10
|
+
* `expectedUiSlots`, NOT the v1 diagnostics-specific hardcoded slot names) against
|
|
11
|
+
* the agent's computed values + the published HTML, and classifies each slot:
|
|
12
|
+
* - computed-but-not-rendered (EV-039) — a value computed yet ABSENT from the
|
|
13
|
+
* HTML → missing-data leak (locus UI, cls B). The operator's exact case;
|
|
14
|
+
* works on the HTML-only path (the v1 OUTSIDER FALLBACK promoted to
|
|
15
|
+
* first-class — no intermediate render-input file required).
|
|
16
|
+
* - orphan slot (EV-039) — the HTML references the slot but NO producer
|
|
17
|
+
* computed it → drawn-but-empty.
|
|
18
|
+
* - faithful — computed AND rendered verbatim → explicitly NOT a leak.
|
|
19
|
+
* Nuanced faithfulness (altered/truncated, EV-040) is left to the judge — this
|
|
20
|
+
* helper flags only verbatim presence/absence (it decides no severity).
|
|
21
|
+
*
|
|
22
|
+
* PURE + deterministic: no clock / random / network; same inputs → same audit.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
/** Classification of one expected UI slot. */
|
|
26
|
+
export const UiSlotKind = {
|
|
27
|
+
/** Computed by the agent but ABSENT from the rendered HTML (EV-039 leak). */
|
|
28
|
+
ComputedNotRendered: "computed-not-rendered",
|
|
29
|
+
/** Referenced in the HTML but NO producer computed it (EV-039 orphan). */
|
|
30
|
+
OrphanSlot: "orphan-slot",
|
|
31
|
+
/** Computed AND rendered verbatim — NOT a leak. */
|
|
32
|
+
Faithful: "faithful",
|
|
33
|
+
/** Neither computed nor referenced in the HTML — out of scope this run. */
|
|
34
|
+
Absent: "absent",
|
|
35
|
+
} as const;
|
|
36
|
+
export type UiSlotKindValue = (typeof UiSlotKind)[keyof typeof UiSlotKind];
|
|
37
|
+
|
|
38
|
+
export interface UiSlotFinding {
|
|
39
|
+
slot: string;
|
|
40
|
+
kind: UiSlotKindValue;
|
|
41
|
+
/** the computed value string (when the agent produced one). */
|
|
42
|
+
computedValue?: string;
|
|
43
|
+
/** whether the slot is referenced (by name) anywhere in the HTML. */
|
|
44
|
+
slotReferenced: boolean;
|
|
45
|
+
/** whether the computed value appears verbatim in the HTML. */
|
|
46
|
+
valueRendered: boolean;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface UiSlotAuditInput {
|
|
50
|
+
/** EV-037 profile-supplied expected UI slots (subject-agnostic). */
|
|
51
|
+
expectedSlots: string[];
|
|
52
|
+
/** slot -> the value the agent COMPUTED (from trace / runMeta / intermediate). */
|
|
53
|
+
computedSlots: Record<string, string>;
|
|
54
|
+
/** the published HTML artifact text (the HTML-only path is supported). */
|
|
55
|
+
renderedHtml: string;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export interface UiSlotAudit {
|
|
59
|
+
findings: UiSlotFinding[];
|
|
60
|
+
/** EV-039 leaks: computed but not in the HTML. */
|
|
61
|
+
computedNotRendered: string[];
|
|
62
|
+
/** EV-039 orphans: referenced in the HTML but never computed. */
|
|
63
|
+
orphanSlots: string[];
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Does the HTML reference this slot by name (a token match)? */
|
|
67
|
+
function slotReferenced(html: string, slot: string): boolean {
|
|
68
|
+
if (slot.length === 0) return false;
|
|
69
|
+
return html.includes(slot);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Does the computed value appear verbatim in the HTML? */
|
|
73
|
+
function valueRendered(html: string, value: string): boolean {
|
|
74
|
+
const v = value.trim();
|
|
75
|
+
if (v.length === 0) return false;
|
|
76
|
+
return html.includes(v);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Audit the expected UI slots against the computed values + the rendered HTML.
|
|
81
|
+
* PURE + deterministic. SUBJECT-AGNOSTIC — the slot list is supplied, never a
|
|
82
|
+
* module constant. Findings are emitted in `expectedSlots` order (stable).
|
|
83
|
+
*/
|
|
84
|
+
export function auditUiSlots(input: UiSlotAuditInput): UiSlotAudit {
|
|
85
|
+
const findings: UiSlotFinding[] = [];
|
|
86
|
+
const computedNotRendered: string[] = [];
|
|
87
|
+
const orphanSlots: string[] = [];
|
|
88
|
+
|
|
89
|
+
for (const slot of input.expectedSlots) {
|
|
90
|
+
const hasComputed = Object.prototype.hasOwnProperty.call(input.computedSlots, slot);
|
|
91
|
+
const computedValue = hasComputed ? input.computedSlots[slot] : undefined;
|
|
92
|
+
const referenced = slotReferenced(input.renderedHtml, slot);
|
|
93
|
+
const rendered =
|
|
94
|
+
computedValue !== undefined && valueRendered(input.renderedHtml, computedValue);
|
|
95
|
+
|
|
96
|
+
let kind: UiSlotKindValue;
|
|
97
|
+
if (hasComputed && !rendered) {
|
|
98
|
+
kind = UiSlotKind.ComputedNotRendered; // EV-039: computed but missing from HTML
|
|
99
|
+
computedNotRendered.push(slot);
|
|
100
|
+
} else if (!hasComputed && referenced) {
|
|
101
|
+
kind = UiSlotKind.OrphanSlot; // EV-039: drawn but no producer
|
|
102
|
+
orphanSlots.push(slot);
|
|
103
|
+
} else if (hasComputed && rendered) {
|
|
104
|
+
kind = UiSlotKind.Faithful;
|
|
105
|
+
} else {
|
|
106
|
+
kind = UiSlotKind.Absent;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
findings.push({
|
|
110
|
+
slot,
|
|
111
|
+
kind,
|
|
112
|
+
...(computedValue !== undefined ? { computedValue } : {}),
|
|
113
|
+
slotReferenced: referenced,
|
|
114
|
+
valueRendered: rendered,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
return { findings, computedNotRendered, orphanSlots };
|
|
119
|
+
}
|