@mutagent/helix 0.1.0-alpha.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/mutagent-helix/SKILL.md +113 -0
- package/.claude/skills/mutagent-helix/scripts/cli/codex-transcode.ts +143 -0
- package/.claude/skills/mutagent-helix/scripts/cli/doctor.ts +111 -0
- package/.claude/skills/mutagent-helix/scripts/cli/init.ts +382 -0
- package/.claude/skills/mutagent-helix/scripts/cli/run.sh +61 -0
- package/CLAUDE.md +193 -0
- package/LICENSE +201 -0
- package/NOTICE +22 -0
- package/README.md +209 -0
- package/assets/adl-loop.svg +162 -0
- package/assets/system-status.svg +422 -0
- package/bin/mutagent-cli.mjs +9950 -0
- package/bin/mutagent-helix.mjs +78 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/CLAUDE.md +28 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/SKILL.md +138 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/assets/templates/agentspec.yaml.tpl +247 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/frameworks/doc-pins.md +53 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/principles.md +91 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/requirements.yaml +295 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/workflows/orchestrator-protocol.md +302 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/doctor.ts +116 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/init.ts +230 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/run.sh +60 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/config/resolve-spec-dir.ts +199 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/contract/agentspec.schema.ts +455 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/setup/detect.ts +67 -0
- package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/validate/validate-spec.ts +93 -0
- package/mutagent-agentspec/LICENSE +201 -0
- package/mutagent-agentspec/NOTICE +19 -0
- package/mutagent-agentspec/README.md +44 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/SKILL.md +100 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/assets/agents/ai-architect.md +312 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/assets/agents/ai-engineer.md +357 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/assets/templates/build-report.md.tpl +39 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/references/principles.md +31 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/references/workflows/build-protocol.md +61 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/references/workflows/improve-handoff.md +23 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/cli/run.sh +60 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/handoff/validate-build-input.ts +72 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/sync-spec/check-sync-spec.ts +499 -0
- package/mutagent-builder/.claude/skills/mutagent-builder/scripts/verify/spec-impl-coverage.ts +182 -0
- package/mutagent-builder/CLAUDE.md +25 -0
- package/mutagent-builder/README.md +26 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/README.md +192 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/SKILL.md +352 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/ai-architect.md +256 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/ai-engineer.md +344 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/diagnostics-analyzer.md +576 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/discovery.md +425 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/audit.json.tpl +24 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/audit.md.tpl +49 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/config.yaml.tpl +100 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/pr-body.md.tpl +56 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/report.html.tpl +327 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/README.md +27 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/iter-N-handover.md.tpl +125 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/self-diagnosis-contract.v0.1.0.yaml.tpl +234 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/spec.yaml.tpl +152 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/team.yaml.tpl +127 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/wave-N-dashboard.html.tpl +1712 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.1-nl-query.md +59 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.2-tier0-progress.md +58 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.3-analysis-progress.md +44 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.4-html-report.md +61 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.5-rca-detail.md +69 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.6-stale-warning.md +53 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.1-welcome.md +60 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.2-source-platform-picker.md +41 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.3-source-auth.md +75 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.4-target-picker.md +48 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.5-ask-tool.md +43 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.6-config-review.md +57 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.7-filter-defaults.md +52 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.8-first-run.md +59 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/examples/sample-findings.json +92 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/examples/sample-meta.json +47 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/adapter-strategy.md +75 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/config-migration.md +84 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/config.md +159 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/filter-search-matrix.md +66 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/harness-knowledge.md +22 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/memory-format.md +78 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/onboarding-decisions.yaml +187 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/operation-inventory.md +59 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/overview.md +167 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/reference.md +119 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/script-index.md +101 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-adapter-contract.md +93 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/claude-code-transcripts.md +66 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/codex-transcripts.md +78 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/codex.md +145 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/install-paths.md +57 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/langfuse.md +178 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/local-jsonl.md +55 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/openobserve.md +102 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/otel.md +63 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/cloud-rest.md +84 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-claude.md +65 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-cloud-agent-sdk.md +92 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-codex.md +72 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-cursor.md +108 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-mastra.md +77 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-opencode.md +70 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/apply-pr-comment-format.md +82 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/diagnostics.md +35 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/handover-contract.md +247 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/onboarding.md +300 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/orchestrator-protocol.md +1861 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/rca.md +331 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/rendering-anatomy.md +314 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/schedule-prep.md +100 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/scope-model.md +128 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/verification-methodology.md +174 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/aggregate/sort-findings.ts +0 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/awareness/blind-spots.ts +167 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/awareness/llm-sample.ts +681 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/doctor.ts +215 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/init.ts +1013 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/install-agents.ts +350 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/run.sh +61 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/shared-templates.ts +79 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/load.ts +392 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/resolve-credential.ts +120 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/schema.ts +641 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/validate.ts +134 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/context/build-diagnosis-context.ts +417 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/context/load-context.ts +105 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/contract/types.ts +166 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/build-render-input.ts +1964 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/corpus-severity.ts +172 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/dismissal-match.ts +270 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/rank-remedies.ts +75 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/hooks/install-hooks.sh +46 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/hooks/pre-push.sh +78 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/invocation/parse-brief.ts +188 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/match.ts +114 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/paths.ts +95 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/store.ts +535 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/types.ts +181 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/lint/template-inline-js.ts +267 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/memory/append.ts +188 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/memory/read.ts +218 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/platforms/entity-context.ts +686 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/read-unitf.ts +267 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/trace.ts +1078 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/unitf-adapter.ts +0 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/unitf-types.ts +213 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/report/persist-selections.ts +174 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/report/render.ts +3254 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/run/diagnose.ts +64 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/run/session.ts +167 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/caps.ts +345 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/deep-read-gate.ts +424 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/representative.ts +584 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/scan/objection.ts +224 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/scan/trajectory.ts +489 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/self-diagnostics/dispatch.ts +135 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/self-diagnostics/probe.ts +170 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/detect.ts +307 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/ensure-cli.ts +315 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/reconfigure.ts +77 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/verify-agents.ts +207 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/slicer.ts +166 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/claude-code.ts +17 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/codex.ts +11 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/langfuse.ts +17 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/local-jsonl.ts +12 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/otel.ts +12 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/index.ts +54 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/registry.ts +45 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/types.ts +86 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/stale-detector.ts +74 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/test/render-playwright.cli.ts +477 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0/claude-code.ts +133 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0/langfuse.ts +143 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0-scan.ts +423 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/completeness-check.ts +436 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/doc-refs.ts +190 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/finalize-gate.ts +417 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/findings-contract.ts +628 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/render-js-syntax.ts +188 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/report-checklist.yaml +217 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/trace-shape.ts +262 -0
- package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/wave6-checklist.ts +326 -0
- package/mutagent-diagnostics/CLAUDE.md +41 -0
- package/mutagent-diagnostics/LICENSE +201 -0
- package/mutagent-diagnostics/NOTICE +18 -0
- package/mutagent-diagnostics/README.md +343 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/SKILL.md +548 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/audit-executor.md +169 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/dataset-builder.md +160 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/discovery.md +425 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/evaluator.md +1108 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/brand/theme.css +213 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/brand/wordmark.html +9 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/context-flow-lens.md +85 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/data-lens.md +47 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/decision-lens.md +48 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/methodology-critic-lens.md +53 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/trajectory-lens.md +44 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/build-review-interface.md +83 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/edd-loop.md +134 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/error-analysis.md +113 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/eval-audit.md +154 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/eval-stage.md +168 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/generate-synthetic-data.md +81 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/grounded-adjudication.md +221 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/memory-format.md +65 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/methodology.md +201 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/operation-inventory.md +197 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/validate-evaluator.md +125 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/workflows/orchestrator-protocol.md +300 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/write-judge-prompt.md +123 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/behavior-tree.schema.yaml +73 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/dataset.schema.yaml +66 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/edd-change-request.schema.yaml +114 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/eval-matrix.schema.yaml +74 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/flow-graph.schema.yaml +69 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/flow-profile.schema.yaml +49 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/methodology-review.schema.yaml +40 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/scorecard.schema.yaml +85 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/agent-dispatch.ts +0 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/aggregate-discover.ts +543 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/artifact-paths.ts +115 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/assemble-scorecard.ts +172 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-dataset.ts +186 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-evals.ts +93 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-review-ui.ts +393 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/check-method-router.ts +170 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/aggregate.ts +112 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/audit-run.ts +182 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/doctor.ts +211 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/dogfood.ts +133 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/init.ts +601 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/methodology-review.ts +122 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/prep.ts +295 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/profile-subject.ts +165 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/run.sh +56 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/variance-check.ts +105 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/code-eval.ts +248 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/code-quality-verdict.ts +285 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/codegen-evals.ts +173 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cold-start-project.ts +151 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cold-start-sampler.ts +267 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/config/load.ts +343 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/config/schema.ts +380 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts +85 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/dataset.ts +149 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-engine.ts +118 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-matrix.ts +577 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-types.ts +1074 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/flow-graph.ts +194 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/types.ts +303 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/validation.ts +193 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/derive-dataset.ts +152 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/determine-outcome.ts +219 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/diff-discriminate.ts +160 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/discover-criteria.ts +348 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/change-request.ts +232 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/edd-types.ts +210 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/variance-gate.ts +186 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/emit-completeness.ts +111 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/eval-engine.ts +160 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/evaluate.ts +333 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/flow-graph.ts +230 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/judge-prompt-template.ts +253 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/judge-provider.ts +135 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/lint-grounding.ts +229 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/lint-uniformity.ts +168 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/living-suite.ts +109 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-bundle.ts +203 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-profile-vocab.ts +64 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-profile.ts +106 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/mask.ts +138 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/materialize-dataset.ts +113 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/matrix-judge.ts +750 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/memory/append.ts +215 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/memory/read.ts +168 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/persist-eval-criteria.ts +232 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts +125 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/profile-subject.ts +310 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/publish-report.ts +131 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/read-unitf-traces.ts +81 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-build-cards.ts +195 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-discover-report.ts +1640 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-eval-report.ts +3838 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-report.ts +212 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/resolve-credential.ts +110 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/resolve-ref.ts +98 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/result-verify.ts +129 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/route-failures.ts +330 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-deterministic.ts +367 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-evaluate.ts +719 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-judge.ts +155 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-pipeline.ts +175 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/sample-traces.ts +210 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/self-audit.ts +387 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/source-map.ts +106 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/subject-profile.ts +134 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/substrate.ts +162 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/sync-eval-criteria.ts +244 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/ui-slots.ts +119 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/unitf-to-evaltrace.ts +284 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/validate-judge.ts +358 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/variance-compare.ts +177 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/behavior-tree.yaml +140 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/eval-matrix.yaml +1270 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/methodology-review.yaml +105 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/audit.workflow.js +82 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/data-leak.workflow.js +236 -0
- package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/variance.workflow.js +163 -0
- package/mutagent-evaluator/CLAUDE.md +45 -0
- package/mutagent-evaluator/LICENSE +201 -0
- package/mutagent-evaluator/NOTICE +23 -0
- package/mutagent-evaluator/README.md +88 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/SKILL.md +287 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/contracts/amend-request.ts +503 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/improve-loop-run.ts +304 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/loop-state-cli.ts +300 -0
- package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/loop-state.ts +440 -0
- package/mutagent-optimize/CLAUDE.md +49 -0
- package/mutagent-optimize/README.md +26 -0
- package/mutagent-orchestrator/CLAUDE.md +102 -0
- package/mutagent-orchestrator/README.md +20 -0
- package/mutagent-orchestrator/assets/agents/discovery.md +425 -0
- package/mutagent-orchestrator/assets/agents/dogfood-monitor.md +266 -0
- package/mutagent-orchestrator/assets/agents/monitor.md +258 -0
- package/mutagent-orchestrator/orchestrator.md +782 -0
- package/mutagent-orchestrator/references/MONITORING.md +209 -0
- package/mutagent-orchestrator/references/config-fields.md +132 -0
- package/mutagent-orchestrator/references/config-migration.md +132 -0
- package/mutagent-orchestrator/references/index-registry.md +92 -0
- package/mutagent-orchestrator/routing.yaml +391 -0
- package/mutagent-orchestrator/scripts/config-schema.ts +1010 -0
- package/mutagent-orchestrator/scripts/dispatch.ts +439 -0
- package/mutagent-orchestrator/scripts/dogfood/extract-feedback.ts +203 -0
- package/mutagent-orchestrator/scripts/dogfood/reconstruct-trajectory.ts +327 -0
- package/mutagent-orchestrator/scripts/dogfood/render-dogfood-report.ts +347 -0
- package/mutagent-orchestrator/scripts/dogfood/resolve-subagents.ts +160 -0
- package/mutagent-orchestrator/scripts/dogfood/run-report.ts +264 -0
- package/mutagent-orchestrator/scripts/dogfood/slack-thread.ts +106 -0
- package/mutagent-orchestrator/scripts/dogfood/types.ts +257 -0
- package/mutagent-orchestrator/scripts/eval-routing.ts +753 -0
- package/mutagent-orchestrator/scripts/gate.ts +219 -0
- package/mutagent-orchestrator/scripts/handover-contract.ts +362 -0
- package/mutagent-orchestrator/scripts/index/build-index.ts +351 -0
- package/mutagent-orchestrator/scripts/index/render-index.ts +110 -0
- package/mutagent-orchestrator/scripts/monitor/slack-notify.ts +111 -0
- package/mutagent-orchestrator/scripts/monitor/triggers.ts +162 -0
- package/mutagent-orchestrator/scripts/onboarding-check.ts +369 -0
- package/mutagent-orchestrator/scripts/render-roster.ts +308 -0
- package/mutagent-orchestrator/scripts/resolve-credential.ts +185 -0
- package/mutagent-orchestrator/scripts/resolve-paths.ts +114 -0
- package/mutagent-orchestrator/scripts/slack/format.ts +63 -0
- package/mutagent-orchestrator/scripts/slack/post.ts +115 -0
- package/mutagent-orchestrator/scripts/sync-index.ts +311 -0
- package/package.json +64 -0
|
@@ -0,0 +1,348 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* scripts/discover-criteria.ts — EV-041 `*discover-evals`.
|
|
3
|
+
* ---------------------------------------------------------------------------
|
|
4
|
+
* Derive emergent BINARY ACTIONABLE eval criteria from LABELED ✓/✗ traces.
|
|
5
|
+
*
|
|
6
|
+
* The clustering itself is the LLM error-analyst's job (deep-read each trace,
|
|
7
|
+
* note the FIRST thing that went wrong, assign an emergent category — never a
|
|
8
|
+
* pre-defined list; error-analysis CORE §2/§3). This is the PURE, deterministic
|
|
9
|
+
* remainder of `*discover-evals`:
|
|
10
|
+
* - aggregate annotations → one criterion per emergent category (support)
|
|
11
|
+
* - compute per-category failure rates (prioritize what matters)
|
|
12
|
+
* - FLAG fixable (infra/dependency) vs eval-worthy (behavioral). Fixables are
|
|
13
|
+
* ROUTED to diagnostics (EV-051), NEVER fixed and NEVER judged on the model.
|
|
14
|
+
* - detect saturation: stop when no NEW category emerges across rounds.
|
|
15
|
+
*
|
|
16
|
+
* "a judge is only a judge" — this module FLAGS, it does not fix. There is no
|
|
17
|
+
* `resolve`/`fix` field on a DiscoveredCriterion by construction.
|
|
18
|
+
*
|
|
19
|
+
* PURE + deterministic: criteria ranked by support desc then id; no clock/random.
|
|
20
|
+
*/
|
|
21
|
+
import {
|
|
22
|
+
AssumptionKind,
|
|
23
|
+
CheckMethod,
|
|
24
|
+
CriterionFlag,
|
|
25
|
+
Generality,
|
|
26
|
+
Grounding,
|
|
27
|
+
JudgeKind,
|
|
28
|
+
MetricDimension,
|
|
29
|
+
MetricLevel,
|
|
30
|
+
MetricSubstrate,
|
|
31
|
+
OutcomeVerdict,
|
|
32
|
+
Severity,
|
|
33
|
+
assertGroundingHonest,
|
|
34
|
+
type CheckMethodValue,
|
|
35
|
+
type CriterionFlagValue,
|
|
36
|
+
type DiscoveredCriterion,
|
|
37
|
+
type DiscoveryRef,
|
|
38
|
+
type GeneralityValue,
|
|
39
|
+
type GroundingValue,
|
|
40
|
+
type JudgeKindValue,
|
|
41
|
+
type MetricDimensionValue,
|
|
42
|
+
type MetricLevelValue,
|
|
43
|
+
type MetricMetadata,
|
|
44
|
+
type MetricSubstrateValue,
|
|
45
|
+
type MinedCriterion,
|
|
46
|
+
type DiscoveryRationale,
|
|
47
|
+
type OutcomeVerdictValue,
|
|
48
|
+
type SeverityValue,
|
|
49
|
+
} from "./contracts/eval-types.ts";
|
|
50
|
+
import type { CodeEvalSpec } from "./code-eval.ts";
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* One annotation: the error-analyst's deep-read of a single trace. For a
|
|
54
|
+
* failure it carries the first-thing-that-went-wrong `note` + the emergent
|
|
55
|
+
* `category` + a classification (`infra` => fixable, else behavioral).
|
|
56
|
+
* `statement`/`judgeKind`/`judgeInputs` are the analyst's criterion proposal,
|
|
57
|
+
* carried on the first annotation that defines a category.
|
|
58
|
+
*/
|
|
59
|
+
export interface TraceAnnotation {
|
|
60
|
+
traceId: string;
|
|
61
|
+
label: OutcomeVerdictValue;
|
|
62
|
+
category?: string;
|
|
63
|
+
note?: string;
|
|
64
|
+
failureClass?: "infra" | "behavioral";
|
|
65
|
+
judgeKind?: JudgeKindValue;
|
|
66
|
+
statement?: string;
|
|
67
|
+
judgeInputs?: string[];
|
|
68
|
+
/**
|
|
69
|
+
* §5c evidence ref — a concrete `trace-id/field` pointer the analyst cited for
|
|
70
|
+
* THIS trace's failure. When present it is used as the grounding ref; else the
|
|
71
|
+
* traceId itself is the ref. ADDITIVE (P2b).
|
|
72
|
+
*/
|
|
73
|
+
evidencePointer?: string;
|
|
74
|
+
/**
|
|
75
|
+
* GA-1 — STRUCTURED grounding refs `{obs,path,value}` the `#mode-discover` leaf
|
|
76
|
+
* cited for THIS trace's failure. PREFERRED over `evidencePointer` (which is
|
|
77
|
+
* synthesized into a best-effort structured ref when `refs` is absent). The
|
|
78
|
+
* `value` is the EXACT cited text so the ref re-resolves against the trace.
|
|
79
|
+
*/
|
|
80
|
+
refs?: DiscoveryRef[];
|
|
81
|
+
/**
|
|
82
|
+
* Optional §5b metadata PROPOSALS the `#mode-discover` leaf may emit (additive
|
|
83
|
+
* agent-contract extension). When present they override the generic defaults
|
|
84
|
+
* `deriveMinedCriteria` would otherwise compute; when absent, defaults apply.
|
|
85
|
+
*/
|
|
86
|
+
dimension?: MetricDimensionValue;
|
|
87
|
+
level?: MetricLevelValue;
|
|
88
|
+
generality?: GeneralityValue;
|
|
89
|
+
severity?: SeverityValue;
|
|
90
|
+
appliesTo?: string[];
|
|
91
|
+
/**
|
|
92
|
+
* The UNIFORM-STANDARD executable code-check the `#mode-discover` leaf EMITS when
|
|
93
|
+
* a mined criterion is deterministically checkable (it picks a registry primitive
|
|
94
|
+
* + field/params and tags the category `code`/`hybrid`). Carried onto the
|
|
95
|
+
* `MinedCriterion.codeEval` so the tier-0 pre-pass runs it (zero judge tokens).
|
|
96
|
+
* Absent for a pure `llm-judge` category. ADDITIVE.
|
|
97
|
+
*/
|
|
98
|
+
codeEval?: CodeEvalSpec;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** Group annotations by their emergent category (uncategorized are dropped). */
|
|
102
|
+
export function groupByCategory(
|
|
103
|
+
annotations: TraceAnnotation[],
|
|
104
|
+
): Map<string, TraceAnnotation[]> {
|
|
105
|
+
const groups = new Map<string, TraceAnnotation[]>();
|
|
106
|
+
for (const a of annotations) {
|
|
107
|
+
if (a.category === undefined || a.category.length === 0) continue;
|
|
108
|
+
const list = groups.get(a.category) ?? [];
|
|
109
|
+
list.push(a);
|
|
110
|
+
groups.set(a.category, list);
|
|
111
|
+
}
|
|
112
|
+
return groups;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
export interface CategoryRate {
|
|
116
|
+
category: string;
|
|
117
|
+
fails: number;
|
|
118
|
+
total: number;
|
|
119
|
+
rate: number;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Per-category failure rate, ranked by fails desc then category asc. */
|
|
123
|
+
export function failureRates(annotations: TraceAnnotation[]): CategoryRate[] {
|
|
124
|
+
const groups = groupByCategory(annotations);
|
|
125
|
+
const rates: CategoryRate[] = [];
|
|
126
|
+
for (const [category, list] of groups) {
|
|
127
|
+
const fails = list.filter((a) => a.label === OutcomeVerdict.Fail).length;
|
|
128
|
+
const total = list.length;
|
|
129
|
+
rates.push({ category, fails, total, rate: total === 0 ? 0 : fails / total });
|
|
130
|
+
}
|
|
131
|
+
return rates.sort((a, b) =>
|
|
132
|
+
b.fails - a.fails !== 0 ? b.fails - a.fails : a.category.localeCompare(b.category),
|
|
133
|
+
);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** Pick the first non-empty value of `key` across a category's annotations. */
|
|
137
|
+
function firstDefined<K extends keyof TraceAnnotation>(
|
|
138
|
+
list: TraceAnnotation[],
|
|
139
|
+
key: K,
|
|
140
|
+
): TraceAnnotation[K] | undefined {
|
|
141
|
+
for (const a of list) {
|
|
142
|
+
if (a[key] !== undefined) return a[key];
|
|
143
|
+
}
|
|
144
|
+
return undefined;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** A category is FIXABLE iff any of its failures is classified infra. */
|
|
148
|
+
function flagFor(list: TraceAnnotation[]): CriterionFlagValue {
|
|
149
|
+
const anyInfra = list.some(
|
|
150
|
+
(a) => a.label === OutcomeVerdict.Fail && a.failureClass === "infra",
|
|
151
|
+
);
|
|
152
|
+
return anyInfra ? CriterionFlag.Fixable : CriterionFlag.EvalWorthy;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Derive one BINARY ACTIONABLE criterion per emergent category. Ranked by
|
|
157
|
+
* support (member count) desc then id asc. PURE — no fix is applied; the
|
|
158
|
+
* returned objects carry no resolution field.
|
|
159
|
+
*/
|
|
160
|
+
export function deriveCriteria(annotations: TraceAnnotation[]): DiscoveredCriterion[] {
|
|
161
|
+
const groups = groupByCategory(annotations);
|
|
162
|
+
const out: DiscoveredCriterion[] = [];
|
|
163
|
+
for (const [category, list] of groups) {
|
|
164
|
+
const statement = firstDefined(list, "statement") ?? `behavior is correct for: ${category}`;
|
|
165
|
+
const judgeKind = firstDefined(list, "judgeKind") ?? JudgeKind.Llm;
|
|
166
|
+
const judgeInputs = firstDefined(list, "judgeInputs") ?? ["prompt", "trajectory", "response"];
|
|
167
|
+
out.push({
|
|
168
|
+
id: category,
|
|
169
|
+
statement,
|
|
170
|
+
judgeInputs,
|
|
171
|
+
judgeKind,
|
|
172
|
+
flag: flagFor(list),
|
|
173
|
+
supportCount: list.length,
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
return out.sort((a, b) =>
|
|
177
|
+
b.supportCount - a.supportCount !== 0
|
|
178
|
+
? b.supportCount - a.supportCount
|
|
179
|
+
: a.id.localeCompare(b.id),
|
|
180
|
+
);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// ════════════════════════════════════════════════════════════════════════════
|
|
184
|
+
// P2 / P2b — mined criteria with §5b metadata + §5c DR-2 (evidence-first)
|
|
185
|
+
// ════════════════════════════════════════════════════════════════════════════
|
|
186
|
+
|
|
187
|
+
/** Map the analyst's JudgeKind → the §5b check_method router (SF-2). */
|
|
188
|
+
function checkMethodFor(judgeKind: JudgeKindValue): CheckMethodValue {
|
|
189
|
+
if (judgeKind === JudgeKind.Code) return CheckMethod.Deterministic;
|
|
190
|
+
if (judgeKind === JudgeKind.Hybrid) return CheckMethod.Hybrid;
|
|
191
|
+
return CheckMethod.LlmJudge;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** §5b substrate is DERIVED from check_method: deterministic→code, else judge. */
|
|
195
|
+
function substrateFor(checkMethod: CheckMethodValue): MetricSubstrateValue {
|
|
196
|
+
return checkMethod === CheckMethod.Deterministic ? MetricSubstrate.Code : MetricSubstrate.Judge;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** Generic generality heuristic: a code/deterministic check is general-structural
|
|
200
|
+
* (reusable on any pipeline); a judge check is specific-semantic. The leaf may
|
|
201
|
+
* override via `annotation.generality`. */
|
|
202
|
+
function generalityFor(checkMethod: CheckMethodValue): GeneralityValue {
|
|
203
|
+
return checkMethod === CheckMethod.Deterministic
|
|
204
|
+
? Generality.GeneralStructural
|
|
205
|
+
: Generality.SpecificSemantic;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Generic severity heuristic from observed failure count (leaf may override). */
|
|
209
|
+
function severityFor(fails: number): SeverityValue {
|
|
210
|
+
if (fails >= 2) return Severity.High;
|
|
211
|
+
if (fails === 1) return Severity.Med;
|
|
212
|
+
return Severity.Low;
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
/** Map the criterion flag → the §5c fix_or_eval routing string. */
|
|
216
|
+
function fixOrEvalFor(flag: CriterionFlagValue): DiscoveryRationale["fix_or_eval"] {
|
|
217
|
+
return flag === CriterionFlag.Fixable ? "fixable->diagnostics" : "eval-worthy";
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/** Synthesize a best-effort structured ref from a legacy `trace-id/field`
|
|
221
|
+
* pointer string + note (GA-1 back-compat: the leaf may still emit a pointer). */
|
|
222
|
+
function synthRef(a: TraceAnnotation): DiscoveryRef {
|
|
223
|
+
const ptr = a.evidencePointer ?? a.traceId;
|
|
224
|
+
const slash = ptr.indexOf("/");
|
|
225
|
+
const obs = slash >= 0 ? ptr.slice(0, slash) : a.traceId;
|
|
226
|
+
const path = slash >= 0 ? ptr.slice(slash + 1) : "";
|
|
227
|
+
// value = the exact cited text (note) when present, else the pointer itself.
|
|
228
|
+
const value = a.note !== undefined && a.note.length > 0 ? a.note : ptr;
|
|
229
|
+
return { obs, path, value };
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** GA-1 — the STRUCTURED evidence refs for a category = the failing annotations'
|
|
233
|
+
* cited structured refs (preferred), else a synthesized ref from the legacy
|
|
234
|
+
* pointer. REAL refs, in input order. */
|
|
235
|
+
function refsFor(list: TraceAnnotation[]): DiscoveryRef[] {
|
|
236
|
+
const out: DiscoveryRef[] = [];
|
|
237
|
+
for (const a of list) {
|
|
238
|
+
if (a.label !== OutcomeVerdict.Fail) continue;
|
|
239
|
+
if (a.refs !== undefined && a.refs.length > 0) out.push(...a.refs);
|
|
240
|
+
else out.push(synthRef(a));
|
|
241
|
+
}
|
|
242
|
+
return out;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Derive the full set of MINED criteria from real annotations: the existing
|
|
247
|
+
* `deriveCriteria` output ENRICHED with the §5b unified metadata + the §5c DR-2
|
|
248
|
+
* discovery-rationale, grounded in the actual support set. EVIDENCE-FIRST:
|
|
249
|
+
* - OBSERVED iff a failure was actually seen (fails>0) — cites the real failing
|
|
250
|
+
* refs + an honest `k/n sampled` prevalence; never promotes inferred→observed.
|
|
251
|
+
* - a pass-only category is INFERRED (good-practice guard, no failure yet).
|
|
252
|
+
* PURE + deterministic (same ranking as deriveCriteria: support desc, id asc).
|
|
253
|
+
*/
|
|
254
|
+
export function deriveMinedCriteria(annotations: TraceAnnotation[]): MinedCriterion[] {
|
|
255
|
+
const groups = groupByCategory(annotations);
|
|
256
|
+
const base = deriveCriteria(annotations); // ranked; carries id/statement/judgeKind/judgeInputs/flag/support
|
|
257
|
+
return base.map((c) => {
|
|
258
|
+
const list = groups.get(c.id) ?? [];
|
|
259
|
+
const fails = list.filter((a) => a.label === OutcomeVerdict.Fail).length;
|
|
260
|
+
const total = list.length;
|
|
261
|
+
const refs = refsFor(list);
|
|
262
|
+
const firstNote = firstDefined(list, "note");
|
|
263
|
+
|
|
264
|
+
const checkMethod = checkMethodFor(c.judgeKind);
|
|
265
|
+
const grounding: GroundingValue = fails > 0 && refs.length > 0 ? Grounding.Observed : Grounding.Inferred;
|
|
266
|
+
const severity = firstDefined(list, "severity") ?? severityFor(fails);
|
|
267
|
+
|
|
268
|
+
const metadata: MetricMetadata = {
|
|
269
|
+
generality: firstDefined(list, "generality") ?? generalityFor(checkMethod),
|
|
270
|
+
dimension: firstDefined(list, "dimension") ?? MetricDimension.OperationCorrectness,
|
|
271
|
+
level: firstDefined(list, "level") ?? MetricLevel.Output,
|
|
272
|
+
check_method: checkMethod,
|
|
273
|
+
substrate: substrateFor(checkMethod),
|
|
274
|
+
applies_to: firstDefined(list, "appliesTo") ?? [],
|
|
275
|
+
severity,
|
|
276
|
+
judge_inputs: c.judgeInputs,
|
|
277
|
+
flag: c.flag,
|
|
278
|
+
};
|
|
279
|
+
|
|
280
|
+
const discovery: DiscoveryRationale = {
|
|
281
|
+
targets: `the '${c.id}' behavior/failure mode`,
|
|
282
|
+
why_problem:
|
|
283
|
+
fails > 0
|
|
284
|
+
? `failure mode observed${firstNote !== undefined ? ` (first-thing-wrong: ${firstNote})` : ""}`
|
|
285
|
+
: "good-practice guard — no failure observed in the sample yet",
|
|
286
|
+
evidence: {
|
|
287
|
+
grounding,
|
|
288
|
+
seen_in_traces: fails > 0,
|
|
289
|
+
prevalence: `${fails}/${total} sampled`,
|
|
290
|
+
refs: grounding === Grounding.Observed ? refs : [],
|
|
291
|
+
},
|
|
292
|
+
reasoning:
|
|
293
|
+
`${fails}/${total} trace(s) in category '${c.id}'` +
|
|
294
|
+
`${firstNote !== undefined ? `; first-thing-wrong: ${firstNote}` : ""} → criterion`,
|
|
295
|
+
assumptions:
|
|
296
|
+
grounding === Grounding.Observed
|
|
297
|
+
? [
|
|
298
|
+
{
|
|
299
|
+
text: "the observed failure recurs under the same template",
|
|
300
|
+
status: "unverified",
|
|
301
|
+
kind: AssumptionKind.FactualIntent,
|
|
302
|
+
},
|
|
303
|
+
]
|
|
304
|
+
: [
|
|
305
|
+
{
|
|
306
|
+
text: "preventive guard; no failure observed yet",
|
|
307
|
+
status: "hypothesis",
|
|
308
|
+
kind: AssumptionKind.Normative,
|
|
309
|
+
},
|
|
310
|
+
],
|
|
311
|
+
expected_impact: {
|
|
312
|
+
severity,
|
|
313
|
+
coverage: `${total} sampled trace(s)`,
|
|
314
|
+
confidence: grounding === Grounding.Observed ? (fails >= 2 ? "high" : "med") : "low",
|
|
315
|
+
},
|
|
316
|
+
fix_or_eval: fixOrEvalFor(c.flag),
|
|
317
|
+
};
|
|
318
|
+
|
|
319
|
+
// UNIFORM STANDARD — carry the leaf-emitted executable code-check (when the
|
|
320
|
+
// category is deterministically checkable) onto the criterion so the tier-0
|
|
321
|
+
// pre-pass can run it. A pure judge category leaves it undefined.
|
|
322
|
+
const codeEval = firstDefined(list, "codeEval");
|
|
323
|
+
|
|
324
|
+
const mined: MinedCriterion = {
|
|
325
|
+
...c,
|
|
326
|
+
metadata,
|
|
327
|
+
discovery,
|
|
328
|
+
...(codeEval !== undefined ? { codeEval } : {}),
|
|
329
|
+
};
|
|
330
|
+
assertGroundingHonest(mined); // defensive — never ship inferred-as-observed
|
|
331
|
+
return mined;
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* Saturation: the next round surfaced NO new category. `prev` is the cumulative
|
|
337
|
+
* category set; `next` is the latest round's categories. Saturated iff every
|
|
338
|
+
* category in `next` is already in `prev`.
|
|
339
|
+
*/
|
|
340
|
+
export function isSaturated(prev: string[], next: string[]): boolean {
|
|
341
|
+
const seen = new Set(prev);
|
|
342
|
+
return next.every((c) => seen.has(c));
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/** The error-analysis target band: aim for 5-10 distinct categories. */
|
|
346
|
+
export function withinSaturationBand(categories: string[]): boolean {
|
|
347
|
+
return categories.length >= 5 && categories.length <= 10;
|
|
348
|
+
}
|
|
@@ -0,0 +1,232 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* scripts/edd/change-request.ts — F18 EDD CLOSURE (the spec↔impl↔eval triad).
|
|
3
|
+
* ---------------------------------------------------------------------------
|
|
4
|
+
* The Evaluator is JUDGE-ONLY (EV-051): it NEVER fixes the subject. To close the
|
|
5
|
+
* EDD loop it instead REQUESTS the AI-Engineer (the *build ACTOR from the SPEC
|
|
6
|
+
* stage) to amend the Agent / AgentSpec — wired over the `SendMessage` tool both
|
|
7
|
+
* agents carry. This module is the structured contract for that hand-off:
|
|
8
|
+
*
|
|
9
|
+
* evaluator --(EddChangeRequest)--> ai-engineer (the failing cases + remedy target)
|
|
10
|
+
* ai-engineer --(ChangeRequestResponse)--> evaluator (amended | rejected)
|
|
11
|
+
* evaluator re-evaluates → loop until full green OR a bounded STOP.
|
|
12
|
+
*
|
|
13
|
+
* The contract is the SEAM: the evaluator emits a request (validated here), the
|
|
14
|
+
* engineer consumes it (validated here), and the loop controller (also here)
|
|
15
|
+
* decides — using ONLY observable state — whether to swing again or terminate.
|
|
16
|
+
* The termination gate is afkloop-legal: full-green is the success terminator, and
|
|
17
|
+
* three bounds (max-swings · max-wallclock · no-improvement-streak) guarantee the
|
|
18
|
+
* loop is NEVER infinite.
|
|
19
|
+
*
|
|
20
|
+
* PURE — no clock / Math.random / network. The wall-clock bound is checked against
|
|
21
|
+
* an INJECTED elapsed value (the caller measures time; this module only decides),
|
|
22
|
+
* keeping the controller deterministic + unit-testable (C-PIN).
|
|
23
|
+
*/
|
|
24
|
+
import { Value } from "@sinclair/typebox/value";
|
|
25
|
+
import {
|
|
26
|
+
ChangeRequestStatus,
|
|
27
|
+
EddPhase,
|
|
28
|
+
EddStopReason,
|
|
29
|
+
EddChangeRequestSchema,
|
|
30
|
+
ChangeRequestResponseSchema,
|
|
31
|
+
RemedyTarget,
|
|
32
|
+
type EddChangeRequestType,
|
|
33
|
+
type ChangeRequestResponseType,
|
|
34
|
+
type EddPhaseValue,
|
|
35
|
+
type EddStopReasonValue,
|
|
36
|
+
type RemedyTargetValue,
|
|
37
|
+
} from "./edd-types.ts";
|
|
38
|
+
|
|
39
|
+
// Re-export the Static types under their public names (edd-types declares the schemas).
|
|
40
|
+
export type { EddChangeRequestType, ChangeRequestResponseType };
|
|
41
|
+
|
|
42
|
+
// ── Validators (fail-loud over the SendMessage seam) ────────────────────────
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Validate an evaluator→engineer change-request. THROWS with the first TypeBox
|
|
46
|
+
* error path on any violation — a malformed request must never cross the seam
|
|
47
|
+
* silently (an ungrounded ask, a missing remedy target, etc. fail loud here).
|
|
48
|
+
*/
|
|
49
|
+
export function validateChangeRequest(req: unknown): EddChangeRequestType {
|
|
50
|
+
if (!Value.Check(EddChangeRequestSchema, req)) {
|
|
51
|
+
const first = [...Value.Errors(EddChangeRequestSchema, req)][0];
|
|
52
|
+
throw new Error(
|
|
53
|
+
`EddChangeRequest invalid${first ? ` at ${first.path}: ${first.message}` : ""}`,
|
|
54
|
+
);
|
|
55
|
+
}
|
|
56
|
+
// Belt-and-braces: every failing case must be GROUNDED (≥1 ref). The schema
|
|
57
|
+
// enforces minItems:1 on refs, but assert the intent explicitly for the seam.
|
|
58
|
+
for (const fc of req.failingCases) {
|
|
59
|
+
if (fc.refs.length === 0) {
|
|
60
|
+
throw new Error(
|
|
61
|
+
`EddChangeRequest invalid: failing case "${fc.caseId}" has no grounding refs (ungrounded ask — EV-051/GA-1)`,
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return req;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Validate an engineer→evaluator response. THROWS on a malformed reply. A
|
|
70
|
+
* `rejected` status MUST carry a `note` (the reason) — enforced by the schema's
|
|
71
|
+
* minLength on `note`; an `amended` status SHOULD echo `amendedTarget`.
|
|
72
|
+
*/
|
|
73
|
+
export function validateChangeResponse(res: unknown): ChangeRequestResponseType {
|
|
74
|
+
if (!Value.Check(ChangeRequestResponseSchema, res)) {
|
|
75
|
+
const first = [...Value.Errors(ChangeRequestResponseSchema, res)][0];
|
|
76
|
+
throw new Error(
|
|
77
|
+
`ChangeRequestResponse invalid${first ? ` at ${first.path}: ${first.message}` : ""}`,
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
return res;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Does this response warrant a re-eval? Only an `amended` response does — a
|
|
85
|
+
* `rejected` one means the engineer declined (the evaluator must re-target or
|
|
86
|
+
* escalate, NOT re-eval an unchanged subject). There is no "trust me, skip
|
|
87
|
+
* re-eval" path: an amend ALWAYS triggers a fresh evaluation swing.
|
|
88
|
+
*/
|
|
89
|
+
export function reEvalWarranted(res: ChangeRequestResponseType): boolean {
|
|
90
|
+
return res.status === ChangeRequestStatus.Amended;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// ── F18 loop control — the bounded EDD terminator ───────────────────────────
|
|
94
|
+
|
|
95
|
+
export interface EddLoopBudget {
|
|
96
|
+
/** hard cap on evaluator→engineer→re-eval swings. */
|
|
97
|
+
maxSwings: number;
|
|
98
|
+
/** hard wall-clock budget in ms (checked against an injected elapsed). */
|
|
99
|
+
maxWallclockMs: number;
|
|
100
|
+
/** stop after this many consecutive swings with no measurable improvement. */
|
|
101
|
+
noImprovementStreakLimit: number;
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export const DEFAULT_EDD_LOOP_BUDGET: EddLoopBudget = {
|
|
105
|
+
maxSwings: 6,
|
|
106
|
+
maxWallclockMs: 30 * 60 * 1000, // 30 min
|
|
107
|
+
noImprovementStreakLimit: 2,
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* The observable loop state the controller reasons over. ALL fields are measured
|
|
112
|
+
* by the caller (the evaluator parent session) and handed in — the controller is
|
|
113
|
+
* a PURE decision function, so the same state always yields the same decision.
|
|
114
|
+
*/
|
|
115
|
+
export interface EddLoopState {
|
|
116
|
+
/** the current EDD phase (variance vs accuracy vs build). */
|
|
117
|
+
phase: EddPhaseValue;
|
|
118
|
+
/** 1-based count of swings COMPLETED so far. */
|
|
119
|
+
swing: number;
|
|
120
|
+
/** F19 — is the per-case variance gate currently passing? */
|
|
121
|
+
varianceStable: boolean;
|
|
122
|
+
/** is accuracy ≥ target over the dataset? (meaningful only post-variance). */
|
|
123
|
+
accuracyMet: boolean;
|
|
124
|
+
/** measured elapsed wall-clock for the whole loop, ms (injected — not read here). */
|
|
125
|
+
elapsedMs: number;
|
|
126
|
+
/**
|
|
127
|
+
* consecutive swings with no measurable improvement (the caller computes this
|
|
128
|
+
* from a monotone progress metric — e.g. fewer flapping/failing cases). 0 ⇒
|
|
129
|
+
* the last swing improved something.
|
|
130
|
+
*/
|
|
131
|
+
noImprovementStreak: number;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export interface EddLoopDecision {
|
|
135
|
+
/** true ⇒ terminate now; false ⇒ swing again. */
|
|
136
|
+
done: boolean;
|
|
137
|
+
/** WHY it stopped — set iff done; null while the loop should continue. */
|
|
138
|
+
reason: EddStopReasonValue | null;
|
|
139
|
+
/** the next phase to run (DONE/STOPPED when terminating). */
|
|
140
|
+
nextPhase: EddPhaseValue;
|
|
141
|
+
/** human one-liner — the convergence delta when bounded, the green when done. */
|
|
142
|
+
detail: string;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* THE termination gate (afkloop-legal — observable + bounded + no infinite loop).
|
|
147
|
+
*
|
|
148
|
+
* Order of checks (success first, then the three bounds):
|
|
149
|
+
* 1. FULL GREEN — varianceStable AND accuracyMet ⇒ DONE (success).
|
|
150
|
+
* 2. MAX SWINGS — swing ≥ budget.maxSwings ⇒ STOP (report delta).
|
|
151
|
+
* 3. MAX WALLCLOCK— elapsedMs ≥ budget.maxWallclockMs ⇒ STOP.
|
|
152
|
+
* 4. NO-IMPROVE — noImprovementStreak ≥ budget.noImprovementStreakLimit ⇒ STOP.
|
|
153
|
+
* else CONTINUE — keep swinging; the next phase is variance until it's stable,
|
|
154
|
+
* then accuracy (F19 ordering is preserved by `nextPhaseAfterVariance`, but the
|
|
155
|
+
* loop controller mirrors it: not-stable ⇒ variance, stable ⇒ accuracy).
|
|
156
|
+
*
|
|
157
|
+
* PURE: no clock read — `elapsedMs` is injected. Same state → same decision.
|
|
158
|
+
*/
|
|
159
|
+
export function decideEddLoop(
|
|
160
|
+
state: EddLoopState,
|
|
161
|
+
budget: EddLoopBudget = DEFAULT_EDD_LOOP_BUDGET,
|
|
162
|
+
): EddLoopDecision {
|
|
163
|
+
// 1. SUCCESS terminator — full green.
|
|
164
|
+
if (state.varianceStable && state.accuracyMet) {
|
|
165
|
+
return {
|
|
166
|
+
done: true,
|
|
167
|
+
reason: EddStopReason.FullGreen,
|
|
168
|
+
nextPhase: EddPhase.Done,
|
|
169
|
+
detail: `full green after ${state.swing} swing(s): variance stable AND accuracy met`,
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
// 2-4. Bounded non-convergence terminators (each reports the convergence delta).
|
|
173
|
+
if (state.swing >= budget.maxSwings) {
|
|
174
|
+
return stopped(
|
|
175
|
+
EddStopReason.MaxSwings,
|
|
176
|
+
`hit max swings (${state.swing}/${budget.maxSwings}); varianceStable=${state.varianceStable} accuracyMet=${state.accuracyMet}`,
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
if (state.elapsedMs >= budget.maxWallclockMs) {
|
|
180
|
+
return stopped(
|
|
181
|
+
EddStopReason.MaxWallclock,
|
|
182
|
+
`hit wall-clock budget (${state.elapsedMs}ms/${budget.maxWallclockMs}ms); varianceStable=${state.varianceStable} accuracyMet=${state.accuracyMet}`,
|
|
183
|
+
);
|
|
184
|
+
}
|
|
185
|
+
if (state.noImprovementStreak >= budget.noImprovementStreakLimit) {
|
|
186
|
+
return stopped(
|
|
187
|
+
EddStopReason.NoImprovementStreak,
|
|
188
|
+
`no improvement for ${state.noImprovementStreak} consecutive swing(s) (limit ${budget.noImprovementStreakLimit})`,
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
// CONTINUE — preserve F19 ordering: stabilize variance before accuracy.
|
|
192
|
+
const nextPhase = state.varianceStable ? EddPhase.Accuracy : EddPhase.Variance;
|
|
193
|
+
return {
|
|
194
|
+
done: false,
|
|
195
|
+
reason: null,
|
|
196
|
+
nextPhase,
|
|
197
|
+
detail: `continue: next phase ${nextPhase} (swing ${state.swing}, variance ${state.varianceStable ? "stable" : "flapping"})`,
|
|
198
|
+
};
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function stopped(reason: EddStopReasonValue, detail: string): EddLoopDecision {
|
|
202
|
+
return { done: true, reason, nextPhase: EddPhase.Stopped, detail };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Build a well-formed change-request for the evaluator to SendMessage. Validates
|
|
207
|
+
* before returning so a malformed request can never leave the evaluator. The
|
|
208
|
+
* caller supplies the swing index + the grounded failing cases + the proposed
|
|
209
|
+
* remedy + the target; this stamps `status: requested`.
|
|
210
|
+
*/
|
|
211
|
+
export function buildChangeRequest(input: {
|
|
212
|
+
requestId: string;
|
|
213
|
+
swing: number;
|
|
214
|
+
subject: string;
|
|
215
|
+
remedyTarget: RemedyTargetValue;
|
|
216
|
+
failingCases: EddChangeRequestType["failingCases"];
|
|
217
|
+
proposedRemedy: string;
|
|
218
|
+
}): EddChangeRequestType {
|
|
219
|
+
const req: EddChangeRequestType = {
|
|
220
|
+
requestId: input.requestId,
|
|
221
|
+
swing: input.swing,
|
|
222
|
+
subject: input.subject,
|
|
223
|
+
remedyTarget: input.remedyTarget,
|
|
224
|
+
failingCases: input.failingCases,
|
|
225
|
+
proposedRemedy: input.proposedRemedy,
|
|
226
|
+
status: ChangeRequestStatus.Requested,
|
|
227
|
+
};
|
|
228
|
+
return validateChangeRequest(req);
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
/** Convenience: the two remedy targets, re-exported for callers building requests. */
|
|
232
|
+
export { RemedyTarget };
|