@mutagent/helix 0.1.0-alpha.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (357) hide show
  1. package/.claude/skills/mutagent-helix/SKILL.md +113 -0
  2. package/.claude/skills/mutagent-helix/scripts/cli/codex-transcode.ts +143 -0
  3. package/.claude/skills/mutagent-helix/scripts/cli/doctor.ts +111 -0
  4. package/.claude/skills/mutagent-helix/scripts/cli/init.ts +382 -0
  5. package/.claude/skills/mutagent-helix/scripts/cli/run.sh +61 -0
  6. package/CLAUDE.md +193 -0
  7. package/LICENSE +201 -0
  8. package/NOTICE +22 -0
  9. package/README.md +209 -0
  10. package/assets/adl-loop.svg +162 -0
  11. package/assets/system-status.svg +422 -0
  12. package/bin/mutagent-cli.mjs +9950 -0
  13. package/bin/mutagent-helix.mjs +78 -0
  14. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/CLAUDE.md +28 -0
  15. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/SKILL.md +138 -0
  16. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/assets/templates/agentspec.yaml.tpl +247 -0
  17. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/frameworks/doc-pins.md +53 -0
  18. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/principles.md +91 -0
  19. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/requirements.yaml +295 -0
  20. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/references/workflows/orchestrator-protocol.md +302 -0
  21. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/doctor.ts +116 -0
  22. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/init.ts +230 -0
  23. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/cli/run.sh +60 -0
  24. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/config/resolve-spec-dir.ts +199 -0
  25. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/contract/agentspec.schema.ts +455 -0
  26. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/setup/detect.ts +67 -0
  27. package/mutagent-agentspec/.claude/skills/mutagent-agentspec/scripts/validate/validate-spec.ts +93 -0
  28. package/mutagent-agentspec/LICENSE +201 -0
  29. package/mutagent-agentspec/NOTICE +19 -0
  30. package/mutagent-agentspec/README.md +44 -0
  31. package/mutagent-builder/.claude/skills/mutagent-builder/SKILL.md +100 -0
  32. package/mutagent-builder/.claude/skills/mutagent-builder/assets/agents/ai-architect.md +312 -0
  33. package/mutagent-builder/.claude/skills/mutagent-builder/assets/agents/ai-engineer.md +357 -0
  34. package/mutagent-builder/.claude/skills/mutagent-builder/assets/templates/build-report.md.tpl +39 -0
  35. package/mutagent-builder/.claude/skills/mutagent-builder/references/principles.md +31 -0
  36. package/mutagent-builder/.claude/skills/mutagent-builder/references/workflows/build-protocol.md +61 -0
  37. package/mutagent-builder/.claude/skills/mutagent-builder/references/workflows/improve-handoff.md +23 -0
  38. package/mutagent-builder/.claude/skills/mutagent-builder/scripts/cli/run.sh +60 -0
  39. package/mutagent-builder/.claude/skills/mutagent-builder/scripts/handoff/validate-build-input.ts +72 -0
  40. package/mutagent-builder/.claude/skills/mutagent-builder/scripts/sync-spec/check-sync-spec.ts +499 -0
  41. package/mutagent-builder/.claude/skills/mutagent-builder/scripts/verify/spec-impl-coverage.ts +182 -0
  42. package/mutagent-builder/CLAUDE.md +25 -0
  43. package/mutagent-builder/README.md +26 -0
  44. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/README.md +192 -0
  45. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/SKILL.md +352 -0
  46. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/ai-architect.md +256 -0
  47. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/ai-engineer.md +344 -0
  48. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/diagnostics-analyzer.md +576 -0
  49. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/agents/discovery.md +425 -0
  50. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/audit.json.tpl +24 -0
  51. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/audit.md.tpl +49 -0
  52. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/config.yaml.tpl +100 -0
  53. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/pr-body.md.tpl +56 -0
  54. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/report.html.tpl +327 -0
  55. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/README.md +27 -0
  56. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/iter-N-handover.md.tpl +125 -0
  57. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/self-diagnosis-contract.v0.1.0.yaml.tpl +234 -0
  58. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/spec.yaml.tpl +152 -0
  59. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/team.yaml.tpl +127 -0
  60. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/templates/shared/wave-N-dashboard.html.tpl +1712 -0
  61. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.1-nl-query.md +59 -0
  62. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.2-tier0-progress.md +58 -0
  63. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.3-analysis-progress.md +44 -0
  64. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.4-html-report.md +61 -0
  65. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.5-rca-detail.md +69 -0
  66. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/diagnostics/WF-2.6-stale-warning.md +53 -0
  67. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.1-welcome.md +60 -0
  68. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.2-source-platform-picker.md +41 -0
  69. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.3-source-auth.md +75 -0
  70. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.4-target-picker.md +48 -0
  71. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.5-ask-tool.md +43 -0
  72. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.6-config-review.md +57 -0
  73. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.7-filter-defaults.md +52 -0
  74. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/assets/wireframes/onboarding/WF-1.8-first-run.md +59 -0
  75. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/examples/sample-findings.json +92 -0
  76. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/examples/sample-meta.json +47 -0
  77. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/adapter-strategy.md +75 -0
  78. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/config-migration.md +84 -0
  79. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/config.md +159 -0
  80. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/filter-search-matrix.md +66 -0
  81. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/harness-knowledge.md +22 -0
  82. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/memory-format.md +78 -0
  83. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/onboarding-decisions.yaml +187 -0
  84. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/operation-inventory.md +59 -0
  85. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/overview.md +167 -0
  86. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/reference.md +119 -0
  87. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/script-index.md +101 -0
  88. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-adapter-contract.md +93 -0
  89. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/claude-code-transcripts.md +66 -0
  90. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/codex-transcripts.md +78 -0
  91. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/codex.md +145 -0
  92. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/install-paths.md +57 -0
  93. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/langfuse.md +178 -0
  94. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/local-jsonl.md +55 -0
  95. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/openobserve.md +102 -0
  96. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/source-platforms/otel.md +63 -0
  97. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/cloud-rest.md +84 -0
  98. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-claude.md +65 -0
  99. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-cloud-agent-sdk.md +92 -0
  100. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-codex.md +72 -0
  101. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-cursor.md +108 -0
  102. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-mastra.md +77 -0
  103. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/target-platforms/local-opencode.md +70 -0
  104. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/apply-pr-comment-format.md +82 -0
  105. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/diagnostics.md +35 -0
  106. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/handover-contract.md +247 -0
  107. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/onboarding.md +300 -0
  108. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/orchestrator-protocol.md +1861 -0
  109. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/rca.md +331 -0
  110. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/rendering-anatomy.md +314 -0
  111. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/schedule-prep.md +100 -0
  112. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/scope-model.md +128 -0
  113. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/references/workflows/verification-methodology.md +174 -0
  114. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/aggregate/sort-findings.ts +0 -0
  115. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/awareness/blind-spots.ts +167 -0
  116. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/awareness/llm-sample.ts +681 -0
  117. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/doctor.ts +215 -0
  118. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/init.ts +1013 -0
  119. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/install-agents.ts +350 -0
  120. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/run.sh +61 -0
  121. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/cli/shared-templates.ts +79 -0
  122. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/load.ts +392 -0
  123. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/resolve-credential.ts +120 -0
  124. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/schema.ts +641 -0
  125. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/config/validate.ts +134 -0
  126. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/context/build-diagnosis-context.ts +417 -0
  127. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/context/load-context.ts +105 -0
  128. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/contract/types.ts +166 -0
  129. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/build-render-input.ts +1964 -0
  130. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/corpus-severity.ts +172 -0
  131. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/dismissal-match.ts +270 -0
  132. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/enrich/rank-remedies.ts +75 -0
  133. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/hooks/install-hooks.sh +46 -0
  134. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/hooks/pre-push.sh +78 -0
  135. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/invocation/parse-brief.ts +188 -0
  136. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/match.ts +114 -0
  137. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/paths.ts +95 -0
  138. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/store.ts +535 -0
  139. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/library/types.ts +181 -0
  140. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/lint/template-inline-js.ts +267 -0
  141. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/memory/append.ts +188 -0
  142. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/memory/read.ts +218 -0
  143. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/platforms/entity-context.ts +686 -0
  144. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/read-unitf.ts +267 -0
  145. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/trace.ts +1078 -0
  146. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/unitf-adapter.ts +0 -0
  147. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/normalize/unitf-types.ts +213 -0
  148. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/report/persist-selections.ts +174 -0
  149. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/report/render.ts +3254 -0
  150. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/run/diagnose.ts +64 -0
  151. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/run/session.ts +167 -0
  152. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/caps.ts +345 -0
  153. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/deep-read-gate.ts +424 -0
  154. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/sample/representative.ts +584 -0
  155. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/scan/objection.ts +224 -0
  156. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/scan/trajectory.ts +489 -0
  157. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/self-diagnostics/dispatch.ts +135 -0
  158. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/self-diagnostics/probe.ts +170 -0
  159. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/detect.ts +307 -0
  160. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/ensure-cli.ts +315 -0
  161. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/reconfigure.ts +77 -0
  162. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/setup/verify-agents.ts +207 -0
  163. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/slicer.ts +166 -0
  164. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/claude-code.ts +17 -0
  165. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/codex.ts +11 -0
  166. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/langfuse.ts +17 -0
  167. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/local-jsonl.ts +12 -0
  168. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/adapters/otel.ts +12 -0
  169. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/index.ts +54 -0
  170. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/registry.ts +45 -0
  171. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/source/types.ts +86 -0
  172. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/stale-detector.ts +74 -0
  173. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/test/render-playwright.cli.ts +477 -0
  174. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0/claude-code.ts +133 -0
  175. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0/langfuse.ts +143 -0
  176. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/tier0-scan.ts +423 -0
  177. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/completeness-check.ts +436 -0
  178. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/doc-refs.ts +190 -0
  179. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/finalize-gate.ts +417 -0
  180. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/findings-contract.ts +628 -0
  181. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/render-js-syntax.ts +188 -0
  182. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/report-checklist.yaml +217 -0
  183. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/trace-shape.ts +262 -0
  184. package/mutagent-diagnostics/.claude/skills/mutagent-diagnostics/scripts/validate/wave6-checklist.ts +326 -0
  185. package/mutagent-diagnostics/CLAUDE.md +41 -0
  186. package/mutagent-diagnostics/LICENSE +201 -0
  187. package/mutagent-diagnostics/NOTICE +18 -0
  188. package/mutagent-diagnostics/README.md +343 -0
  189. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/SKILL.md +548 -0
  190. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/audit-executor.md +169 -0
  191. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/dataset-builder.md +160 -0
  192. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/discovery.md +425 -0
  193. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/agents/evaluator.md +1108 -0
  194. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/brand/theme.css +213 -0
  195. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/assets/brand/wordmark.html +9 -0
  196. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/context-flow-lens.md +85 -0
  197. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/data-lens.md +47 -0
  198. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/decision-lens.md +48 -0
  199. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/methodology-critic-lens.md +53 -0
  200. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/lenses/trajectory-lens.md +44 -0
  201. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/build-review-interface.md +83 -0
  202. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/edd-loop.md +134 -0
  203. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/error-analysis.md +113 -0
  204. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/eval-audit.md +154 -0
  205. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/eval-stage.md +168 -0
  206. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/generate-synthetic-data.md +81 -0
  207. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/grounded-adjudication.md +221 -0
  208. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/memory-format.md +65 -0
  209. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/methodology.md +201 -0
  210. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/operation-inventory.md +197 -0
  211. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/validate-evaluator.md +125 -0
  212. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/workflows/orchestrator-protocol.md +300 -0
  213. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/references/write-judge-prompt.md +123 -0
  214. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/behavior-tree.schema.yaml +73 -0
  215. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/dataset.schema.yaml +66 -0
  216. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/edd-change-request.schema.yaml +114 -0
  217. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/eval-matrix.schema.yaml +74 -0
  218. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/flow-graph.schema.yaml +69 -0
  219. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/flow-profile.schema.yaml +49 -0
  220. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/methodology-review.schema.yaml +40 -0
  221. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/schemas/scorecard.schema.yaml +85 -0
  222. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/agent-dispatch.ts +0 -0
  223. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/aggregate-discover.ts +543 -0
  224. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/artifact-paths.ts +115 -0
  225. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/assemble-scorecard.ts +172 -0
  226. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-dataset.ts +186 -0
  227. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-evals.ts +93 -0
  228. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/build-review-ui.ts +393 -0
  229. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/check-method-router.ts +170 -0
  230. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/aggregate.ts +112 -0
  231. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/audit-run.ts +182 -0
  232. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/doctor.ts +211 -0
  233. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/dogfood.ts +133 -0
  234. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/init.ts +601 -0
  235. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/methodology-review.ts +122 -0
  236. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/prep.ts +295 -0
  237. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/profile-subject.ts +165 -0
  238. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/run.sh +56 -0
  239. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cli/variance-check.ts +105 -0
  240. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/code-eval.ts +248 -0
  241. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/code-quality-verdict.ts +285 -0
  242. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/codegen-evals.ts +173 -0
  243. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cold-start-project.ts +151 -0
  244. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/cold-start-sampler.ts +267 -0
  245. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/config/load.ts +343 -0
  246. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/config/schema.ts +380 -0
  247. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/agentspec-evals.ts +85 -0
  248. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/dataset.ts +149 -0
  249. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-engine.ts +118 -0
  250. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-matrix.ts +577 -0
  251. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/eval-types.ts +1074 -0
  252. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/flow-graph.ts +194 -0
  253. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/types.ts +303 -0
  254. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/contracts/validation.ts +193 -0
  255. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/derive-dataset.ts +152 -0
  256. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/determine-outcome.ts +219 -0
  257. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/diff-discriminate.ts +160 -0
  258. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/discover-criteria.ts +348 -0
  259. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/change-request.ts +232 -0
  260. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/edd-types.ts +210 -0
  261. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/edd/variance-gate.ts +186 -0
  262. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/emit-completeness.ts +111 -0
  263. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/eval-engine.ts +160 -0
  264. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/evaluate.ts +333 -0
  265. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/flow-graph.ts +230 -0
  266. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/judge-prompt-template.ts +253 -0
  267. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/judge-provider.ts +135 -0
  268. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/lint-grounding.ts +229 -0
  269. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/lint-uniformity.ts +168 -0
  270. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/living-suite.ts +109 -0
  271. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-bundle.ts +203 -0
  272. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-profile-vocab.ts +64 -0
  273. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/load-profile.ts +106 -0
  274. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/mask.ts +138 -0
  275. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/materialize-dataset.ts +113 -0
  276. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/matrix-judge.ts +750 -0
  277. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/memory/append.ts +215 -0
  278. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/memory/read.ts +168 -0
  279. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/persist-eval-criteria.ts +232 -0
  280. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/prep-tasks.ts +125 -0
  281. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/profile-subject.ts +310 -0
  282. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/publish-report.ts +131 -0
  283. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/read-unitf-traces.ts +81 -0
  284. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-build-cards.ts +195 -0
  285. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-discover-report.ts +1640 -0
  286. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-eval-report.ts +3838 -0
  287. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/render-report.ts +212 -0
  288. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/resolve-credential.ts +110 -0
  289. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/resolve-ref.ts +98 -0
  290. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/result-verify.ts +129 -0
  291. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/route-failures.ts +330 -0
  292. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-deterministic.ts +367 -0
  293. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-evaluate.ts +719 -0
  294. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-judge.ts +155 -0
  295. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/run-pipeline.ts +175 -0
  296. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/sample-traces.ts +210 -0
  297. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/self-audit.ts +387 -0
  298. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/source-map.ts +106 -0
  299. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/subject-profile.ts +134 -0
  300. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/substrate.ts +162 -0
  301. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/sync-eval-criteria.ts +244 -0
  302. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/ui-slots.ts +119 -0
  303. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/unitf-to-evaltrace.ts +284 -0
  304. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/validate-judge.ts +358 -0
  305. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/scripts/variance-compare.ts +177 -0
  306. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/behavior-tree.yaml +140 -0
  307. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/eval-matrix.yaml +1270 -0
  308. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/subjects/mutagent-diagnostics/methodology-review.yaml +105 -0
  309. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/audit.workflow.js +82 -0
  310. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/data-leak.workflow.js +236 -0
  311. package/mutagent-evaluator/.claude/skills/mutagent-evaluator/workflows/variance.workflow.js +163 -0
  312. package/mutagent-evaluator/CLAUDE.md +45 -0
  313. package/mutagent-evaluator/LICENSE +201 -0
  314. package/mutagent-evaluator/NOTICE +23 -0
  315. package/mutagent-evaluator/README.md +88 -0
  316. package/mutagent-optimize/.claude/skills/mutagent-optimize/SKILL.md +287 -0
  317. package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/contracts/amend-request.ts +503 -0
  318. package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/improve-loop-run.ts +304 -0
  319. package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/loop-state-cli.ts +300 -0
  320. package/mutagent-optimize/.claude/skills/mutagent-optimize/scripts/loop-state.ts +440 -0
  321. package/mutagent-optimize/CLAUDE.md +49 -0
  322. package/mutagent-optimize/README.md +26 -0
  323. package/mutagent-orchestrator/CLAUDE.md +102 -0
  324. package/mutagent-orchestrator/README.md +20 -0
  325. package/mutagent-orchestrator/assets/agents/discovery.md +425 -0
  326. package/mutagent-orchestrator/assets/agents/dogfood-monitor.md +266 -0
  327. package/mutagent-orchestrator/assets/agents/monitor.md +258 -0
  328. package/mutagent-orchestrator/orchestrator.md +782 -0
  329. package/mutagent-orchestrator/references/MONITORING.md +209 -0
  330. package/mutagent-orchestrator/references/config-fields.md +132 -0
  331. package/mutagent-orchestrator/references/config-migration.md +132 -0
  332. package/mutagent-orchestrator/references/index-registry.md +92 -0
  333. package/mutagent-orchestrator/routing.yaml +391 -0
  334. package/mutagent-orchestrator/scripts/config-schema.ts +1010 -0
  335. package/mutagent-orchestrator/scripts/dispatch.ts +439 -0
  336. package/mutagent-orchestrator/scripts/dogfood/extract-feedback.ts +203 -0
  337. package/mutagent-orchestrator/scripts/dogfood/reconstruct-trajectory.ts +327 -0
  338. package/mutagent-orchestrator/scripts/dogfood/render-dogfood-report.ts +347 -0
  339. package/mutagent-orchestrator/scripts/dogfood/resolve-subagents.ts +160 -0
  340. package/mutagent-orchestrator/scripts/dogfood/run-report.ts +264 -0
  341. package/mutagent-orchestrator/scripts/dogfood/slack-thread.ts +106 -0
  342. package/mutagent-orchestrator/scripts/dogfood/types.ts +257 -0
  343. package/mutagent-orchestrator/scripts/eval-routing.ts +753 -0
  344. package/mutagent-orchestrator/scripts/gate.ts +219 -0
  345. package/mutagent-orchestrator/scripts/handover-contract.ts +362 -0
  346. package/mutagent-orchestrator/scripts/index/build-index.ts +351 -0
  347. package/mutagent-orchestrator/scripts/index/render-index.ts +110 -0
  348. package/mutagent-orchestrator/scripts/monitor/slack-notify.ts +111 -0
  349. package/mutagent-orchestrator/scripts/monitor/triggers.ts +162 -0
  350. package/mutagent-orchestrator/scripts/onboarding-check.ts +369 -0
  351. package/mutagent-orchestrator/scripts/render-roster.ts +308 -0
  352. package/mutagent-orchestrator/scripts/resolve-credential.ts +185 -0
  353. package/mutagent-orchestrator/scripts/resolve-paths.ts +114 -0
  354. package/mutagent-orchestrator/scripts/slack/format.ts +63 -0
  355. package/mutagent-orchestrator/scripts/slack/post.ts +115 -0
  356. package/mutagent-orchestrator/scripts/sync-index.ts +311 -0
  357. package/package.json +64 -0
@@ -0,0 +1,348 @@
1
+ /**
2
+ * scripts/discover-criteria.ts — EV-041 `*discover-evals`.
3
+ * ---------------------------------------------------------------------------
4
+ * Derive emergent BINARY ACTIONABLE eval criteria from LABELED ✓/✗ traces.
5
+ *
6
+ * The clustering itself is the LLM error-analyst's job (deep-read each trace,
7
+ * note the FIRST thing that went wrong, assign an emergent category — never a
8
+ * pre-defined list; error-analysis CORE §2/§3). This is the PURE, deterministic
9
+ * remainder of `*discover-evals`:
10
+ * - aggregate annotations → one criterion per emergent category (support)
11
+ * - compute per-category failure rates (prioritize what matters)
12
+ * - FLAG fixable (infra/dependency) vs eval-worthy (behavioral). Fixables are
13
+ * ROUTED to diagnostics (EV-051), NEVER fixed and NEVER judged on the model.
14
+ * - detect saturation: stop when no NEW category emerges across rounds.
15
+ *
16
+ * "a judge is only a judge" — this module FLAGS, it does not fix. There is no
17
+ * `resolve`/`fix` field on a DiscoveredCriterion by construction.
18
+ *
19
+ * PURE + deterministic: criteria ranked by support desc then id; no clock/random.
20
+ */
21
+ import {
22
+ AssumptionKind,
23
+ CheckMethod,
24
+ CriterionFlag,
25
+ Generality,
26
+ Grounding,
27
+ JudgeKind,
28
+ MetricDimension,
29
+ MetricLevel,
30
+ MetricSubstrate,
31
+ OutcomeVerdict,
32
+ Severity,
33
+ assertGroundingHonest,
34
+ type CheckMethodValue,
35
+ type CriterionFlagValue,
36
+ type DiscoveredCriterion,
37
+ type DiscoveryRef,
38
+ type GeneralityValue,
39
+ type GroundingValue,
40
+ type JudgeKindValue,
41
+ type MetricDimensionValue,
42
+ type MetricLevelValue,
43
+ type MetricMetadata,
44
+ type MetricSubstrateValue,
45
+ type MinedCriterion,
46
+ type DiscoveryRationale,
47
+ type OutcomeVerdictValue,
48
+ type SeverityValue,
49
+ } from "./contracts/eval-types.ts";
50
+ import type { CodeEvalSpec } from "./code-eval.ts";
51
+
52
+ /**
53
+ * One annotation: the error-analyst's deep-read of a single trace. For a
54
+ * failure it carries the first-thing-that-went-wrong `note` + the emergent
55
+ * `category` + a classification (`infra` => fixable, else behavioral).
56
+ * `statement`/`judgeKind`/`judgeInputs` are the analyst's criterion proposal,
57
+ * carried on the first annotation that defines a category.
58
+ */
59
+ export interface TraceAnnotation {
60
+ traceId: string;
61
+ label: OutcomeVerdictValue;
62
+ category?: string;
63
+ note?: string;
64
+ failureClass?: "infra" | "behavioral";
65
+ judgeKind?: JudgeKindValue;
66
+ statement?: string;
67
+ judgeInputs?: string[];
68
+ /**
69
+ * §5c evidence ref — a concrete `trace-id/field` pointer the analyst cited for
70
+ * THIS trace's failure. When present it is used as the grounding ref; else the
71
+ * traceId itself is the ref. ADDITIVE (P2b).
72
+ */
73
+ evidencePointer?: string;
74
+ /**
75
+ * GA-1 — STRUCTURED grounding refs `{obs,path,value}` the `#mode-discover` leaf
76
+ * cited for THIS trace's failure. PREFERRED over `evidencePointer` (which is
77
+ * synthesized into a best-effort structured ref when `refs` is absent). The
78
+ * `value` is the EXACT cited text so the ref re-resolves against the trace.
79
+ */
80
+ refs?: DiscoveryRef[];
81
+ /**
82
+ * Optional §5b metadata PROPOSALS the `#mode-discover` leaf may emit (additive
83
+ * agent-contract extension). When present they override the generic defaults
84
+ * `deriveMinedCriteria` would otherwise compute; when absent, defaults apply.
85
+ */
86
+ dimension?: MetricDimensionValue;
87
+ level?: MetricLevelValue;
88
+ generality?: GeneralityValue;
89
+ severity?: SeverityValue;
90
+ appliesTo?: string[];
91
+ /**
92
+ * The UNIFORM-STANDARD executable code-check the `#mode-discover` leaf EMITS when
93
+ * a mined criterion is deterministically checkable (it picks a registry primitive
94
+ * + field/params and tags the category `code`/`hybrid`). Carried onto the
95
+ * `MinedCriterion.codeEval` so the tier-0 pre-pass runs it (zero judge tokens).
96
+ * Absent for a pure `llm-judge` category. ADDITIVE.
97
+ */
98
+ codeEval?: CodeEvalSpec;
99
+ }
100
+
101
+ /** Group annotations by their emergent category (uncategorized are dropped). */
102
+ export function groupByCategory(
103
+ annotations: TraceAnnotation[],
104
+ ): Map<string, TraceAnnotation[]> {
105
+ const groups = new Map<string, TraceAnnotation[]>();
106
+ for (const a of annotations) {
107
+ if (a.category === undefined || a.category.length === 0) continue;
108
+ const list = groups.get(a.category) ?? [];
109
+ list.push(a);
110
+ groups.set(a.category, list);
111
+ }
112
+ return groups;
113
+ }
114
+
115
+ export interface CategoryRate {
116
+ category: string;
117
+ fails: number;
118
+ total: number;
119
+ rate: number;
120
+ }
121
+
122
+ /** Per-category failure rate, ranked by fails desc then category asc. */
123
+ export function failureRates(annotations: TraceAnnotation[]): CategoryRate[] {
124
+ const groups = groupByCategory(annotations);
125
+ const rates: CategoryRate[] = [];
126
+ for (const [category, list] of groups) {
127
+ const fails = list.filter((a) => a.label === OutcomeVerdict.Fail).length;
128
+ const total = list.length;
129
+ rates.push({ category, fails, total, rate: total === 0 ? 0 : fails / total });
130
+ }
131
+ return rates.sort((a, b) =>
132
+ b.fails - a.fails !== 0 ? b.fails - a.fails : a.category.localeCompare(b.category),
133
+ );
134
+ }
135
+
136
+ /** Pick the first non-empty value of `key` across a category's annotations. */
137
+ function firstDefined<K extends keyof TraceAnnotation>(
138
+ list: TraceAnnotation[],
139
+ key: K,
140
+ ): TraceAnnotation[K] | undefined {
141
+ for (const a of list) {
142
+ if (a[key] !== undefined) return a[key];
143
+ }
144
+ return undefined;
145
+ }
146
+
147
+ /** A category is FIXABLE iff any of its failures is classified infra. */
148
+ function flagFor(list: TraceAnnotation[]): CriterionFlagValue {
149
+ const anyInfra = list.some(
150
+ (a) => a.label === OutcomeVerdict.Fail && a.failureClass === "infra",
151
+ );
152
+ return anyInfra ? CriterionFlag.Fixable : CriterionFlag.EvalWorthy;
153
+ }
154
+
155
+ /**
156
+ * Derive one BINARY ACTIONABLE criterion per emergent category. Ranked by
157
+ * support (member count) desc then id asc. PURE — no fix is applied; the
158
+ * returned objects carry no resolution field.
159
+ */
160
+ export function deriveCriteria(annotations: TraceAnnotation[]): DiscoveredCriterion[] {
161
+ const groups = groupByCategory(annotations);
162
+ const out: DiscoveredCriterion[] = [];
163
+ for (const [category, list] of groups) {
164
+ const statement = firstDefined(list, "statement") ?? `behavior is correct for: ${category}`;
165
+ const judgeKind = firstDefined(list, "judgeKind") ?? JudgeKind.Llm;
166
+ const judgeInputs = firstDefined(list, "judgeInputs") ?? ["prompt", "trajectory", "response"];
167
+ out.push({
168
+ id: category,
169
+ statement,
170
+ judgeInputs,
171
+ judgeKind,
172
+ flag: flagFor(list),
173
+ supportCount: list.length,
174
+ });
175
+ }
176
+ return out.sort((a, b) =>
177
+ b.supportCount - a.supportCount !== 0
178
+ ? b.supportCount - a.supportCount
179
+ : a.id.localeCompare(b.id),
180
+ );
181
+ }
182
+
183
+ // ════════════════════════════════════════════════════════════════════════════
184
+ // P2 / P2b — mined criteria with §5b metadata + §5c DR-2 (evidence-first)
185
+ // ════════════════════════════════════════════════════════════════════════════
186
+
187
+ /** Map the analyst's JudgeKind → the §5b check_method router (SF-2). */
188
+ function checkMethodFor(judgeKind: JudgeKindValue): CheckMethodValue {
189
+ if (judgeKind === JudgeKind.Code) return CheckMethod.Deterministic;
190
+ if (judgeKind === JudgeKind.Hybrid) return CheckMethod.Hybrid;
191
+ return CheckMethod.LlmJudge;
192
+ }
193
+
194
+ /** §5b substrate is DERIVED from check_method: deterministic→code, else judge. */
195
+ function substrateFor(checkMethod: CheckMethodValue): MetricSubstrateValue {
196
+ return checkMethod === CheckMethod.Deterministic ? MetricSubstrate.Code : MetricSubstrate.Judge;
197
+ }
198
+
199
+ /** Generic generality heuristic: a code/deterministic check is general-structural
200
+ * (reusable on any pipeline); a judge check is specific-semantic. The leaf may
201
+ * override via `annotation.generality`. */
202
+ function generalityFor(checkMethod: CheckMethodValue): GeneralityValue {
203
+ return checkMethod === CheckMethod.Deterministic
204
+ ? Generality.GeneralStructural
205
+ : Generality.SpecificSemantic;
206
+ }
207
+
208
+ /** Generic severity heuristic from observed failure count (leaf may override). */
209
+ function severityFor(fails: number): SeverityValue {
210
+ if (fails >= 2) return Severity.High;
211
+ if (fails === 1) return Severity.Med;
212
+ return Severity.Low;
213
+ }
214
+
215
+ /** Map the criterion flag → the §5c fix_or_eval routing string. */
216
+ function fixOrEvalFor(flag: CriterionFlagValue): DiscoveryRationale["fix_or_eval"] {
217
+ return flag === CriterionFlag.Fixable ? "fixable->diagnostics" : "eval-worthy";
218
+ }
219
+
220
+ /** Synthesize a best-effort structured ref from a legacy `trace-id/field`
221
+ * pointer string + note (GA-1 back-compat: the leaf may still emit a pointer). */
222
+ function synthRef(a: TraceAnnotation): DiscoveryRef {
223
+ const ptr = a.evidencePointer ?? a.traceId;
224
+ const slash = ptr.indexOf("/");
225
+ const obs = slash >= 0 ? ptr.slice(0, slash) : a.traceId;
226
+ const path = slash >= 0 ? ptr.slice(slash + 1) : "";
227
+ // value = the exact cited text (note) when present, else the pointer itself.
228
+ const value = a.note !== undefined && a.note.length > 0 ? a.note : ptr;
229
+ return { obs, path, value };
230
+ }
231
+
232
+ /** GA-1 — the STRUCTURED evidence refs for a category = the failing annotations'
233
+ * cited structured refs (preferred), else a synthesized ref from the legacy
234
+ * pointer. REAL refs, in input order. */
235
+ function refsFor(list: TraceAnnotation[]): DiscoveryRef[] {
236
+ const out: DiscoveryRef[] = [];
237
+ for (const a of list) {
238
+ if (a.label !== OutcomeVerdict.Fail) continue;
239
+ if (a.refs !== undefined && a.refs.length > 0) out.push(...a.refs);
240
+ else out.push(synthRef(a));
241
+ }
242
+ return out;
243
+ }
244
+
245
+ /**
246
+ * Derive the full set of MINED criteria from real annotations: the existing
247
+ * `deriveCriteria` output ENRICHED with the §5b unified metadata + the §5c DR-2
248
+ * discovery-rationale, grounded in the actual support set. EVIDENCE-FIRST:
249
+ * - OBSERVED iff a failure was actually seen (fails>0) — cites the real failing
250
+ * refs + an honest `k/n sampled` prevalence; never promotes inferred→observed.
251
+ * - a pass-only category is INFERRED (good-practice guard, no failure yet).
252
+ * PURE + deterministic (same ranking as deriveCriteria: support desc, id asc).
253
+ */
254
+ export function deriveMinedCriteria(annotations: TraceAnnotation[]): MinedCriterion[] {
255
+ const groups = groupByCategory(annotations);
256
+ const base = deriveCriteria(annotations); // ranked; carries id/statement/judgeKind/judgeInputs/flag/support
257
+ return base.map((c) => {
258
+ const list = groups.get(c.id) ?? [];
259
+ const fails = list.filter((a) => a.label === OutcomeVerdict.Fail).length;
260
+ const total = list.length;
261
+ const refs = refsFor(list);
262
+ const firstNote = firstDefined(list, "note");
263
+
264
+ const checkMethod = checkMethodFor(c.judgeKind);
265
+ const grounding: GroundingValue = fails > 0 && refs.length > 0 ? Grounding.Observed : Grounding.Inferred;
266
+ const severity = firstDefined(list, "severity") ?? severityFor(fails);
267
+
268
+ const metadata: MetricMetadata = {
269
+ generality: firstDefined(list, "generality") ?? generalityFor(checkMethod),
270
+ dimension: firstDefined(list, "dimension") ?? MetricDimension.OperationCorrectness,
271
+ level: firstDefined(list, "level") ?? MetricLevel.Output,
272
+ check_method: checkMethod,
273
+ substrate: substrateFor(checkMethod),
274
+ applies_to: firstDefined(list, "appliesTo") ?? [],
275
+ severity,
276
+ judge_inputs: c.judgeInputs,
277
+ flag: c.flag,
278
+ };
279
+
280
+ const discovery: DiscoveryRationale = {
281
+ targets: `the '${c.id}' behavior/failure mode`,
282
+ why_problem:
283
+ fails > 0
284
+ ? `failure mode observed${firstNote !== undefined ? ` (first-thing-wrong: ${firstNote})` : ""}`
285
+ : "good-practice guard — no failure observed in the sample yet",
286
+ evidence: {
287
+ grounding,
288
+ seen_in_traces: fails > 0,
289
+ prevalence: `${fails}/${total} sampled`,
290
+ refs: grounding === Grounding.Observed ? refs : [],
291
+ },
292
+ reasoning:
293
+ `${fails}/${total} trace(s) in category '${c.id}'` +
294
+ `${firstNote !== undefined ? `; first-thing-wrong: ${firstNote}` : ""} → criterion`,
295
+ assumptions:
296
+ grounding === Grounding.Observed
297
+ ? [
298
+ {
299
+ text: "the observed failure recurs under the same template",
300
+ status: "unverified",
301
+ kind: AssumptionKind.FactualIntent,
302
+ },
303
+ ]
304
+ : [
305
+ {
306
+ text: "preventive guard; no failure observed yet",
307
+ status: "hypothesis",
308
+ kind: AssumptionKind.Normative,
309
+ },
310
+ ],
311
+ expected_impact: {
312
+ severity,
313
+ coverage: `${total} sampled trace(s)`,
314
+ confidence: grounding === Grounding.Observed ? (fails >= 2 ? "high" : "med") : "low",
315
+ },
316
+ fix_or_eval: fixOrEvalFor(c.flag),
317
+ };
318
+
319
+ // UNIFORM STANDARD — carry the leaf-emitted executable code-check (when the
320
+ // category is deterministically checkable) onto the criterion so the tier-0
321
+ // pre-pass can run it. A pure judge category leaves it undefined.
322
+ const codeEval = firstDefined(list, "codeEval");
323
+
324
+ const mined: MinedCriterion = {
325
+ ...c,
326
+ metadata,
327
+ discovery,
328
+ ...(codeEval !== undefined ? { codeEval } : {}),
329
+ };
330
+ assertGroundingHonest(mined); // defensive — never ship inferred-as-observed
331
+ return mined;
332
+ });
333
+ }
334
+
335
+ /**
336
+ * Saturation: the next round surfaced NO new category. `prev` is the cumulative
337
+ * category set; `next` is the latest round's categories. Saturated iff every
338
+ * category in `next` is already in `prev`.
339
+ */
340
+ export function isSaturated(prev: string[], next: string[]): boolean {
341
+ const seen = new Set(prev);
342
+ return next.every((c) => seen.has(c));
343
+ }
344
+
345
+ /** The error-analysis target band: aim for 5-10 distinct categories. */
346
+ export function withinSaturationBand(categories: string[]): boolean {
347
+ return categories.length >= 5 && categories.length <= 10;
348
+ }
@@ -0,0 +1,232 @@
1
+ /**
2
+ * scripts/edd/change-request.ts — F18 EDD CLOSURE (the spec↔impl↔eval triad).
3
+ * ---------------------------------------------------------------------------
4
+ * The Evaluator is JUDGE-ONLY (EV-051): it NEVER fixes the subject. To close the
5
+ * EDD loop it instead REQUESTS the AI-Engineer (the *build ACTOR from the SPEC
6
+ * stage) to amend the Agent / AgentSpec — wired over the `SendMessage` tool both
7
+ * agents carry. This module is the structured contract for that hand-off:
8
+ *
9
+ * evaluator --(EddChangeRequest)--> ai-engineer (the failing cases + remedy target)
10
+ * ai-engineer --(ChangeRequestResponse)--> evaluator (amended | rejected)
11
+ * evaluator re-evaluates → loop until full green OR a bounded STOP.
12
+ *
13
+ * The contract is the SEAM: the evaluator emits a request (validated here), the
14
+ * engineer consumes it (validated here), and the loop controller (also here)
15
+ * decides — using ONLY observable state — whether to swing again or terminate.
16
+ * The termination gate is afkloop-legal: full-green is the success terminator, and
17
+ * three bounds (max-swings · max-wallclock · no-improvement-streak) guarantee the
18
+ * loop is NEVER infinite.
19
+ *
20
+ * PURE — no clock / Math.random / network. The wall-clock bound is checked against
21
+ * an INJECTED elapsed value (the caller measures time; this module only decides),
22
+ * keeping the controller deterministic + unit-testable (C-PIN).
23
+ */
24
+ import { Value } from "@sinclair/typebox/value";
25
+ import {
26
+ ChangeRequestStatus,
27
+ EddPhase,
28
+ EddStopReason,
29
+ EddChangeRequestSchema,
30
+ ChangeRequestResponseSchema,
31
+ RemedyTarget,
32
+ type EddChangeRequestType,
33
+ type ChangeRequestResponseType,
34
+ type EddPhaseValue,
35
+ type EddStopReasonValue,
36
+ type RemedyTargetValue,
37
+ } from "./edd-types.ts";
38
+
39
+ // Re-export the Static types under their public names (edd-types declares the schemas).
40
+ export type { EddChangeRequestType, ChangeRequestResponseType };
41
+
42
+ // ── Validators (fail-loud over the SendMessage seam) ────────────────────────
43
+
44
+ /**
45
+ * Validate an evaluator→engineer change-request. THROWS with the first TypeBox
46
+ * error path on any violation — a malformed request must never cross the seam
47
+ * silently (an ungrounded ask, a missing remedy target, etc. fail loud here).
48
+ */
49
+ export function validateChangeRequest(req: unknown): EddChangeRequestType {
50
+ if (!Value.Check(EddChangeRequestSchema, req)) {
51
+ const first = [...Value.Errors(EddChangeRequestSchema, req)][0];
52
+ throw new Error(
53
+ `EddChangeRequest invalid${first ? ` at ${first.path}: ${first.message}` : ""}`,
54
+ );
55
+ }
56
+ // Belt-and-braces: every failing case must be GROUNDED (≥1 ref). The schema
57
+ // enforces minItems:1 on refs, but assert the intent explicitly for the seam.
58
+ for (const fc of req.failingCases) {
59
+ if (fc.refs.length === 0) {
60
+ throw new Error(
61
+ `EddChangeRequest invalid: failing case "${fc.caseId}" has no grounding refs (ungrounded ask — EV-051/GA-1)`,
62
+ );
63
+ }
64
+ }
65
+ return req;
66
+ }
67
+
68
+ /**
69
+ * Validate an engineer→evaluator response. THROWS on a malformed reply. A
70
+ * `rejected` status MUST carry a `note` (the reason) — enforced by the schema's
71
+ * minLength on `note`; an `amended` status SHOULD echo `amendedTarget`.
72
+ */
73
+ export function validateChangeResponse(res: unknown): ChangeRequestResponseType {
74
+ if (!Value.Check(ChangeRequestResponseSchema, res)) {
75
+ const first = [...Value.Errors(ChangeRequestResponseSchema, res)][0];
76
+ throw new Error(
77
+ `ChangeRequestResponse invalid${first ? ` at ${first.path}: ${first.message}` : ""}`,
78
+ );
79
+ }
80
+ return res;
81
+ }
82
+
83
+ /**
84
+ * Does this response warrant a re-eval? Only an `amended` response does — a
85
+ * `rejected` one means the engineer declined (the evaluator must re-target or
86
+ * escalate, NOT re-eval an unchanged subject). There is no "trust me, skip
87
+ * re-eval" path: an amend ALWAYS triggers a fresh evaluation swing.
88
+ */
89
+ export function reEvalWarranted(res: ChangeRequestResponseType): boolean {
90
+ return res.status === ChangeRequestStatus.Amended;
91
+ }
92
+
93
+ // ── F18 loop control — the bounded EDD terminator ───────────────────────────
94
+
95
+ export interface EddLoopBudget {
96
+ /** hard cap on evaluator→engineer→re-eval swings. */
97
+ maxSwings: number;
98
+ /** hard wall-clock budget in ms (checked against an injected elapsed). */
99
+ maxWallclockMs: number;
100
+ /** stop after this many consecutive swings with no measurable improvement. */
101
+ noImprovementStreakLimit: number;
102
+ }
103
+
104
+ export const DEFAULT_EDD_LOOP_BUDGET: EddLoopBudget = {
105
+ maxSwings: 6,
106
+ maxWallclockMs: 30 * 60 * 1000, // 30 min
107
+ noImprovementStreakLimit: 2,
108
+ };
109
+
110
+ /**
111
+ * The observable loop state the controller reasons over. ALL fields are measured
112
+ * by the caller (the evaluator parent session) and handed in — the controller is
113
+ * a PURE decision function, so the same state always yields the same decision.
114
+ */
115
+ export interface EddLoopState {
116
+ /** the current EDD phase (variance vs accuracy vs build). */
117
+ phase: EddPhaseValue;
118
+ /** 1-based count of swings COMPLETED so far. */
119
+ swing: number;
120
+ /** F19 — is the per-case variance gate currently passing? */
121
+ varianceStable: boolean;
122
+ /** is accuracy ≥ target over the dataset? (meaningful only post-variance). */
123
+ accuracyMet: boolean;
124
+ /** measured elapsed wall-clock for the whole loop, ms (injected — not read here). */
125
+ elapsedMs: number;
126
+ /**
127
+ * consecutive swings with no measurable improvement (the caller computes this
128
+ * from a monotone progress metric — e.g. fewer flapping/failing cases). 0 ⇒
129
+ * the last swing improved something.
130
+ */
131
+ noImprovementStreak: number;
132
+ }
133
+
134
+ export interface EddLoopDecision {
135
+ /** true ⇒ terminate now; false ⇒ swing again. */
136
+ done: boolean;
137
+ /** WHY it stopped — set iff done; null while the loop should continue. */
138
+ reason: EddStopReasonValue | null;
139
+ /** the next phase to run (DONE/STOPPED when terminating). */
140
+ nextPhase: EddPhaseValue;
141
+ /** human one-liner — the convergence delta when bounded, the green when done. */
142
+ detail: string;
143
+ }
144
+
145
+ /**
146
+ * THE termination gate (afkloop-legal — observable + bounded + no infinite loop).
147
+ *
148
+ * Order of checks (success first, then the three bounds):
149
+ * 1. FULL GREEN — varianceStable AND accuracyMet ⇒ DONE (success).
150
+ * 2. MAX SWINGS — swing ≥ budget.maxSwings ⇒ STOP (report delta).
151
+ * 3. MAX WALLCLOCK— elapsedMs ≥ budget.maxWallclockMs ⇒ STOP.
152
+ * 4. NO-IMPROVE — noImprovementStreak ≥ budget.noImprovementStreakLimit ⇒ STOP.
153
+ * else CONTINUE — keep swinging; the next phase is variance until it's stable,
154
+ * then accuracy (F19 ordering is preserved by `nextPhaseAfterVariance`, but the
155
+ * loop controller mirrors it: not-stable ⇒ variance, stable ⇒ accuracy).
156
+ *
157
+ * PURE: no clock read — `elapsedMs` is injected. Same state → same decision.
158
+ */
159
+ export function decideEddLoop(
160
+ state: EddLoopState,
161
+ budget: EddLoopBudget = DEFAULT_EDD_LOOP_BUDGET,
162
+ ): EddLoopDecision {
163
+ // 1. SUCCESS terminator — full green.
164
+ if (state.varianceStable && state.accuracyMet) {
165
+ return {
166
+ done: true,
167
+ reason: EddStopReason.FullGreen,
168
+ nextPhase: EddPhase.Done,
169
+ detail: `full green after ${state.swing} swing(s): variance stable AND accuracy met`,
170
+ };
171
+ }
172
+ // 2-4. Bounded non-convergence terminators (each reports the convergence delta).
173
+ if (state.swing >= budget.maxSwings) {
174
+ return stopped(
175
+ EddStopReason.MaxSwings,
176
+ `hit max swings (${state.swing}/${budget.maxSwings}); varianceStable=${state.varianceStable} accuracyMet=${state.accuracyMet}`,
177
+ );
178
+ }
179
+ if (state.elapsedMs >= budget.maxWallclockMs) {
180
+ return stopped(
181
+ EddStopReason.MaxWallclock,
182
+ `hit wall-clock budget (${state.elapsedMs}ms/${budget.maxWallclockMs}ms); varianceStable=${state.varianceStable} accuracyMet=${state.accuracyMet}`,
183
+ );
184
+ }
185
+ if (state.noImprovementStreak >= budget.noImprovementStreakLimit) {
186
+ return stopped(
187
+ EddStopReason.NoImprovementStreak,
188
+ `no improvement for ${state.noImprovementStreak} consecutive swing(s) (limit ${budget.noImprovementStreakLimit})`,
189
+ );
190
+ }
191
+ // CONTINUE — preserve F19 ordering: stabilize variance before accuracy.
192
+ const nextPhase = state.varianceStable ? EddPhase.Accuracy : EddPhase.Variance;
193
+ return {
194
+ done: false,
195
+ reason: null,
196
+ nextPhase,
197
+ detail: `continue: next phase ${nextPhase} (swing ${state.swing}, variance ${state.varianceStable ? "stable" : "flapping"})`,
198
+ };
199
+ }
200
+
201
+ function stopped(reason: EddStopReasonValue, detail: string): EddLoopDecision {
202
+ return { done: true, reason, nextPhase: EddPhase.Stopped, detail };
203
+ }
204
+
205
+ /**
206
+ * Build a well-formed change-request for the evaluator to SendMessage. Validates
207
+ * before returning so a malformed request can never leave the evaluator. The
208
+ * caller supplies the swing index + the grounded failing cases + the proposed
209
+ * remedy + the target; this stamps `status: requested`.
210
+ */
211
+ export function buildChangeRequest(input: {
212
+ requestId: string;
213
+ swing: number;
214
+ subject: string;
215
+ remedyTarget: RemedyTargetValue;
216
+ failingCases: EddChangeRequestType["failingCases"];
217
+ proposedRemedy: string;
218
+ }): EddChangeRequestType {
219
+ const req: EddChangeRequestType = {
220
+ requestId: input.requestId,
221
+ swing: input.swing,
222
+ subject: input.subject,
223
+ remedyTarget: input.remedyTarget,
224
+ failingCases: input.failingCases,
225
+ proposedRemedy: input.proposedRemedy,
226
+ status: ChangeRequestStatus.Requested,
227
+ };
228
+ return validateChangeRequest(req);
229
+ }
230
+
231
+ /** Convenience: the two remedy targets, re-exported for callers building requests. */
232
+ export { RemedyTarget };