@tangle-network/agent-eval 0.93.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +44 -30
- package/dist/action-policy.d.ts +24 -0
- package/dist/action-policy.d.ts.map +1 -0
- package/dist/action-policy.test.d.ts +2 -0
- package/dist/action-policy.test.d.ts.map +1 -0
- package/dist/active-learning.d.ts +41 -0
- package/dist/active-learning.d.ts.map +1 -0
- package/dist/adapters/http.d.ts +15 -21
- package/dist/adapters/http.d.ts.map +1 -0
- package/dist/adapters/http.js.map +1 -1
- package/dist/adapters/langchain.d.ts +7 -13
- package/dist/adapters/langchain.d.ts.map +1 -0
- package/dist/adapters/otel.d.ts +13 -24
- package/dist/adapters/otel.d.ts.map +1 -0
- package/dist/agent-profile-cell.d.ts +101 -0
- package/dist/agent-profile-cell.d.ts.map +1 -0
- package/dist/agent-profile.d.ts +27 -0
- package/dist/agent-profile.d.ts.map +1 -0
- package/dist/agent-profile.test.d.ts +2 -0
- package/dist/agent-profile.test.d.ts.map +1 -0
- package/dist/analyst/adapters.d.ts +62 -0
- package/dist/analyst/adapters.d.ts.map +1 -0
- package/dist/analyst/analyst.test.d.ts +2 -0
- package/dist/analyst/analyst.test.d.ts.map +1 -0
- package/dist/analyst/ax-service.d.ts +27 -0
- package/dist/analyst/ax-service.d.ts.map +1 -0
- package/dist/analyst/behavioral-analyst.d.ts +28 -0
- package/dist/analyst/behavioral-analyst.d.ts.map +1 -0
- package/dist/analyst/chat-client.d.ts +91 -0
- package/dist/analyst/chat-client.d.ts.map +1 -0
- package/dist/analyst/default-registry.d.ts +27 -0
- package/dist/analyst/default-registry.d.ts.map +1 -0
- package/dist/analyst/default-registry.test.d.ts +2 -0
- package/dist/analyst/default-registry.test.d.ts.map +1 -0
- package/dist/analyst/finding-signature.d.ts +48 -0
- package/dist/analyst/finding-signature.d.ts.map +1 -0
- package/dist/analyst/finding-subject.d.ts +146 -0
- package/dist/analyst/finding-subject.d.ts.map +1 -0
- package/dist/analyst/finding-subject.test.d.ts +2 -0
- package/dist/analyst/finding-subject.test.d.ts.map +1 -0
- package/dist/analyst/findings-store.d.ts +75 -0
- package/dist/analyst/findings-store.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +28 -256
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +6 -66
- package/dist/analyst/index.js.map +1 -1
- package/dist/{kind-factory-5b7xXXOr.d.ts → analyst/kind-factory.d.ts} +11 -63
- package/dist/analyst/kind-factory.d.ts.map +1 -0
- package/dist/analyst/kinds/failure-mode.d.ts +19 -0
- package/dist/analyst/kinds/failure-mode.d.ts.map +1 -0
- package/dist/analyst/kinds/improvement.d.ts +23 -0
- package/dist/analyst/kinds/improvement.d.ts.map +1 -0
- package/dist/analyst/kinds/index.d.ts +22 -0
- package/dist/analyst/kinds/index.d.ts.map +1 -0
- package/dist/analyst/kinds/kinds.test.d.ts +2 -0
- package/dist/analyst/kinds/kinds.test.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts +28 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +22 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.d.ts +84 -0
- package/dist/analyst/kinds/skill-usage.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts +2 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.d.ts +26 -0
- package/dist/analyst/parse-tolerant.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.test.d.ts +2 -0
- package/dist/analyst/parse-tolerant.test.d.ts.map +1 -0
- package/dist/analyst/registry.budget.test.d.ts +2 -0
- package/dist/analyst/registry.budget.test.d.ts.map +1 -0
- package/dist/{default-registry-zoGHUQEH.d.ts → analyst/registry.d.ts} +8 -37
- package/dist/analyst/registry.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.d.ts +35 -0
- package/dist/analyst/steer-firewall.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.test.d.ts +2 -0
- package/dist/analyst/steer-firewall.test.d.ts.map +1 -0
- package/dist/analyst/structure-findings.d.ts +37 -0
- package/dist/analyst/structure-findings.d.ts.map +1 -0
- package/dist/analyst/structure-findings.test.d.ts +2 -0
- package/dist/analyst/structure-findings.test.d.ts.map +1 -0
- package/dist/analyst/tool-groups.d.ts +34 -0
- package/dist/analyst/tool-groups.d.ts.map +1 -0
- package/dist/{types-2VVIL04s.d.ts → analyst/types.d.ts} +19 -112
- package/dist/analyst/types.d.ts.map +1 -0
- package/dist/anti-slop.d.ts +59 -0
- package/dist/anti-slop.d.ts.map +1 -0
- package/dist/artifact-validator.d.ts +74 -0
- package/dist/artifact-validator.d.ts.map +1 -0
- package/dist/attestation.d.ts +63 -0
- package/dist/attestation.d.ts.map +1 -0
- package/dist/attestation.test.d.ts +2 -0
- package/dist/attestation.test.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +15 -16
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.test.d.ts +2 -0
- package/dist/authenticity/index.test.d.ts.map +1 -0
- package/dist/auto-pr.d.ts +120 -0
- package/dist/auto-pr.d.ts.map +1 -0
- package/dist/{baseline-DE36-Np7.d.ts → baseline.d.ts} +8 -44
- package/dist/baseline.d.ts.map +1 -0
- package/dist/behavior-dsl.d.ts +73 -0
- package/dist/behavior-dsl.d.ts.map +1 -0
- package/dist/belief-state/calibration.d.ts +10 -0
- package/dist/belief-state/calibration.d.ts.map +1 -0
- package/dist/belief-state/calibration.test.d.ts +2 -0
- package/dist/belief-state/calibration.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.d.ts +66 -0
- package/dist/belief-state/code-agent-corpus.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts +2 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.d.ts +22 -0
- package/dist/belief-state/code-agent-evidence.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts +2 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/extract.d.ts +7 -0
- package/dist/belief-state/extract.d.ts.map +1 -0
- package/dist/belief-state/extract.test.d.ts +2 -0
- package/dist/belief-state/extract.test.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +14 -604
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1 -1
- package/dist/belief-state/ope.d.ts +17 -0
- package/dist/belief-state/ope.d.ts.map +1 -0
- package/dist/belief-state/ope.test.d.ts +2 -0
- package/dist/belief-state/ope.test.d.ts.map +1 -0
- package/dist/belief-state/phase0-measurement.d.ts +55 -0
- package/dist/belief-state/phase0-measurement.d.ts.map +1 -0
- package/dist/belief-state/report.d.ts +17 -0
- package/dist/belief-state/report.d.ts.map +1 -0
- package/dist/belief-state/report.test.d.ts +2 -0
- package/dist/belief-state/report.test.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.d.ts +23 -0
- package/dist/belief-state/research-evidence.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.test.d.ts +2 -0
- package/dist/belief-state/research-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +32 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.d.ts +87 -0
- package/dist/belief-state/runtime-hooks.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.test.d.ts +2 -0
- package/dist/belief-state/runtime-hooks.test.d.ts.map +1 -0
- package/dist/belief-state/selective.d.ts +15 -0
- package/dist/belief-state/selective.d.ts.map +1 -0
- package/dist/belief-state/selective.test.d.ts +2 -0
- package/dist/belief-state/selective.test.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.d.ts +80 -0
- package/dist/belief-state/shadow-probe.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.test.d.ts +2 -0
- package/dist/belief-state/shadow-probe.test.d.ts.map +1 -0
- package/dist/belief-state/types.d.ts +195 -0
- package/dist/belief-state/types.d.ts.map +1 -0
- package/dist/belief-state/types.test.d.ts +2 -0
- package/dist/belief-state/types.test.d.ts.map +1 -0
- package/dist/benchmark.d.ts +14 -0
- package/dist/benchmark.d.ts.map +1 -0
- package/dist/benchmarks/index.d.ts +23 -4
- package/dist/benchmarks/index.d.ts.map +1 -0
- package/dist/benchmarks/routing/dataset.d.ts +34 -0
- package/dist/benchmarks/routing/dataset.d.ts.map +1 -0
- package/dist/benchmarks/routing/index.d.ts +34 -0
- package/dist/benchmarks/routing/index.d.ts.map +1 -0
- package/dist/benchmarks/types.d.ts +49 -0
- package/dist/benchmarks/types.d.ts.map +1 -0
- package/dist/bisector.d.ts +81 -0
- package/dist/bisector.d.ts.map +1 -0
- package/dist/budget-guard.d.ts +31 -0
- package/dist/budget-guard.d.ts.map +1 -0
- package/dist/builder-eval/builder-session.d.ts +111 -0
- package/dist/builder-eval/builder-session.d.ts.map +1 -0
- package/dist/builder-eval/correlation.d.ts +32 -0
- package/dist/builder-eval/correlation.d.ts.map +1 -0
- package/dist/builder-eval/index.d.ts +5 -250
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/project-registry.d.ts +51 -0
- package/dist/builder-eval/project-registry.d.ts.map +1 -0
- package/dist/builder-eval/three-layer-eval.d.ts +55 -0
- package/dist/builder-eval/three-layer-eval.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.d.ts +108 -0
- package/dist/campaign/analyst-surface.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.test.d.ts +2 -0
- package/dist/campaign/analyst-surface.test.d.ts.map +1 -0
- package/dist/campaign/auto-pr.d.ts +46 -0
- package/dist/campaign/auto-pr.d.ts.map +1 -0
- package/dist/campaign/distillation/agreement-judge.d.ts +69 -0
- package/dist/campaign/distillation/agreement-judge.d.ts.map +1 -0
- package/dist/campaign/distillation/cli.d.ts +35 -0
- package/dist/campaign/distillation/cli.d.ts.map +1 -0
- package/dist/campaign/distillation/distillation.test.d.ts +2 -0
- package/dist/campaign/distillation/distillation.test.d.ts.map +1 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts +54 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +1 -0
- package/dist/campaign/distillation/run-distillation.d.ts +119 -0
- package/dist/campaign/distillation/run-distillation.d.ts.map +1 -0
- package/dist/campaign/gates/compose.d.ts +12 -0
- package/dist/campaign/gates/compose.d.ts.map +1 -0
- package/dist/campaign/gates/default-production-gate.d.ts +58 -0
- package/dist/campaign/gates/default-production-gate.d.ts.map +1 -0
- package/dist/campaign/gates/heldout-gate.d.ts +12 -0
- package/dist/campaign/gates/heldout-gate.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.d.ts +125 -0
- package/dist/campaign/gates/promotion-policy.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts +2 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.d.ts +146 -0
- package/dist/campaign/gates/sequential.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.test.d.ts +2 -0
- package/dist/campaign/gates/sequential.test.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.d.ts +99 -0
- package/dist/campaign/gates/statistical-heldout.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts +2 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +1 -0
- package/dist/campaign/index.d.ts +38 -1341
- package/dist/campaign/index.d.ts.map +1 -0
- package/dist/campaign/index.js +1865 -1318
- package/dist/campaign/index.js.map +1 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +59 -0
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +1 -0
- package/dist/campaign/presets/compare-proposers.d.ts +146 -0
- package/dist/campaign/presets/compare-proposers.d.ts.map +1 -0
- package/dist/campaign/presets/playback.d.ts +120 -0
- package/dist/campaign/presets/playback.d.ts.map +1 -0
- package/dist/campaign/presets/playback.test.d.ts +2 -0
- package/dist/campaign/presets/playback.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-eval.d.ts +14 -0
- package/dist/campaign/presets/run-eval.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts +63 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +2 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-optimization.d.ts +92 -0
- package/dist/campaign/presets/run-optimization.d.ts.map +1 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts +151 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +1 -0
- package/dist/campaign/presets/run-skill-opt.d.ts +96 -0
- package/dist/campaign/presets/run-skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/_findings-text.d.ts +22 -0
- package/dist/campaign/proposers/_findings-text.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.d.ts +33 -0
- package/dist/campaign/proposers/ace.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.test.d.ts +2 -0
- package/dist/campaign/proposers/ace.test.d.ts.map +1 -0
- package/dist/campaign/proposers/analysis-edit.d.ts +32 -0
- package/dist/campaign/proposers/analysis-edit.d.ts.map +1 -0
- package/dist/campaign/proposers/evolutionary.d.ts +20 -0
- package/dist/campaign/proposers/evolutionary.d.ts.map +1 -0
- package/dist/campaign/proposers/fapo.d.ts +120 -0
- package/dist/campaign/proposers/fapo.d.ts.map +1 -0
- package/dist/campaign/proposers/gepa.d.ts +86 -0
- package/dist/campaign/proposers/gepa.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.d.ts +44 -0
- package/dist/campaign/proposers/halo.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.test.d.ts +2 -0
- package/dist/campaign/proposers/halo.test.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.d.ts +47 -0
- package/dist/campaign/proposers/memory.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.test.d.ts +2 -0
- package/dist/campaign/proposers/memory.test.d.ts.map +1 -0
- package/dist/campaign/proposers/skill-opt.d.ts +88 -0
- package/dist/campaign/proposers/skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.d.ts +48 -0
- package/dist/campaign/proposers/trace-analyst.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts +2 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +1 -0
- package/dist/campaign/provenance.d.ts +185 -0
- package/dist/campaign/provenance.d.ts.map +1 -0
- package/dist/campaign/run-campaign.d.ts +90 -0
- package/dist/campaign/run-campaign.d.ts.map +1 -0
- package/dist/campaign/score-utils.d.ts +26 -0
- package/dist/campaign/score-utils.d.ts.map +1 -0
- package/dist/campaign/skill-patch.d.ts +62 -0
- package/dist/campaign/skill-patch.d.ts.map +1 -0
- package/dist/campaign/storage.d.ts +38 -0
- package/dist/campaign/storage.d.ts.map +1 -0
- package/dist/{types-BU-7W85F.d.ts → campaign/types.d.ts} +98 -99
- package/dist/campaign/types.d.ts.map +1 -0
- package/dist/campaign/worktree/index.d.ts +53 -0
- package/dist/campaign/worktree/index.d.ts.map +1 -0
- package/dist/canary.d.ts +101 -0
- package/dist/canary.d.ts.map +1 -0
- package/dist/causal-attribution.d.ts +45 -0
- package/dist/causal-attribution.d.ts.map +1 -0
- package/dist/{chunk-TWS7AZEY.js → chunk-2T4EZACH.js} +2 -2
- package/dist/chunk-2T4EZACH.js.map +1 -0
- package/dist/{chunk-LMZQ2Z4U.js → chunk-56GC6VYO.js} +126 -1
- package/dist/chunk-56GC6VYO.js.map +1 -0
- package/dist/{chunk-QG2OVF2D.js → chunk-77T4STFI.js} +154 -112
- package/dist/chunk-77T4STFI.js.map +1 -0
- package/dist/{chunk-ZFIBGEOL.js → chunk-7QTQKIDD.js} +179 -178
- package/dist/chunk-7QTQKIDD.js.map +1 -0
- package/dist/{chunk-UMMZHCPB.js → chunk-AQ5WQAIV.js} +26 -9
- package/dist/chunk-AQ5WQAIV.js.map +1 -0
- package/dist/{chunk-CVVHBFGN.js → chunk-CWNP4DV4.js} +53 -5
- package/dist/chunk-CWNP4DV4.js.map +1 -0
- package/dist/{chunk-QS3RBQPI.js → chunk-D5KBVULY.js} +2 -2
- package/dist/chunk-DJWX3GVS.js +81 -0
- package/dist/chunk-DJWX3GVS.js.map +1 -0
- package/dist/{chunk-S6OZEZQK.js → chunk-HMA63UEO.js} +37 -9
- package/dist/{chunk-S6OZEZQK.js.map → chunk-HMA63UEO.js.map} +1 -1
- package/dist/{chunk-UHMJT4T7.js → chunk-IZCEK2HR.js} +2 -2
- package/dist/{chunk-6SOJM3VR.js → chunk-KKWJD5E6.js} +21 -21
- package/dist/chunk-KKWJD5E6.js.map +1 -0
- package/dist/{chunk-KWRRMR3J.js → chunk-LO6IOIJ2.js} +10 -10
- package/dist/chunk-LO6IOIJ2.js.map +1 -0
- package/dist/{chunk-L3JOU6XM.js → chunk-NACM5RS7.js} +3 -3
- package/dist/{chunk-E4GH6USR.js → chunk-NZEQVRH5.js} +2 -2
- package/dist/chunk-NZEQVRH5.js.map +1 -0
- package/dist/{chunk-XY4DDNEG.js → chunk-PSWWQXHF.js} +14 -89
- package/dist/chunk-PSWWQXHF.js.map +1 -0
- package/dist/{chunk-4FBZZIYD.js → chunk-QTMYW64F.js} +2 -2
- package/dist/{chunk-D3V5B42D.js → chunk-S4SYLDFX.js} +7 -4
- package/dist/chunk-S4SYLDFX.js.map +1 -0
- package/dist/{chunk-47X6LRCE.js → chunk-UFSG7ACU.js} +10 -7
- package/dist/chunk-UFSG7ACU.js.map +1 -0
- package/dist/{chunk-CY6U5S3X.js → chunk-URWVKRCS.js} +2 -2
- package/dist/{chunk-GSH6QNNS.js → chunk-X74V6ESX.js} +102 -79
- package/dist/chunk-X74V6ESX.js.map +1 -0
- package/dist/{chunk-QAY5UIJO.js → chunk-YBIGNSCZ.js} +178 -59
- package/dist/chunk-YBIGNSCZ.js.map +1 -0
- package/dist/{chunk-Y47J2LJ3.js → chunk-Z6L6YSU6.js} +5 -5
- package/dist/{chunk-T375SUOZ.js → chunk-ZOPLCJSY.js} +86 -31
- package/dist/chunk-ZOPLCJSY.js.map +1 -0
- package/dist/ci-gate.d.ts +44 -0
- package/dist/ci-gate.d.ts.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client.d.ts +77 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.test.d.ts +2 -0
- package/dist/client.test.d.ts.map +1 -0
- package/dist/command-runner.d.ts +74 -0
- package/dist/command-runner.d.ts.map +1 -0
- package/dist/command-runner.test.d.ts +2 -0
- package/dist/command-runner.test.d.ts.map +1 -0
- package/dist/completion-verifier.d.ts +147 -0
- package/dist/completion-verifier.d.ts.map +1 -0
- package/dist/completion-verifier.test.d.ts +9 -0
- package/dist/completion-verifier.test.d.ts.map +1 -0
- package/dist/concurrency.d.ts +23 -0
- package/dist/concurrency.d.ts.map +1 -0
- package/dist/contamination-guard.d.ts +81 -0
- package/dist/contamination-guard.d.ts.map +1 -0
- package/dist/{analyze-runs-DwCEkpO_.d.ts → contract/analyze-runs.d.ts} +9 -10
- package/dist/contract/analyze-runs.d.ts.map +1 -0
- package/dist/contract/define-agent-eval.d.ts +52 -0
- package/dist/contract/define-agent-eval.d.ts.map +1 -0
- package/dist/contract/diff.d.ts +114 -0
- package/dist/contract/diff.d.ts.map +1 -0
- package/dist/contract/index.d.ts +106 -640
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +131 -21
- package/dist/contract/index.js.map +1 -1
- package/dist/{insight-report-BBwvOh6x.d.ts → contract/insight-report.d.ts} +16 -19
- package/dist/contract/insight-report.d.ts.map +1 -0
- package/dist/contract/insight-types-fwd.d.ts +7 -0
- package/dist/contract/insight-types-fwd.d.ts.map +1 -0
- package/dist/contract/intake/agent-trace.d.ts +97 -0
- package/dist/contract/intake/agent-trace.d.ts.map +1 -0
- package/dist/{code-agent-session-BO8nCnv3.d.ts → contract/intake/code-agent-session.d.ts} +15 -17
- package/dist/contract/intake/code-agent-session.d.ts.map +1 -0
- package/dist/contract/intake/feedback-table.d.ts +87 -0
- package/dist/contract/intake/feedback-table.d.ts.map +1 -0
- package/dist/contract/intake/index.d.ts +22 -0
- package/dist/contract/intake/index.d.ts.map +1 -0
- package/dist/contract/intake/otel-spans.d.ts +33 -0
- package/dist/contract/intake/otel-spans.d.ts.map +1 -0
- package/dist/contract/self-improve.d.ts +284 -0
- package/dist/contract/self-improve.d.ts.map +1 -0
- package/dist/{control-runtime-DuFBYg7A.d.ts → control-runtime.d.ts} +23 -26
- package/dist/control-runtime.d.ts.map +1 -0
- package/dist/control-runtime.test.d.ts +2 -0
- package/dist/control-runtime.test.d.ts.map +1 -0
- package/dist/control.d.ts +11 -9
- package/dist/control.d.ts.map +1 -0
- package/dist/control.js +2 -2
- package/dist/convergence.d.ts +29 -0
- package/dist/convergence.d.ts.map +1 -0
- package/dist/{cost-ledger-DuSqlw5B.d.ts → cost-ledger.d.ts} +10 -11
- package/dist/cost-ledger.d.ts.map +1 -0
- package/dist/cost-ledger.test.d.ts +2 -0
- package/dist/cost-ledger.test.d.ts.map +1 -0
- package/dist/cost-report.d.ts +43 -0
- package/dist/cost-report.d.ts.map +1 -0
- package/dist/cost-report.test.d.ts +2 -0
- package/dist/cost-report.test.d.ts.map +1 -0
- package/dist/cost-tracker.d.ts +76 -0
- package/dist/cost-tracker.d.ts.map +1 -0
- package/dist/{counterfactual-Dwibr5IW.d.ts → counterfactual.d.ts} +12 -13
- package/dist/counterfactual.d.ts.map +1 -0
- package/dist/cross-trace-diff.d.ts +56 -0
- package/dist/cross-trace-diff.d.ts.map +1 -0
- package/dist/{dataset-BbGkaN2I.d.ts → dataset.d.ts} +11 -14
- package/dist/dataset.d.ts.map +1 -0
- package/dist/deploy-gate-layer.d.ts +125 -0
- package/dist/deploy-gate-layer.d.ts.map +1 -0
- package/dist/deploy-gate-layer.test.d.ts +2 -0
- package/dist/deploy-gate-layer.test.d.ts.map +1 -0
- package/dist/description-length-gate.d.ts +119 -0
- package/dist/description-length-gate.d.ts.map +1 -0
- package/dist/detectors/edge.test.d.ts +2 -0
- package/dist/detectors/edge.test.d.ts.map +1 -0
- package/dist/detectors/index.d.ts +81 -0
- package/dist/detectors/index.d.ts.map +1 -0
- package/dist/detectors/index.test.d.ts +2 -0
- package/dist/detectors/index.test.d.ts.map +1 -0
- package/dist/diagnose/causal-sweep.d.ts +100 -0
- package/dist/diagnose/causal-sweep.d.ts.map +1 -0
- package/dist/diagnose/index.d.ts +36 -0
- package/dist/diagnose/index.d.ts.map +1 -0
- package/dist/diagnose/remediation.d.ts +68 -0
- package/dist/diagnose/remediation.d.ts.map +1 -0
- package/dist/diagnose/repair.d.ts +77 -0
- package/dist/diagnose/repair.d.ts.map +1 -0
- package/dist/diagnose.d.ts +1 -251
- package/dist/diagnose.js +2 -2
- package/dist/discover-personas.d.ts +35 -0
- package/dist/discover-personas.d.ts.map +1 -0
- package/dist/driver.d.ts +95 -0
- package/dist/driver.d.ts.map +1 -0
- package/dist/driver.test.d.ts +8 -0
- package/dist/driver.test.d.ts.map +1 -0
- package/dist/dual-agent-bench.d.ts +81 -0
- package/dist/dual-agent-bench.d.ts.map +1 -0
- package/dist/error-count-extractor.d.ts +47 -0
- package/dist/error-count-extractor.d.ts.map +1 -0
- package/dist/error-count-extractor.test.d.ts +2 -0
- package/dist/error-count-extractor.test.d.ts.map +1 -0
- package/dist/{errors-CzMUYo7b.d.ts → errors.d.ts} +10 -11
- package/dist/errors.d.ts.map +1 -0
- package/dist/{researcher-DE6Gpnb4.d.ts → eval-campaign.d.ts} +34 -156
- package/dist/eval-campaign.d.ts.map +1 -0
- package/dist/eval-campaign.test.d.ts +2 -0
- package/dist/eval-campaign.test.d.ts.map +1 -0
- package/dist/eval-tools.d.ts +55 -0
- package/dist/eval-tools.d.ts.map +1 -0
- package/dist/eval-trace-store.d.ts +107 -0
- package/dist/eval-trace-store.d.ts.map +1 -0
- package/dist/eval-trace-store.test.d.ts +2 -0
- package/dist/eval-trace-store.test.d.ts.map +1 -0
- package/dist/executor.d.ts +38 -0
- package/dist/executor.d.ts.map +1 -0
- package/dist/executor.test.d.ts +10 -0
- package/dist/executor.test.d.ts.map +1 -0
- package/dist/experiment-tracker.d.ts +178 -0
- package/dist/experiment-tracker.d.ts.map +1 -0
- package/dist/experiment-tracker.test.d.ts +2 -0
- package/dist/experiment-tracker.test.d.ts.map +1 -0
- package/dist/failure-taxonomy.d.ts +38 -0
- package/dist/failure-taxonomy.d.ts.map +1 -0
- package/dist/{feedback-trajectory-D9OVLrg9.d.ts → feedback-trajectory.d.ts} +36 -38
- package/dist/feedback-trajectory.d.ts.map +1 -0
- package/dist/feedback-trajectory.test.d.ts +2 -0
- package/dist/feedback-trajectory.test.d.ts.map +1 -0
- package/dist/flow-layer.d.ts +90 -0
- package/dist/flow-layer.d.ts.map +1 -0
- package/dist/flow-layer.test.d.ts +2 -0
- package/dist/flow-layer.test.d.ts.map +1 -0
- package/dist/fuzz/capsule.d.ts +46 -0
- package/dist/fuzz/capsule.d.ts.map +1 -0
- package/dist/fuzz/cube.d.ts +36 -0
- package/dist/fuzz/cube.d.ts.map +1 -0
- package/dist/fuzz/explorer-cost.test.d.ts +2 -0
- package/dist/fuzz/explorer-cost.test.d.ts.map +1 -0
- package/dist/fuzz/explorer.d.ts +64 -0
- package/dist/fuzz/explorer.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.d.ts +16 -0
- package/dist/fuzz/fuzz-agent.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.test.d.ts +2 -0
- package/dist/fuzz/fuzz-agent.test.d.ts.map +1 -0
- package/dist/fuzz/gates.d.ts +33 -0
- package/dist/fuzz/gates.d.ts.map +1 -0
- package/dist/fuzz/index.d.ts +26 -0
- package/dist/fuzz/index.d.ts.map +1 -0
- package/dist/fuzz/policies.d.ts +28 -0
- package/dist/fuzz/policies.d.ts.map +1 -0
- package/dist/fuzz/tools.d.ts +20 -0
- package/dist/fuzz/tools.d.ts.map +1 -0
- package/dist/fuzz/types.d.ts +307 -0
- package/dist/fuzz/types.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -547
- package/dist/golden-matcher.d.ts +71 -0
- package/dist/golden-matcher.d.ts.map +1 -0
- package/dist/governance/eu-ai-act.d.ts +37 -0
- package/dist/governance/eu-ai-act.d.ts.map +1 -0
- package/dist/governance/index.d.ts +5 -135
- package/dist/governance/index.d.ts.map +1 -0
- package/dist/governance/nist-ai-rmf.d.ts +15 -0
- package/dist/governance/nist-ai-rmf.d.ts.map +1 -0
- package/dist/governance/soc2.d.ts +12 -0
- package/dist/governance/soc2.d.ts.map +1 -0
- package/dist/governance/types.d.ts +66 -0
- package/dist/governance/types.d.ts.map +1 -0
- package/dist/harness-optimizer.d.ts +82 -0
- package/dist/harness-optimizer.d.ts.map +1 -0
- package/dist/held-out-gate.d.ts +135 -0
- package/dist/held-out-gate.d.ts.map +1 -0
- package/dist/hosted/client.d.ts +73 -0
- package/dist/hosted/client.d.ts.map +1 -0
- package/dist/hosted/from-env.test.d.ts +8 -0
- package/dist/hosted/from-env.test.d.ts.map +1 -0
- package/dist/hosted/index.d.ts +10 -238
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/types.d.ts +159 -0
- package/dist/hosted/types.d.ts.map +1 -0
- package/dist/index.d.ts +277 -5643
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +317 -108
- package/dist/index.js.map +1 -1
- package/dist/integrity/backend-integrity.d.ts +71 -0
- package/dist/integrity/backend-integrity.d.ts.map +1 -0
- package/dist/integrity/preflight.d.ts +72 -0
- package/dist/integrity/preflight.d.ts.map +1 -0
- package/dist/integrity/preflight.test.d.ts +2 -0
- package/dist/integrity/preflight.test.d.ts.map +1 -0
- package/dist/integrity/single-backend.d.ts +67 -0
- package/dist/integrity/single-backend.d.ts.map +1 -0
- package/dist/intent-match-judge.d.ts +69 -0
- package/dist/intent-match-judge.d.ts.map +1 -0
- package/dist/intent-match-judge.test.d.ts +2 -0
- package/dist/intent-match-judge.test.d.ts.map +1 -0
- package/dist/{judge-calibration-0p2QcWNE.d.ts → judge-calibration.d.ts} +16 -17
- package/dist/judge-calibration.d.ts.map +1 -0
- package/dist/judge-ensemble.d.ts +66 -0
- package/dist/judge-ensemble.d.ts.map +1 -0
- package/dist/judge-ensemble.test.d.ts +8 -0
- package/dist/judge-ensemble.test.d.ts.map +1 -0
- package/dist/judge-families.d.ts +38 -0
- package/dist/judge-families.d.ts.map +1 -0
- package/dist/judge-panel.d.ts +65 -0
- package/dist/judge-panel.d.ts.map +1 -0
- package/dist/judge-retry.d.ts +70 -0
- package/dist/judge-retry.d.ts.map +1 -0
- package/dist/judge-runner.d.ts +36 -0
- package/dist/judge-runner.d.ts.map +1 -0
- package/dist/judge-runner.test.d.ts +2 -0
- package/dist/judge-runner.test.d.ts.map +1 -0
- package/dist/judges.d.ts +74 -0
- package/dist/judges.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.d.ts +89 -0
- package/dist/keyword-coverage-judge.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.test.d.ts +2 -0
- package/dist/keyword-coverage-judge.test.d.ts.map +1 -0
- package/dist/knowledge/index.d.ts +3 -103
- package/dist/knowledge/index.d.ts.map +1 -0
- package/dist/knowledge/readiness.d.ts +26 -0
- package/dist/knowledge/readiness.d.ts.map +1 -0
- package/dist/knowledge/types.d.ts +75 -0
- package/dist/knowledge/types.d.ts.map +1 -0
- package/dist/live-proof.d.ts +62 -0
- package/dist/live-proof.d.ts.map +1 -0
- package/dist/{llm-client-BeEcAokY.d.ts → llm-client.d.ts} +52 -23
- package/dist/llm-client.d.ts.map +1 -0
- package/dist/llm-client.test.d.ts +2 -0
- package/dist/llm-client.test.d.ts.map +1 -0
- package/dist/locked-jsonl-appender.d.ts +19 -0
- package/dist/locked-jsonl-appender.d.ts.map +1 -0
- package/dist/matrix/aggregation.d.ts +16 -0
- package/dist/matrix/aggregation.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +12 -30
- package/dist/matrix/index.d.ts.map +1 -0
- package/dist/matrix/runner.d.ts +15 -0
- package/dist/matrix/runner.d.ts.map +1 -0
- package/dist/{types-mn5Aqk7x.d.ts → matrix/types.d.ts} +11 -15
- package/dist/matrix/types.d.ts.map +1 -0
- package/dist/meta-eval/calibration.d.ts +47 -0
- package/dist/meta-eval/calibration.d.ts.map +1 -0
- package/dist/meta-eval/correlation-study.d.ts +53 -0
- package/dist/meta-eval/correlation-study.d.ts.map +1 -0
- package/dist/meta-eval/index.d.ts +6 -181
- package/dist/meta-eval/index.d.ts.map +1 -0
- package/dist/meta-eval/index.js +1 -1
- package/dist/{outcome-store-rnXLEqSn.d.ts → meta-eval/outcome-store.d.ts} +7 -8
- package/dist/meta-eval/outcome-store.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → meta-eval/rubric-predictive-validity.d.ts} +8 -11
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +1 -0
- package/dist/meta-eval/sentinel.d.ts +169 -0
- package/dist/meta-eval/sentinel.d.ts.map +1 -0
- package/dist/metrics.d.ts +63 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/model-seats.d.ts +71 -0
- package/dist/model-seats.d.ts.map +1 -0
- package/dist/model-seats.test.d.ts +2 -0
- package/dist/model-seats.test.d.ts.map +1 -0
- package/dist/muffled-gate-scanner.d.ts +102 -0
- package/dist/muffled-gate-scanner.d.ts.map +1 -0
- package/dist/{multi-layer-verifier-DUZXrPDA.d.ts → multi-layer-verifier.d.ts} +12 -15
- package/dist/multi-layer-verifier.d.ts.map +1 -0
- package/dist/multi-layer-verifier.test.d.ts +2 -0
- package/dist/multi-layer-verifier.test.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.d.ts +80 -0
- package/dist/multi-toolchain-layer.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.test.d.ts +2 -0
- package/dist/multi-toolchain-layer.test.d.ts.map +1 -0
- package/dist/multishot/default-tools.d.ts +34 -0
- package/dist/multishot/default-tools.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +7 -290
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js.map +1 -1
- package/dist/multishot/judges.d.ts +32 -0
- package/dist/multishot/judges.d.ts.map +1 -0
- package/dist/multishot/matrix.d.ts +107 -0
- package/dist/multishot/matrix.d.ts.map +1 -0
- package/dist/multishot/multishot.d.ts +23 -0
- package/dist/multishot/multishot.d.ts.map +1 -0
- package/dist/multishot/router.d.ts +37 -0
- package/dist/multishot/router.d.ts.map +1 -0
- package/dist/multishot/types.d.ts +60 -0
- package/dist/multishot/types.d.ts.map +1 -0
- package/dist/observability.d.ts +71 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/oracle.d.ts +55 -0
- package/dist/oracle.d.ts.map +1 -0
- package/dist/orthogonality.d.ts +35 -0
- package/dist/orthogonality.d.ts.map +1 -0
- package/dist/otel-pipeline.d.ts +31 -0
- package/dist/otel-pipeline.d.ts.map +1 -0
- package/dist/paraphrase.d.ts +107 -0
- package/dist/paraphrase.d.ts.map +1 -0
- package/dist/{pareto-E-pembql.d.ts → pareto.d.ts} +9 -10
- package/dist/pareto.d.ts.map +1 -0
- package/dist/partition-held-out.d.ts +70 -0
- package/dist/partition-held-out.d.ts.map +1 -0
- package/dist/partition-held-out.test.d.ts +2 -0
- package/dist/partition-held-out.test.d.ts.map +1 -0
- package/dist/perf/index.d.ts +13 -119
- package/dist/perf/index.d.ts.map +1 -0
- package/dist/perf/integrity.d.ts +30 -0
- package/dist/perf/integrity.d.ts.map +1 -0
- package/dist/perf/journey.d.ts +45 -0
- package/dist/perf/journey.d.ts.map +1 -0
- package/dist/perf/ratchet.d.ts +47 -0
- package/dist/perf/ratchet.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.d.ts +31 -0
- package/dist/pipelines/budget-breach.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.test.d.ts +2 -0
- package/dist/pipelines/budget-breach.test.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.d.ts +38 -0
- package/dist/pipelines/failure-cluster.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.test.d.ts +2 -0
- package/dist/pipelines/failure-cluster.test.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.d.ts +26 -0
- package/dist/pipelines/first-divergence.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.test.d.ts +2 -0
- package/dist/pipelines/first-divergence.test.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +8 -173
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +40 -15
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pipelines/judge-agreement.d.ts +26 -0
- package/dist/pipelines/judge-agreement.d.ts.map +1 -0
- package/dist/pipelines/judge-agreement.test.d.ts +2 -0
- package/dist/pipelines/judge-agreement.test.d.ts.map +1 -0
- package/dist/pipelines/regression.d.ts +23 -0
- package/dist/pipelines/regression.d.ts.map +1 -0
- package/dist/pipelines/regression.test.d.ts +2 -0
- package/dist/pipelines/regression.test.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.d.ts +32 -0
- package/dist/pipelines/stuck-loop.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.test.d.ts +2 -0
- package/dist/pipelines/stuck-loop.test.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.d.ts +34 -0
- package/dist/pipelines/tool-waste.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.test.d.ts +2 -0
- package/dist/pipelines/tool-waste.test.d.ts.map +1 -0
- package/dist/playbook.d.ts +16 -0
- package/dist/playbook.d.ts.map +1 -0
- package/dist/pr-review-benchmark.d.ts +88 -0
- package/dist/pr-review-benchmark.d.ts.map +1 -0
- package/dist/pr-review-benchmark.test.d.ts +2 -0
- package/dist/pr-review-benchmark.test.d.ts.map +1 -0
- package/dist/pre-registration.d.ts +125 -0
- package/dist/pre-registration.d.ts.map +1 -0
- package/dist/prm/builtin-rubrics.d.ts +33 -0
- package/dist/prm/builtin-rubrics.d.ts.map +1 -0
- package/dist/prm/index.d.ts +5 -100
- package/dist/prm/index.d.ts.map +1 -0
- package/dist/prm/index.js +55 -12
- package/dist/prm/index.js.map +1 -1
- package/dist/prm/inference.d.ts +29 -0
- package/dist/prm/inference.d.ts.map +1 -0
- package/dist/prm/inference.test.d.ts +2 -0
- package/dist/prm/inference.test.d.ts.map +1 -0
- package/dist/{rubric-BOfxn4ja.d.ts → prm/rubric.d.ts} +10 -13
- package/dist/prm/rubric.d.ts.map +1 -0
- package/dist/prm/training-export.d.ts +38 -0
- package/dist/prm/training-export.d.ts.map +1 -0
- package/dist/produced-state.d.ts +63 -0
- package/dist/produced-state.d.ts.map +1 -0
- package/dist/produced-state.test.d.ts +8 -0
- package/dist/produced-state.test.d.ts.map +1 -0
- package/dist/profile/baselines.d.ts +37 -0
- package/dist/profile/baselines.d.ts.map +1 -0
- package/dist/profile/index.d.ts +105 -0
- package/dist/profile/index.d.ts.map +1 -0
- package/dist/promotion-gate.d.ts +93 -0
- package/dist/promotion-gate.d.ts.map +1 -0
- package/dist/prompt-registry.d.ts +41 -0
- package/dist/prompt-registry.d.ts.map +1 -0
- package/dist/propose-review-control.d.ts +49 -0
- package/dist/propose-review-control.d.ts.map +1 -0
- package/dist/propose-review-control.test.d.ts +2 -0
- package/dist/propose-review-control.test.d.ts.map +1 -0
- package/dist/propose-review.d.ts +155 -0
- package/dist/propose-review.d.ts.map +1 -0
- package/dist/{red-team-BXHil6c8.d.ts → red-team.d.ts} +14 -16
- package/dist/red-team.d.ts.map +1 -0
- package/dist/reference-replay-steering.d.ts +11 -0
- package/dist/reference-replay-steering.d.ts.map +1 -0
- package/dist/reference-replay.d.ts +176 -0
- package/dist/reference-replay.d.ts.map +1 -0
- package/dist/reflective-mutation.d.ts +79 -0
- package/dist/reflective-mutation.d.ts.map +1 -0
- package/dist/registry.d.ts +31 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/release-confidence.d.ts +128 -0
- package/dist/release-confidence.d.ts.map +1 -0
- package/dist/release-report.d.ts +11 -0
- package/dist/release-report.d.ts.map +1 -0
- package/dist/replay.d.ts +120 -0
- package/dist/replay.d.ts.map +1 -0
- package/dist/reporter.d.ts +14 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporting.d.ts +15 -15
- package/dist/reporting.d.ts.map +1 -0
- package/dist/reporting.js +3 -3
- package/dist/researcher.d.ts +140 -0
- package/dist/researcher.d.ts.map +1 -0
- package/dist/reviewer.d.ts +118 -0
- package/dist/reviewer.d.ts.map +1 -0
- package/dist/reviewer.test.d.ts +2 -0
- package/dist/reviewer.test.d.ts.map +1 -0
- package/dist/reward-model-export.d.ts +60 -0
- package/dist/reward-model-export.d.ts.map +1 -0
- package/dist/rl/active-curriculum.d.ts +110 -0
- package/dist/rl/active-curriculum.d.ts.map +1 -0
- package/dist/rl/adaptation-eval.d.ts +109 -0
- package/dist/rl/adaptation-eval.d.ts.map +1 -0
- package/dist/{adversarial-DIVcDoI_.d.ts → rl/adversarial.d.ts} +6 -7
- package/dist/rl/adversarial.d.ts.map +1 -0
- package/dist/rl/compute-curves.d.ts +127 -0
- package/dist/rl/compute-curves.d.ts.map +1 -0
- package/dist/rl/contamination.d.ts +117 -0
- package/dist/rl/contamination.d.ts.map +1 -0
- package/dist/rl/corpus.d.ts +55 -0
- package/dist/rl/corpus.d.ts.map +1 -0
- package/dist/rl/corpus.test.d.ts +2 -0
- package/dist/rl/corpus.test.d.ts.map +1 -0
- package/dist/rl/dataset.d.ts +102 -0
- package/dist/rl/dataset.d.ts.map +1 -0
- package/dist/rl/dataset.test.d.ts +2 -0
- package/dist/rl/dataset.test.d.ts.map +1 -0
- package/dist/rl/exporters.d.ts +141 -0
- package/dist/rl/exporters.d.ts.map +1 -0
- package/dist/rl/index.d.ts +49 -0
- package/dist/rl/index.d.ts.map +1 -0
- package/dist/{off-policy-DiwuKKg7.d.ts → rl/off-policy.d.ts} +8 -9
- package/dist/rl/off-policy.d.ts.map +1 -0
- package/dist/rl/predictive-validity-researcher.d.ts +69 -0
- package/dist/rl/predictive-validity-researcher.d.ts.map +1 -0
- package/dist/rl/preferences.d.ts +141 -0
- package/dist/rl/preferences.d.ts.map +1 -0
- package/dist/rl/process-reward.d.ts +122 -0
- package/dist/rl/process-reward.d.ts.map +1 -0
- package/dist/rl/reward-hacking.d.ts +104 -0
- package/dist/rl/reward-hacking.d.ts.map +1 -0
- package/dist/rl/rl-campaign.d.ts +85 -0
- package/dist/rl/rl-campaign.d.ts.map +1 -0
- package/dist/rl/run-record-adapters.d.ts +56 -0
- package/dist/rl/run-record-adapters.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.d.ts +166 -0
- package/dist/rl/sim-fidelity.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.test.d.ts +2 -0
- package/dist/rl/sim-fidelity.test.d.ts.map +1 -0
- package/dist/rl/tournament.d.ts +115 -0
- package/dist/rl/tournament.d.ts.map +1 -0
- package/dist/rl/verifiable-reward.d.ts +124 -0
- package/dist/rl/verifiable-reward.d.ts.map +1 -0
- package/dist/rl.d.ts +1 -1192
- package/dist/rl.js +615 -615
- package/dist/rl.js.map +1 -1
- package/dist/{run-campaign-RDGAM5KJ.js → run-campaign-WXY7KI67.js} +3 -3
- package/dist/run-critic.d.ts +23 -0
- package/dist/run-critic.d.ts.map +1 -0
- package/dist/run-evidence.d.ts +32 -0
- package/dist/run-evidence.d.ts.map +1 -0
- package/dist/{run-record-e7vj1uZQ.d.ts → run-record.d.ts} +16 -122
- package/dist/run-record.d.ts.map +1 -0
- package/dist/run-record.test.d.ts +2 -0
- package/dist/run-record.test.d.ts.map +1 -0
- package/dist/run-score.d.ts +31 -0
- package/dist/run-score.d.ts.map +1 -0
- package/dist/runtime-trajectory.d.ts +47 -0
- package/dist/runtime-trajectory.d.ts.map +1 -0
- package/dist/{test-graded-scenario-BdVaPyHT.d.ts → sandbox-harness.d.ts} +34 -60
- package/dist/sandbox-harness.d.ts.map +1 -0
- package/dist/sandbox-harness.test.d.ts +2 -0
- package/dist/sandbox-harness.test.d.ts.map +1 -0
- package/dist/sandbox-pool.d.ts +74 -0
- package/dist/sandbox-pool.d.ts.map +1 -0
- package/dist/sandbox-pool.test.d.ts +2 -0
- package/dist/sandbox-pool.test.d.ts.map +1 -0
- package/dist/scorecard.d.ts +133 -0
- package/dist/scorecard.d.ts.map +1 -0
- package/dist/scorecard.test.d.ts +2 -0
- package/dist/scorecard.test.d.ts.map +1 -0
- package/dist/self-play.d.ts +69 -0
- package/dist/self-play.d.ts.map +1 -0
- package/dist/semantic-concept-judge.d.ts +135 -0
- package/dist/semantic-concept-judge.d.ts.map +1 -0
- package/dist/semantic-concept-judge.test.d.ts +2 -0
- package/dist/semantic-concept-judge.test.d.ts.map +1 -0
- package/dist/{sequential-5iSVfzl2.d.ts → sequential.d.ts} +9 -10
- package/dist/sequential.d.ts.map +1 -0
- package/dist/{series-convergence-D5OWMBg6.d.ts → series-convergence.d.ts} +4 -5
- package/dist/series-convergence.d.ts.map +1 -0
- package/dist/slo.d.ts +48 -0
- package/dist/slo.d.ts.map +1 -0
- package/dist/state-continuity.d.ts +47 -0
- package/dist/state-continuity.d.ts.map +1 -0
- package/dist/{statistics-C7PozGrZ.d.ts → statistics.d.ts} +152 -39
- package/dist/statistics.d.ts.map +1 -0
- package/dist/statistics.test.d.ts +2 -0
- package/dist/statistics.test.d.ts.map +1 -0
- package/dist/steering-optimizer.d.ts +58 -0
- package/dist/steering-optimizer.d.ts.map +1 -0
- package/dist/steering.d.ts +24 -0
- package/dist/steering.d.ts.map +1 -0
- package/dist/storyboard/code-edit.d.ts +64 -0
- package/dist/storyboard/code-edit.d.ts.map +1 -0
- package/dist/storyboard/code-edit.test.d.ts +2 -0
- package/dist/storyboard/code-edit.test.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +16 -81
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.test.d.ts +2 -0
- package/dist/storyboard/index.test.d.ts.map +1 -0
- package/dist/{summary-report-DGmUucwQ.d.ts → summary-report.d.ts} +23 -160
- package/dist/summary-report.d.ts.map +1 -0
- package/dist/telemetry/client.d.ts +35 -0
- package/dist/telemetry/client.d.ts.map +1 -0
- package/dist/telemetry/index.d.ts +17 -35
- package/dist/telemetry/index.d.ts.map +1 -0
- package/dist/telemetry/schema.d.ts +61 -0
- package/dist/telemetry/schema.d.ts.map +1 -0
- package/dist/telemetry/sink-fetch.d.ts +39 -0
- package/dist/telemetry/sink-fetch.d.ts.map +1 -0
- package/dist/telemetry/{file.d.ts → sink-file.d.ts} +5 -7
- package/dist/telemetry/sink-file.d.ts.map +1 -0
- package/dist/test-graded-scenario.d.ts +42 -0
- package/dist/test-graded-scenario.d.ts.map +1 -0
- package/dist/testing.d.ts +5 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +8 -0
- package/dist/testing.js.map +1 -0
- package/dist/tool-use-metrics.d.ts +35 -0
- package/dist/tool-use-metrics.d.ts.map +1 -0
- package/dist/trace/capture-fetch.d.ts +48 -0
- package/dist/trace/capture-fetch.d.ts.map +1 -0
- package/dist/trace/capture-fetch.test.d.ts +2 -0
- package/dist/trace/capture-fetch.test.d.ts.map +1 -0
- package/dist/{emitter-DEZwY14K.d.ts → trace/emitter.d.ts} +9 -12
- package/dist/trace/emitter.d.ts.map +1 -0
- package/dist/trace/extract-usage.d.ts +46 -0
- package/dist/trace/extract-usage.d.ts.map +1 -0
- package/dist/trace/extract-usage.test.d.ts +2 -0
- package/dist/trace/extract-usage.test.d.ts.map +1 -0
- package/dist/trace/index.d.ts +15 -0
- package/dist/trace/index.d.ts.map +1 -0
- package/dist/{integrity-VJ9A7aST.d.ts → trace/integrity.d.ts} +11 -14
- package/dist/trace/integrity.d.ts.map +1 -0
- package/dist/trace/otel-bridge.d.ts +29 -0
- package/dist/trace/otel-bridge.d.ts.map +1 -0
- package/dist/trace/otel-export.d.ts +52 -0
- package/dist/trace/otel-export.d.ts.map +1 -0
- package/dist/trace/otel.d.ts +57 -0
- package/dist/trace/otel.d.ts.map +1 -0
- package/dist/trace/otlp-attributes.d.ts +17 -0
- package/dist/trace/otlp-attributes.d.ts.map +1 -0
- package/dist/trace/query.d.ts +29 -0
- package/dist/trace/query.d.ts.map +1 -0
- package/dist/trace/query.test.d.ts +2 -0
- package/dist/trace/query.test.d.ts.map +1 -0
- package/dist/{raw-provider-sink-C46HDghv.d.ts → trace/raw-provider-sink.d.ts} +13 -14
- package/dist/trace/raw-provider-sink.d.ts.map +1 -0
- package/dist/{redact-B40YG2M_.d.ts → trace/redact.d.ts} +7 -8
- package/dist/trace/redact.d.ts.map +1 -0
- package/dist/{schema-m0gsnbt3.d.ts → trace/schema.d.ts} +29 -30
- package/dist/trace/schema.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.d.ts +72 -0
- package/dist/trace/store-to-otlp.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.test.d.ts +2 -0
- package/dist/trace/store-to-otlp.test.d.ts.map +1 -0
- package/dist/{store-CKUAgsJz.d.ts → trace/store.d.ts} +37 -13
- package/dist/trace/store.d.ts.map +1 -0
- package/dist/trace/store.test.d.ts +2 -0
- package/dist/trace/store.test.d.ts.map +1 -0
- package/dist/{analyst-C8HHvfJp.d.ts → trace-analyst/analyst.d.ts} +12 -14
- package/dist/trace-analyst/analyst.d.ts.map +1 -0
- package/dist/trace-analyst/analyst.test.d.ts +2 -0
- package/dist/trace-analyst/analyst.test.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts +40 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +2 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +1 -0
- package/dist/trace-analyst/hook.d.ts +55 -0
- package/dist/trace-analyst/hook.d.ts.map +1 -0
- package/dist/trace-analyst/index.d.ts +18 -0
- package/dist/trace-analyst/index.d.ts.map +1 -0
- package/dist/trace-analyst/insights.d.ts +71 -0
- package/dist/trace-analyst/insights.d.ts.map +1 -0
- package/dist/trace-analyst/insights.test.d.ts +2 -0
- package/dist/trace-analyst/insights.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-flatten.d.ts +42 -0
- package/dist/trace-analyst/otlp-flatten.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.d.ts +85 -0
- package/dist/trace-analyst/otlp-span.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-span.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts +115 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/prompts.d.ts +6 -0
- package/dist/trace-analyst/prompts.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.d.ts +126 -0
- package/dist/trace-analyst/store-otlp.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.test.d.ts +8 -0
- package/dist/trace-analyst/store-otlp.test.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/store.d.ts +63 -0
- package/dist/trace-analyst/store.d.ts.map +1 -0
- package/dist/trace-analyst/tools.d.ts +44 -0
- package/dist/trace-analyst/tools.d.ts.map +1 -0
- package/dist/trace-analyst/tools.test.d.ts +10 -0
- package/dist/trace-analyst/tools.test.d.ts.map +1 -0
- package/dist/{store-C1YxJDEK.d.ts → trace-analyst/types.d.ts} +18 -74
- package/dist/trace-analyst/types.d.ts.map +1 -0
- package/dist/trace-contracts.d.ts +180 -0
- package/dist/trace-contracts.d.ts.map +1 -0
- package/dist/traced-analyst.d.ts +26 -0
- package/dist/traced-analyst.d.ts.map +1 -0
- package/dist/traced-judges.d.ts +27 -0
- package/dist/traced-judges.d.ts.map +1 -0
- package/dist/traces.d.ts +4 -947
- package/dist/traces.d.ts.map +1 -0
- package/dist/traces.js +42 -12
- package/dist/{trajectory-GEdXJCL5.d.ts → trajectory.d.ts} +8 -9
- package/dist/trajectory.d.ts.map +1 -0
- package/dist/{types-Croy5h7V.d.ts → types.d.ts} +33 -33
- package/dist/types.d.ts.map +1 -0
- package/dist/ui-finding.d.ts +104 -0
- package/dist/ui-finding.d.ts.map +1 -0
- package/dist/verdict-cache.d.ts +78 -0
- package/dist/verdict-cache.d.ts.map +1 -0
- package/dist/verdict-cache.test.d.ts +2 -0
- package/dist/verdict-cache.test.d.ts.map +1 -0
- package/dist/{verdict-C9MlYujm.d.ts → verdict.d.ts} +2 -3
- package/dist/verdict.d.ts.map +1 -0
- package/dist/visual-diff.d.ts +32 -0
- package/dist/visual-diff.d.ts.map +1 -0
- package/dist/wire/handlers.d.ts +54 -0
- package/dist/wire/handlers.d.ts.map +1 -0
- package/dist/wire/index.d.ts +13 -570
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -2
- package/dist/wire/openapi.d.ts +3 -0
- package/dist/wire/openapi.d.ts.map +1 -0
- package/dist/wire/rpc.d.ts +21 -0
- package/dist/wire/rpc.d.ts.map +1 -0
- package/dist/wire/rubrics.d.ts +34 -0
- package/dist/wire/rubrics.d.ts.map +1 -0
- package/dist/wire/schemas.d.ts +410 -0
- package/dist/wire/schemas.d.ts.map +1 -0
- package/dist/wire/server.d.ts +60 -0
- package/dist/wire/server.d.ts.map +1 -0
- package/dist/workflow/event-schema.d.ts +5 -0
- package/dist/workflow/event-schema.d.ts.map +1 -0
- package/dist/workflow/feedback-pack.d.ts +99 -0
- package/dist/workflow/feedback-pack.d.ts.map +1 -0
- package/dist/workflow/index.d.ts +22 -495
- package/dist/workflow/index.d.ts.map +1 -0
- package/dist/workflow/index.js +2 -2
- package/dist/workflow/intelligence-export.d.ts +62 -0
- package/dist/workflow/intelligence-export.d.ts.map +1 -0
- package/dist/workflow/partner-report.d.ts +49 -0
- package/dist/workflow/partner-report.d.ts.map +1 -0
- package/dist/workflow/phase-graph.d.ts +43 -0
- package/dist/workflow/phase-graph.d.ts.map +1 -0
- package/dist/workflow/promotion-gate.d.ts +61 -0
- package/dist/workflow/promotion-gate.d.ts.map +1 -0
- package/dist/workflow/run-record.d.ts +12 -0
- package/dist/workflow/run-record.d.ts.map +1 -0
- package/dist/workflow/runtime-adapter.d.ts +20 -0
- package/dist/workflow/runtime-adapter.d.ts.map +1 -0
- package/dist/workflow/sanitize.d.ts +21 -0
- package/dist/workflow/sanitize.d.ts.map +1 -0
- package/dist/workflow/schema.d.ts +5 -0
- package/dist/workflow/schema.d.ts.map +1 -0
- package/dist/workflow/summary.d.ts +43 -0
- package/dist/workflow/summary.d.ts.map +1 -0
- package/dist/workflow/trace-event-fields.d.ts +6 -0
- package/dist/workflow/trace-event-fields.d.ts.map +1 -0
- package/dist/workflow/trajectory.d.ts +15 -0
- package/dist/workflow/trajectory.d.ts.map +1 -0
- package/dist/workflow/types.d.ts +68 -0
- package/dist/workflow/types.d.ts.map +1 -0
- package/dist/workspace-inspector.d.ts +67 -0
- package/dist/workspace-inspector.d.ts.map +1 -0
- package/dist/wrangler-deploy-runner.test.d.ts +2 -0
- package/dist/wrangler-deploy-runner.test.d.ts.map +1 -0
- package/docs/campaign-proposers.md +170 -0
- package/docs/concepts.md +8 -4
- package/docs/customer-journeys.md +15 -13
- package/docs/design/loop-taxonomy.md +34 -66
- package/docs/distributed-driver.md +14 -14
- package/docs/feature-guide.md +1 -1
- package/docs/hosted-ingest-spec.md +2 -3
- package/docs/multi-shot-optimization.md +8 -8
- package/docs/product-eval-adoption.md +1 -1
- package/docs/self-improvement-map.md +33 -29
- package/package.json +9 -15
- package/dist/calibration-Cpr3WaX3.d.ts +0 -101
- package/dist/chunk-47X6LRCE.js.map +0 -1
- package/dist/chunk-6SOJM3VR.js.map +0 -1
- package/dist/chunk-CVVHBFGN.js.map +0 -1
- package/dist/chunk-D3V5B42D.js.map +0 -1
- package/dist/chunk-E4GH6USR.js.map +0 -1
- package/dist/chunk-GSH6QNNS.js.map +0 -1
- package/dist/chunk-KWRRMR3J.js.map +0 -1
- package/dist/chunk-LMZQ2Z4U.js.map +0 -1
- package/dist/chunk-QAY5UIJO.js.map +0 -1
- package/dist/chunk-QG2OVF2D.js.map +0 -1
- package/dist/chunk-T375SUOZ.js.map +0 -1
- package/dist/chunk-TWS7AZEY.js.map +0 -1
- package/dist/chunk-UMMZHCPB.js.map +0 -1
- package/dist/chunk-XY4DDNEG.js.map +0 -1
- package/dist/chunk-ZFIBGEOL.js.map +0 -1
- package/dist/control-_Qb7skHX.d.ts +0 -259
- package/dist/corpus-BoR-041R.d.ts +0 -560
- package/dist/failure-cluster-CL7IVgkJ.d.ts +0 -76
- package/dist/harness-optimizer-EnEnQPsr.d.ts +0 -106
- package/dist/index-Bx3gZ8xl.d.ts +0 -159
- package/dist/pre-registration-mAnCugl9.d.ts +0 -523
- package/dist/provenance-LnqRT0sS.d.ts +0 -441
- package/dist/query-CqTxMwDw.d.ts +0 -31
- package/dist/release-report-euXIV_Sk.d.ts +0 -233
- package/dist/run-critic-BAIjX99r.d.ts +0 -56
- package/dist/run-improvement-loop-5z_l5zDz.d.ts +0 -427
- package/dist/runtime-trajectory-BDgfGZSr.d.ts +0 -49
- package/dist/semantic-concept-judge-Dn8Z6KEG.d.ts +0 -624
- package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
- package/docs/design/external-agent-wedge.md +0 -89
- package/docs/design/phase-d-rfc.md +0 -125
- package/docs/design/phase4-consumer-migration.md +0 -70
- package/docs/design/primitives-integration-spec.md +0 -393
- package/docs/design/product-self-improvement-loop.md +0 -146
- package/docs/design/self-improvement-engine.md +0 -140
- package/docs/design/self-improvement-protocol.md +0 -223
- package/docs/design/self-improvement-roadmap.md +0 -106
- package/docs/design/substrate-gaps.md +0 -118
- package/docs/phase-b-pairing-kit.md +0 -188
- package/docs/phase-b-runbook.md +0 -176
- package/docs/pilot/README.md +0 -62
- package/docs/pilot/customer-checklist.md +0 -90
- package/docs/pilot/integration-foreign-stack.md +0 -296
- package/docs/pilot/integration-tangle-stack.md +0 -248
- package/docs/pilot/one-pager.md +0 -161
- package/docs/pilot/sample-insight-report.json +0 -172
- package/docs/quickstart-external.md +0 -229
- package/docs/research/belief-state-agent-eval-roadmap.md +0 -593
- package/docs/research/research-roadmap.md +0 -205
- package/docs/specs/driver-honest-spec.md +0 -251
- package/docs/specs/hermes-self-improvement-audit.md +0 -93
- package/docs/specs/profile-versioning.md +0 -291
- package/docs/three-package-architecture.md +0 -168
- /package/dist/{chunk-QS3RBQPI.js.map → chunk-D5KBVULY.js.map} +0 -0
- /package/dist/{chunk-UHMJT4T7.js.map → chunk-IZCEK2HR.js.map} +0 -0
- /package/dist/{chunk-L3JOU6XM.js.map → chunk-NACM5RS7.js.map} +0 -0
- /package/dist/{chunk-4FBZZIYD.js.map → chunk-QTMYW64F.js.map} +0 -0
- /package/dist/{chunk-CY6U5S3X.js.map → chunk-URWVKRCS.js.map} +0 -0
- /package/dist/{chunk-Y47J2LJ3.js.map → chunk-Z6L6YSU6.js.map} +0 -0
- /package/dist/{run-campaign-RDGAM5KJ.js.map → run-campaign-WXY7KI67.js.map} +0 -0
package/dist/campaign/index.js
CHANGED
|
@@ -4,18 +4,20 @@ import {
|
|
|
4
4
|
defaultProductionGate,
|
|
5
5
|
detectScale,
|
|
6
6
|
dimensionRegressions,
|
|
7
|
-
|
|
7
|
+
evolutionaryProposer,
|
|
8
8
|
heldoutSignificance,
|
|
9
9
|
pairHoldout,
|
|
10
10
|
paretoPolicy,
|
|
11
11
|
paretoSignificanceGate,
|
|
12
12
|
runEval
|
|
13
|
-
} from "../chunk-
|
|
13
|
+
} from "../chunk-KKWJD5E6.js";
|
|
14
14
|
import {
|
|
15
15
|
agentProfileHash,
|
|
16
|
+
agentProfileId,
|
|
17
|
+
agentProfileModelId,
|
|
16
18
|
extractProducedState,
|
|
17
19
|
verifyCompletion
|
|
18
|
-
} from "../chunk-
|
|
20
|
+
} from "../chunk-HMA63UEO.js";
|
|
19
21
|
import {
|
|
20
22
|
buildLoopProvenanceRecord,
|
|
21
23
|
campaignBreakdown,
|
|
@@ -24,7 +26,7 @@ import {
|
|
|
24
26
|
defaultRenderDiff,
|
|
25
27
|
emitLoopProvenance,
|
|
26
28
|
extractH2Sections,
|
|
27
|
-
|
|
29
|
+
gepaProposer,
|
|
28
30
|
heldOutGate,
|
|
29
31
|
isProposedCandidate,
|
|
30
32
|
labelTrustRank,
|
|
@@ -37,14 +39,14 @@ import {
|
|
|
37
39
|
runOptimization,
|
|
38
40
|
surfaceContentHash,
|
|
39
41
|
surfaceHash
|
|
40
|
-
} from "../chunk-
|
|
42
|
+
} from "../chunk-7QTQKIDD.js";
|
|
41
43
|
import {
|
|
42
44
|
assertRealBackend,
|
|
43
45
|
fsCampaignStorage,
|
|
44
46
|
inMemoryCampaignStorage,
|
|
45
47
|
runCampaign,
|
|
46
48
|
summarizeBackendIntegrity
|
|
47
|
-
} from "../chunk-
|
|
49
|
+
} from "../chunk-2T4EZACH.js";
|
|
48
50
|
import {
|
|
49
51
|
estimateCost,
|
|
50
52
|
isModelPriced
|
|
@@ -53,28 +55,28 @@ import {
|
|
|
53
55
|
AnalystRegistry,
|
|
54
56
|
DEFAULT_TRACE_ANALYST_KINDS,
|
|
55
57
|
createTraceAnalystKind
|
|
56
|
-
} from "../chunk-
|
|
58
|
+
} from "../chunk-S4SYLDFX.js";
|
|
57
59
|
import "../chunk-5HRORJQY.js";
|
|
58
60
|
import {
|
|
59
61
|
callLlm
|
|
60
|
-
} from "../chunk-
|
|
62
|
+
} from "../chunk-CWNP4DV4.js";
|
|
61
63
|
import "../chunk-45EEMHTC.js";
|
|
62
64
|
import {
|
|
63
65
|
eProcess,
|
|
64
66
|
mulberry32,
|
|
65
67
|
pairedBootstrap
|
|
66
|
-
} from "../chunk-
|
|
67
|
-
import "../chunk-GGE4NNQT.js";
|
|
68
|
+
} from "../chunk-56GC6VYO.js";
|
|
68
69
|
import {
|
|
69
70
|
analyzeTraces
|
|
70
|
-
} from "../chunk-
|
|
71
|
+
} from "../chunk-IZCEK2HR.js";
|
|
72
|
+
import "../chunk-GGE4NNQT.js";
|
|
71
73
|
import {
|
|
72
74
|
OtlpFileTraceStore
|
|
73
|
-
} from "../chunk-
|
|
75
|
+
} from "../chunk-YBIGNSCZ.js";
|
|
74
76
|
import "../chunk-PC4UYEBM.js";
|
|
75
77
|
import {
|
|
76
78
|
validateRunRecord
|
|
77
|
-
} from "../chunk-
|
|
79
|
+
} from "../chunk-LO6IOIJ2.js";
|
|
78
80
|
import {
|
|
79
81
|
canonicalize
|
|
80
82
|
} from "../chunk-VSMTAMNK.js";
|
|
@@ -145,383 +147,1068 @@ function failureModeRecallJudge(opts = {}) {
|
|
|
145
147
|
};
|
|
146
148
|
}
|
|
147
149
|
|
|
148
|
-
// src/campaign/
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
if (
|
|
152
|
-
|
|
153
|
-
const cand = o.recommended_action ?? o.claim ?? o.lesson ?? o.text ?? o.message;
|
|
154
|
-
if (typeof cand === "string" && cand.trim()) return cand.trim();
|
|
150
|
+
// src/campaign/gates/sequential.ts
|
|
151
|
+
import { createHash } from "crypto";
|
|
152
|
+
function verifyManifestSync(m) {
|
|
153
|
+
if (m.algo !== void 0 && m.algo !== "sha256-content") {
|
|
154
|
+
throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
|
|
155
155
|
}
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
function surfaceToText2(surface) {
|
|
162
|
-
if (typeof surface === "string") return surface;
|
|
163
|
-
throw new Error(
|
|
164
|
-
`curator driver: surface must be a string prompt, got a ${surface.kind}-tier surface (${surface.worktreeRef}) \u2014 curation is prompt-tier`
|
|
165
|
-
);
|
|
166
|
-
}
|
|
167
|
-
function extractBlockBody(text, startMarker, endMarker) {
|
|
168
|
-
const start = text.indexOf(startMarker);
|
|
169
|
-
const end = text.indexOf(endMarker);
|
|
170
|
-
if (start === -1 || end === -1 || end < start) return "";
|
|
171
|
-
return text.slice(start + startMarker.length, end);
|
|
172
|
-
}
|
|
173
|
-
function stripBlock(text, startMarker, endMarker) {
|
|
174
|
-
const start = text.indexOf(startMarker);
|
|
175
|
-
const end = text.indexOf(endMarker);
|
|
176
|
-
if (start === -1 || end === -1 || end < start) return text.trimEnd();
|
|
177
|
-
return (text.slice(0, start) + text.slice(end + endMarker.length)).trimEnd();
|
|
156
|
+
const { contentHash, algo: _algo, ...rest } = m;
|
|
157
|
+
void _algo;
|
|
158
|
+
const bytes = JSON.stringify(canonicalize(rest));
|
|
159
|
+
const hash = createHash("sha256").update(bytes, "utf8").digest("hex");
|
|
160
|
+
return hash === contentHash;
|
|
178
161
|
}
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
162
|
+
function resolveConfig(opts) {
|
|
163
|
+
const m = opts.preRegistration;
|
|
164
|
+
let alpha;
|
|
165
|
+
let maxN;
|
|
166
|
+
let direction;
|
|
167
|
+
let minEffect;
|
|
168
|
+
if (m) {
|
|
169
|
+
if (!verifyManifestSync(m)) {
|
|
170
|
+
throw new Error(
|
|
171
|
+
`sequentialPairedGate: pre-registration manifest '${m.id}' content hash mismatch (tampered)`
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
if (opts.alpha !== void 0 && opts.alpha !== m.alpha) {
|
|
175
|
+
throw new Error(
|
|
176
|
+
`sequentialPairedGate: alpha ${opts.alpha} conflicts with pre-registered alpha ${m.alpha} \u2014 the registered statistic is the only one anytime validity covers`
|
|
177
|
+
);
|
|
178
|
+
}
|
|
179
|
+
if (opts.maxN !== void 0 && opts.maxN !== m.preRegisteredN) {
|
|
180
|
+
throw new Error(
|
|
181
|
+
`sequentialPairedGate: maxN ${opts.maxN} conflicts with pre-registered N ${m.preRegisteredN}`
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
alpha = m.alpha;
|
|
185
|
+
maxN = m.preRegisteredN;
|
|
186
|
+
direction = m.direction;
|
|
187
|
+
minEffect = m.minEffect;
|
|
188
|
+
} else {
|
|
189
|
+
alpha = opts.alpha ?? 0.05;
|
|
190
|
+
if (opts.maxN === void 0) {
|
|
191
|
+
throw new Error(
|
|
192
|
+
"sequentialPairedGate: maxN is required (or bind a preRegistration manifest whose preRegisteredN is the budget) \u2014 an unbounded stream has no pre-registered budget"
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
maxN = opts.maxN;
|
|
196
|
+
direction = "increase";
|
|
197
|
+
minEffect = 0;
|
|
194
198
|
}
|
|
195
|
-
|
|
199
|
+
const scale = opts.scale ?? 1;
|
|
200
|
+
if (!Number.isFinite(scale) || scale <= 0) {
|
|
201
|
+
throw new Error(`sequentialPairedGate: scale must be > 0, got ${scale}`);
|
|
202
|
+
}
|
|
203
|
+
if (!Number.isInteger(maxN) || maxN < 1) {
|
|
204
|
+
throw new Error(`sequentialPairedGate: maxN must be a positive integer, got ${maxN}`);
|
|
205
|
+
}
|
|
206
|
+
const minN = opts.minN ?? 5;
|
|
207
|
+
if (!Number.isInteger(minN) || minN < 1 || minN > maxN) {
|
|
208
|
+
throw new Error(
|
|
209
|
+
`sequentialPairedGate: minN must be an integer in [1, maxN=${maxN}], got ${minN}`
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
if (!Number.isFinite(minEffect) || minEffect < 0 || minEffect >= scale) {
|
|
213
|
+
throw new Error(
|
|
214
|
+
`sequentialPairedGate: minEffect must be in [0, scale=${scale}), got ${minEffect}`
|
|
215
|
+
);
|
|
216
|
+
}
|
|
217
|
+
const nullMean = 0.5 + minEffect / (2 * scale);
|
|
218
|
+
return {
|
|
219
|
+
alpha,
|
|
220
|
+
minN,
|
|
221
|
+
maxN,
|
|
222
|
+
maxBet: opts.maxBet ?? 0.5,
|
|
223
|
+
scale,
|
|
224
|
+
shuffleSeed: opts.shuffleSeed ?? 1337,
|
|
225
|
+
direction,
|
|
226
|
+
nullMean,
|
|
227
|
+
minEffect
|
|
228
|
+
};
|
|
196
229
|
}
|
|
197
|
-
function
|
|
198
|
-
const
|
|
199
|
-
|
|
200
|
-
|
|
230
|
+
function makeStream(cfg) {
|
|
231
|
+
const proc = eProcess({ alpha: cfg.alpha, maxBet: cfg.maxBet, nullMean: cfg.nullMean });
|
|
232
|
+
const threshold = 1 / cfg.alpha;
|
|
233
|
+
let terminal;
|
|
201
234
|
return {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
const fresh = [];
|
|
208
|
-
for (const f of ctx.findings ?? []) {
|
|
209
|
-
const lesson = findingToLesson(f);
|
|
210
|
-
if (!lesson) continue;
|
|
211
|
-
const k = normKey(lesson);
|
|
212
|
-
if (!k || seen.has(k)) continue;
|
|
213
|
-
seen.add(k);
|
|
214
|
-
fresh.push({ gen: ctx.generation, text: lesson });
|
|
235
|
+
observe(delta) {
|
|
236
|
+
if (terminal === "undecided-at-maxN") {
|
|
237
|
+
throw new Error(
|
|
238
|
+
`sequentialPairedGate: pre-registered maxN=${cfg.maxN} exhausted \u2014 extending the stream after seeing the result reopens optional stopping; start a NEW pre-registered test`
|
|
239
|
+
);
|
|
215
240
|
}
|
|
216
|
-
if (
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
241
|
+
if (!Number.isFinite(delta) || Math.abs(delta) > cfg.scale) {
|
|
242
|
+
throw new Error(
|
|
243
|
+
`sequentialPairedGate: delta ${delta} outside \xB1scale=${cfg.scale} \u2014 pass the judge's native scale explicitly (detectScale helps pick 1 vs 100)`
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
const d = cfg.direction === "decrease" ? -delta : delta;
|
|
247
|
+
const step = proc.update((d / cfg.scale + 1) / 2);
|
|
248
|
+
if (terminal === void 0 && step.n >= cfg.minN && step.wealth >= threshold) {
|
|
249
|
+
terminal = "promote";
|
|
250
|
+
}
|
|
251
|
+
if (terminal === "promote") {
|
|
252
|
+
return {
|
|
253
|
+
decision: "promote",
|
|
254
|
+
eValue: step.wealth,
|
|
255
|
+
n: step.n,
|
|
256
|
+
reason: `e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} at n=${step.n} (minN=${cfg.minN}): the paired improvement exceeds ${cfg.minEffect} at anytime-valid level \u03B1=${cfg.alpha}`
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
if (step.n >= cfg.maxN) {
|
|
260
|
+
terminal = "undecided-at-maxN";
|
|
261
|
+
return {
|
|
262
|
+
decision: "undecided-at-maxN",
|
|
263
|
+
eValue: step.wealth,
|
|
264
|
+
n: step.n,
|
|
265
|
+
reason: `undecided at pre-registered maxN=${cfg.maxN} (e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)}). This is NOT evidence of no effect \u2014 the effect may be real but smaller than this budget can detect; re-register with a larger N to test that`
|
|
266
|
+
};
|
|
267
|
+
}
|
|
268
|
+
return {
|
|
269
|
+
decision: "continue",
|
|
270
|
+
eValue: step.wealth,
|
|
271
|
+
n: step.n,
|
|
272
|
+
reason: `e-value ${step.wealth.toFixed(2)} < 1/\u03B1=${threshold.toFixed(2)} at n=${step.n}/${cfg.maxN} \u2014 keep observing`
|
|
273
|
+
};
|
|
274
|
+
},
|
|
275
|
+
state() {
|
|
276
|
+
return { ...proc.state(), decision: terminal ?? "continue" };
|
|
235
277
|
}
|
|
236
278
|
};
|
|
237
279
|
}
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
var APPLY_SYSTEM = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
|
|
248
|
-
function surfaceToPromptText(surface) {
|
|
249
|
-
return typeof surface === "string" ? surface : JSON.stringify(surface);
|
|
280
|
+
function seededShuffle(items, seed) {
|
|
281
|
+
const rng = mulberry32(seed);
|
|
282
|
+
for (let i = items.length - 1; i > 0; i--) {
|
|
283
|
+
const j = Math.floor(rng() * (i + 1));
|
|
284
|
+
const tmp = items[i];
|
|
285
|
+
items[i] = items[j];
|
|
286
|
+
items[j] = tmp;
|
|
287
|
+
}
|
|
288
|
+
return items;
|
|
250
289
|
}
|
|
251
|
-
function
|
|
290
|
+
function sequentialPairedGate(options) {
|
|
291
|
+
const cfg = resolveConfig(options);
|
|
292
|
+
const name = options.name ?? "sequentialPairedGate";
|
|
293
|
+
const manifest = options.preRegistration;
|
|
294
|
+
const observeStream = makeStream(cfg);
|
|
252
295
|
return {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
const report = await opts.analyze(tracePath, ctx);
|
|
263
|
-
const applied = await callLlm(
|
|
264
|
-
{
|
|
265
|
-
model: opts.applyModel,
|
|
266
|
-
messages: [
|
|
267
|
-
{ role: "system", content: APPLY_SYSTEM },
|
|
268
|
-
{
|
|
269
|
-
role: "user",
|
|
270
|
-
content: `CURRENT PROMPT:
|
|
271
|
-
${parent}
|
|
272
|
-
|
|
273
|
-
TRACE-ANALYSIS REPORT:
|
|
274
|
-
${report}
|
|
275
|
-
|
|
276
|
-
Return the full revised prompt.`
|
|
277
|
-
}
|
|
278
|
-
]
|
|
279
|
-
},
|
|
280
|
-
{ baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl }
|
|
281
|
-
);
|
|
282
|
-
const text = applied.content.trim();
|
|
283
|
-
if (!text || text === parent) return [];
|
|
284
|
-
return [{ surface: text, label: opts.label, rationale: opts.rationale(report) }];
|
|
285
|
-
}
|
|
286
|
-
};
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
// src/campaign/drivers/halo.ts
|
|
290
|
-
var execFileAsync = promisify(execFile);
|
|
291
|
-
var DEFAULT_ANALYSIS_PROMPT = "Diagnose the failures in these agent execution traces \u2014 hallucinated tool calls, redundant tool arguments, refusal loops, and semantic-correctness errors \u2014 and suggest concrete, generalizable fixes to the agent instructions.";
|
|
292
|
-
function haloDriver(opts) {
|
|
293
|
-
const haloBin = opts.haloBin ?? "halo";
|
|
294
|
-
const model = opts.model ?? "gpt-5.4-mini";
|
|
295
|
-
return analysisEditDriver({
|
|
296
|
-
kind: "halo",
|
|
297
|
-
label: "halo",
|
|
298
|
-
baseUrl: opts.baseUrl,
|
|
299
|
-
apiKey: opts.apiKey,
|
|
300
|
-
applyModel: opts.applyModel ?? model,
|
|
301
|
-
fetchImpl: opts.fetchImpl,
|
|
302
|
-
resolveTraces: opts.resolveTraces,
|
|
303
|
-
noTracesError: "haloDriver: resolveTraces returned no OTLP traces \u2014 the halo engine has nothing to analyze",
|
|
304
|
-
// HALO's real findings are preserved verbatim in the rationale (attribution).
|
|
305
|
-
rationale: (findings) => `halo-engine findings:
|
|
306
|
-
${findings.slice(0, 800)}`,
|
|
307
|
-
analyze: async (tracePath, ctx) => {
|
|
308
|
-
const args = [
|
|
309
|
-
tracePath,
|
|
310
|
-
"-p",
|
|
311
|
-
opts.analysisPrompt ?? DEFAULT_ANALYSIS_PROMPT,
|
|
312
|
-
"-m",
|
|
313
|
-
model,
|
|
314
|
-
...opts.maxDepth !== void 0 ? ["--max-depth", String(opts.maxDepth)] : [],
|
|
315
|
-
...opts.maxTurns !== void 0 ? ["--max-turns", String(opts.maxTurns)] : []
|
|
316
|
-
];
|
|
317
|
-
let findings;
|
|
318
|
-
try {
|
|
319
|
-
const { stdout } = await execFileAsync(haloBin, args, {
|
|
320
|
-
maxBuffer: 64 * 1024 * 1024,
|
|
321
|
-
signal: ctx.signal,
|
|
322
|
-
env: {
|
|
323
|
-
...process.env,
|
|
324
|
-
...opts.apiKey ? { OPENAI_API_KEY: opts.apiKey } : {},
|
|
325
|
-
OPENAI_BASE_URL: opts.baseUrl
|
|
326
|
-
}
|
|
327
|
-
});
|
|
328
|
-
findings = stdout.trim();
|
|
329
|
-
} catch (e) {
|
|
296
|
+
name,
|
|
297
|
+
observe(delta) {
|
|
298
|
+
return observeStream.observe(delta);
|
|
299
|
+
},
|
|
300
|
+
state() {
|
|
301
|
+
return observeStream.state();
|
|
302
|
+
},
|
|
303
|
+
async decide(ctx) {
|
|
304
|
+
if (!ctx.baselineJudgeScores) {
|
|
330
305
|
throw new Error(
|
|
331
|
-
|
|
306
|
+
`${name}: ctx.baselineJudgeScores is required \u2014 falling back to the candidate's own scores would compare the candidate against itself (delta 0, silent no-op)`
|
|
332
307
|
);
|
|
333
308
|
}
|
|
334
|
-
|
|
335
|
-
|
|
309
|
+
const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
|
|
310
|
+
const paired = pairHoldout(
|
|
311
|
+
ctx.judgeScores,
|
|
312
|
+
ctx.baselineJudgeScores,
|
|
313
|
+
scenarioIds,
|
|
314
|
+
(s) => s.composite
|
|
315
|
+
);
|
|
316
|
+
const deltas = paired.after.map((a, i) => a - paired.before[i]);
|
|
317
|
+
seededShuffle(deltas, cfg.shuffleSeed);
|
|
318
|
+
const stream = makeStream(cfg);
|
|
319
|
+
let last;
|
|
320
|
+
for (const d of deltas) {
|
|
321
|
+
last = stream.observe(d);
|
|
322
|
+
if (last.decision !== "continue") break;
|
|
323
|
+
}
|
|
324
|
+
const detail = {
|
|
325
|
+
...stream.state(),
|
|
326
|
+
minN: cfg.minN,
|
|
327
|
+
maxN: cfg.maxN,
|
|
328
|
+
scale: cfg.scale,
|
|
329
|
+
shuffleSeed: cfg.shuffleSeed,
|
|
330
|
+
direction: cfg.direction,
|
|
331
|
+
minEffect: cfg.minEffect,
|
|
332
|
+
pairedN: deltas.length,
|
|
333
|
+
preRegisteredId: manifest?.id,
|
|
334
|
+
metric: manifest?.metric
|
|
335
|
+
};
|
|
336
|
+
const meanDelta = deltas.length === 0 ? void 0 : deltas.reduce((s, d) => s + d, 0) / deltas.length;
|
|
337
|
+
if (last === void 0) {
|
|
338
|
+
return {
|
|
339
|
+
decision: "need_more_work",
|
|
340
|
+
reasons: [`${name}: no paired holdout observations \u2014 nothing to test`],
|
|
341
|
+
contributingGates: [{ name, passed: false, detail }]
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
const decision = last.decision === "promote" ? "ship" : last.decision === "continue" ? "need_more_work" : "hold";
|
|
345
|
+
return {
|
|
346
|
+
decision,
|
|
347
|
+
reasons: [`${name}: ${last.reason}`],
|
|
348
|
+
contributingGates: [{ name, passed: decision === "ship", detail }],
|
|
349
|
+
delta: meanDelta
|
|
350
|
+
};
|
|
336
351
|
}
|
|
337
|
-
}
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
// src/campaign/drivers/memory.ts
|
|
341
|
-
var BLOCK_START2 = "<!-- BEGIN curated-memory (auto-managed by memoryCurationDriver) -->";
|
|
342
|
-
var BLOCK_END2 = "<!-- END curated-memory -->";
|
|
343
|
-
var DEFAULT_HEADING2 = "## Learned from prior runs (curated memory)";
|
|
344
|
-
var DISTILL_SYSTEM = 'You compress raw trace-analysis findings into crisp, generalizable agent guidance. Output ONLY a JSON array of strings, each one imperative lesson the agent should follow (e.g. "Always fetch a resource before mutating it"). No prose outside the JSON. Deduplicate; keep the most actionable and general; drop case-specific noise.';
|
|
345
|
-
function extractExistingLessons(text) {
|
|
346
|
-
return extractBlockBody(text, BLOCK_START2, BLOCK_END2).split("\n").map((l) => l.replace(/^\s*-\s+/, "").trim()).filter((l) => l && !l.startsWith("#"));
|
|
352
|
+
};
|
|
347
353
|
}
|
|
348
|
-
|
|
349
|
-
const
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
(x) => typeof x === "string" && x.trim().length > 0
|
|
354
|
+
function sequentialDecide(options = {}) {
|
|
355
|
+
const alpha = options.alpha ?? 0.05;
|
|
356
|
+
const minN = options.minN ?? 5;
|
|
357
|
+
const scale = options.scale ?? 1;
|
|
358
|
+
if (!Number.isFinite(scale) || scale <= 0) {
|
|
359
|
+
throw new Error(`sequentialDecide: scale must be > 0, got ${scale}`);
|
|
360
|
+
}
|
|
361
|
+
const proc = eProcess({ alpha, maxBet: options.maxBet ?? 0.5, nullMean: 0.5 });
|
|
362
|
+
const threshold = 1 / alpha;
|
|
363
|
+
let processedGenerations = 0;
|
|
364
|
+
let reference;
|
|
365
|
+
let stopped;
|
|
366
|
+
const topCandidate = (record) => {
|
|
367
|
+
if (record.candidates.length === 0) {
|
|
368
|
+
throw new Error(
|
|
369
|
+
`sequentialDecide: generation ${record.generationIndex} has no candidates \u2014 cannot extract a top candidate`
|
|
365
370
|
);
|
|
366
|
-
if (lessons.length > 0) return lessons;
|
|
367
371
|
}
|
|
368
|
-
|
|
369
|
-
}
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
kind: "memory-curation",
|
|
377
|
-
async propose(ctx) {
|
|
378
|
-
const parent = surfaceToText2(ctx.currentSurface);
|
|
379
|
-
const fresh = [];
|
|
380
|
-
for (const f of ctx.findings ?? []) {
|
|
381
|
-
const l = findingToLesson(f);
|
|
382
|
-
if (l) fresh.push(l);
|
|
383
|
-
}
|
|
384
|
-
const carried = extractExistingLessons(parent);
|
|
385
|
-
if (fresh.length === 0 && carried.length === 0) return [];
|
|
386
|
-
const distilled = opts.distill && fresh.length > 0 ? await distillLessons(fresh, opts.distill) : fresh;
|
|
387
|
-
const byKey = /* @__PURE__ */ new Map();
|
|
388
|
-
for (const l of carried) {
|
|
389
|
-
const k = normKey(l);
|
|
390
|
-
if (k) byKey.set(k, { text: l, count: 1 });
|
|
391
|
-
}
|
|
392
|
-
for (const l of distilled) {
|
|
393
|
-
const k = normKey(l);
|
|
394
|
-
if (!k) continue;
|
|
395
|
-
const e = byKey.get(k);
|
|
396
|
-
if (e) e.count += 1;
|
|
397
|
-
else byKey.set(k, { text: l, count: 1 });
|
|
398
|
-
}
|
|
399
|
-
const ranked = [...byKey.values()].sort((a, b) => b.count - a.count || a.text.localeCompare(b.text)).slice(0, maxEntries);
|
|
400
|
-
if (ranked.length === 0) return [];
|
|
401
|
-
const block = [BLOCK_START2, heading, ...ranked.map((e) => `- ${e.text}`), BLOCK_END2].join(
|
|
402
|
-
"\n"
|
|
372
|
+
return record.candidates.reduce((best, c) => c.composite > best.composite ? c : best);
|
|
373
|
+
};
|
|
374
|
+
const decide = ({ history }) => {
|
|
375
|
+
if (stopped) return stopped;
|
|
376
|
+
if (history.length === 0) return { stop: false };
|
|
377
|
+
if (reference === void 0) {
|
|
378
|
+
reference = new Map(
|
|
379
|
+
topCandidate(history[0]).scenarios.map((s) => [s.scenarioId, s.composite])
|
|
403
380
|
);
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
{
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
381
|
+
}
|
|
382
|
+
for (let g = Math.max(1, processedGenerations); g < history.length; g++) {
|
|
383
|
+
const top = topCandidate(history[g]);
|
|
384
|
+
const byScenario = new Map(top.scenarios.map((s) => [s.scenarioId, s.composite]));
|
|
385
|
+
for (const [scenarioId, refComposite] of reference) {
|
|
386
|
+
const candComposite = byScenario.get(scenarioId);
|
|
387
|
+
if (candComposite === void 0) {
|
|
388
|
+
throw new Error(
|
|
389
|
+
`sequentialDecide: generation ${history[g].generationIndex} top candidate is missing scenario '${scenarioId}' \u2014 generations must score the same scenario set to pair`
|
|
390
|
+
);
|
|
414
391
|
}
|
|
415
|
-
|
|
392
|
+
const delta = candComposite - refComposite;
|
|
393
|
+
if (!Number.isFinite(delta) || Math.abs(delta) > scale) {
|
|
394
|
+
throw new Error(
|
|
395
|
+
`sequentialDecide: paired delta ${delta} outside \xB1scale=${scale} on scenario '${scenarioId}' \u2014 pass the composite scale explicitly`
|
|
396
|
+
);
|
|
397
|
+
}
|
|
398
|
+
const step = proc.update((delta / scale + 1) / 2);
|
|
399
|
+
if (step.n >= minN && step.wealth >= threshold) {
|
|
400
|
+
stopped = {
|
|
401
|
+
stop: true,
|
|
402
|
+
reason: `sequential e-process decided at generation ${history[g].generationIndex}: e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} after n=${step.n} paired deltas vs the generation-0 incumbent \u2014 the improvement is real at \u03B1=${alpha}; stop exploring and promote via the gate`
|
|
403
|
+
};
|
|
404
|
+
processedGenerations = history.length;
|
|
405
|
+
return stopped;
|
|
406
|
+
}
|
|
407
|
+
}
|
|
416
408
|
}
|
|
409
|
+
processedGenerations = history.length;
|
|
410
|
+
return { stop: false };
|
|
417
411
|
};
|
|
412
|
+
decide.state = () => proc.state();
|
|
413
|
+
return decide;
|
|
418
414
|
}
|
|
419
415
|
|
|
420
|
-
// src/campaign/
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
416
|
+
// src/campaign/labeled-store/fs-adapter.ts
|
|
417
|
+
import { createHash as createHash2 } from "crypto";
|
|
418
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
|
|
419
|
+
import { join } from "path";
|
|
420
|
+
var LabeledScenarioStoreError = class extends Error {
|
|
421
|
+
constructor(code, message) {
|
|
422
|
+
super(message);
|
|
423
|
+
this.code = code;
|
|
424
|
+
this.name = "LabeledScenarioStoreError";
|
|
425
|
+
}
|
|
426
|
+
code;
|
|
427
|
+
};
|
|
428
|
+
var FsLabeledScenarioStore = class {
|
|
429
|
+
constructor(options) {
|
|
430
|
+
this.options = options;
|
|
431
|
+
if (!existsSync(options.root)) mkdirSync(options.root, { recursive: true });
|
|
432
|
+
this.now = options.now ?? Date.now;
|
|
433
|
+
}
|
|
434
|
+
options;
|
|
435
|
+
now;
|
|
436
|
+
rateLimits = /* @__PURE__ */ new Map();
|
|
437
|
+
async observe(write) {
|
|
438
|
+
this.assertProvenance(write);
|
|
439
|
+
this.assertRateLimit(write);
|
|
440
|
+
const record = this.toRecord(write);
|
|
441
|
+
const path = this.pathForSource(write.source);
|
|
442
|
+
const line = `${JSON.stringify(record)}
|
|
443
|
+
`;
|
|
444
|
+
appendLine(path, line);
|
|
445
|
+
}
|
|
446
|
+
async sample(args) {
|
|
447
|
+
if (!args.split) {
|
|
448
|
+
throw new LabeledScenarioStoreError(
|
|
449
|
+
"split_required",
|
|
450
|
+
"sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
|
|
451
|
+
);
|
|
452
|
+
}
|
|
453
|
+
if (!args.capturedBefore) {
|
|
454
|
+
throw new LabeledScenarioStoreError(
|
|
455
|
+
"capturedBefore_required",
|
|
456
|
+
"sample() requires an explicit `capturedBefore` timestamp for temporal-split discipline"
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
const all = [];
|
|
460
|
+
for (const source of ALL_SOURCES) {
|
|
461
|
+
if (args.split === "train" && source === "production-trace") {
|
|
462
|
+
const explicit = sourceFilterContains(args.filter?.source, "production-trace");
|
|
463
|
+
if (!explicit) continue;
|
|
464
|
+
}
|
|
465
|
+
const path = this.pathForSource(source);
|
|
466
|
+
if (!existsSync(path)) continue;
|
|
467
|
+
const lines = readFileSync(path, "utf8").split("\n").filter(Boolean);
|
|
468
|
+
for (const line of lines) {
|
|
469
|
+
let record;
|
|
470
|
+
try {
|
|
471
|
+
record = JSON.parse(line);
|
|
472
|
+
} catch {
|
|
473
|
+
continue;
|
|
474
|
+
}
|
|
475
|
+
if (!matchesFilter(record, args, source)) continue;
|
|
476
|
+
all.push(record);
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
all.sort((a, b) => {
|
|
480
|
+
if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
|
|
481
|
+
return a.recordHash.localeCompare(b.recordHash);
|
|
482
|
+
});
|
|
483
|
+
return all.slice(0, args.count);
|
|
484
|
+
}
|
|
485
|
+
async size() {
|
|
486
|
+
const bySource = {};
|
|
487
|
+
const byTrust = {
|
|
488
|
+
unverified: 0,
|
|
489
|
+
"verified-signal": 0,
|
|
490
|
+
"human-rated": 0
|
|
491
|
+
};
|
|
492
|
+
let total = 0;
|
|
493
|
+
for (const source of ALL_SOURCES) {
|
|
494
|
+
const path = this.pathForSource(source);
|
|
495
|
+
if (!existsSync(path)) {
|
|
496
|
+
bySource[source] = 0;
|
|
497
|
+
continue;
|
|
498
|
+
}
|
|
499
|
+
const lines = readFileSync(path, "utf8").split("\n").filter(Boolean);
|
|
500
|
+
bySource[source] = lines.length;
|
|
501
|
+
total += lines.length;
|
|
502
|
+
for (const line of lines) {
|
|
503
|
+
let trust = "unverified";
|
|
504
|
+
try {
|
|
505
|
+
trust = JSON.parse(line).labelTrust ?? "unverified";
|
|
506
|
+
} catch {
|
|
507
|
+
}
|
|
508
|
+
byTrust[trust] += 1;
|
|
462
509
|
}
|
|
463
|
-
lines = [...lines.slice(0, idx), ...op.text.split("\n"), ...lines.slice(idx + 1)];
|
|
464
|
-
applied++;
|
|
465
510
|
}
|
|
511
|
+
return { train: total, test: total, bySource, byTrust };
|
|
466
512
|
}
|
|
467
|
-
|
|
513
|
+
assertProvenance(write) {
|
|
514
|
+
if (!write.source) {
|
|
515
|
+
throw new LabeledScenarioStoreError(
|
|
516
|
+
"missing_source",
|
|
517
|
+
"LabeledScenarioWrite requires `source`"
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
|
|
521
|
+
throw new LabeledScenarioStoreError(
|
|
522
|
+
"missing_source_version",
|
|
523
|
+
"LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
if (!write.capturedAt) {
|
|
527
|
+
throw new LabeledScenarioStoreError(
|
|
528
|
+
"missing_captured_at",
|
|
529
|
+
"LabeledScenarioWrite requires `capturedAt` ISO timestamp"
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
if (!write.redactionStatus) {
|
|
533
|
+
throw new LabeledScenarioStoreError(
|
|
534
|
+
"missing_redaction_status",
|
|
535
|
+
"LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
|
|
536
|
+
);
|
|
537
|
+
}
|
|
538
|
+
if (!ALL_SOURCES.includes(write.source)) {
|
|
539
|
+
throw new LabeledScenarioStoreError(
|
|
540
|
+
"unknown_source",
|
|
541
|
+
`LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
|
|
542
|
+
);
|
|
543
|
+
}
|
|
544
|
+
}
|
|
545
|
+
assertRateLimit(write) {
|
|
546
|
+
const cap = this.options.maxWritesPerMinutePerBucket;
|
|
547
|
+
if (!cap || !write.rateLimitBucket) return;
|
|
548
|
+
const now = this.now();
|
|
549
|
+
const windowMs = 6e4;
|
|
550
|
+
let state = this.rateLimits.get(write.rateLimitBucket);
|
|
551
|
+
if (!state || now - state.windowStartMs >= windowMs) {
|
|
552
|
+
state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
|
|
553
|
+
this.rateLimits.set(write.rateLimitBucket, state);
|
|
554
|
+
}
|
|
555
|
+
if (state.count >= cap) {
|
|
556
|
+
throw new LabeledScenarioStoreError(
|
|
557
|
+
"rate_limit_exceeded",
|
|
558
|
+
`LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
|
|
559
|
+
);
|
|
560
|
+
}
|
|
561
|
+
state.count += 1;
|
|
562
|
+
}
|
|
563
|
+
toRecord(write) {
|
|
564
|
+
const recordHash = sha256(
|
|
565
|
+
JSON.stringify({
|
|
566
|
+
id: write.scenario.id,
|
|
567
|
+
src: write.source,
|
|
568
|
+
at: write.capturedAt,
|
|
569
|
+
ver: write.sourceVersionHash
|
|
570
|
+
})
|
|
571
|
+
);
|
|
572
|
+
return {
|
|
573
|
+
...write,
|
|
574
|
+
recordHash,
|
|
575
|
+
split: "train"
|
|
576
|
+
};
|
|
577
|
+
}
|
|
578
|
+
pathForSource(source) {
|
|
579
|
+
return join(this.options.root, `${source}.jsonl`);
|
|
580
|
+
}
|
|
581
|
+
};
|
|
582
|
+
var ALL_SOURCES = [
|
|
583
|
+
"production-trace",
|
|
584
|
+
"eval-run",
|
|
585
|
+
"manual",
|
|
586
|
+
"red-team",
|
|
587
|
+
"synthetic"
|
|
588
|
+
];
|
|
589
|
+
function sourceFilterContains(filter, needle) {
|
|
590
|
+
if (!filter) return false;
|
|
591
|
+
if (Array.isArray(filter)) return filter.includes(needle);
|
|
592
|
+
return filter === needle;
|
|
468
593
|
}
|
|
469
|
-
function
|
|
470
|
-
|
|
594
|
+
function matchesFilter(record, args, source) {
|
|
595
|
+
if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
|
|
596
|
+
if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
|
|
597
|
+
const f = args.filter;
|
|
598
|
+
if (!f) return true;
|
|
599
|
+
if (f.kind && record.scenario.kind !== f.kind) return false;
|
|
600
|
+
if (f.source) {
|
|
601
|
+
const sources = Array.isArray(f.source) ? f.source : [f.source];
|
|
602
|
+
if (!sources.includes(source)) return false;
|
|
603
|
+
}
|
|
604
|
+
if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
|
|
605
|
+
const composites = Object.values(record.judgeScores).map((s) => s.composite);
|
|
606
|
+
const max = composites.length === 0 ? 0 : Math.max(...composites);
|
|
607
|
+
if (f.minComposite !== void 0 && max < f.minComposite) return false;
|
|
608
|
+
if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
|
|
609
|
+
}
|
|
610
|
+
if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
|
|
611
|
+
return false;
|
|
612
|
+
}
|
|
613
|
+
return true;
|
|
471
614
|
}
|
|
472
|
-
function
|
|
473
|
-
return
|
|
615
|
+
function sha256(input) {
|
|
616
|
+
return createHash2("sha256").update(input).digest("hex").slice(0, 16);
|
|
617
|
+
}
|
|
618
|
+
function appendLine(path, line) {
|
|
619
|
+
if (existsSync(path)) {
|
|
620
|
+
const existing = readFileSync(path, "utf8");
|
|
621
|
+
writeFileSync(path, existing + line);
|
|
622
|
+
} else {
|
|
623
|
+
writeFileSync(path, line);
|
|
624
|
+
}
|
|
474
625
|
}
|
|
475
626
|
|
|
476
|
-
// src/campaign/
|
|
477
|
-
var
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
findingsNote: args.findingsNote,
|
|
490
|
-
count: args.count
|
|
491
|
-
});
|
|
492
|
-
const result = await callLlm(
|
|
493
|
-
{
|
|
494
|
-
model: opts.model,
|
|
495
|
-
messages: [
|
|
496
|
-
{ role: "system", content: SKILLOPT_SYSTEM },
|
|
497
|
-
{ role: "user", content: userPrompt }
|
|
498
|
-
],
|
|
499
|
-
jsonMode: true,
|
|
500
|
-
temperature: opts.temperature ?? 0.6,
|
|
501
|
-
maxTokens: opts.maxTokens ?? 4e3
|
|
502
|
-
},
|
|
503
|
-
opts.llm
|
|
504
|
-
);
|
|
505
|
-
return parseSkillPatchResponse(result.content, args.count, args.editBudget);
|
|
627
|
+
// src/campaign/proposers/fapo.ts
|
|
628
|
+
var FAPO_LEVELS = ["prompt", "parameter", "structural"];
|
|
629
|
+
var MAX_FINDING_DEPTH = 16;
|
|
630
|
+
var UNSAFE_JSON_KEYS = /* @__PURE__ */ new Set(["__proto__", "prototype", "constructor"]);
|
|
631
|
+
function fapoProposer(opts) {
|
|
632
|
+
const proposers = levelProposers(opts);
|
|
633
|
+
const allowed = allowedLevels(opts, proposers);
|
|
634
|
+
const plateauWindow = opts.plateauWindow ?? 3;
|
|
635
|
+
const minDistinctStrategies = opts.minDistinctStrategies ?? 3;
|
|
636
|
+
const minImprovement = opts.minImprovement ?? 0;
|
|
637
|
+
const proposalsPerCycle = opts.proposalsPerCycle ?? 1;
|
|
638
|
+
if (allowed.length === 0) {
|
|
639
|
+
throw new Error("fapoProposer: at least one allowed level must have a proposer");
|
|
506
640
|
}
|
|
641
|
+
if (plateauWindow < 1) throw new Error("fapoProposer: plateauWindow must be >= 1");
|
|
642
|
+
if (minDistinctStrategies < 1) {
|
|
643
|
+
throw new Error("fapoProposer: minDistinctStrategies must be >= 1");
|
|
644
|
+
}
|
|
645
|
+
if (proposalsPerCycle < 1) throw new Error("fapoProposer: proposalsPerCycle must be >= 1");
|
|
507
646
|
return {
|
|
508
|
-
kind: "
|
|
509
|
-
proposePatches,
|
|
647
|
+
kind: "fapo",
|
|
510
648
|
async propose(ctx) {
|
|
511
|
-
|
|
649
|
+
const state = policyState(ctx.history, ctx.findings, {
|
|
650
|
+
plateauWindow,
|
|
651
|
+
minDistinctStrategies,
|
|
652
|
+
minImprovement
|
|
653
|
+
});
|
|
654
|
+
const decision = chooseLevel({
|
|
655
|
+
allowed,
|
|
656
|
+
available: allowed.filter((level) => !state.exhausted[level]),
|
|
657
|
+
proposers,
|
|
658
|
+
state,
|
|
659
|
+
promptFirst: opts.promptFirst ?? true,
|
|
660
|
+
parameterBeforeStructural: opts.parameterBeforeStructural ?? true
|
|
661
|
+
});
|
|
662
|
+
if (!decision) return [];
|
|
663
|
+
const proposer = proposers[decision.level];
|
|
664
|
+
if (!proposer) {
|
|
665
|
+
throw new Error(`fapoProposer: selected ${decision.level} but no proposer is configured`);
|
|
666
|
+
}
|
|
667
|
+
const requested = Math.min(ctx.populationSize, proposalsPerCycle);
|
|
668
|
+
const raw = await proposer.propose({ ...ctx, populationSize: requested });
|
|
669
|
+
const wrapped = raw.map((candidate) => normalizeProposal(candidate)).map((candidate) => wrapProposal(candidate, decision.level, decision.reason));
|
|
670
|
+
const reviewed = [];
|
|
671
|
+
for (const candidate of wrapped) {
|
|
672
|
+
const review = opts.reviewCandidate ? await opts.reviewCandidate({
|
|
673
|
+
level: decision.level,
|
|
674
|
+
candidate,
|
|
675
|
+
context: ctx,
|
|
676
|
+
reason: decision.reason
|
|
677
|
+
}) : { verdict: "pass" };
|
|
678
|
+
if (review.verdict === "fail") continue;
|
|
679
|
+
const warn = review.verdict === "warn" ? renderReviewWarnings(review) : "";
|
|
680
|
+
reviewed.push(
|
|
681
|
+
warn ? { ...candidate, rationale: `${candidate.rationale}
|
|
682
|
+
${warn}` } : candidate
|
|
683
|
+
);
|
|
684
|
+
}
|
|
685
|
+
if (wrapped.length > 0 && reviewed.length === 0) {
|
|
512
686
|
throw new Error(
|
|
513
|
-
|
|
687
|
+
`fapoProposer: reviewer blocked every ${decision.level} candidate; refusing to evaluate an unreviewed variant`
|
|
514
688
|
);
|
|
515
689
|
}
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
690
|
+
return reviewed;
|
|
691
|
+
},
|
|
692
|
+
decide({ history }) {
|
|
693
|
+
const last = history[history.length - 1];
|
|
694
|
+
if (last && last.candidates.length === 0) {
|
|
695
|
+
return { stop: true, reason: "FAPO produced no scoped candidate in the prior generation" };
|
|
696
|
+
}
|
|
697
|
+
const state = policyState(history, [], {
|
|
698
|
+
plateauWindow,
|
|
699
|
+
minDistinctStrategies,
|
|
700
|
+
minImprovement
|
|
701
|
+
});
|
|
702
|
+
const everyAllowedLevelExhausted = allowed.every((level) => state.exhausted[level]);
|
|
703
|
+
return everyAllowedLevelExhausted ? { stop: true, reason: `all FAPO levels exhausted (${allowed.join(", ")})` } : { stop: false };
|
|
704
|
+
}
|
|
705
|
+
};
|
|
706
|
+
}
|
|
707
|
+
function levelProposers(opts) {
|
|
708
|
+
return {
|
|
709
|
+
...opts.proposers ?? {},
|
|
710
|
+
...opts.promptProposer ? { prompt: opts.promptProposer } : {},
|
|
711
|
+
...opts.parameterProposer ? { parameter: opts.parameterProposer } : {},
|
|
712
|
+
...opts.structuralProposer ? { structural: opts.structuralProposer } : {}
|
|
713
|
+
};
|
|
714
|
+
}
|
|
715
|
+
function allowedLevels(opts, proposers) {
|
|
716
|
+
const explicit = opts.scope?.allowedLevels;
|
|
717
|
+
const forbidden = new Set(opts.scope?.forbiddenLevels ?? []);
|
|
718
|
+
return FAPO_LEVELS.filter((level) => {
|
|
719
|
+
if (!proposers[level]) return false;
|
|
720
|
+
if (forbidden.has(level)) return false;
|
|
721
|
+
return explicit ? explicit.includes(level) : true;
|
|
722
|
+
});
|
|
723
|
+
}
|
|
724
|
+
function policyState(history, findings, opts) {
|
|
725
|
+
const attempts = extractAttempts(history);
|
|
726
|
+
return {
|
|
727
|
+
attempts,
|
|
728
|
+
exhausted: {
|
|
729
|
+
prompt: isLevelExhausted("prompt", attempts, opts),
|
|
730
|
+
parameter: isLevelExhausted("parameter", attempts, opts),
|
|
731
|
+
structural: isLevelExhausted("structural", attempts, opts)
|
|
732
|
+
},
|
|
733
|
+
signals: extractFapoAttributionSignals(findings)
|
|
734
|
+
};
|
|
735
|
+
}
|
|
736
|
+
function extractAttempts(history) {
|
|
737
|
+
const attempts = [];
|
|
738
|
+
for (const generation of history) {
|
|
739
|
+
for (const candidate of generation.candidates) {
|
|
740
|
+
const level = candidateLevel(candidate);
|
|
741
|
+
if (!level) continue;
|
|
742
|
+
attempts.push({
|
|
743
|
+
level,
|
|
744
|
+
composite: candidate.composite,
|
|
745
|
+
strategy: candidateStrategy(candidate)
|
|
746
|
+
});
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
return attempts;
|
|
750
|
+
}
|
|
751
|
+
function candidateLevel(candidate) {
|
|
752
|
+
const label = candidate.label ?? "";
|
|
753
|
+
const rationale = candidate.rationale ?? "";
|
|
754
|
+
return parseLevel(label) ?? parseLevel(rationale);
|
|
755
|
+
}
|
|
756
|
+
function parseLevel(text) {
|
|
757
|
+
const match = /\bfapo:(prompt|parameter|structural)\b/.exec(text);
|
|
758
|
+
return match ? match[1] : null;
|
|
759
|
+
}
|
|
760
|
+
function candidateStrategy(candidate) {
|
|
761
|
+
const label = candidate.label ?? candidate.rationale ?? candidate.surfaceHash;
|
|
762
|
+
return label.replace(/\bfapo:(prompt|parameter|structural):?/g, "").trim() || candidate.surfaceHash;
|
|
763
|
+
}
|
|
764
|
+
function isLevelExhausted(level, attempts, opts) {
|
|
765
|
+
const own = attempts.filter((attempt) => attempt.level === level);
|
|
766
|
+
if (own.length < opts.plateauWindow) return false;
|
|
767
|
+
const distinctStrategies = new Set(own.map((attempt) => attempt.strategy)).size;
|
|
768
|
+
if (distinctStrategies < opts.minDistinctStrategies) return false;
|
|
769
|
+
let best = Number.NEGATIVE_INFINITY;
|
|
770
|
+
let nonImproving = 0;
|
|
771
|
+
for (const attempt of own) {
|
|
772
|
+
if (attempt.composite > best + opts.minImprovement) {
|
|
773
|
+
best = attempt.composite;
|
|
774
|
+
nonImproving = 0;
|
|
775
|
+
} else {
|
|
776
|
+
nonImproving += 1;
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
return nonImproving >= opts.plateauWindow;
|
|
780
|
+
}
|
|
781
|
+
function chooseLevel(args) {
|
|
782
|
+
const { available, state } = args;
|
|
783
|
+
if (available.length === 0) return null;
|
|
784
|
+
const triedPrompt = state.attempts.some((attempt) => attempt.level === "prompt");
|
|
785
|
+
if (args.promptFirst && available.includes("prompt") && !triedPrompt) {
|
|
786
|
+
return {
|
|
787
|
+
level: "prompt",
|
|
788
|
+
reason: "prompt-first policy: no prompt-level variant has been tried yet"
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
const supported = available.filter((level) => state.signals.counts[level] > 0);
|
|
792
|
+
if (supported.includes("prompt")) {
|
|
793
|
+
return {
|
|
794
|
+
level: "prompt",
|
|
795
|
+
reason: `attribution has ${state.signals.counts.prompt} prompt-addressable failure(s)`
|
|
796
|
+
};
|
|
797
|
+
}
|
|
798
|
+
const strongest = supported.sort(
|
|
799
|
+
(a, b) => state.signals.counts[b] - state.signals.counts[a] || levelRank(a) - levelRank(b)
|
|
800
|
+
)[0];
|
|
801
|
+
if (strongest) {
|
|
802
|
+
if (strongest === "structural" && args.parameterBeforeStructural && available.includes("parameter") && args.proposers.parameter) {
|
|
803
|
+
return {
|
|
804
|
+
level: "parameter",
|
|
805
|
+
reason: "attribution indicates a non-prompt bottleneck; trying parameter/config edits before structural edits"
|
|
806
|
+
};
|
|
807
|
+
}
|
|
808
|
+
return {
|
|
809
|
+
level: strongest,
|
|
810
|
+
reason: `attribution has ${state.signals.counts[strongest]} ${strongest}-addressable failure(s)`
|
|
811
|
+
};
|
|
812
|
+
}
|
|
813
|
+
if (state.attempts.length === 0) {
|
|
814
|
+
const first = available.slice().sort((a, b) => levelRank(a) - levelRank(b))[0];
|
|
815
|
+
return { level: first, reason: "no attribution yet; starting at the cheapest allowed level" };
|
|
816
|
+
}
|
|
817
|
+
return null;
|
|
818
|
+
}
|
|
819
|
+
function levelRank(level) {
|
|
820
|
+
return level === "prompt" ? 0 : level === "parameter" ? 1 : 2;
|
|
821
|
+
}
|
|
822
|
+
function normalizeProposal(value) {
|
|
823
|
+
return isProposedCandidate(value) ? value : { surface: value, label: "candidate", rationale: "bare candidate from level proposer" };
|
|
824
|
+
}
|
|
825
|
+
function wrapProposal(candidate, level, reason) {
|
|
826
|
+
const label = candidate.label ? `fapo:${level}:${candidate.label}` : `fapo:${level}`;
|
|
827
|
+
return {
|
|
828
|
+
...candidate,
|
|
829
|
+
label,
|
|
830
|
+
rationale: [
|
|
831
|
+
`fapo:${level} selected by reviewed escalation policy`,
|
|
832
|
+
`Reason: ${reason}`,
|
|
833
|
+
candidate.rationale ? `Level proposer rationale: ${candidate.rationale}` : ""
|
|
834
|
+
].filter(Boolean).join("\n")
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
function renderReviewWarnings(review) {
|
|
838
|
+
const issues = review.issues?.filter((issue) => issue.severity === "warn") ?? [];
|
|
839
|
+
if (issues.length === 0) return "";
|
|
840
|
+
return `Reviewer warnings:
|
|
841
|
+
${issues.map((issue) => `- ${issue.checkName}: ${issue.description}`).join("\n")}`;
|
|
842
|
+
}
|
|
843
|
+
function extractFapoAttributionSignals(findings) {
|
|
844
|
+
const signals = {
|
|
845
|
+
counts: { prompt: 0, parameter: 0, structural: 0 },
|
|
846
|
+
clusters: []
|
|
847
|
+
};
|
|
848
|
+
const seen = /* @__PURE__ */ new WeakSet();
|
|
849
|
+
for (const finding of findings) {
|
|
850
|
+
collectFinding(signals, finding, seen, 0);
|
|
851
|
+
}
|
|
852
|
+
return signals;
|
|
853
|
+
}
|
|
854
|
+
function collectFinding(signals, finding, seen, depth) {
|
|
855
|
+
if (depth > MAX_FINDING_DEPTH) return;
|
|
856
|
+
if (!finding) return;
|
|
857
|
+
if (typeof finding === "string") {
|
|
858
|
+
const level = inferLevelFromText(finding);
|
|
859
|
+
if (level) addCluster(signals, { label: finding, level, count: 1 });
|
|
860
|
+
return;
|
|
861
|
+
}
|
|
862
|
+
if (Array.isArray(finding)) {
|
|
863
|
+
if (seen.has(finding)) return;
|
|
864
|
+
seen.add(finding);
|
|
865
|
+
for (const item of finding) collectFinding(signals, item, seen, depth + 1);
|
|
866
|
+
return;
|
|
867
|
+
}
|
|
868
|
+
if (typeof finding !== "object") return;
|
|
869
|
+
if (seen.has(finding)) return;
|
|
870
|
+
seen.add(finding);
|
|
871
|
+
const obj = finding;
|
|
872
|
+
collectLevelPartition(signals, obj.level_partition ?? obj.levelPartition, seen, depth + 1);
|
|
873
|
+
collectCounts(signals, obj);
|
|
874
|
+
collectClusters(signals, obj.clusters, true, seen, depth + 1);
|
|
875
|
+
const explicit = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel);
|
|
876
|
+
const text = textFields(obj);
|
|
877
|
+
const inferred = explicit ?? inferLevelFromText(text);
|
|
878
|
+
if (inferred) {
|
|
879
|
+
addCluster(signals, {
|
|
880
|
+
label: text || inferred,
|
|
881
|
+
level: inferred,
|
|
882
|
+
count: numberField(obj.count) ?? 1,
|
|
883
|
+
confidence: confidenceField(obj.confidence),
|
|
884
|
+
suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
|
|
885
|
+
caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
|
|
886
|
+
});
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
function collectLevelPartition(signals, raw, seen, depth) {
|
|
890
|
+
if (!raw || typeof raw !== "object") return;
|
|
891
|
+
if (seen.has(raw)) return;
|
|
892
|
+
seen.add(raw);
|
|
893
|
+
const partition = raw;
|
|
894
|
+
for (const level of FAPO_LEVELS) {
|
|
895
|
+
const bucket = partition[level];
|
|
896
|
+
if (!bucket || typeof bucket !== "object") continue;
|
|
897
|
+
const obj = bucket;
|
|
898
|
+
const count = numberField(obj.count) ?? 0;
|
|
899
|
+
if (count > 0) signals.counts[level] += count;
|
|
900
|
+
collectClusters(signals, obj.clusters, false, seen, depth + 1);
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
function collectCounts(signals, obj) {
|
|
904
|
+
const prompt = numberField(obj.prompt_addressable ?? obj.promptAddressable);
|
|
905
|
+
const structural = numberField(obj.structural_addressable ?? obj.structuralAddressable);
|
|
906
|
+
const tool = numberField(obj.tool_addressable ?? obj.toolAddressable);
|
|
907
|
+
if (prompt) signals.counts.prompt += prompt;
|
|
908
|
+
if (structural) signals.counts.structural += structural;
|
|
909
|
+
if (tool) signals.counts.structural += tool;
|
|
910
|
+
}
|
|
911
|
+
function collectClusters(signals, raw, countClusters, seen, depth) {
|
|
912
|
+
if (!Array.isArray(raw)) return;
|
|
913
|
+
if (seen.has(raw)) return;
|
|
914
|
+
seen.add(raw);
|
|
915
|
+
for (const item of raw) {
|
|
916
|
+
if (countClusters) {
|
|
917
|
+
collectFinding(signals, item, seen, depth + 1);
|
|
918
|
+
continue;
|
|
919
|
+
}
|
|
920
|
+
const cluster = parseCluster(item);
|
|
921
|
+
if (cluster) signals.clusters.push(cluster);
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
function addCluster(signals, cluster) {
|
|
925
|
+
signals.counts[cluster.level] += Math.max(1, cluster.count);
|
|
926
|
+
signals.clusters.push(cluster);
|
|
927
|
+
}
|
|
928
|
+
function parseCluster(raw) {
|
|
929
|
+
if (!raw || typeof raw !== "object") return null;
|
|
930
|
+
const obj = raw;
|
|
931
|
+
const text = textFields(obj);
|
|
932
|
+
const level = parseFindingLevel(obj.level ?? obj.optimization_level ?? obj.optimizationLevel) ?? inferLevelFromText(text);
|
|
933
|
+
if (!level) return null;
|
|
934
|
+
return {
|
|
935
|
+
label: text || level,
|
|
936
|
+
level,
|
|
937
|
+
count: numberField(obj.count) ?? 1,
|
|
938
|
+
confidence: confidenceField(obj.confidence),
|
|
939
|
+
suggestedFix: stringField(obj.suggested_fix ?? obj.suggestedFix ?? obj.recommended_action),
|
|
940
|
+
caseIds: stringArrayField(obj.case_ids ?? obj.caseIds)
|
|
941
|
+
};
|
|
942
|
+
}
|
|
943
|
+
function parseFindingLevel(raw) {
|
|
944
|
+
if (typeof raw !== "string") return null;
|
|
945
|
+
const lower = raw.toLowerCase();
|
|
946
|
+
if (lower === "prompt" || lower === "parameter" || lower === "structural") return lower;
|
|
947
|
+
if (lower === "chain" || lower === "tool" || lower === "code") return "structural";
|
|
948
|
+
if (lower === "config" || lower === "params") return "parameter";
|
|
949
|
+
return null;
|
|
950
|
+
}
|
|
951
|
+
function inferLevelFromText(text) {
|
|
952
|
+
const lower = text.toLowerCase();
|
|
953
|
+
if (/\b(retrieval_k|temperature|top_p|max_tokens|max_completion_tokens|reasoning_effort|config|parameter)\b/.test(
|
|
954
|
+
lower
|
|
955
|
+
)) {
|
|
956
|
+
return "parameter";
|
|
957
|
+
}
|
|
958
|
+
if (/\b(retriev\w*|search\w*|bm25|hop|evidence|cascade|tool|node|chain|state|structur\w*|topology|import)\b/.test(
|
|
959
|
+
lower
|
|
960
|
+
)) {
|
|
961
|
+
return "structural";
|
|
962
|
+
}
|
|
963
|
+
if (/\b(format\w*|verbose|brevity|abstain\w*|reasoning|instruction|prompt|output|answer)\b/.test(
|
|
964
|
+
lower
|
|
965
|
+
)) {
|
|
966
|
+
return "prompt";
|
|
967
|
+
}
|
|
968
|
+
return null;
|
|
969
|
+
}
|
|
970
|
+
function textFields(obj) {
|
|
971
|
+
return [
|
|
972
|
+
obj.label,
|
|
973
|
+
obj.claim,
|
|
974
|
+
obj.recommended_action,
|
|
975
|
+
obj.suggested_fix,
|
|
976
|
+
obj.suggestedFix,
|
|
977
|
+
obj.message,
|
|
978
|
+
obj.text,
|
|
979
|
+
obj.heuristic,
|
|
980
|
+
obj.area
|
|
981
|
+
].filter((value) => typeof value === "string" && value.trim().length > 0).join(" ");
|
|
982
|
+
}
|
|
983
|
+
function numberField(value) {
|
|
984
|
+
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
985
|
+
}
|
|
986
|
+
function stringField(value) {
|
|
987
|
+
return typeof value === "string" && value.trim() ? value.trim() : void 0;
|
|
988
|
+
}
|
|
989
|
+
function confidenceField(value) {
|
|
990
|
+
return value === "high" || value === "medium" || value === "low" ? value : void 0;
|
|
991
|
+
}
|
|
992
|
+
function stringArrayField(value) {
|
|
993
|
+
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : void 0;
|
|
994
|
+
}
|
|
995
|
+
function parameterSweepProposer(opts) {
|
|
996
|
+
if (opts.candidates.length === 0) {
|
|
997
|
+
throw new Error("parameterSweepProposer: candidates must not be empty");
|
|
998
|
+
}
|
|
999
|
+
return {
|
|
1000
|
+
kind: "parameter-sweep",
|
|
1001
|
+
async propose(ctx) {
|
|
1002
|
+
if (typeof ctx.currentSurface !== "string") {
|
|
1003
|
+
throw new Error(
|
|
1004
|
+
"parameterSweepProposer: currentSurface must be a JSON string config surface"
|
|
1005
|
+
);
|
|
1006
|
+
}
|
|
1007
|
+
const parse = opts.parse ?? parseJsonObject;
|
|
1008
|
+
const stringify = opts.stringify ?? ((config) => JSON.stringify(config, null, 2));
|
|
1009
|
+
const current = parse(ctx.currentSurface);
|
|
1010
|
+
const currentCanonical = canonicalJson(current);
|
|
1011
|
+
const tried = triedLabels(ctx.history);
|
|
1012
|
+
const out = [];
|
|
1013
|
+
for (const candidate of opts.candidates) {
|
|
1014
|
+
if (tried.has(candidate.label)) continue;
|
|
1015
|
+
const next = applyParameterCandidate(current, candidate);
|
|
1016
|
+
const surface = stringify(next);
|
|
1017
|
+
if (surface === ctx.currentSurface || canonicalJson(next) === currentCanonical) continue;
|
|
1018
|
+
out.push({ surface, label: candidate.label, rationale: candidate.rationale });
|
|
1019
|
+
if (out.length >= ctx.populationSize) break;
|
|
1020
|
+
}
|
|
1021
|
+
return out;
|
|
1022
|
+
}
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
1025
|
+
function parseJsonObject(surface) {
|
|
1026
|
+
let parsed;
|
|
1027
|
+
try {
|
|
1028
|
+
parsed = JSON.parse(surface);
|
|
1029
|
+
} catch (error) {
|
|
1030
|
+
throw new Error(
|
|
1031
|
+
`parameterSweepProposer: currentSurface must be valid JSON object config (${error instanceof Error ? error.message : String(error)})`
|
|
1032
|
+
);
|
|
1033
|
+
}
|
|
1034
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
1035
|
+
throw new Error("parameterSweepProposer: JSON surface must parse to an object");
|
|
1036
|
+
}
|
|
1037
|
+
return parsed;
|
|
1038
|
+
}
|
|
1039
|
+
function triedLabels(history) {
|
|
1040
|
+
const tried = /* @__PURE__ */ new Set();
|
|
1041
|
+
for (const generation of history) {
|
|
1042
|
+
for (const candidate of generation.candidates) {
|
|
1043
|
+
if (!candidate.label) continue;
|
|
1044
|
+
tried.add(candidate.label);
|
|
1045
|
+
tried.add(candidateStrategy(candidate));
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
return tried;
|
|
1049
|
+
}
|
|
1050
|
+
function applyParameterCandidate(current, candidate) {
|
|
1051
|
+
const next = cloneJsonObject(current);
|
|
1052
|
+
if (candidate.patch) deepMerge(next, candidate.patch);
|
|
1053
|
+
for (const change of candidate.changes ?? []) {
|
|
1054
|
+
setPath(
|
|
1055
|
+
next,
|
|
1056
|
+
typeof change.path === "string" ? change.path.split(".") : [...change.path],
|
|
1057
|
+
change.value
|
|
1058
|
+
);
|
|
1059
|
+
}
|
|
1060
|
+
return next;
|
|
1061
|
+
}
|
|
1062
|
+
function cloneJsonObject(value) {
|
|
1063
|
+
return JSON.parse(JSON.stringify(value));
|
|
1064
|
+
}
|
|
1065
|
+
function deepMerge(target, patch) {
|
|
1066
|
+
for (const [key, value] of Object.entries(patch)) {
|
|
1067
|
+
assertSafeJsonKey(key);
|
|
1068
|
+
if (isPlainObject(value) && isPlainObject(target[key])) {
|
|
1069
|
+
deepMerge(target[key], value);
|
|
1070
|
+
} else {
|
|
1071
|
+
target[key] = value;
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
function setPath(target, path, value) {
|
|
1076
|
+
if (path.length === 0) throw new Error("parameterSweepProposer: change path must not be empty");
|
|
1077
|
+
for (const part of path) assertSafeJsonKey(part);
|
|
1078
|
+
let cursor = target;
|
|
1079
|
+
for (const part of path.slice(0, -1)) {
|
|
1080
|
+
const existing = cursor[part];
|
|
1081
|
+
if (!isPlainObject(existing)) {
|
|
1082
|
+
cursor[part] = {};
|
|
1083
|
+
}
|
|
1084
|
+
cursor = cursor[part];
|
|
1085
|
+
}
|
|
1086
|
+
cursor[path[path.length - 1]] = value;
|
|
1087
|
+
}
|
|
1088
|
+
function assertSafeJsonKey(key) {
|
|
1089
|
+
if (!key.trim()) throw new Error("parameterSweepProposer: change path contains an empty key");
|
|
1090
|
+
if (UNSAFE_JSON_KEYS.has(key)) {
|
|
1091
|
+
throw new Error(`parameterSweepProposer: unsafe JSON key "${key}" is not allowed`);
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
function canonicalJson(value) {
|
|
1095
|
+
if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item)).join(",")}]`;
|
|
1096
|
+
if (isPlainObject(value)) {
|
|
1097
|
+
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`;
|
|
1098
|
+
}
|
|
1099
|
+
return JSON.stringify(value);
|
|
1100
|
+
}
|
|
1101
|
+
function isPlainObject(value) {
|
|
1102
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
1103
|
+
const proto = Object.getPrototypeOf(value);
|
|
1104
|
+
return proto === Object.prototype || proto === null;
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
// src/campaign/skill-patch.ts
|
|
1108
|
+
function applySkillPatch(surface, patch) {
|
|
1109
|
+
let lines = surface.split("\n");
|
|
1110
|
+
let applied = 0;
|
|
1111
|
+
const rejected = [];
|
|
1112
|
+
const findLine = (anchor) => lines.findIndex((l) => l.includes(anchor));
|
|
1113
|
+
for (const op of patch.ops) {
|
|
1114
|
+
if (op.op === "add") {
|
|
1115
|
+
if (typeof op.text !== "string" || op.text.trim() === "") {
|
|
1116
|
+
rejected.push({ op, reason: "empty add text" });
|
|
1117
|
+
continue;
|
|
1118
|
+
}
|
|
1119
|
+
const insert = op.text.split("\n");
|
|
1120
|
+
if (op.after === void 0 || op.after === "") {
|
|
1121
|
+
lines = [...lines, ...insert];
|
|
1122
|
+
applied++;
|
|
1123
|
+
continue;
|
|
1124
|
+
}
|
|
1125
|
+
const idx = findLine(op.after);
|
|
1126
|
+
if (idx === -1) {
|
|
1127
|
+
rejected.push({ op, reason: `add anchor not found: ${truncate(op.after)}` });
|
|
1128
|
+
continue;
|
|
1129
|
+
}
|
|
1130
|
+
lines = [...lines.slice(0, idx + 1), ...insert, ...lines.slice(idx + 1)];
|
|
1131
|
+
applied++;
|
|
1132
|
+
} else if (op.op === "delete") {
|
|
1133
|
+
const idx = findLine(op.anchor);
|
|
1134
|
+
if (idx === -1) {
|
|
1135
|
+
rejected.push({ op, reason: `delete anchor not found: ${truncate(op.anchor)}` });
|
|
1136
|
+
continue;
|
|
1137
|
+
}
|
|
1138
|
+
lines = [...lines.slice(0, idx), ...lines.slice(idx + 1)];
|
|
1139
|
+
applied++;
|
|
1140
|
+
} else {
|
|
1141
|
+
const idx = findLine(op.anchor);
|
|
1142
|
+
if (idx === -1) {
|
|
1143
|
+
rejected.push({ op, reason: `replace anchor not found: ${truncate(op.anchor)}` });
|
|
1144
|
+
continue;
|
|
1145
|
+
}
|
|
1146
|
+
if (typeof op.text !== "string") {
|
|
1147
|
+
rejected.push({ op, reason: "replace text missing" });
|
|
1148
|
+
continue;
|
|
1149
|
+
}
|
|
1150
|
+
lines = [...lines.slice(0, idx), ...op.text.split("\n"), ...lines.slice(idx + 1)];
|
|
1151
|
+
applied++;
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
return { surface: lines.join("\n"), applied, rejected };
|
|
1155
|
+
}
|
|
1156
|
+
function patchEditCount(patch) {
|
|
1157
|
+
return patch.ops.length;
|
|
1158
|
+
}
|
|
1159
|
+
function truncate(s, max = 48) {
|
|
1160
|
+
return s.length <= max ? s : `${s.slice(0, max)}\u2026`;
|
|
1161
|
+
}
|
|
1162
|
+
|
|
1163
|
+
// src/campaign/proposers/skill-opt.ts
|
|
1164
|
+
var SKILLOPT_SYSTEM = 'You are a SkillOpt optimizer. You improve ONE skill document by proposing BOUNDED, anchored edits \u2014 never a full rewrite. Output ONLY a JSON object of shape {"patches":[{"label":string,"rationale":string,"ops":[op,...]}]} where each op is one of: {"op":"add","after":<exact substring of an existing line, or omit to append>,"text":<new line(s)>}, {"op":"delete","anchor":<exact substring of the line to remove>}, {"op":"replace","anchor":<exact substring of the line to replace>,"text":<replacement line(s)>}. Anchors MUST be verbatim substrings of lines that exist in the document. No prose outside JSON.';
|
|
1165
|
+
function skillOptProposer(opts) {
|
|
1166
|
+
const evidenceK = opts.evidenceK ?? 3;
|
|
1167
|
+
const defaultBudget = opts.editBudget ?? 3;
|
|
1168
|
+
async function proposePatches(args) {
|
|
1169
|
+
const userPrompt = buildPatchPrompt({
|
|
1170
|
+
target: opts.target,
|
|
1171
|
+
surface: args.surface,
|
|
1172
|
+
evidence: args.evidence,
|
|
1173
|
+
editBudget: args.editBudget,
|
|
1174
|
+
rejectedBuffer: args.rejectedBuffer,
|
|
1175
|
+
metaNote: args.metaNote,
|
|
1176
|
+
findingsNote: args.findingsNote,
|
|
1177
|
+
count: args.count
|
|
1178
|
+
});
|
|
1179
|
+
const result = await callLlm(
|
|
1180
|
+
{
|
|
1181
|
+
model: opts.model,
|
|
1182
|
+
messages: [
|
|
1183
|
+
{ role: "system", content: SKILLOPT_SYSTEM },
|
|
1184
|
+
{ role: "user", content: userPrompt }
|
|
1185
|
+
],
|
|
1186
|
+
jsonMode: true,
|
|
1187
|
+
temperature: opts.temperature ?? 0.6,
|
|
1188
|
+
maxTokens: opts.maxTokens ?? 4e3
|
|
1189
|
+
},
|
|
1190
|
+
opts.llm
|
|
1191
|
+
);
|
|
1192
|
+
return parseSkillPatchResponse(result.content, args.count, args.editBudget);
|
|
1193
|
+
}
|
|
1194
|
+
return {
|
|
1195
|
+
kind: "skill-opt",
|
|
1196
|
+
proposePatches,
|
|
1197
|
+
async propose(ctx) {
|
|
1198
|
+
if (typeof ctx.currentSurface !== "string") {
|
|
1199
|
+
throw new Error(
|
|
1200
|
+
"skillOptProposer: surface must be a string skill document (got a CodeSurface). SkillOpt patches text."
|
|
1201
|
+
);
|
|
1202
|
+
}
|
|
1203
|
+
const surface = ctx.currentSurface;
|
|
1204
|
+
const patches = await proposePatches({
|
|
1205
|
+
surface,
|
|
1206
|
+
evidence: evidenceFromHistory(ctx, evidenceK),
|
|
1207
|
+
editBudget: defaultBudget,
|
|
1208
|
+
rejectedBuffer: [],
|
|
1209
|
+
findingsNote: renderAnalystEvidence(ctx.findings, ctx.report) ?? void 0,
|
|
1210
|
+
count: ctx.populationSize,
|
|
1211
|
+
signal: ctx.signal
|
|
525
1212
|
});
|
|
526
1213
|
const out = [];
|
|
527
1214
|
const seen = /* @__PURE__ */ new Set();
|
|
@@ -655,770 +1342,236 @@ function snippet(s, max = 120) {
|
|
|
655
1342
|
return t.length <= max ? t : `${t.slice(0, max)}\u2026`;
|
|
656
1343
|
}
|
|
657
1344
|
|
|
658
|
-
// src/campaign/
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
return `${i + 1}. [${f.severity}/${f.area}]${subject} ${f.claim}${action}`;
|
|
666
|
-
}).join("\n");
|
|
667
|
-
}
|
|
668
|
-
function traceAnalystDriver(opts) {
|
|
669
|
-
if (!opts.apiKey) throw new Error("traceAnalystDriver: apiKey is required");
|
|
670
|
-
if (!opts.model) throw new Error("traceAnalystDriver: model is required");
|
|
671
|
-
const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS;
|
|
672
|
-
const produceFindings = opts.analyze ?? (async (path, c) => {
|
|
673
|
-
const aiService = ai({
|
|
674
|
-
name: opts.provider ?? "openai",
|
|
675
|
-
apiKey: opts.apiKey,
|
|
676
|
-
apiURL: opts.baseUrl,
|
|
677
|
-
config: { model: opts.model }
|
|
678
|
-
});
|
|
679
|
-
const registry = new AnalystRegistry();
|
|
680
|
-
for (const spec of kinds) {
|
|
681
|
-
registry.register(createTraceAnalystKind(spec, { ai: aiService, model: opts.model }));
|
|
682
|
-
}
|
|
683
|
-
const result = await registry.run(
|
|
684
|
-
`trace-analyst-gen-${c.generation}`,
|
|
685
|
-
{ traceStore: new OtlpFileTraceStore({ path }) },
|
|
686
|
-
{ signal: c.signal }
|
|
1345
|
+
// src/campaign/presets/run-skill-opt.ts
|
|
1346
|
+
async function runSkillOpt(opts) {
|
|
1347
|
+
if (opts.trainScenarios.length === 0) throw new Error("runSkillOpt: trainScenarios is empty");
|
|
1348
|
+
if (opts.holdoutScenarios.length === 0) throw new Error("runSkillOpt: holdoutScenarios is empty");
|
|
1349
|
+
if (!opts.judges || opts.judges.length === 0) {
|
|
1350
|
+
throw new Error(
|
|
1351
|
+
"runSkillOpt: at least one judge is required \u2014 scoring (and therefore acceptance) is meaningless without one, and would report a silent zero lift."
|
|
687
1352
|
);
|
|
688
|
-
return result.findings;
|
|
689
|
-
});
|
|
690
|
-
return analysisEditDriver({
|
|
691
|
-
kind: "trace-analyst",
|
|
692
|
-
label: "trace-analyst",
|
|
693
|
-
baseUrl: opts.baseUrl,
|
|
694
|
-
apiKey: opts.apiKey,
|
|
695
|
-
applyModel: opts.applyModel ?? opts.model,
|
|
696
|
-
fetchImpl: opts.fetchImpl,
|
|
697
|
-
resolveTraces: opts.resolveTraces,
|
|
698
|
-
noTracesError: "traceAnalystDriver: resolveTraces returned no OTLP traces \u2014 the analyst has nothing to read",
|
|
699
|
-
rationale: (report) => `trace-analyst findings:
|
|
700
|
-
${report.slice(0, 800)}`,
|
|
701
|
-
analyze: async (tracePath, ctx) => {
|
|
702
|
-
let findings;
|
|
703
|
-
try {
|
|
704
|
-
findings = await produceFindings(tracePath, ctx);
|
|
705
|
-
} catch (e) {
|
|
706
|
-
throw new Error(
|
|
707
|
-
`traceAnalystDriver: analyst engine failed \u2014 ${e instanceof Error ? e.message : String(e)}`
|
|
708
|
-
);
|
|
709
|
-
}
|
|
710
|
-
if (findings.length === 0) {
|
|
711
|
-
throw new Error("traceAnalystDriver: analyst engine produced no findings");
|
|
712
|
-
}
|
|
713
|
-
return renderFindings(findings);
|
|
714
|
-
}
|
|
715
|
-
});
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
// src/campaign/gates/sequential.ts
|
|
719
|
-
import { createHash } from "crypto";
|
|
720
|
-
function verifyManifestSync(m) {
|
|
721
|
-
if (m.algo !== void 0 && m.algo !== "sha256-content") {
|
|
722
|
-
throw new Error(`sequentialPairedGate: unrecognized manifest hash algo '${m.algo}'`);
|
|
723
|
-
}
|
|
724
|
-
const { contentHash, algo: _algo, ...rest } = m;
|
|
725
|
-
void _algo;
|
|
726
|
-
const bytes = JSON.stringify(canonicalize(rest));
|
|
727
|
-
const hash = createHash("sha256").update(bytes, "utf8").digest("hex");
|
|
728
|
-
return hash === contentHash;
|
|
729
|
-
}
|
|
730
|
-
function resolveConfig(opts) {
|
|
731
|
-
const m = opts.preRegistration;
|
|
732
|
-
let alpha;
|
|
733
|
-
let maxN;
|
|
734
|
-
let direction;
|
|
735
|
-
let minEffect;
|
|
736
|
-
if (m) {
|
|
737
|
-
if (!verifyManifestSync(m)) {
|
|
738
|
-
throw new Error(
|
|
739
|
-
`sequentialPairedGate: pre-registration manifest '${m.id}' content hash mismatch (tampered)`
|
|
740
|
-
);
|
|
741
|
-
}
|
|
742
|
-
if (opts.alpha !== void 0 && opts.alpha !== m.alpha) {
|
|
743
|
-
throw new Error(
|
|
744
|
-
`sequentialPairedGate: alpha ${opts.alpha} conflicts with pre-registered alpha ${m.alpha} \u2014 the registered statistic is the only one anytime validity covers`
|
|
745
|
-
);
|
|
746
|
-
}
|
|
747
|
-
if (opts.maxN !== void 0 && opts.maxN !== m.preRegisteredN) {
|
|
748
|
-
throw new Error(
|
|
749
|
-
`sequentialPairedGate: maxN ${opts.maxN} conflicts with pre-registered N ${m.preRegisteredN}`
|
|
750
|
-
);
|
|
751
|
-
}
|
|
752
|
-
alpha = m.alpha;
|
|
753
|
-
maxN = m.preRegisteredN;
|
|
754
|
-
direction = m.direction;
|
|
755
|
-
minEffect = m.minEffect;
|
|
756
|
-
} else {
|
|
757
|
-
alpha = opts.alpha ?? 0.05;
|
|
758
|
-
if (opts.maxN === void 0) {
|
|
759
|
-
throw new Error(
|
|
760
|
-
"sequentialPairedGate: maxN is required (or bind a preRegistration manifest whose preRegisteredN is the budget) \u2014 an unbounded stream has no pre-registered budget"
|
|
761
|
-
);
|
|
762
|
-
}
|
|
763
|
-
maxN = opts.maxN;
|
|
764
|
-
direction = "increase";
|
|
765
|
-
minEffect = 0;
|
|
766
|
-
}
|
|
767
|
-
const scale = opts.scale ?? 1;
|
|
768
|
-
if (!Number.isFinite(scale) || scale <= 0) {
|
|
769
|
-
throw new Error(`sequentialPairedGate: scale must be > 0, got ${scale}`);
|
|
770
|
-
}
|
|
771
|
-
if (!Number.isInteger(maxN) || maxN < 1) {
|
|
772
|
-
throw new Error(`sequentialPairedGate: maxN must be a positive integer, got ${maxN}`);
|
|
773
1353
|
}
|
|
774
|
-
const
|
|
775
|
-
|
|
1354
|
+
const holdoutIds = new Set(opts.holdoutScenarios.map((s) => s.id));
|
|
1355
|
+
const overlap = opts.trainScenarios.filter((s) => holdoutIds.has(s.id)).map((s) => s.id);
|
|
1356
|
+
if (overlap.length > 0) {
|
|
776
1357
|
throw new Error(
|
|
777
|
-
`
|
|
1358
|
+
`runSkillOpt: trainScenarios and holdoutScenarios must be disjoint (overlap: [${overlap.join(
|
|
1359
|
+
", "
|
|
1360
|
+
)}]) \u2014 a shared scenario leaks the held-out acceptance axis into the proposal evidence.`
|
|
778
1361
|
);
|
|
779
1362
|
}
|
|
780
|
-
|
|
1363
|
+
const patchesPerEpoch = opts.patchesPerEpoch ?? 2;
|
|
1364
|
+
const initialBudget = opts.editBudget ?? 3;
|
|
1365
|
+
const minImprovement = opts.minImprovement ?? 0;
|
|
1366
|
+
if (minImprovement < 0) {
|
|
781
1367
|
throw new Error(
|
|
782
|
-
|
|
1368
|
+
"runSkillOpt: minImprovement must be >= 0 \u2014 a negative threshold would accept held-out regressions, breaking the monotonic-lift contract."
|
|
783
1369
|
);
|
|
784
1370
|
}
|
|
785
|
-
const
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
nullMean,
|
|
795
|
-
minEffect
|
|
1371
|
+
const patience = opts.patience ?? opts.maxEpochs;
|
|
1372
|
+
const budgetAnneal = opts.budgetAnneal ?? true;
|
|
1373
|
+
const rejectedBufferSize = opts.rejectedBufferSize ?? 12;
|
|
1374
|
+
const slowMetaEvery = opts.slowMetaEvery ?? 2;
|
|
1375
|
+
let totalCostUsd = 0;
|
|
1376
|
+
const scoreHoldout = async (surface, tag) => {
|
|
1377
|
+
const campaign = await runScoringCampaign(opts, opts.holdoutScenarios, surface, tag);
|
|
1378
|
+
totalCostUsd += campaign.aggregates.totalCostUsd;
|
|
1379
|
+
return campaignMeanComposite(campaign);
|
|
796
1380
|
};
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
1381
|
+
const evidenceK = opts.evidenceK ?? 3;
|
|
1382
|
+
const trainEvidence = async (surface, tag) => {
|
|
1383
|
+
const campaign = await runScoringCampaign(opts, opts.trainScenarios, surface, tag);
|
|
1384
|
+
totalCostUsd += campaign.aggregates.totalCostUsd;
|
|
1385
|
+
return toEvidence(campaign, evidenceK);
|
|
1386
|
+
};
|
|
1387
|
+
let current = opts.baselineSurface;
|
|
1388
|
+
let currentEvidence = await trainEvidence(current, "baseline-train");
|
|
1389
|
+
const baselineHoldout = await scoreHoldout(current, "baseline-holdout");
|
|
1390
|
+
let currentHoldout = baselineHoldout;
|
|
1391
|
+
const buffer = [];
|
|
1392
|
+
const acceptedEdits = [];
|
|
1393
|
+
const rejectedAll = [];
|
|
1394
|
+
const history = [];
|
|
1395
|
+
let budget = initialBudget;
|
|
1396
|
+
let sinceAccept = 0;
|
|
1397
|
+
let metaNote;
|
|
1398
|
+
let epochsRun = 0;
|
|
1399
|
+
for (let epoch = 0; epoch < opts.maxEpochs; epoch++) {
|
|
1400
|
+
epochsRun++;
|
|
1401
|
+
const patches = await opts.proposer.proposePatches({
|
|
1402
|
+
surface: current,
|
|
1403
|
+
evidence: currentEvidence,
|
|
1404
|
+
editBudget: budget,
|
|
1405
|
+
rejectedBuffer: buffer,
|
|
1406
|
+
metaNote,
|
|
1407
|
+
count: patchesPerEpoch,
|
|
1408
|
+
signal: opts.signal ?? new AbortController().signal
|
|
1409
|
+
});
|
|
1410
|
+
let accepted = null;
|
|
1411
|
+
const rejectedThisEpoch = [];
|
|
1412
|
+
for (let i = 0; i < patches.length; i++) {
|
|
1413
|
+
const patch = patches[i];
|
|
1414
|
+
const { surface: candidate, applied } = applySkillPatch(current, patch);
|
|
1415
|
+
if (applied === 0 || candidate === current) {
|
|
1416
|
+
rejectedThisEpoch.push({
|
|
1417
|
+
label: patch.label,
|
|
1418
|
+
rationale: patch.rationale,
|
|
1419
|
+
reason: "no-op (unanchored or zero-change)"
|
|
1420
|
+
});
|
|
1421
|
+
continue;
|
|
826
1422
|
}
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
1423
|
+
const candidateHoldout = await scoreHoldout(candidate, `epoch-${epoch}-cand-${i}-holdout`);
|
|
1424
|
+
if (candidateHoldout > currentHoldout + minImprovement) {
|
|
1425
|
+
accepted = {
|
|
1426
|
+
epoch,
|
|
1427
|
+
label: patch.label,
|
|
1428
|
+
rationale: patch.rationale,
|
|
1429
|
+
holdoutDelta: candidateHoldout - currentHoldout
|
|
834
1430
|
};
|
|
1431
|
+
current = candidate;
|
|
1432
|
+
currentHoldout = candidateHoldout;
|
|
1433
|
+
currentEvidence = await trainEvidence(current, `epoch-${epoch}-train`);
|
|
1434
|
+
break;
|
|
835
1435
|
}
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
1436
|
+
rejectedThisEpoch.push({
|
|
1437
|
+
label: patch.label,
|
|
1438
|
+
rationale: patch.rationale,
|
|
1439
|
+
reason: `held-out ${candidateHoldout.toFixed(3)} \u2264 current ${currentHoldout.toFixed(3)}`
|
|
1440
|
+
});
|
|
1441
|
+
}
|
|
1442
|
+
if (accepted) {
|
|
1443
|
+
acceptedEdits.push(accepted);
|
|
1444
|
+
sinceAccept = 0;
|
|
1445
|
+
} else {
|
|
1446
|
+
sinceAccept++;
|
|
1447
|
+
if (budgetAnneal && sinceAccept >= 2 && budget > 1) budget--;
|
|
1448
|
+
}
|
|
1449
|
+
for (const r of rejectedThisEpoch) {
|
|
1450
|
+
buffer.push(r);
|
|
1451
|
+
rejectedAll.push(r);
|
|
1452
|
+
}
|
|
1453
|
+
while (buffer.length > rejectedBufferSize) buffer.shift();
|
|
1454
|
+
if (slowMetaEvery > 0 && (epoch + 1) % slowMetaEvery === 0) {
|
|
1455
|
+
metaNote = buildMetaNote(acceptedEdits, buffer);
|
|
845
1456
|
}
|
|
1457
|
+
history.push({
|
|
1458
|
+
epoch,
|
|
1459
|
+
editBudget: budget,
|
|
1460
|
+
proposed: patches.length,
|
|
1461
|
+
accepted,
|
|
1462
|
+
rejected: rejectedThisEpoch,
|
|
1463
|
+
holdoutComposite: currentHoldout
|
|
1464
|
+
});
|
|
1465
|
+
if (sinceAccept >= patience) break;
|
|
1466
|
+
}
|
|
1467
|
+
return {
|
|
1468
|
+
winnerSurface: current,
|
|
1469
|
+
baselineHoldoutComposite: baselineHoldout,
|
|
1470
|
+
winnerHoldoutComposite: currentHoldout,
|
|
1471
|
+
lift: currentHoldout - baselineHoldout,
|
|
1472
|
+
acceptedEdits,
|
|
1473
|
+
rejectedEdits: rejectedAll,
|
|
1474
|
+
epochsRun,
|
|
1475
|
+
history,
|
|
1476
|
+
totalCostUsd
|
|
846
1477
|
};
|
|
847
1478
|
}
|
|
848
|
-
function
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
}
|
|
856
|
-
return items;
|
|
1479
|
+
function runScoringCampaign(opts, scenarios, surface, tag) {
|
|
1480
|
+
return runCampaign({
|
|
1481
|
+
...opts,
|
|
1482
|
+
scenarios,
|
|
1483
|
+
dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
|
|
1484
|
+
runDir: `${opts.runDir}/${tag}`
|
|
1485
|
+
});
|
|
857
1486
|
}
|
|
858
|
-
function
|
|
859
|
-
const
|
|
860
|
-
const
|
|
861
|
-
const
|
|
862
|
-
|
|
863
|
-
return {
|
|
864
|
-
name,
|
|
865
|
-
observe(delta) {
|
|
866
|
-
return observeStream.observe(delta);
|
|
867
|
-
},
|
|
868
|
-
state() {
|
|
869
|
-
return observeStream.state();
|
|
870
|
-
},
|
|
871
|
-
async decide(ctx) {
|
|
872
|
-
if (!ctx.baselineJudgeScores) {
|
|
873
|
-
throw new Error(
|
|
874
|
-
`${name}: ctx.baselineJudgeScores is required \u2014 falling back to the candidate's own scores would compare the candidate against itself (delta 0, silent no-op)`
|
|
875
|
-
);
|
|
876
|
-
}
|
|
877
|
-
const scenarioIds = new Set(ctx.scenarios.map((s) => s.id));
|
|
878
|
-
const paired = pairHoldout(
|
|
879
|
-
ctx.judgeScores,
|
|
880
|
-
ctx.baselineJudgeScores,
|
|
881
|
-
scenarioIds,
|
|
882
|
-
(s) => s.composite
|
|
883
|
-
);
|
|
884
|
-
const deltas = paired.after.map((a, i) => a - paired.before[i]);
|
|
885
|
-
seededShuffle(deltas, cfg.shuffleSeed);
|
|
886
|
-
const stream = makeStream(cfg);
|
|
887
|
-
let last;
|
|
888
|
-
for (const d of deltas) {
|
|
889
|
-
last = stream.observe(d);
|
|
890
|
-
if (last.decision !== "continue") break;
|
|
891
|
-
}
|
|
892
|
-
const detail = {
|
|
893
|
-
...stream.state(),
|
|
894
|
-
minN: cfg.minN,
|
|
895
|
-
maxN: cfg.maxN,
|
|
896
|
-
scale: cfg.scale,
|
|
897
|
-
shuffleSeed: cfg.shuffleSeed,
|
|
898
|
-
direction: cfg.direction,
|
|
899
|
-
minEffect: cfg.minEffect,
|
|
900
|
-
pairedN: deltas.length,
|
|
901
|
-
preRegisteredId: manifest?.id,
|
|
902
|
-
metric: manifest?.metric
|
|
903
|
-
};
|
|
904
|
-
const meanDelta = deltas.length === 0 ? void 0 : deltas.reduce((s, d) => s + d, 0) / deltas.length;
|
|
905
|
-
if (last === void 0) {
|
|
906
|
-
return {
|
|
907
|
-
decision: "need_more_work",
|
|
908
|
-
reasons: [`${name}: no paired holdout observations \u2014 nothing to test`],
|
|
909
|
-
contributingGates: [{ name, passed: false, detail }]
|
|
910
|
-
};
|
|
911
|
-
}
|
|
912
|
-
const decision = last.decision === "promote" ? "ship" : last.decision === "continue" ? "need_more_work" : "hold";
|
|
913
|
-
return {
|
|
914
|
-
decision,
|
|
915
|
-
reasons: [`${name}: ${last.reason}`],
|
|
916
|
-
contributingGates: [{ name, passed: decision === "ship", detail }],
|
|
917
|
-
delta: meanDelta
|
|
918
|
-
};
|
|
919
|
-
}
|
|
920
|
-
};
|
|
1487
|
+
function toEvidence(campaign, k) {
|
|
1488
|
+
const { dimensions, scenarios } = campaignBreakdown(campaign);
|
|
1489
|
+
const weakScenarios = [...scenarios].sort((a, b) => a.composite - b.composite).slice(0, k);
|
|
1490
|
+
const weakDimensions = Object.entries(dimensions).sort((a, b) => a[1] - b[1]).slice(0, k).map(([dimension, score]) => ({ dimension, score }));
|
|
1491
|
+
return { weakScenarios, weakDimensions };
|
|
921
1492
|
}
|
|
922
|
-
function
|
|
923
|
-
const
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
1493
|
+
function buildMetaNote(accepted, rejected) {
|
|
1494
|
+
const parts = [];
|
|
1495
|
+
if (accepted.length > 0) {
|
|
1496
|
+
parts.push(
|
|
1497
|
+
`Edits that improved held-out so far: ${accepted.map((a) => `"${a.label}" (+${a.holdoutDelta.toFixed(3)})`).join("; ")}. Build on these.`
|
|
1498
|
+
);
|
|
928
1499
|
}
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
if (record.candidates.length === 0) {
|
|
936
|
-
throw new Error(
|
|
937
|
-
`sequentialDecide: generation ${record.generationIndex} has no candidates \u2014 cannot extract a top candidate`
|
|
938
|
-
);
|
|
939
|
-
}
|
|
940
|
-
return record.candidates.reduce((best, c) => c.composite > best.composite ? c : best);
|
|
941
|
-
};
|
|
942
|
-
const decide = ({ history }) => {
|
|
943
|
-
if (stopped) return stopped;
|
|
944
|
-
if (history.length === 0) return { stop: false };
|
|
945
|
-
if (reference === void 0) {
|
|
946
|
-
reference = new Map(
|
|
947
|
-
topCandidate(history[0]).scenarios.map((s) => [s.scenarioId, s.composite])
|
|
948
|
-
);
|
|
949
|
-
}
|
|
950
|
-
for (let g = Math.max(1, processedGenerations); g < history.length; g++) {
|
|
951
|
-
const top = topCandidate(history[g]);
|
|
952
|
-
const byScenario = new Map(top.scenarios.map((s) => [s.scenarioId, s.composite]));
|
|
953
|
-
for (const [scenarioId, refComposite] of reference) {
|
|
954
|
-
const candComposite = byScenario.get(scenarioId);
|
|
955
|
-
if (candComposite === void 0) {
|
|
956
|
-
throw new Error(
|
|
957
|
-
`sequentialDecide: generation ${history[g].generationIndex} top candidate is missing scenario '${scenarioId}' \u2014 generations must score the same scenario set to pair`
|
|
958
|
-
);
|
|
959
|
-
}
|
|
960
|
-
const delta = candComposite - refComposite;
|
|
961
|
-
if (!Number.isFinite(delta) || Math.abs(delta) > scale) {
|
|
962
|
-
throw new Error(
|
|
963
|
-
`sequentialDecide: paired delta ${delta} outside \xB1scale=${scale} on scenario '${scenarioId}' \u2014 pass the composite scale explicitly`
|
|
964
|
-
);
|
|
965
|
-
}
|
|
966
|
-
const step = proc.update((delta / scale + 1) / 2);
|
|
967
|
-
if (step.n >= minN && step.wealth >= threshold) {
|
|
968
|
-
stopped = {
|
|
969
|
-
stop: true,
|
|
970
|
-
reason: `sequential e-process decided at generation ${history[g].generationIndex}: e-value ${step.wealth.toFixed(2)} \u2265 1/\u03B1=${threshold.toFixed(2)} after n=${step.n} paired deltas vs the generation-0 incumbent \u2014 the improvement is real at \u03B1=${alpha}; stop exploring and promote via the gate`
|
|
971
|
-
};
|
|
972
|
-
processedGenerations = history.length;
|
|
973
|
-
return stopped;
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
}
|
|
977
|
-
processedGenerations = history.length;
|
|
978
|
-
return { stop: false };
|
|
979
|
-
};
|
|
980
|
-
decide.state = () => proc.state();
|
|
981
|
-
return decide;
|
|
1500
|
+
if (rejected.length > 0) {
|
|
1501
|
+
const labels = [...new Set(rejected.map((r) => r.label))].slice(0, 5);
|
|
1502
|
+
parts.push(`Dead ends to avoid: ${labels.join(", ")}. Try a different anchor or rule.`);
|
|
1503
|
+
}
|
|
1504
|
+
parts.push("Keep edits small and anchored to existing lines.");
|
|
1505
|
+
return parts.join(" ");
|
|
982
1506
|
}
|
|
983
1507
|
|
|
984
|
-
// src/campaign/
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
now;
|
|
1004
|
-
rateLimits = /* @__PURE__ */ new Map();
|
|
1005
|
-
async observe(write) {
|
|
1006
|
-
this.assertProvenance(write);
|
|
1007
|
-
this.assertRateLimit(write);
|
|
1008
|
-
const record = this.toRecord(write);
|
|
1009
|
-
const path = this.pathForSource(write.source);
|
|
1010
|
-
const line = `${JSON.stringify(record)}
|
|
1011
|
-
`;
|
|
1012
|
-
appendLine(path, line);
|
|
1013
|
-
}
|
|
1014
|
-
async sample(args) {
|
|
1015
|
-
if (!args.split) {
|
|
1016
|
-
throw new LabeledScenarioStoreError(
|
|
1017
|
-
"split_required",
|
|
1018
|
-
"sample() requires an explicit `split` (train | test) \u2014 substrate refuses ambiguous reads"
|
|
1019
|
-
);
|
|
1508
|
+
// src/campaign/presets/compare-proposers.ts
|
|
1509
|
+
async function compareProposers(opts) {
|
|
1510
|
+
return compareProposerEntries(opts);
|
|
1511
|
+
}
|
|
1512
|
+
async function compareProposerEntries(opts) {
|
|
1513
|
+
if (opts.proposers.length === 0) throw new Error("compareProposers: no proposers to compare");
|
|
1514
|
+
const seed = opts.seed ?? 42;
|
|
1515
|
+
const resamples = opts.resamples ?? 2e3;
|
|
1516
|
+
const confidence = opts.confidence ?? 0.95;
|
|
1517
|
+
const scoreOnHoldout = async (surface, tag) => {
|
|
1518
|
+
const campaign = await runCampaign({
|
|
1519
|
+
...opts,
|
|
1520
|
+
scenarios: opts.holdoutScenarios,
|
|
1521
|
+
dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
|
|
1522
|
+
runDir: `${opts.runDir}/${tag}`
|
|
1523
|
+
});
|
|
1524
|
+
const byScenario = {};
|
|
1525
|
+
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) {
|
|
1526
|
+
byScenario[scenarioId] = composite;
|
|
1020
1527
|
}
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1528
|
+
return byScenario;
|
|
1529
|
+
};
|
|
1530
|
+
const scenarioIds = [...new Set(opts.holdoutScenarios.map((s) => s.id))].sort();
|
|
1531
|
+
if (scenarioIds.length === 0) throw new Error("compareProposers: holdoutScenarios is empty");
|
|
1532
|
+
const align = (byScenario, label) => {
|
|
1533
|
+
const missing = scenarioIds.filter((id) => !(id in byScenario));
|
|
1534
|
+
if (missing.length > 0) {
|
|
1535
|
+
throw new Error(
|
|
1536
|
+
`compareProposers: ${label} produced no held-out score for scenario(s) [${missing.join(
|
|
1537
|
+
", "
|
|
1538
|
+
)}] \u2014 a cell errored or its judges returned nothing. Refusing to fabricate a 0 (it would corrupt the lift comparison). Fix the dispatch/judge or drop the scenario.`
|
|
1025
1539
|
);
|
|
1026
1540
|
}
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
if (!matchesFilter(record, args, source)) continue;
|
|
1044
|
-
all.push(record);
|
|
1045
|
-
}
|
|
1046
|
-
}
|
|
1047
|
-
all.sort((a, b) => {
|
|
1048
|
-
if (a.capturedAt !== b.capturedAt) return a.capturedAt.localeCompare(b.capturedAt);
|
|
1049
|
-
return a.recordHash.localeCompare(b.recordHash);
|
|
1541
|
+
return scenarioIds.map((id) => byScenario[id]);
|
|
1542
|
+
};
|
|
1543
|
+
const baselineArr = align(
|
|
1544
|
+
await scoreOnHoldout(opts.baselineSurface, "compare-baseline"),
|
|
1545
|
+
"baseline"
|
|
1546
|
+
);
|
|
1547
|
+
const winners = [];
|
|
1548
|
+
for (const proposer of opts.proposers) {
|
|
1549
|
+
const out = await proposer.optimize();
|
|
1550
|
+
const byScenario = await scoreOnHoldout(out.winnerSurface, `compare-${slug(proposer.name)}`);
|
|
1551
|
+
winners.push({
|
|
1552
|
+
name: proposer.name,
|
|
1553
|
+
winnerSurface: out.winnerSurface,
|
|
1554
|
+
costUsd: out.costUsd,
|
|
1555
|
+
durationMs: out.durationMs,
|
|
1556
|
+
arr: align(byScenario, `proposer "${proposer.name}"`)
|
|
1050
1557
|
});
|
|
1051
|
-
return all.slice(0, args.count);
|
|
1052
1558
|
}
|
|
1053
|
-
|
|
1054
|
-
const
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
"
|
|
1059
|
-
};
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
total += lines.length;
|
|
1070
|
-
for (const line of lines) {
|
|
1071
|
-
let trust = "unverified";
|
|
1072
|
-
try {
|
|
1073
|
-
trust = JSON.parse(line).labelTrust ?? "unverified";
|
|
1074
|
-
} catch {
|
|
1075
|
-
}
|
|
1076
|
-
byTrust[trust] += 1;
|
|
1077
|
-
}
|
|
1078
|
-
}
|
|
1079
|
-
return { train: total, test: total, bySource, byTrust };
|
|
1080
|
-
}
|
|
1081
|
-
assertProvenance(write) {
|
|
1082
|
-
if (!write.source) {
|
|
1083
|
-
throw new LabeledScenarioStoreError(
|
|
1084
|
-
"missing_source",
|
|
1085
|
-
"LabeledScenarioWrite requires `source`"
|
|
1086
|
-
);
|
|
1087
|
-
}
|
|
1088
|
-
if (!write.sourceVersionHash || write.sourceVersionHash.length === 0) {
|
|
1089
|
-
throw new LabeledScenarioStoreError(
|
|
1090
|
-
"missing_source_version",
|
|
1091
|
-
"LabeledScenarioWrite requires `sourceVersionHash` (git sha or substrate version)"
|
|
1092
|
-
);
|
|
1093
|
-
}
|
|
1094
|
-
if (!write.capturedAt) {
|
|
1095
|
-
throw new LabeledScenarioStoreError(
|
|
1096
|
-
"missing_captured_at",
|
|
1097
|
-
"LabeledScenarioWrite requires `capturedAt` ISO timestamp"
|
|
1098
|
-
);
|
|
1099
|
-
}
|
|
1100
|
-
if (!write.redactionStatus) {
|
|
1101
|
-
throw new LabeledScenarioStoreError(
|
|
1102
|
-
"missing_redaction_status",
|
|
1103
|
-
"LabeledScenarioWrite requires explicit `redactionStatus` \u2014 raw / redacted-pii / redacted-secrets / fully-redacted"
|
|
1104
|
-
);
|
|
1105
|
-
}
|
|
1106
|
-
if (!ALL_SOURCES.includes(write.source)) {
|
|
1107
|
-
throw new LabeledScenarioStoreError(
|
|
1108
|
-
"unknown_source",
|
|
1109
|
-
`LabeledScenarioWrite.source must be one of: ${ALL_SOURCES.join(", ")}`
|
|
1110
|
-
);
|
|
1111
|
-
}
|
|
1112
|
-
}
|
|
1113
|
-
assertRateLimit(write) {
|
|
1114
|
-
const cap = this.options.maxWritesPerMinutePerBucket;
|
|
1115
|
-
if (!cap || !write.rateLimitBucket) return;
|
|
1116
|
-
const now = this.now();
|
|
1117
|
-
const windowMs = 6e4;
|
|
1118
|
-
let state = this.rateLimits.get(write.rateLimitBucket);
|
|
1119
|
-
if (!state || now - state.windowStartMs >= windowMs) {
|
|
1120
|
-
state = { bucket: write.rateLimitBucket, windowStartMs: now, count: 0 };
|
|
1121
|
-
this.rateLimits.set(write.rateLimitBucket, state);
|
|
1122
|
-
}
|
|
1123
|
-
if (state.count >= cap) {
|
|
1124
|
-
throw new LabeledScenarioStoreError(
|
|
1125
|
-
"rate_limit_exceeded",
|
|
1126
|
-
`LabeledScenarioStore: bucket ${write.rateLimitBucket} exceeded ${cap} writes/min`
|
|
1127
|
-
);
|
|
1128
|
-
}
|
|
1129
|
-
state.count += 1;
|
|
1130
|
-
}
|
|
1131
|
-
toRecord(write) {
|
|
1132
|
-
const recordHash = sha256(
|
|
1133
|
-
JSON.stringify({
|
|
1134
|
-
id: write.scenario.id,
|
|
1135
|
-
src: write.source,
|
|
1136
|
-
at: write.capturedAt,
|
|
1137
|
-
ver: write.sourceVersionHash
|
|
1138
|
-
})
|
|
1139
|
-
);
|
|
1140
|
-
return {
|
|
1141
|
-
...write,
|
|
1142
|
-
recordHash,
|
|
1143
|
-
split: "train"
|
|
1144
|
-
};
|
|
1145
|
-
}
|
|
1146
|
-
pathForSource(source) {
|
|
1147
|
-
return join2(this.options.root, `${source}.jsonl`);
|
|
1148
|
-
}
|
|
1149
|
-
};
|
|
1150
|
-
var ALL_SOURCES = [
|
|
1151
|
-
"production-trace",
|
|
1152
|
-
"eval-run",
|
|
1153
|
-
"manual",
|
|
1154
|
-
"red-team",
|
|
1155
|
-
"synthetic"
|
|
1156
|
-
];
|
|
1157
|
-
function sourceFilterContains(filter, needle) {
|
|
1158
|
-
if (!filter) return false;
|
|
1159
|
-
if (Array.isArray(filter)) return filter.includes(needle);
|
|
1160
|
-
return filter === needle;
|
|
1161
|
-
}
|
|
1162
|
-
function matchesFilter(record, args, source) {
|
|
1163
|
-
if (args.split === "train" && record.capturedAt >= args.capturedBefore) return false;
|
|
1164
|
-
if (args.split === "test" && record.capturedAt < args.capturedBefore) return false;
|
|
1165
|
-
const f = args.filter;
|
|
1166
|
-
if (!f) return true;
|
|
1167
|
-
if (f.kind && record.scenario.kind !== f.kind) return false;
|
|
1168
|
-
if (f.source) {
|
|
1169
|
-
const sources = Array.isArray(f.source) ? f.source : [f.source];
|
|
1170
|
-
if (!sources.includes(source)) return false;
|
|
1171
|
-
}
|
|
1172
|
-
if (f.minComposite !== void 0 || f.maxComposite !== void 0) {
|
|
1173
|
-
const composites = Object.values(record.judgeScores).map((s) => s.composite);
|
|
1174
|
-
const max = composites.length === 0 ? 0 : Math.max(...composites);
|
|
1175
|
-
if (f.minComposite !== void 0 && max < f.minComposite) return false;
|
|
1176
|
-
if (f.maxComposite !== void 0 && max > f.maxComposite) return false;
|
|
1177
|
-
}
|
|
1178
|
-
if (f.minTrust !== void 0 && labelTrustRank(record.labelTrust) < labelTrustRank(f.minTrust)) {
|
|
1179
|
-
return false;
|
|
1180
|
-
}
|
|
1181
|
-
return true;
|
|
1182
|
-
}
|
|
1183
|
-
function sha256(input) {
|
|
1184
|
-
return createHash2("sha256").update(input).digest("hex").slice(0, 16);
|
|
1185
|
-
}
|
|
1186
|
-
function appendLine(path, line) {
|
|
1187
|
-
if (existsSync(path)) {
|
|
1188
|
-
const existing = readFileSync(path, "utf8");
|
|
1189
|
-
writeFileSync2(path, existing + line);
|
|
1190
|
-
} else {
|
|
1191
|
-
writeFileSync2(path, line);
|
|
1192
|
-
}
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
// src/campaign/presets/run-skill-opt.ts
|
|
1196
|
-
async function runSkillOpt(opts) {
|
|
1197
|
-
if (opts.trainScenarios.length === 0) throw new Error("runSkillOpt: trainScenarios is empty");
|
|
1198
|
-
if (opts.holdoutScenarios.length === 0) throw new Error("runSkillOpt: holdoutScenarios is empty");
|
|
1199
|
-
if (!opts.judges || opts.judges.length === 0) {
|
|
1200
|
-
throw new Error(
|
|
1201
|
-
"runSkillOpt: at least one judge is required \u2014 scoring (and therefore acceptance) is meaningless without one, and would report a silent zero lift."
|
|
1202
|
-
);
|
|
1203
|
-
}
|
|
1204
|
-
const holdoutIds = new Set(opts.holdoutScenarios.map((s) => s.id));
|
|
1205
|
-
const overlap = opts.trainScenarios.filter((s) => holdoutIds.has(s.id)).map((s) => s.id);
|
|
1206
|
-
if (overlap.length > 0) {
|
|
1207
|
-
throw new Error(
|
|
1208
|
-
`runSkillOpt: trainScenarios and holdoutScenarios must be disjoint (overlap: [${overlap.join(
|
|
1209
|
-
", "
|
|
1210
|
-
)}]) \u2014 a shared scenario leaks the held-out acceptance axis into the proposal evidence.`
|
|
1211
|
-
);
|
|
1212
|
-
}
|
|
1213
|
-
const patchesPerEpoch = opts.patchesPerEpoch ?? 2;
|
|
1214
|
-
const initialBudget = opts.editBudget ?? 3;
|
|
1215
|
-
const minImprovement = opts.minImprovement ?? 0;
|
|
1216
|
-
if (minImprovement < 0) {
|
|
1217
|
-
throw new Error(
|
|
1218
|
-
"runSkillOpt: minImprovement must be >= 0 \u2014 a negative threshold would accept held-out regressions, breaking the monotonic-lift contract."
|
|
1219
|
-
);
|
|
1220
|
-
}
|
|
1221
|
-
const patience = opts.patience ?? opts.maxEpochs;
|
|
1222
|
-
const budgetAnneal = opts.budgetAnneal ?? true;
|
|
1223
|
-
const rejectedBufferSize = opts.rejectedBufferSize ?? 12;
|
|
1224
|
-
const slowMetaEvery = opts.slowMetaEvery ?? 2;
|
|
1225
|
-
let totalCostUsd = 0;
|
|
1226
|
-
const scoreHoldout = async (surface, tag) => {
|
|
1227
|
-
const campaign = await runScoringCampaign(opts, opts.holdoutScenarios, surface, tag);
|
|
1228
|
-
totalCostUsd += campaign.aggregates.totalCostUsd;
|
|
1229
|
-
return campaignMeanComposite(campaign);
|
|
1230
|
-
};
|
|
1231
|
-
const evidenceK = opts.evidenceK ?? 3;
|
|
1232
|
-
const trainEvidence = async (surface, tag) => {
|
|
1233
|
-
const campaign = await runScoringCampaign(opts, opts.trainScenarios, surface, tag);
|
|
1234
|
-
totalCostUsd += campaign.aggregates.totalCostUsd;
|
|
1235
|
-
return toEvidence(campaign, evidenceK);
|
|
1236
|
-
};
|
|
1237
|
-
let current = opts.baselineSurface;
|
|
1238
|
-
let currentEvidence = await trainEvidence(current, "baseline-train");
|
|
1239
|
-
const baselineHoldout = await scoreHoldout(current, "baseline-holdout");
|
|
1240
|
-
let currentHoldout = baselineHoldout;
|
|
1241
|
-
const buffer = [];
|
|
1242
|
-
const acceptedEdits = [];
|
|
1243
|
-
const rejectedAll = [];
|
|
1244
|
-
const history = [];
|
|
1245
|
-
let budget = initialBudget;
|
|
1246
|
-
let sinceAccept = 0;
|
|
1247
|
-
let metaNote;
|
|
1248
|
-
let epochsRun = 0;
|
|
1249
|
-
for (let epoch = 0; epoch < opts.maxEpochs; epoch++) {
|
|
1250
|
-
epochsRun++;
|
|
1251
|
-
const patches = await opts.driver.proposePatches({
|
|
1252
|
-
surface: current,
|
|
1253
|
-
evidence: currentEvidence,
|
|
1254
|
-
editBudget: budget,
|
|
1255
|
-
rejectedBuffer: buffer,
|
|
1256
|
-
metaNote,
|
|
1257
|
-
count: patchesPerEpoch,
|
|
1258
|
-
signal: opts.signal ?? new AbortController().signal
|
|
1259
|
-
});
|
|
1260
|
-
let accepted = null;
|
|
1261
|
-
const rejectedThisEpoch = [];
|
|
1262
|
-
for (let i = 0; i < patches.length; i++) {
|
|
1263
|
-
const patch = patches[i];
|
|
1264
|
-
const { surface: candidate, applied } = applySkillPatch(current, patch);
|
|
1265
|
-
if (applied === 0 || candidate === current) {
|
|
1266
|
-
rejectedThisEpoch.push({
|
|
1267
|
-
label: patch.label,
|
|
1268
|
-
rationale: patch.rationale,
|
|
1269
|
-
reason: "no-op (unanchored or zero-change)"
|
|
1270
|
-
});
|
|
1271
|
-
continue;
|
|
1272
|
-
}
|
|
1273
|
-
const candidateHoldout = await scoreHoldout(candidate, `epoch-${epoch}-cand-${i}-holdout`);
|
|
1274
|
-
if (candidateHoldout > currentHoldout + minImprovement) {
|
|
1275
|
-
accepted = {
|
|
1276
|
-
epoch,
|
|
1277
|
-
label: patch.label,
|
|
1278
|
-
rationale: patch.rationale,
|
|
1279
|
-
holdoutDelta: candidateHoldout - currentHoldout
|
|
1280
|
-
};
|
|
1281
|
-
current = candidate;
|
|
1282
|
-
currentHoldout = candidateHoldout;
|
|
1283
|
-
currentEvidence = await trainEvidence(current, `epoch-${epoch}-train`);
|
|
1284
|
-
break;
|
|
1285
|
-
}
|
|
1286
|
-
rejectedThisEpoch.push({
|
|
1287
|
-
label: patch.label,
|
|
1288
|
-
rationale: patch.rationale,
|
|
1289
|
-
reason: `held-out ${candidateHoldout.toFixed(3)} \u2264 current ${currentHoldout.toFixed(3)}`
|
|
1290
|
-
});
|
|
1291
|
-
}
|
|
1292
|
-
if (accepted) {
|
|
1293
|
-
acceptedEdits.push(accepted);
|
|
1294
|
-
sinceAccept = 0;
|
|
1295
|
-
} else {
|
|
1296
|
-
sinceAccept++;
|
|
1297
|
-
if (budgetAnneal && sinceAccept >= 2 && budget > 1) budget--;
|
|
1298
|
-
}
|
|
1299
|
-
for (const r of rejectedThisEpoch) {
|
|
1300
|
-
buffer.push(r);
|
|
1301
|
-
rejectedAll.push(r);
|
|
1302
|
-
}
|
|
1303
|
-
while (buffer.length > rejectedBufferSize) buffer.shift();
|
|
1304
|
-
if (slowMetaEvery > 0 && (epoch + 1) % slowMetaEvery === 0) {
|
|
1305
|
-
metaNote = buildMetaNote(acceptedEdits, buffer);
|
|
1306
|
-
}
|
|
1307
|
-
history.push({
|
|
1308
|
-
epoch,
|
|
1309
|
-
editBudget: budget,
|
|
1310
|
-
proposed: patches.length,
|
|
1311
|
-
accepted,
|
|
1312
|
-
rejected: rejectedThisEpoch,
|
|
1313
|
-
holdoutComposite: currentHoldout
|
|
1314
|
-
});
|
|
1315
|
-
if (sinceAccept >= patience) break;
|
|
1316
|
-
}
|
|
1317
|
-
return {
|
|
1318
|
-
winnerSurface: current,
|
|
1319
|
-
baselineHoldoutComposite: baselineHoldout,
|
|
1320
|
-
winnerHoldoutComposite: currentHoldout,
|
|
1321
|
-
lift: currentHoldout - baselineHoldout,
|
|
1322
|
-
acceptedEdits,
|
|
1323
|
-
rejectedEdits: rejectedAll,
|
|
1324
|
-
epochsRun,
|
|
1325
|
-
history,
|
|
1326
|
-
totalCostUsd
|
|
1327
|
-
};
|
|
1328
|
-
}
|
|
1329
|
-
function runScoringCampaign(opts, scenarios, surface, tag) {
|
|
1330
|
-
return runCampaign({
|
|
1331
|
-
...opts,
|
|
1332
|
-
scenarios,
|
|
1333
|
-
dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
|
|
1334
|
-
runDir: `${opts.runDir}/${tag}`
|
|
1335
|
-
});
|
|
1336
|
-
}
|
|
1337
|
-
function toEvidence(campaign, k) {
|
|
1338
|
-
const { dimensions, scenarios } = campaignBreakdown(campaign);
|
|
1339
|
-
const weakScenarios = [...scenarios].sort((a, b) => a.composite - b.composite).slice(0, k);
|
|
1340
|
-
const weakDimensions = Object.entries(dimensions).sort((a, b) => a[1] - b[1]).slice(0, k).map(([dimension, score]) => ({ dimension, score }));
|
|
1341
|
-
return { weakScenarios, weakDimensions };
|
|
1342
|
-
}
|
|
1343
|
-
function buildMetaNote(accepted, rejected) {
|
|
1344
|
-
const parts = [];
|
|
1345
|
-
if (accepted.length > 0) {
|
|
1346
|
-
parts.push(
|
|
1347
|
-
`Edits that improved held-out so far: ${accepted.map((a) => `"${a.label}" (+${a.holdoutDelta.toFixed(3)})`).join("; ")}. Build on these.`
|
|
1348
|
-
);
|
|
1349
|
-
}
|
|
1350
|
-
if (rejected.length > 0) {
|
|
1351
|
-
const labels = [...new Set(rejected.map((r) => r.label))].slice(0, 5);
|
|
1352
|
-
parts.push(`Dead ends to avoid: ${labels.join(", ")}. Try a different anchor or rule.`);
|
|
1353
|
-
}
|
|
1354
|
-
parts.push("Keep edits small and anchored to existing lines.");
|
|
1355
|
-
return parts.join(" ");
|
|
1356
|
-
}
|
|
1357
|
-
|
|
1358
|
-
// src/campaign/presets/compare-drivers.ts
|
|
1359
|
-
async function compareDrivers(opts) {
|
|
1360
|
-
if (opts.drivers.length === 0) throw new Error("compareDrivers: no drivers to compare");
|
|
1361
|
-
const seed = opts.seed ?? 42;
|
|
1362
|
-
const resamples = opts.resamples ?? 2e3;
|
|
1363
|
-
const confidence = opts.confidence ?? 0.95;
|
|
1364
|
-
const scoreOnHoldout = async (surface, tag) => {
|
|
1365
|
-
const campaign = await runCampaign({
|
|
1366
|
-
...opts,
|
|
1367
|
-
scenarios: opts.holdoutScenarios,
|
|
1368
|
-
dispatch: (scenario, ctx) => opts.dispatchWithSurface(surface, scenario, ctx),
|
|
1369
|
-
runDir: `${opts.runDir}/${tag}`
|
|
1370
|
-
});
|
|
1371
|
-
const byScenario = {};
|
|
1372
|
-
for (const { scenarioId, composite } of campaignBreakdown(campaign).scenarios) {
|
|
1373
|
-
byScenario[scenarioId] = composite;
|
|
1374
|
-
}
|
|
1375
|
-
return byScenario;
|
|
1376
|
-
};
|
|
1377
|
-
const scenarioIds = [...new Set(opts.holdoutScenarios.map((s) => s.id))].sort();
|
|
1378
|
-
if (scenarioIds.length === 0) throw new Error("compareDrivers: holdoutScenarios is empty");
|
|
1379
|
-
const align = (byScenario, label) => {
|
|
1380
|
-
const missing = scenarioIds.filter((id) => !(id in byScenario));
|
|
1381
|
-
if (missing.length > 0) {
|
|
1382
|
-
throw new Error(
|
|
1383
|
-
`compareDrivers: ${label} produced no held-out score for scenario(s) [${missing.join(
|
|
1384
|
-
", "
|
|
1385
|
-
)}] \u2014 a cell errored or its judges returned nothing. Refusing to fabricate a 0 (it would corrupt the lift comparison). Fix the dispatch/judge or drop the scenario.`
|
|
1386
|
-
);
|
|
1387
|
-
}
|
|
1388
|
-
return scenarioIds.map((id) => byScenario[id]);
|
|
1389
|
-
};
|
|
1390
|
-
const baselineArr = align(
|
|
1391
|
-
await scoreOnHoldout(opts.baselineSurface, "compare-baseline"),
|
|
1392
|
-
"baseline"
|
|
1393
|
-
);
|
|
1394
|
-
const winners = [];
|
|
1395
|
-
for (const d of opts.drivers) {
|
|
1396
|
-
const out = await d.optimize();
|
|
1397
|
-
const byScenario = await scoreOnHoldout(out.winnerSurface, `compare-${slug(d.name)}`);
|
|
1398
|
-
winners.push({
|
|
1399
|
-
name: d.name,
|
|
1400
|
-
winnerSurface: out.winnerSurface,
|
|
1401
|
-
costUsd: out.costUsd,
|
|
1402
|
-
durationMs: out.durationMs,
|
|
1403
|
-
arr: align(byScenario, `driver "${d.name}"`)
|
|
1404
|
-
});
|
|
1405
|
-
}
|
|
1406
|
-
const scores = winners.map((w) => {
|
|
1407
|
-
const boot = pairedBootstrap(baselineArr, w.arr, {
|
|
1408
|
-
seed,
|
|
1409
|
-
resamples,
|
|
1410
|
-
confidence,
|
|
1411
|
-
statistic: "mean"
|
|
1412
|
-
});
|
|
1413
|
-
const score = {
|
|
1414
|
-
name: w.name,
|
|
1415
|
-
baselineComposite: mean(baselineArr),
|
|
1416
|
-
winnerComposite: mean(w.arr),
|
|
1417
|
-
lift: boot.mean,
|
|
1418
|
-
liftCi: { low: boot.low, high: boot.high },
|
|
1419
|
-
costUsd: w.costUsd,
|
|
1420
|
-
winnerSurface: w.winnerSurface,
|
|
1421
|
-
rank: 0
|
|
1559
|
+
const scores = winners.map((w) => {
|
|
1560
|
+
const boot = pairedBootstrap(baselineArr, w.arr, {
|
|
1561
|
+
seed,
|
|
1562
|
+
resamples,
|
|
1563
|
+
confidence,
|
|
1564
|
+
statistic: "mean"
|
|
1565
|
+
});
|
|
1566
|
+
const score = {
|
|
1567
|
+
name: w.name,
|
|
1568
|
+
baselineComposite: mean(baselineArr),
|
|
1569
|
+
winnerComposite: mean(w.arr),
|
|
1570
|
+
lift: boot.mean,
|
|
1571
|
+
liftCi: { low: boot.low, high: boot.high },
|
|
1572
|
+
costUsd: w.costUsd,
|
|
1573
|
+
winnerSurface: w.winnerSurface,
|
|
1574
|
+
rank: 0
|
|
1422
1575
|
};
|
|
1423
1576
|
if (w.durationMs !== void 0) score.durationMs = w.durationMs;
|
|
1424
1577
|
return score;
|
|
@@ -1467,7 +1620,7 @@ function gepaEntry(config, combineParents, name) {
|
|
|
1467
1620
|
name,
|
|
1468
1621
|
async optimize() {
|
|
1469
1622
|
const started = Date.now();
|
|
1470
|
-
const
|
|
1623
|
+
const proposer = gepaProposer({
|
|
1471
1624
|
llm: config.llm,
|
|
1472
1625
|
model: config.model,
|
|
1473
1626
|
target: config.target,
|
|
@@ -1480,7 +1633,7 @@ function gepaEntry(config, combineParents, name) {
|
|
|
1480
1633
|
baselineSurface: config.baselineSurface,
|
|
1481
1634
|
dispatchWithSurface: config.dispatchWithSurface,
|
|
1482
1635
|
judges: config.judges,
|
|
1483
|
-
|
|
1636
|
+
proposer,
|
|
1484
1637
|
populationSize: config.populationSize ?? 2,
|
|
1485
1638
|
maxGenerations: config.maxGenerations ?? 3,
|
|
1486
1639
|
gate: defaultProductionGate({
|
|
@@ -1490,9 +1643,9 @@ function gepaEntry(config, combineParents, name) {
|
|
|
1490
1643
|
autoOnPromote: "none",
|
|
1491
1644
|
runDir: `${config.runDir}/${slug(name)}-loop`,
|
|
1492
1645
|
...config.seed !== void 0 ? { seed: config.seed } : {},
|
|
1493
|
-
//
|
|
1494
|
-
//
|
|
1495
|
-
//
|
|
1646
|
+
// Flow findings to the proposer's propose(). These reach runOptimization
|
|
1647
|
+
// unchanged (runImprovementLoop extends RunOptimizationOptions and
|
|
1648
|
+
// forwards {...opts}); ctx.findings/report/analyzeGeneration are consumed there.
|
|
1496
1649
|
...config.findings !== void 0 ? { findings: config.findings } : {},
|
|
1497
1650
|
...config.analyzeGeneration ? { analyzeGeneration: config.analyzeGeneration } : {},
|
|
1498
1651
|
...config.report !== void 0 ? { report: config.report } : {}
|
|
@@ -1510,12 +1663,16 @@ function skillOptEntry(config, name = "skill-opt") {
|
|
|
1510
1663
|
name,
|
|
1511
1664
|
async optimize() {
|
|
1512
1665
|
const started = Date.now();
|
|
1513
|
-
const
|
|
1666
|
+
const proposer = skillOptProposer({
|
|
1667
|
+
llm: config.llm,
|
|
1668
|
+
model: config.model,
|
|
1669
|
+
target: config.target
|
|
1670
|
+
});
|
|
1514
1671
|
const result = await runSkillOpt({
|
|
1515
1672
|
baselineSurface: config.baselineSurface,
|
|
1516
1673
|
dispatchWithSurface: config.dispatchWithSurface,
|
|
1517
1674
|
judges: config.judges,
|
|
1518
|
-
|
|
1675
|
+
proposer,
|
|
1519
1676
|
trainScenarios: config.trainScenarios,
|
|
1520
1677
|
holdoutScenarios: config.holdoutScenarios,
|
|
1521
1678
|
maxEpochs: config.maxEpochs ?? 6,
|
|
@@ -1530,6 +1687,54 @@ function skillOptEntry(config, name = "skill-opt") {
|
|
|
1530
1687
|
}
|
|
1531
1688
|
};
|
|
1532
1689
|
}
|
|
1690
|
+
function fapoEscalationEntry(config, name = "fapo-escalation") {
|
|
1691
|
+
return {
|
|
1692
|
+
name,
|
|
1693
|
+
async optimize() {
|
|
1694
|
+
const started = Date.now();
|
|
1695
|
+
const promptProposer = config.promptProposer ?? gepaProposer({
|
|
1696
|
+
llm: config.llm,
|
|
1697
|
+
model: config.model,
|
|
1698
|
+
target: config.target,
|
|
1699
|
+
combineParents: true,
|
|
1700
|
+
...config.mutationPrimitives ? { mutationPrimitives: config.mutationPrimitives } : {}
|
|
1701
|
+
});
|
|
1702
|
+
const parameterProposer = config.parameterProposer ?? (config.parameterCandidates ? parameterSweepProposer({ candidates: config.parameterCandidates }) : void 0);
|
|
1703
|
+
const structuralProposer = config.structuralProposer;
|
|
1704
|
+
const proposer = fapoProposer({
|
|
1705
|
+
...config.fapo ?? {},
|
|
1706
|
+
promptProposer,
|
|
1707
|
+
...parameterProposer ? { parameterProposer } : {},
|
|
1708
|
+
...structuralProposer ? { structuralProposer } : {}
|
|
1709
|
+
});
|
|
1710
|
+
const result = await runImprovementLoop({
|
|
1711
|
+
scenarios: config.trainScenarios,
|
|
1712
|
+
holdoutScenarios: config.holdoutScenarios,
|
|
1713
|
+
baselineSurface: config.baselineSurface,
|
|
1714
|
+
dispatchWithSurface: config.dispatchWithSurface,
|
|
1715
|
+
judges: config.judges,
|
|
1716
|
+
proposer,
|
|
1717
|
+
populationSize: config.populationSize ?? 2,
|
|
1718
|
+
maxGenerations: config.maxGenerations ?? 3,
|
|
1719
|
+
gate: defaultProductionGate({
|
|
1720
|
+
holdoutScenarios: config.holdoutScenarios,
|
|
1721
|
+
deltaThreshold: 0
|
|
1722
|
+
}),
|
|
1723
|
+
autoOnPromote: "none",
|
|
1724
|
+
runDir: `${config.runDir}/${slug(name)}-loop`,
|
|
1725
|
+
...config.seed !== void 0 ? { seed: config.seed } : {},
|
|
1726
|
+
...config.findings !== void 0 ? { findings: config.findings } : {},
|
|
1727
|
+
...config.analyzeGeneration ? { analyzeGeneration: config.analyzeGeneration } : {},
|
|
1728
|
+
...config.report !== void 0 ? { report: config.report } : {}
|
|
1729
|
+
});
|
|
1730
|
+
const costUsd = result.baselineCampaign.aggregates.totalCostUsd + result.generations.reduce(
|
|
1731
|
+
(sum, g) => sum + g.surfaces.reduce((s, sf) => s + sf.campaign.aggregates.totalCostUsd, 0),
|
|
1732
|
+
0
|
|
1733
|
+
);
|
|
1734
|
+
return { winnerSurface: result.winnerSurface, costUsd, durationMs: Date.now() - started };
|
|
1735
|
+
}
|
|
1736
|
+
};
|
|
1737
|
+
}
|
|
1533
1738
|
|
|
1534
1739
|
// src/campaign/presets/playback.ts
|
|
1535
1740
|
function makePlaybackDispatch(driver) {
|
|
@@ -1626,236 +1831,574 @@ function renderScoreboardMarkdown(rows, opts = {}) {
|
|
|
1626
1831
|
"| Requirement | Status | Evidence |",
|
|
1627
1832
|
"| --- | --- | --- |"
|
|
1628
1833
|
);
|
|
1629
|
-
for (const r of storyRows) {
|
|
1630
|
-
out.push(
|
|
1631
|
-
`| ${escapeCell(r.reqTitle)} | ${r.status === "PASS" ? "\u2705 PASS" : "\u274C FAIL"} | ${ev(r.evidence)} |`
|
|
1632
|
-
);
|
|
1633
|
-
}
|
|
1634
|
-
out.push("");
|
|
1834
|
+
for (const r of storyRows) {
|
|
1835
|
+
out.push(
|
|
1836
|
+
`| ${escapeCell(r.reqTitle)} | ${r.status === "PASS" ? "\u2705 PASS" : "\u274C FAIL"} | ${ev(r.evidence)} |`
|
|
1837
|
+
);
|
|
1838
|
+
}
|
|
1839
|
+
out.push("");
|
|
1840
|
+
}
|
|
1841
|
+
return out.join("\n");
|
|
1842
|
+
}
|
|
1843
|
+
|
|
1844
|
+
// src/campaign/presets/run-profile-matrix.ts
|
|
1845
|
+
import { createHash as createHash3 } from "crypto";
|
|
1846
|
+
import { join as join2 } from "path";
|
|
1847
|
+
var ProfileMatrixError = class extends AgentEvalError {
|
|
1848
|
+
constructor(message) {
|
|
1849
|
+
super("profile_matrix", message);
|
|
1850
|
+
}
|
|
1851
|
+
};
|
|
1852
|
+
function sanitize(id) {
|
|
1853
|
+
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
1854
|
+
}
|
|
1855
|
+
function sha(input) {
|
|
1856
|
+
return createHash3("sha256").update(JSON.stringify(input)).digest("hex");
|
|
1857
|
+
}
|
|
1858
|
+
function mean2(xs) {
|
|
1859
|
+
return xs.length === 0 ? 0 : xs.reduce((a, b) => a + b, 0) / xs.length;
|
|
1860
|
+
}
|
|
1861
|
+
function cellComposite(cell) {
|
|
1862
|
+
const composites = Object.values(cell.judgeScores).map((s) => s.composite);
|
|
1863
|
+
return composites.length === 0 ? 0 : mean2(composites);
|
|
1864
|
+
}
|
|
1865
|
+
function buildRunRecord(args) {
|
|
1866
|
+
const { cell, profile, profileHash, configHash, experimentId, splitTag, commitSha, matrixId } = args;
|
|
1867
|
+
const profileId = agentProfileId(profile);
|
|
1868
|
+
const model = agentProfileModelId(profile);
|
|
1869
|
+
const composite = cellComposite(cell);
|
|
1870
|
+
const raw = { composite };
|
|
1871
|
+
const perJudge = {};
|
|
1872
|
+
const dimAccum = {};
|
|
1873
|
+
const notes = [];
|
|
1874
|
+
for (const [judgeName, js] of Object.entries(cell.judgeScores)) {
|
|
1875
|
+
perJudge[judgeName] = { ...js.dimensions };
|
|
1876
|
+
for (const [dim, value] of Object.entries(js.dimensions)) {
|
|
1877
|
+
raw[`${judgeName}.${dim}`] = value;
|
|
1878
|
+
dimAccum[dim] ??= [];
|
|
1879
|
+
dimAccum[dim].push(value);
|
|
1880
|
+
}
|
|
1881
|
+
if (js.notes) notes.push(`${judgeName}: ${js.notes}`);
|
|
1882
|
+
}
|
|
1883
|
+
const perDimMean = {};
|
|
1884
|
+
for (const [dim, values] of Object.entries(dimAccum)) perDimMean[dim] = mean2(values);
|
|
1885
|
+
let costUsd = cell.costUsd;
|
|
1886
|
+
let costEstimated = false;
|
|
1887
|
+
if (costUsd === 0 && cell.tokenUsage.output > 0 && isModelPriced(model)) {
|
|
1888
|
+
costUsd = estimateCost(cell.tokenUsage.input, cell.tokenUsage.output, model);
|
|
1889
|
+
costEstimated = costUsd > 0;
|
|
1890
|
+
}
|
|
1891
|
+
raw.cost_usd = costUsd;
|
|
1892
|
+
raw.cost_estimated = costEstimated ? 1 : 0;
|
|
1893
|
+
raw.tokens_input = cell.tokenUsage.input;
|
|
1894
|
+
raw.tokens_output = cell.tokenUsage.output;
|
|
1895
|
+
if (typeof cell.tokenUsage.cached === "number") raw.tokens_cached = cell.tokenUsage.cached;
|
|
1896
|
+
raw.latency_ms = cell.durationMs;
|
|
1897
|
+
if (costUsd > 0) {
|
|
1898
|
+
raw.tokens_per_dollar = (cell.tokenUsage.input + cell.tokenUsage.output) / costUsd;
|
|
1899
|
+
}
|
|
1900
|
+
if (composite > 0.01) raw.cost_per_quality = costUsd / composite;
|
|
1901
|
+
const outcome = splitTag === "holdout" ? { holdoutScore: composite, raw } : { searchScore: composite, raw };
|
|
1902
|
+
if (Object.keys(perJudge).length > 0) {
|
|
1903
|
+
outcome.judgeScores = {
|
|
1904
|
+
perJudge,
|
|
1905
|
+
perDimMean,
|
|
1906
|
+
composite,
|
|
1907
|
+
...notes.length > 0 ? { notes: notes.join(" | ") } : {}
|
|
1908
|
+
};
|
|
1909
|
+
}
|
|
1910
|
+
const record = {
|
|
1911
|
+
runId: `${matrixId}:${profileId}:${cell.cellId}`,
|
|
1912
|
+
experimentId,
|
|
1913
|
+
candidateId: profileId,
|
|
1914
|
+
seed: cell.seed,
|
|
1915
|
+
model,
|
|
1916
|
+
promptHash: profileHash,
|
|
1917
|
+
configHash,
|
|
1918
|
+
commitSha,
|
|
1919
|
+
wallMs: cell.durationMs,
|
|
1920
|
+
costUsd,
|
|
1921
|
+
tokenUsage: cell.tokenUsage,
|
|
1922
|
+
outcome,
|
|
1923
|
+
splitTag,
|
|
1924
|
+
scenarioId: cell.scenarioId,
|
|
1925
|
+
...cell.error ? { failureMode: cell.error } : {}
|
|
1926
|
+
};
|
|
1927
|
+
if (args.corpusText && args.scenario) {
|
|
1928
|
+
try {
|
|
1929
|
+
const text = args.corpusText(cell.artifact, args.scenario);
|
|
1930
|
+
if (text && typeof text.prompt === "string" && typeof text.completion === "string") {
|
|
1931
|
+
record.prompt = text.prompt;
|
|
1932
|
+
record.completion = text.completion;
|
|
1933
|
+
}
|
|
1934
|
+
} catch {
|
|
1935
|
+
}
|
|
1936
|
+
}
|
|
1937
|
+
return record;
|
|
1938
|
+
}
|
|
1939
|
+
async function runProfileMatrix(opts) {
|
|
1940
|
+
if (opts.profiles.length === 0) throw new ProfileMatrixError("profiles must not be empty");
|
|
1941
|
+
if (opts.scenarios.length === 0) throw new ProfileMatrixError("scenarios must not be empty");
|
|
1942
|
+
const splitTag = opts.splitTag ?? "search";
|
|
1943
|
+
const seed = opts.seed ?? 42;
|
|
1944
|
+
const validate = opts.validate ?? true;
|
|
1945
|
+
const integrityMode = opts.integrity ?? "assert";
|
|
1946
|
+
const profileIds = opts.profiles.map(agentProfileId);
|
|
1947
|
+
const experimentId = opts.experimentId ?? `pm_${sha({ profileIds, scenarios: opts.scenarios.map((s) => s.id) }).slice(0, 16)}`;
|
|
1948
|
+
const matrixId = `mtx_${sha({ experimentId, profileIds, seed, splitTag }).slice(0, 16)}`;
|
|
1949
|
+
const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s]));
|
|
1950
|
+
for (const profile of opts.profiles) {
|
|
1951
|
+
const profileHash = agentProfileHash(profile);
|
|
1952
|
+
const profileId = agentProfileId(profile);
|
|
1953
|
+
const model = agentProfileModelId(profile);
|
|
1954
|
+
try {
|
|
1955
|
+
validateRunRecord({
|
|
1956
|
+
runId: `${matrixId}:${profileId}:probe`,
|
|
1957
|
+
experimentId,
|
|
1958
|
+
candidateId: profileId,
|
|
1959
|
+
seed,
|
|
1960
|
+
model,
|
|
1961
|
+
promptHash: profileHash,
|
|
1962
|
+
configHash: profileHash,
|
|
1963
|
+
commitSha: opts.commitSha,
|
|
1964
|
+
wallMs: 0,
|
|
1965
|
+
costUsd: 0,
|
|
1966
|
+
tokenUsage: { input: 0, output: 0 },
|
|
1967
|
+
outcome: splitTag === "holdout" ? { holdoutScore: 0, raw: {} } : { searchScore: 0, raw: {} },
|
|
1968
|
+
splitTag
|
|
1969
|
+
});
|
|
1970
|
+
} catch (err) {
|
|
1971
|
+
throw new ProfileMatrixError(
|
|
1972
|
+
`profile '${profileId}' is not recordable: ${err instanceof Error ? err.message : String(err)}`
|
|
1973
|
+
);
|
|
1974
|
+
}
|
|
1975
|
+
}
|
|
1976
|
+
const records = [];
|
|
1977
|
+
const campaigns = {};
|
|
1978
|
+
const byProfile = {};
|
|
1979
|
+
for (const profile of opts.profiles) {
|
|
1980
|
+
const profileHash = agentProfileHash(profile);
|
|
1981
|
+
const profileId = agentProfileId(profile);
|
|
1982
|
+
const model = agentProfileModelId(profile);
|
|
1983
|
+
const configHash = sha({
|
|
1984
|
+
profile: profileHash,
|
|
1985
|
+
judges: (opts.judges ?? []).map((j) => j.name),
|
|
1986
|
+
seed,
|
|
1987
|
+
splitTag
|
|
1988
|
+
});
|
|
1989
|
+
const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
|
|
1990
|
+
Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profileId)}` });
|
|
1991
|
+
const campaign = await runCampaign({
|
|
1992
|
+
scenarios: opts.scenarios,
|
|
1993
|
+
dispatch,
|
|
1994
|
+
judges: opts.judges,
|
|
1995
|
+
seed,
|
|
1996
|
+
reps: opts.reps,
|
|
1997
|
+
maxConcurrency: opts.maxConcurrency,
|
|
1998
|
+
costCeiling: opts.costCeiling,
|
|
1999
|
+
labeledStore: opts.labeledStore,
|
|
2000
|
+
captureSource: opts.captureSource,
|
|
2001
|
+
storage: opts.storage,
|
|
2002
|
+
now: opts.now,
|
|
2003
|
+
runDir: join2(opts.runDir, sanitize(profileId))
|
|
2004
|
+
});
|
|
2005
|
+
const profileRecords = [];
|
|
2006
|
+
for (const cell of campaign.cells) {
|
|
2007
|
+
const record = buildRunRecord({
|
|
2008
|
+
cell,
|
|
2009
|
+
profile,
|
|
2010
|
+
profileHash,
|
|
2011
|
+
configHash,
|
|
2012
|
+
experimentId,
|
|
2013
|
+
splitTag,
|
|
2014
|
+
commitSha: opts.commitSha,
|
|
2015
|
+
matrixId,
|
|
2016
|
+
scenario: scenarioById.get(cell.scenarioId),
|
|
2017
|
+
corpusText: opts.corpusText
|
|
2018
|
+
});
|
|
2019
|
+
if (validate) validateRunRecord(record);
|
|
2020
|
+
profileRecords.push(record);
|
|
2021
|
+
records.push(record);
|
|
2022
|
+
}
|
|
2023
|
+
const pricedTotalCostUsd = profileRecords.reduce((a, r) => a + r.costUsd, 0);
|
|
2024
|
+
campaigns[profileId] = {
|
|
2025
|
+
...campaign,
|
|
2026
|
+
aggregates: { ...campaign.aggregates, totalCostUsd: pricedTotalCostUsd }
|
|
2027
|
+
};
|
|
2028
|
+
byProfile[profileId] = {
|
|
2029
|
+
profileId,
|
|
2030
|
+
profileHash,
|
|
2031
|
+
model,
|
|
2032
|
+
records: profileRecords.length,
|
|
2033
|
+
meanComposite: mean2(profileRecords.map(compositeOf)),
|
|
2034
|
+
totalCostUsd: pricedTotalCostUsd,
|
|
2035
|
+
integrity: summarizeBackendIntegrity(profileRecords)
|
|
2036
|
+
};
|
|
2037
|
+
}
|
|
2038
|
+
const integrity = summarizeBackendIntegrity(records);
|
|
2039
|
+
if (integrityMode === "assert") {
|
|
2040
|
+
assertRealBackend(records, { allowMixed: opts.allowMixed ?? true });
|
|
2041
|
+
} else if (integrityMode === "warn" && integrity.verdict !== "real") {
|
|
2042
|
+
console.warn(
|
|
2043
|
+
`[runProfileMatrix] backend integrity: ${integrity.verdict} \u2014 ${integrity.diagnosis}`
|
|
2044
|
+
);
|
|
1635
2045
|
}
|
|
1636
|
-
|
|
2046
|
+
const byScenario = rollup(records, (r) => r.scenarioId);
|
|
2047
|
+
const byPersona = opts.personaOf ? rollupByPersona(records, opts.scenarios, opts.personaOf) : void 0;
|
|
2048
|
+
return { matrixId, experimentId, records, byProfile, byScenario, byPersona, integrity, campaigns };
|
|
2049
|
+
}
|
|
2050
|
+
function compositeOf(r) {
|
|
2051
|
+
return r.outcome.holdoutScore ?? r.outcome.searchScore ?? 0;
|
|
2052
|
+
}
|
|
2053
|
+
function rollup(records, keyOf) {
|
|
2054
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2055
|
+
for (const r of records) {
|
|
2056
|
+
const key = keyOf(r);
|
|
2057
|
+
if (key === void 0) continue;
|
|
2058
|
+
const arr = groups.get(key) ?? [];
|
|
2059
|
+
arr.push(compositeOf(r));
|
|
2060
|
+
groups.set(key, arr);
|
|
2061
|
+
}
|
|
2062
|
+
const out = {};
|
|
2063
|
+
for (const [key, xs] of groups) out[key] = { meanComposite: mean2(xs), n: xs.length };
|
|
2064
|
+
return out;
|
|
2065
|
+
}
|
|
2066
|
+
function rollupByPersona(records, scenarios, personaOf) {
|
|
2067
|
+
const personaByScenarioId = /* @__PURE__ */ new Map();
|
|
2068
|
+
for (const s of scenarios) personaByScenarioId.set(s.id, personaOf(s));
|
|
2069
|
+
return rollup(records, (r) => r.scenarioId ? personaByScenarioId.get(r.scenarioId) : void 0);
|
|
1637
2070
|
}
|
|
1638
2071
|
|
|
1639
|
-
// src/campaign/
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
2072
|
+
// src/campaign/proposers/_findings-text.ts
|
|
2073
|
+
function findingToLesson(f) {
|
|
2074
|
+
if (typeof f === "string") return f.trim() || null;
|
|
2075
|
+
if (f && typeof f === "object") {
|
|
2076
|
+
const o = f;
|
|
2077
|
+
const cand = o.recommended_action ?? o.claim ?? o.lesson ?? o.text ?? o.message;
|
|
2078
|
+
if (typeof cand === "string" && cand.trim()) return cand.trim();
|
|
1645
2079
|
}
|
|
1646
|
-
|
|
1647
|
-
function sanitize(id) {
|
|
1648
|
-
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
2080
|
+
return null;
|
|
1649
2081
|
}
|
|
1650
|
-
function
|
|
1651
|
-
return
|
|
2082
|
+
function normKey(s) {
|
|
2083
|
+
return s.toLowerCase().replace(/\s+/g, " ").replace(/[.;:!?\s]+$/, "").trim();
|
|
1652
2084
|
}
|
|
1653
|
-
function
|
|
1654
|
-
|
|
2085
|
+
function surfaceToText2(surface) {
|
|
2086
|
+
if (typeof surface === "string") return surface;
|
|
2087
|
+
throw new Error(
|
|
2088
|
+
`curator proposer: surface must be a string prompt, got a ${surface.kind}-tier surface (${surface.worktreeRef}) \u2014 curation is prompt-tier`
|
|
2089
|
+
);
|
|
1655
2090
|
}
|
|
1656
|
-
function
|
|
1657
|
-
const
|
|
1658
|
-
|
|
2091
|
+
function extractBlockBody(text, startMarker, endMarker) {
|
|
2092
|
+
const start = text.indexOf(startMarker);
|
|
2093
|
+
const end = text.indexOf(endMarker);
|
|
2094
|
+
if (start === -1 || end === -1 || end < start) return "";
|
|
2095
|
+
return text.slice(start + startMarker.length, end);
|
|
1659
2096
|
}
|
|
1660
|
-
function
|
|
1661
|
-
const
|
|
1662
|
-
const
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
costUsd = estimateCost(cell.tokenUsage.input, cell.tokenUsage.output, profile.model);
|
|
1682
|
-
costEstimated = costUsd > 0;
|
|
1683
|
-
}
|
|
1684
|
-
raw.cost_usd = costUsd;
|
|
1685
|
-
raw.cost_estimated = costEstimated ? 1 : 0;
|
|
1686
|
-
raw.tokens_input = cell.tokenUsage.input;
|
|
1687
|
-
raw.tokens_output = cell.tokenUsage.output;
|
|
1688
|
-
if (typeof cell.tokenUsage.cached === "number") raw.tokens_cached = cell.tokenUsage.cached;
|
|
1689
|
-
raw.latency_ms = cell.durationMs;
|
|
1690
|
-
if (costUsd > 0) {
|
|
1691
|
-
raw.tokens_per_dollar = (cell.tokenUsage.input + cell.tokenUsage.output) / costUsd;
|
|
1692
|
-
}
|
|
1693
|
-
if (composite > 0.01) raw.cost_per_quality = costUsd / composite;
|
|
1694
|
-
const outcome = splitTag === "holdout" ? { holdoutScore: composite, raw } : { searchScore: composite, raw };
|
|
1695
|
-
if (Object.keys(perJudge).length > 0) {
|
|
1696
|
-
outcome.judgeScores = {
|
|
1697
|
-
perJudge,
|
|
1698
|
-
perDimMean,
|
|
1699
|
-
composite,
|
|
1700
|
-
...notes.length > 0 ? { notes: notes.join(" | ") } : {}
|
|
1701
|
-
};
|
|
2097
|
+
function stripBlock(text, startMarker, endMarker) {
|
|
2098
|
+
const start = text.indexOf(startMarker);
|
|
2099
|
+
const end = text.indexOf(endMarker);
|
|
2100
|
+
if (start === -1 || end === -1 || end < start) return text.trimEnd();
|
|
2101
|
+
return (text.slice(0, start) + text.slice(end + endMarker.length)).trimEnd();
|
|
2102
|
+
}
|
|
2103
|
+
|
|
2104
|
+
// src/campaign/proposers/ace.ts
|
|
2105
|
+
var BLOCK_START = "<!-- BEGIN ace-playbook (auto-managed by aceProposer) -->";
|
|
2106
|
+
var BLOCK_END = "<!-- END ace-playbook -->";
|
|
2107
|
+
var DEFAULT_HEADING = "## Playbook (accumulated lessons \u2014 append-only)";
|
|
2108
|
+
function parsePlaybook(surface) {
|
|
2109
|
+
const body = extractBlockBody(surface, BLOCK_START, BLOCK_END);
|
|
2110
|
+
const out = [];
|
|
2111
|
+
for (const raw of body.split("\n")) {
|
|
2112
|
+
const line = raw.trim();
|
|
2113
|
+
if (!line.startsWith("- ")) continue;
|
|
2114
|
+
const item = line.slice(2).trim();
|
|
2115
|
+
const tag = /^\[g(-?\d+)\]\s*(.*)$/.exec(item);
|
|
2116
|
+
if (tag) out.push({ gen: Number(tag[1]), text: tag[2].trim() });
|
|
2117
|
+
else out.push({ gen: -1, text: item });
|
|
1702
2118
|
}
|
|
1703
|
-
|
|
1704
|
-
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
1718
|
-
|
|
2119
|
+
return out;
|
|
2120
|
+
}
|
|
2121
|
+
function aceProposer(opts = {}) {
|
|
2122
|
+
const maxEntries = opts.maxEntries ?? 50;
|
|
2123
|
+
if (maxEntries < 1) throw new Error("aceProposer: maxEntries must be >= 1");
|
|
2124
|
+
const heading = opts.sectionHeading ?? DEFAULT_HEADING;
|
|
2125
|
+
return {
|
|
2126
|
+
kind: "ace",
|
|
2127
|
+
async propose(ctx) {
|
|
2128
|
+
const parent = surfaceToText2(ctx.currentSurface);
|
|
2129
|
+
const existing = parsePlaybook(parent);
|
|
2130
|
+
const seen = new Set(existing.map((b) => normKey(b.text)));
|
|
2131
|
+
const fresh = [];
|
|
2132
|
+
for (const f of ctx.findings ?? []) {
|
|
2133
|
+
const lesson = findingToLesson(f);
|
|
2134
|
+
if (!lesson) continue;
|
|
2135
|
+
const k = normKey(lesson);
|
|
2136
|
+
if (!k || seen.has(k)) continue;
|
|
2137
|
+
seen.add(k);
|
|
2138
|
+
fresh.push({ gen: ctx.generation, text: lesson });
|
|
2139
|
+
}
|
|
2140
|
+
if (fresh.length === 0) return [];
|
|
2141
|
+
const all = [...existing, ...fresh].slice(-maxEntries);
|
|
2142
|
+
const block = [
|
|
2143
|
+
BLOCK_START,
|
|
2144
|
+
heading,
|
|
2145
|
+
...all.map((b) => `- [g${b.gen}] ${b.text}`),
|
|
2146
|
+
BLOCK_END
|
|
2147
|
+
].join("\n");
|
|
2148
|
+
const base = stripBlock(parent, BLOCK_START, BLOCK_END);
|
|
2149
|
+
const surface = base ? `${base}
|
|
2150
|
+
|
|
2151
|
+
${block}` : block;
|
|
2152
|
+
return [
|
|
2153
|
+
{
|
|
2154
|
+
surface,
|
|
2155
|
+
label: `ace-playbook +${fresh.length}`,
|
|
2156
|
+
rationale: `appended ${fresh.length} new lesson(s) from gen ${ctx.generation} findings (playbook now ${all.length} bullet(s), append-only)`
|
|
2157
|
+
}
|
|
2158
|
+
];
|
|
2159
|
+
}
|
|
1719
2160
|
};
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
2161
|
+
}
|
|
2162
|
+
|
|
2163
|
+
// src/campaign/proposers/halo.ts
|
|
2164
|
+
import { execFile } from "child_process";
|
|
2165
|
+
import { promisify } from "util";
|
|
2166
|
+
|
|
2167
|
+
// src/campaign/proposers/analysis-edit.ts
|
|
2168
|
+
import { mkdtempSync, writeFileSync as writeFileSync2 } from "fs";
|
|
2169
|
+
import { tmpdir } from "os";
|
|
2170
|
+
import { join as join3 } from "path";
|
|
2171
|
+
var APPLY_SYSTEM = "You apply a trace-analysis report to an agent instruction prompt. Output ONLY the full revised prompt \u2014 no preamble, no commentary, no code fences. Make the minimal edits that address the report findings; preserve everything else verbatim.";
|
|
2172
|
+
function surfaceToPromptText(surface) {
|
|
2173
|
+
return typeof surface === "string" ? surface : JSON.stringify(surface);
|
|
2174
|
+
}
|
|
2175
|
+
function analysisEditProposer(opts) {
|
|
2176
|
+
return {
|
|
2177
|
+
kind: opts.kind,
|
|
2178
|
+
async propose(ctx) {
|
|
2179
|
+
const parent = surfaceToPromptText(ctx.currentSurface);
|
|
2180
|
+
const traces = await opts.resolveTraces(ctx) ?? "";
|
|
2181
|
+
if (!traces.trim()) throw new Error(opts.noTracesError);
|
|
2182
|
+
const dir = mkdtempSync(join3(tmpdir(), `${opts.kind}-proposer-`));
|
|
2183
|
+
const tracePath = join3(dir, "traces.jsonl");
|
|
2184
|
+
writeFileSync2(tracePath, traces.endsWith("\n") ? traces : `${traces}
|
|
2185
|
+
`);
|
|
2186
|
+
const report = await opts.analyze(tracePath, ctx);
|
|
2187
|
+
const applied = await callLlm(
|
|
2188
|
+
{
|
|
2189
|
+
model: opts.applyModel,
|
|
2190
|
+
messages: [
|
|
2191
|
+
{ role: "system", content: APPLY_SYSTEM },
|
|
2192
|
+
{
|
|
2193
|
+
role: "user",
|
|
2194
|
+
content: `CURRENT PROMPT:
|
|
2195
|
+
${parent}
|
|
2196
|
+
|
|
2197
|
+
TRACE-ANALYSIS REPORT:
|
|
2198
|
+
${report}
|
|
2199
|
+
|
|
2200
|
+
Return the full revised prompt.`
|
|
2201
|
+
}
|
|
2202
|
+
]
|
|
2203
|
+
},
|
|
2204
|
+
{ baseUrl: opts.baseUrl, apiKey: opts.apiKey, fetch: opts.fetchImpl }
|
|
2205
|
+
);
|
|
2206
|
+
const text = applied.content.trim();
|
|
2207
|
+
if (!text || text === parent) return [];
|
|
2208
|
+
return [{ surface: text, label: opts.label, rationale: opts.rationale(report) }];
|
|
2209
|
+
}
|
|
2210
|
+
};
|
|
2211
|
+
}
|
|
2212
|
+
|
|
2213
|
+
// src/campaign/proposers/halo.ts
|
|
2214
|
+
var execFileAsync = promisify(execFile);
|
|
2215
|
+
var DEFAULT_ANALYSIS_PROMPT = "Diagnose the failures in these agent execution traces \u2014 hallucinated tool calls, redundant tool arguments, refusal loops, and semantic-correctness errors \u2014 and suggest concrete, generalizable fixes to the agent instructions.";
|
|
2216
|
+
function haloProposer(opts) {
|
|
2217
|
+
const haloBin = opts.haloBin ?? "halo";
|
|
2218
|
+
const model = opts.model ?? "gpt-5.4-mini";
|
|
2219
|
+
return analysisEditProposer({
|
|
2220
|
+
kind: "halo",
|
|
2221
|
+
label: "halo",
|
|
2222
|
+
baseUrl: opts.baseUrl,
|
|
2223
|
+
apiKey: opts.apiKey,
|
|
2224
|
+
applyModel: opts.applyModel ?? model,
|
|
2225
|
+
fetchImpl: opts.fetchImpl,
|
|
2226
|
+
resolveTraces: opts.resolveTraces,
|
|
2227
|
+
noTracesError: "haloProposer: resolveTraces returned no OTLP traces \u2014 the halo engine has nothing to analyze",
|
|
2228
|
+
// HALO's real findings are preserved verbatim in the rationale (attribution).
|
|
2229
|
+
rationale: (findings) => `halo-engine findings:
|
|
2230
|
+
${findings.slice(0, 800)}`,
|
|
2231
|
+
analyze: async (tracePath, ctx) => {
|
|
2232
|
+
const args = [
|
|
2233
|
+
tracePath,
|
|
2234
|
+
"-p",
|
|
2235
|
+
opts.analysisPrompt ?? DEFAULT_ANALYSIS_PROMPT,
|
|
2236
|
+
"-m",
|
|
2237
|
+
model,
|
|
2238
|
+
...opts.maxDepth !== void 0 ? ["--max-depth", String(opts.maxDepth)] : [],
|
|
2239
|
+
...opts.maxTurns !== void 0 ? ["--max-turns", String(opts.maxTurns)] : []
|
|
2240
|
+
];
|
|
2241
|
+
let findings;
|
|
2242
|
+
try {
|
|
2243
|
+
const { stdout } = await execFileAsync(haloBin, args, {
|
|
2244
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
2245
|
+
signal: ctx.signal,
|
|
2246
|
+
env: {
|
|
2247
|
+
...process.env,
|
|
2248
|
+
...opts.apiKey ? { OPENAI_API_KEY: opts.apiKey } : {},
|
|
2249
|
+
OPENAI_BASE_URL: opts.baseUrl
|
|
2250
|
+
}
|
|
2251
|
+
});
|
|
2252
|
+
findings = stdout.trim();
|
|
2253
|
+
} catch (e) {
|
|
2254
|
+
throw new Error(
|
|
2255
|
+
`haloProposer: halo-engine ('${haloBin}') failed \u2014 ${e instanceof Error ? e.message : String(e)}`
|
|
2256
|
+
);
|
|
1726
2257
|
}
|
|
1727
|
-
|
|
2258
|
+
if (!findings) throw new Error("haloProposer: halo-engine produced no findings");
|
|
2259
|
+
return findings;
|
|
1728
2260
|
}
|
|
1729
|
-
}
|
|
1730
|
-
return record;
|
|
2261
|
+
});
|
|
1731
2262
|
}
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
const
|
|
1743
|
-
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
|
|
1755
|
-
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
outcome: splitTag === "holdout" ? { holdoutScore: 0, raw: {} } : { searchScore: 0, raw: {} },
|
|
1759
|
-
splitTag
|
|
1760
|
-
});
|
|
1761
|
-
} catch (err) {
|
|
1762
|
-
throw new ProfileMatrixError(
|
|
1763
|
-
`profile '${profile.id}' is not recordable: ${err instanceof Error ? err.message : String(err)}`
|
|
2263
|
+
|
|
2264
|
+
// src/campaign/proposers/memory.ts
|
|
2265
|
+
var BLOCK_START2 = "<!-- BEGIN curated-memory (auto-managed by memoryCurationProposer) -->";
|
|
2266
|
+
var BLOCK_END2 = "<!-- END curated-memory -->";
|
|
2267
|
+
var DEFAULT_HEADING2 = "## Learned from prior runs (curated memory)";
|
|
2268
|
+
var DISTILL_SYSTEM = 'You compress raw trace-analysis findings into crisp, generalizable agent guidance. Output ONLY a JSON array of strings, each one imperative lesson the agent should follow (e.g. "Always fetch a resource before mutating it"). No prose outside the JSON. Deduplicate; keep the most actionable and general; drop case-specific noise.';
|
|
2269
|
+
function extractExistingLessons(text) {
|
|
2270
|
+
return extractBlockBody(text, BLOCK_START2, BLOCK_END2).split("\n").map((l) => l.replace(/^\s*-\s+/, "").trim()).filter((l) => l && !l.startsWith("#"));
|
|
2271
|
+
}
|
|
2272
|
+
async function distillLessons(raw, distill) {
|
|
2273
|
+
const res = await callLlm(
|
|
2274
|
+
{
|
|
2275
|
+
model: distill.model,
|
|
2276
|
+
messages: [
|
|
2277
|
+
{ role: "system", content: DISTILL_SYSTEM },
|
|
2278
|
+
{ role: "user", content: `Findings:
|
|
2279
|
+
${raw.map((r) => `- ${r}`).join("\n")}` }
|
|
2280
|
+
]
|
|
2281
|
+
},
|
|
2282
|
+
{ baseUrl: distill.baseUrl, apiKey: distill.apiKey, fetch: distill.fetchImpl }
|
|
2283
|
+
);
|
|
2284
|
+
try {
|
|
2285
|
+
const parsed = JSON.parse(res.content.trim());
|
|
2286
|
+
if (Array.isArray(parsed)) {
|
|
2287
|
+
const lessons = parsed.filter(
|
|
2288
|
+
(x) => typeof x === "string" && x.trim().length > 0
|
|
1764
2289
|
);
|
|
2290
|
+
if (lessons.length > 0) return lessons;
|
|
1765
2291
|
}
|
|
2292
|
+
} catch {
|
|
1766
2293
|
}
|
|
1767
|
-
|
|
1768
|
-
const campaigns = {};
|
|
1769
|
-
const byProfile = {};
|
|
1770
|
-
for (const profile of opts.profiles) {
|
|
1771
|
-
const profileHash = agentProfileHash(profile);
|
|
1772
|
-
const configHash = sha({
|
|
1773
|
-
profile: profileHash,
|
|
1774
|
-
judges: (opts.judges ?? []).map((j) => j.name),
|
|
1775
|
-
seed,
|
|
1776
|
-
splitTag
|
|
1777
|
-
});
|
|
1778
|
-
const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
|
|
1779
|
-
Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profile.id)}` });
|
|
1780
|
-
const campaign = await runCampaign({
|
|
1781
|
-
scenarios: opts.scenarios,
|
|
1782
|
-
dispatch,
|
|
1783
|
-
judges: opts.judges,
|
|
1784
|
-
seed,
|
|
1785
|
-
reps: opts.reps,
|
|
1786
|
-
maxConcurrency: opts.maxConcurrency,
|
|
1787
|
-
costCeiling: opts.costCeiling,
|
|
1788
|
-
labeledStore: opts.labeledStore,
|
|
1789
|
-
captureSource: opts.captureSource,
|
|
1790
|
-
storage: opts.storage,
|
|
1791
|
-
now: opts.now,
|
|
1792
|
-
runDir: join3(opts.runDir, sanitize(profile.id))
|
|
1793
|
-
});
|
|
1794
|
-
const profileRecords = [];
|
|
1795
|
-
for (const cell of campaign.cells) {
|
|
1796
|
-
const record = buildRunRecord({
|
|
1797
|
-
cell,
|
|
1798
|
-
profile,
|
|
1799
|
-
profileHash,
|
|
1800
|
-
configHash,
|
|
1801
|
-
experimentId,
|
|
1802
|
-
splitTag,
|
|
1803
|
-
commitSha: opts.commitSha,
|
|
1804
|
-
matrixId,
|
|
1805
|
-
scenario: scenarioById.get(cell.scenarioId),
|
|
1806
|
-
corpusText: opts.corpusText
|
|
1807
|
-
});
|
|
1808
|
-
if (validate) validateRunRecord(record);
|
|
1809
|
-
profileRecords.push(record);
|
|
1810
|
-
records.push(record);
|
|
1811
|
-
}
|
|
1812
|
-
const pricedTotalCostUsd = profileRecords.reduce((a, r) => a + r.costUsd, 0);
|
|
1813
|
-
campaigns[profile.id] = {
|
|
1814
|
-
...campaign,
|
|
1815
|
-
aggregates: { ...campaign.aggregates, totalCostUsd: pricedTotalCostUsd }
|
|
1816
|
-
};
|
|
1817
|
-
byProfile[profile.id] = {
|
|
1818
|
-
profileId: profile.id,
|
|
1819
|
-
profileHash,
|
|
1820
|
-
model: profile.model,
|
|
1821
|
-
records: profileRecords.length,
|
|
1822
|
-
meanComposite: mean2(profileRecords.map(compositeOf)),
|
|
1823
|
-
totalCostUsd: pricedTotalCostUsd,
|
|
1824
|
-
integrity: summarizeBackendIntegrity(profileRecords)
|
|
1825
|
-
};
|
|
1826
|
-
}
|
|
1827
|
-
const integrity = summarizeBackendIntegrity(records);
|
|
1828
|
-
if (integrityMode === "assert") {
|
|
1829
|
-
assertRealBackend(records, { allowMixed: opts.allowMixed ?? true });
|
|
1830
|
-
} else if (integrityMode === "warn" && integrity.verdict !== "real") {
|
|
1831
|
-
console.warn(
|
|
1832
|
-
`[runProfileMatrix] backend integrity: ${integrity.verdict} \u2014 ${integrity.diagnosis}`
|
|
1833
|
-
);
|
|
1834
|
-
}
|
|
1835
|
-
const byScenario = rollup(records, (r) => r.scenarioId);
|
|
1836
|
-
const byPersona = opts.personaOf ? rollupByPersona(records, opts.scenarios, opts.personaOf) : void 0;
|
|
1837
|
-
return { matrixId, experimentId, records, byProfile, byScenario, byPersona, integrity, campaigns };
|
|
2294
|
+
return raw;
|
|
1838
2295
|
}
|
|
1839
|
-
function
|
|
1840
|
-
|
|
2296
|
+
function memoryCurationProposer(opts = {}) {
|
|
2297
|
+
const maxEntries = opts.maxEntries ?? 12;
|
|
2298
|
+
const heading = opts.sectionHeading ?? DEFAULT_HEADING2;
|
|
2299
|
+
return {
|
|
2300
|
+
kind: "memory-curation",
|
|
2301
|
+
async propose(ctx) {
|
|
2302
|
+
const parent = surfaceToText2(ctx.currentSurface);
|
|
2303
|
+
const fresh = [];
|
|
2304
|
+
for (const f of ctx.findings ?? []) {
|
|
2305
|
+
const l = findingToLesson(f);
|
|
2306
|
+
if (l) fresh.push(l);
|
|
2307
|
+
}
|
|
2308
|
+
const carried = extractExistingLessons(parent);
|
|
2309
|
+
if (fresh.length === 0 && carried.length === 0) return [];
|
|
2310
|
+
const distilled = opts.distill && fresh.length > 0 ? await distillLessons(fresh, opts.distill) : fresh;
|
|
2311
|
+
const byKey = /* @__PURE__ */ new Map();
|
|
2312
|
+
for (const l of carried) {
|
|
2313
|
+
const k = normKey(l);
|
|
2314
|
+
if (k) byKey.set(k, { text: l, count: 1 });
|
|
2315
|
+
}
|
|
2316
|
+
for (const l of distilled) {
|
|
2317
|
+
const k = normKey(l);
|
|
2318
|
+
if (!k) continue;
|
|
2319
|
+
const e = byKey.get(k);
|
|
2320
|
+
if (e) e.count += 1;
|
|
2321
|
+
else byKey.set(k, { text: l, count: 1 });
|
|
2322
|
+
}
|
|
2323
|
+
const ranked = [...byKey.values()].sort((a, b) => b.count - a.count || a.text.localeCompare(b.text)).slice(0, maxEntries);
|
|
2324
|
+
if (ranked.length === 0) return [];
|
|
2325
|
+
const block = [BLOCK_START2, heading, ...ranked.map((e) => `- ${e.text}`), BLOCK_END2].join(
|
|
2326
|
+
"\n"
|
|
2327
|
+
);
|
|
2328
|
+
const next = `${stripBlock(parent, BLOCK_START2, BLOCK_END2)}
|
|
2329
|
+
|
|
2330
|
+
${block}
|
|
2331
|
+
`;
|
|
2332
|
+
if (next === parent) return [];
|
|
2333
|
+
return [
|
|
2334
|
+
{
|
|
2335
|
+
surface: next,
|
|
2336
|
+
label: "memory-curation",
|
|
2337
|
+
rationale: `curated ${ranked.length} lessons (from ${fresh.length} new finding(s) + ${carried.length} carried)`
|
|
2338
|
+
}
|
|
2339
|
+
];
|
|
2340
|
+
}
|
|
2341
|
+
};
|
|
1841
2342
|
}
|
|
1842
|
-
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
const
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
for (const [key, xs] of groups) out[key] = { meanComposite: mean2(xs), n: xs.length };
|
|
1853
|
-
return out;
|
|
2343
|
+
|
|
2344
|
+
// src/campaign/proposers/trace-analyst.ts
|
|
2345
|
+
import { ai } from "@ax-llm/ax";
|
|
2346
|
+
function renderFindings(findings) {
|
|
2347
|
+
return findings.map((f, i) => {
|
|
2348
|
+
const action = f.recommended_action ? `
|
|
2349
|
+
FIX: ${f.recommended_action}` : "";
|
|
2350
|
+
const subject = f.subject ? ` (${f.subject})` : "";
|
|
2351
|
+
return `${i + 1}. [${f.severity}/${f.area}]${subject} ${f.claim}${action}`;
|
|
2352
|
+
}).join("\n");
|
|
1854
2353
|
}
|
|
1855
|
-
function
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
2354
|
+
function traceAnalystProposer(opts) {
|
|
2355
|
+
if (!opts.apiKey) throw new Error("traceAnalystProposer: apiKey is required");
|
|
2356
|
+
if (!opts.model) throw new Error("traceAnalystProposer: model is required");
|
|
2357
|
+
const kinds = opts.kinds ?? DEFAULT_TRACE_ANALYST_KINDS;
|
|
2358
|
+
const produceFindings = opts.analyze ?? (async (path, c) => {
|
|
2359
|
+
const aiService = ai({
|
|
2360
|
+
name: opts.provider ?? "openai",
|
|
2361
|
+
apiKey: opts.apiKey,
|
|
2362
|
+
apiURL: opts.baseUrl,
|
|
2363
|
+
config: { model: opts.model }
|
|
2364
|
+
});
|
|
2365
|
+
const registry = new AnalystRegistry();
|
|
2366
|
+
for (const spec of kinds) {
|
|
2367
|
+
registry.register(createTraceAnalystKind(spec, { ai: aiService, model: opts.model }));
|
|
2368
|
+
}
|
|
2369
|
+
const result = await registry.run(
|
|
2370
|
+
`trace-analyst-gen-${c.generation}`,
|
|
2371
|
+
{ traceStore: new OtlpFileTraceStore({ path }) },
|
|
2372
|
+
{ signal: c.signal }
|
|
2373
|
+
);
|
|
2374
|
+
return result.findings;
|
|
2375
|
+
});
|
|
2376
|
+
return analysisEditProposer({
|
|
2377
|
+
kind: "trace-analyst",
|
|
2378
|
+
label: "trace-analyst",
|
|
2379
|
+
baseUrl: opts.baseUrl,
|
|
2380
|
+
apiKey: opts.apiKey,
|
|
2381
|
+
applyModel: opts.applyModel ?? opts.model,
|
|
2382
|
+
fetchImpl: opts.fetchImpl,
|
|
2383
|
+
resolveTraces: opts.resolveTraces,
|
|
2384
|
+
noTracesError: "traceAnalystProposer: resolveTraces returned no OTLP traces \u2014 the analyst has nothing to read",
|
|
2385
|
+
rationale: (report) => `trace-analyst findings:
|
|
2386
|
+
${report.slice(0, 800)}`,
|
|
2387
|
+
analyze: async (tracePath, ctx) => {
|
|
2388
|
+
let findings;
|
|
2389
|
+
try {
|
|
2390
|
+
findings = await produceFindings(tracePath, ctx);
|
|
2391
|
+
} catch (e) {
|
|
2392
|
+
throw new Error(
|
|
2393
|
+
`traceAnalystProposer: analyst engine failed \u2014 ${e instanceof Error ? e.message : String(e)}`
|
|
2394
|
+
);
|
|
2395
|
+
}
|
|
2396
|
+
if (findings.length === 0) {
|
|
2397
|
+
throw new Error("traceAnalystProposer: analyst engine produced no findings");
|
|
2398
|
+
}
|
|
2399
|
+
return renderFindings(findings);
|
|
2400
|
+
}
|
|
2401
|
+
});
|
|
1859
2402
|
}
|
|
1860
2403
|
|
|
1861
2404
|
// src/campaign/worktree/index.ts
|
|
@@ -1923,14 +2466,14 @@ export {
|
|
|
1923
2466
|
ProfileMatrixError,
|
|
1924
2467
|
SkillPatchParseError,
|
|
1925
2468
|
WorktreeAdapterError,
|
|
1926
|
-
|
|
2469
|
+
aceProposer,
|
|
1927
2470
|
applySkillPatch,
|
|
1928
2471
|
buildAnalystSurfaceDispatch,
|
|
1929
2472
|
buildEvidenceVector,
|
|
1930
2473
|
buildLoopProvenanceRecord,
|
|
1931
2474
|
campaignBreakdown,
|
|
1932
2475
|
campaignMeanComposite,
|
|
1933
|
-
|
|
2476
|
+
compareProposers,
|
|
1934
2477
|
composeGate,
|
|
1935
2478
|
countSentenceEdits,
|
|
1936
2479
|
defaultProductionGate,
|
|
@@ -1938,15 +2481,18 @@ export {
|
|
|
1938
2481
|
detectScale,
|
|
1939
2482
|
dimensionRegressions,
|
|
1940
2483
|
emitLoopProvenance,
|
|
1941
|
-
|
|
2484
|
+
evolutionaryProposer,
|
|
2485
|
+
extractFapoAttributionSignals,
|
|
1942
2486
|
extractH2Sections,
|
|
1943
2487
|
failureModeRecallJudge,
|
|
2488
|
+
fapoEscalationEntry,
|
|
2489
|
+
fapoProposer,
|
|
1944
2490
|
fsCampaignStorage,
|
|
1945
|
-
gepaDriver,
|
|
1946
2491
|
gepaParetoEntry,
|
|
2492
|
+
gepaProposer,
|
|
1947
2493
|
gepaReflectionEntry,
|
|
1948
2494
|
gitWorktreeAdapter,
|
|
1949
|
-
|
|
2495
|
+
haloProposer,
|
|
1950
2496
|
heldOutGate,
|
|
1951
2497
|
heldoutSignificance,
|
|
1952
2498
|
inMemoryCampaignStorage,
|
|
@@ -1954,9 +2500,10 @@ export {
|
|
|
1954
2500
|
labelTrustRank,
|
|
1955
2501
|
loopProvenanceSpans,
|
|
1956
2502
|
makePlaybackDispatch,
|
|
1957
|
-
|
|
2503
|
+
memoryCurationProposer,
|
|
1958
2504
|
openAutoPr,
|
|
1959
2505
|
pairHoldout,
|
|
2506
|
+
parameterSweepProposer,
|
|
1960
2507
|
paretoPolicy,
|
|
1961
2508
|
paretoSignificanceGate,
|
|
1962
2509
|
parseSkillPatchResponse,
|
|
@@ -1975,11 +2522,11 @@ export {
|
|
|
1975
2522
|
scoreboardSummary,
|
|
1976
2523
|
sequentialDecide,
|
|
1977
2524
|
sequentialPairedGate,
|
|
1978
|
-
skillOptDriver,
|
|
1979
2525
|
skillOptEntry,
|
|
2526
|
+
skillOptProposer,
|
|
1980
2527
|
surfaceContentHash,
|
|
1981
2528
|
surfaceHash,
|
|
1982
|
-
|
|
2529
|
+
traceAnalystProposer,
|
|
1983
2530
|
userStoryScoreboard
|
|
1984
2531
|
};
|
|
1985
2532
|
//# sourceMappingURL=index.js.map
|