@tangle-network/agent-eval 0.93.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +44 -30
- package/dist/action-policy.d.ts +24 -0
- package/dist/action-policy.d.ts.map +1 -0
- package/dist/action-policy.test.d.ts +2 -0
- package/dist/action-policy.test.d.ts.map +1 -0
- package/dist/active-learning.d.ts +41 -0
- package/dist/active-learning.d.ts.map +1 -0
- package/dist/adapters/http.d.ts +15 -21
- package/dist/adapters/http.d.ts.map +1 -0
- package/dist/adapters/http.js.map +1 -1
- package/dist/adapters/langchain.d.ts +7 -13
- package/dist/adapters/langchain.d.ts.map +1 -0
- package/dist/adapters/otel.d.ts +13 -24
- package/dist/adapters/otel.d.ts.map +1 -0
- package/dist/agent-profile-cell.d.ts +101 -0
- package/dist/agent-profile-cell.d.ts.map +1 -0
- package/dist/agent-profile.d.ts +27 -0
- package/dist/agent-profile.d.ts.map +1 -0
- package/dist/agent-profile.test.d.ts +2 -0
- package/dist/agent-profile.test.d.ts.map +1 -0
- package/dist/analyst/adapters.d.ts +62 -0
- package/dist/analyst/adapters.d.ts.map +1 -0
- package/dist/analyst/analyst.test.d.ts +2 -0
- package/dist/analyst/analyst.test.d.ts.map +1 -0
- package/dist/analyst/ax-service.d.ts +27 -0
- package/dist/analyst/ax-service.d.ts.map +1 -0
- package/dist/analyst/behavioral-analyst.d.ts +28 -0
- package/dist/analyst/behavioral-analyst.d.ts.map +1 -0
- package/dist/analyst/chat-client.d.ts +91 -0
- package/dist/analyst/chat-client.d.ts.map +1 -0
- package/dist/analyst/default-registry.d.ts +27 -0
- package/dist/analyst/default-registry.d.ts.map +1 -0
- package/dist/analyst/default-registry.test.d.ts +2 -0
- package/dist/analyst/default-registry.test.d.ts.map +1 -0
- package/dist/analyst/finding-signature.d.ts +48 -0
- package/dist/analyst/finding-signature.d.ts.map +1 -0
- package/dist/analyst/finding-subject.d.ts +146 -0
- package/dist/analyst/finding-subject.d.ts.map +1 -0
- package/dist/analyst/finding-subject.test.d.ts +2 -0
- package/dist/analyst/finding-subject.test.d.ts.map +1 -0
- package/dist/analyst/findings-store.d.ts +75 -0
- package/dist/analyst/findings-store.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +28 -256
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +6 -66
- package/dist/analyst/index.js.map +1 -1
- package/dist/{kind-factory-5b7xXXOr.d.ts → analyst/kind-factory.d.ts} +11 -63
- package/dist/analyst/kind-factory.d.ts.map +1 -0
- package/dist/analyst/kinds/failure-mode.d.ts +19 -0
- package/dist/analyst/kinds/failure-mode.d.ts.map +1 -0
- package/dist/analyst/kinds/improvement.d.ts +23 -0
- package/dist/analyst/kinds/improvement.d.ts.map +1 -0
- package/dist/analyst/kinds/index.d.ts +22 -0
- package/dist/analyst/kinds/index.d.ts.map +1 -0
- package/dist/analyst/kinds/kinds.test.d.ts +2 -0
- package/dist/analyst/kinds/kinds.test.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts +28 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +22 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.d.ts +84 -0
- package/dist/analyst/kinds/skill-usage.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts +2 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.d.ts +26 -0
- package/dist/analyst/parse-tolerant.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.test.d.ts +2 -0
- package/dist/analyst/parse-tolerant.test.d.ts.map +1 -0
- package/dist/analyst/registry.budget.test.d.ts +2 -0
- package/dist/analyst/registry.budget.test.d.ts.map +1 -0
- package/dist/{default-registry-zoGHUQEH.d.ts → analyst/registry.d.ts} +8 -37
- package/dist/analyst/registry.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.d.ts +35 -0
- package/dist/analyst/steer-firewall.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.test.d.ts +2 -0
- package/dist/analyst/steer-firewall.test.d.ts.map +1 -0
- package/dist/analyst/structure-findings.d.ts +37 -0
- package/dist/analyst/structure-findings.d.ts.map +1 -0
- package/dist/analyst/structure-findings.test.d.ts +2 -0
- package/dist/analyst/structure-findings.test.d.ts.map +1 -0
- package/dist/analyst/tool-groups.d.ts +34 -0
- package/dist/analyst/tool-groups.d.ts.map +1 -0
- package/dist/{types-2VVIL04s.d.ts → analyst/types.d.ts} +19 -112
- package/dist/analyst/types.d.ts.map +1 -0
- package/dist/anti-slop.d.ts +59 -0
- package/dist/anti-slop.d.ts.map +1 -0
- package/dist/artifact-validator.d.ts +74 -0
- package/dist/artifact-validator.d.ts.map +1 -0
- package/dist/attestation.d.ts +63 -0
- package/dist/attestation.d.ts.map +1 -0
- package/dist/attestation.test.d.ts +2 -0
- package/dist/attestation.test.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +15 -16
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.test.d.ts +2 -0
- package/dist/authenticity/index.test.d.ts.map +1 -0
- package/dist/auto-pr.d.ts +120 -0
- package/dist/auto-pr.d.ts.map +1 -0
- package/dist/{baseline-DE36-Np7.d.ts → baseline.d.ts} +8 -44
- package/dist/baseline.d.ts.map +1 -0
- package/dist/behavior-dsl.d.ts +73 -0
- package/dist/behavior-dsl.d.ts.map +1 -0
- package/dist/belief-state/calibration.d.ts +10 -0
- package/dist/belief-state/calibration.d.ts.map +1 -0
- package/dist/belief-state/calibration.test.d.ts +2 -0
- package/dist/belief-state/calibration.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.d.ts +66 -0
- package/dist/belief-state/code-agent-corpus.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts +2 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.d.ts +22 -0
- package/dist/belief-state/code-agent-evidence.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts +2 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/extract.d.ts +7 -0
- package/dist/belief-state/extract.d.ts.map +1 -0
- package/dist/belief-state/extract.test.d.ts +2 -0
- package/dist/belief-state/extract.test.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +14 -604
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1 -1
- package/dist/belief-state/ope.d.ts +17 -0
- package/dist/belief-state/ope.d.ts.map +1 -0
- package/dist/belief-state/ope.test.d.ts +2 -0
- package/dist/belief-state/ope.test.d.ts.map +1 -0
- package/dist/belief-state/phase0-measurement.d.ts +55 -0
- package/dist/belief-state/phase0-measurement.d.ts.map +1 -0
- package/dist/belief-state/report.d.ts +17 -0
- package/dist/belief-state/report.d.ts.map +1 -0
- package/dist/belief-state/report.test.d.ts +2 -0
- package/dist/belief-state/report.test.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.d.ts +23 -0
- package/dist/belief-state/research-evidence.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.test.d.ts +2 -0
- package/dist/belief-state/research-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +32 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.d.ts +87 -0
- package/dist/belief-state/runtime-hooks.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.test.d.ts +2 -0
- package/dist/belief-state/runtime-hooks.test.d.ts.map +1 -0
- package/dist/belief-state/selective.d.ts +15 -0
- package/dist/belief-state/selective.d.ts.map +1 -0
- package/dist/belief-state/selective.test.d.ts +2 -0
- package/dist/belief-state/selective.test.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.d.ts +80 -0
- package/dist/belief-state/shadow-probe.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.test.d.ts +2 -0
- package/dist/belief-state/shadow-probe.test.d.ts.map +1 -0
- package/dist/belief-state/types.d.ts +195 -0
- package/dist/belief-state/types.d.ts.map +1 -0
- package/dist/belief-state/types.test.d.ts +2 -0
- package/dist/belief-state/types.test.d.ts.map +1 -0
- package/dist/benchmark.d.ts +14 -0
- package/dist/benchmark.d.ts.map +1 -0
- package/dist/benchmarks/index.d.ts +23 -4
- package/dist/benchmarks/index.d.ts.map +1 -0
- package/dist/benchmarks/routing/dataset.d.ts +34 -0
- package/dist/benchmarks/routing/dataset.d.ts.map +1 -0
- package/dist/benchmarks/routing/index.d.ts +34 -0
- package/dist/benchmarks/routing/index.d.ts.map +1 -0
- package/dist/benchmarks/types.d.ts +49 -0
- package/dist/benchmarks/types.d.ts.map +1 -0
- package/dist/bisector.d.ts +81 -0
- package/dist/bisector.d.ts.map +1 -0
- package/dist/budget-guard.d.ts +31 -0
- package/dist/budget-guard.d.ts.map +1 -0
- package/dist/builder-eval/builder-session.d.ts +111 -0
- package/dist/builder-eval/builder-session.d.ts.map +1 -0
- package/dist/builder-eval/correlation.d.ts +32 -0
- package/dist/builder-eval/correlation.d.ts.map +1 -0
- package/dist/builder-eval/index.d.ts +5 -250
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/project-registry.d.ts +51 -0
- package/dist/builder-eval/project-registry.d.ts.map +1 -0
- package/dist/builder-eval/three-layer-eval.d.ts +55 -0
- package/dist/builder-eval/three-layer-eval.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.d.ts +108 -0
- package/dist/campaign/analyst-surface.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.test.d.ts +2 -0
- package/dist/campaign/analyst-surface.test.d.ts.map +1 -0
- package/dist/campaign/auto-pr.d.ts +46 -0
- package/dist/campaign/auto-pr.d.ts.map +1 -0
- package/dist/campaign/distillation/agreement-judge.d.ts +69 -0
- package/dist/campaign/distillation/agreement-judge.d.ts.map +1 -0
- package/dist/campaign/distillation/cli.d.ts +35 -0
- package/dist/campaign/distillation/cli.d.ts.map +1 -0
- package/dist/campaign/distillation/distillation.test.d.ts +2 -0
- package/dist/campaign/distillation/distillation.test.d.ts.map +1 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts +54 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +1 -0
- package/dist/campaign/distillation/run-distillation.d.ts +119 -0
- package/dist/campaign/distillation/run-distillation.d.ts.map +1 -0
- package/dist/campaign/gates/compose.d.ts +12 -0
- package/dist/campaign/gates/compose.d.ts.map +1 -0
- package/dist/campaign/gates/default-production-gate.d.ts +58 -0
- package/dist/campaign/gates/default-production-gate.d.ts.map +1 -0
- package/dist/campaign/gates/heldout-gate.d.ts +12 -0
- package/dist/campaign/gates/heldout-gate.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.d.ts +125 -0
- package/dist/campaign/gates/promotion-policy.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts +2 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.d.ts +146 -0
- package/dist/campaign/gates/sequential.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.test.d.ts +2 -0
- package/dist/campaign/gates/sequential.test.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.d.ts +99 -0
- package/dist/campaign/gates/statistical-heldout.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts +2 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +1 -0
- package/dist/campaign/index.d.ts +38 -1341
- package/dist/campaign/index.d.ts.map +1 -0
- package/dist/campaign/index.js +1865 -1318
- package/dist/campaign/index.js.map +1 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +59 -0
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +1 -0
- package/dist/campaign/presets/compare-proposers.d.ts +146 -0
- package/dist/campaign/presets/compare-proposers.d.ts.map +1 -0
- package/dist/campaign/presets/playback.d.ts +120 -0
- package/dist/campaign/presets/playback.d.ts.map +1 -0
- package/dist/campaign/presets/playback.test.d.ts +2 -0
- package/dist/campaign/presets/playback.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-eval.d.ts +14 -0
- package/dist/campaign/presets/run-eval.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts +63 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +2 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-optimization.d.ts +92 -0
- package/dist/campaign/presets/run-optimization.d.ts.map +1 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts +151 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +1 -0
- package/dist/campaign/presets/run-skill-opt.d.ts +96 -0
- package/dist/campaign/presets/run-skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/_findings-text.d.ts +22 -0
- package/dist/campaign/proposers/_findings-text.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.d.ts +33 -0
- package/dist/campaign/proposers/ace.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.test.d.ts +2 -0
- package/dist/campaign/proposers/ace.test.d.ts.map +1 -0
- package/dist/campaign/proposers/analysis-edit.d.ts +32 -0
- package/dist/campaign/proposers/analysis-edit.d.ts.map +1 -0
- package/dist/campaign/proposers/evolutionary.d.ts +20 -0
- package/dist/campaign/proposers/evolutionary.d.ts.map +1 -0
- package/dist/campaign/proposers/fapo.d.ts +120 -0
- package/dist/campaign/proposers/fapo.d.ts.map +1 -0
- package/dist/campaign/proposers/gepa.d.ts +86 -0
- package/dist/campaign/proposers/gepa.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.d.ts +44 -0
- package/dist/campaign/proposers/halo.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.test.d.ts +2 -0
- package/dist/campaign/proposers/halo.test.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.d.ts +47 -0
- package/dist/campaign/proposers/memory.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.test.d.ts +2 -0
- package/dist/campaign/proposers/memory.test.d.ts.map +1 -0
- package/dist/campaign/proposers/skill-opt.d.ts +88 -0
- package/dist/campaign/proposers/skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.d.ts +48 -0
- package/dist/campaign/proposers/trace-analyst.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts +2 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +1 -0
- package/dist/campaign/provenance.d.ts +185 -0
- package/dist/campaign/provenance.d.ts.map +1 -0
- package/dist/campaign/run-campaign.d.ts +90 -0
- package/dist/campaign/run-campaign.d.ts.map +1 -0
- package/dist/campaign/score-utils.d.ts +26 -0
- package/dist/campaign/score-utils.d.ts.map +1 -0
- package/dist/campaign/skill-patch.d.ts +62 -0
- package/dist/campaign/skill-patch.d.ts.map +1 -0
- package/dist/campaign/storage.d.ts +38 -0
- package/dist/campaign/storage.d.ts.map +1 -0
- package/dist/{types-BU-7W85F.d.ts → campaign/types.d.ts} +98 -99
- package/dist/campaign/types.d.ts.map +1 -0
- package/dist/campaign/worktree/index.d.ts +53 -0
- package/dist/campaign/worktree/index.d.ts.map +1 -0
- package/dist/canary.d.ts +101 -0
- package/dist/canary.d.ts.map +1 -0
- package/dist/causal-attribution.d.ts +45 -0
- package/dist/causal-attribution.d.ts.map +1 -0
- package/dist/{chunk-TWS7AZEY.js → chunk-2T4EZACH.js} +2 -2
- package/dist/chunk-2T4EZACH.js.map +1 -0
- package/dist/{chunk-LMZQ2Z4U.js → chunk-56GC6VYO.js} +126 -1
- package/dist/chunk-56GC6VYO.js.map +1 -0
- package/dist/{chunk-QG2OVF2D.js → chunk-77T4STFI.js} +154 -112
- package/dist/chunk-77T4STFI.js.map +1 -0
- package/dist/{chunk-ZFIBGEOL.js → chunk-7QTQKIDD.js} +179 -178
- package/dist/chunk-7QTQKIDD.js.map +1 -0
- package/dist/{chunk-UMMZHCPB.js → chunk-AQ5WQAIV.js} +26 -9
- package/dist/chunk-AQ5WQAIV.js.map +1 -0
- package/dist/{chunk-CVVHBFGN.js → chunk-CWNP4DV4.js} +53 -5
- package/dist/chunk-CWNP4DV4.js.map +1 -0
- package/dist/{chunk-QS3RBQPI.js → chunk-D5KBVULY.js} +2 -2
- package/dist/chunk-DJWX3GVS.js +81 -0
- package/dist/chunk-DJWX3GVS.js.map +1 -0
- package/dist/{chunk-S6OZEZQK.js → chunk-HMA63UEO.js} +37 -9
- package/dist/{chunk-S6OZEZQK.js.map → chunk-HMA63UEO.js.map} +1 -1
- package/dist/{chunk-UHMJT4T7.js → chunk-IZCEK2HR.js} +2 -2
- package/dist/{chunk-6SOJM3VR.js → chunk-KKWJD5E6.js} +21 -21
- package/dist/chunk-KKWJD5E6.js.map +1 -0
- package/dist/{chunk-KWRRMR3J.js → chunk-LO6IOIJ2.js} +10 -10
- package/dist/chunk-LO6IOIJ2.js.map +1 -0
- package/dist/{chunk-L3JOU6XM.js → chunk-NACM5RS7.js} +3 -3
- package/dist/{chunk-E4GH6USR.js → chunk-NZEQVRH5.js} +2 -2
- package/dist/chunk-NZEQVRH5.js.map +1 -0
- package/dist/{chunk-XY4DDNEG.js → chunk-PSWWQXHF.js} +14 -89
- package/dist/chunk-PSWWQXHF.js.map +1 -0
- package/dist/{chunk-4FBZZIYD.js → chunk-QTMYW64F.js} +2 -2
- package/dist/{chunk-D3V5B42D.js → chunk-S4SYLDFX.js} +7 -4
- package/dist/chunk-S4SYLDFX.js.map +1 -0
- package/dist/{chunk-47X6LRCE.js → chunk-UFSG7ACU.js} +10 -7
- package/dist/chunk-UFSG7ACU.js.map +1 -0
- package/dist/{chunk-CY6U5S3X.js → chunk-URWVKRCS.js} +2 -2
- package/dist/{chunk-GSH6QNNS.js → chunk-X74V6ESX.js} +102 -79
- package/dist/chunk-X74V6ESX.js.map +1 -0
- package/dist/{chunk-QAY5UIJO.js → chunk-YBIGNSCZ.js} +178 -59
- package/dist/chunk-YBIGNSCZ.js.map +1 -0
- package/dist/{chunk-Y47J2LJ3.js → chunk-Z6L6YSU6.js} +5 -5
- package/dist/{chunk-T375SUOZ.js → chunk-ZOPLCJSY.js} +86 -31
- package/dist/chunk-ZOPLCJSY.js.map +1 -0
- package/dist/ci-gate.d.ts +44 -0
- package/dist/ci-gate.d.ts.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client.d.ts +77 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.test.d.ts +2 -0
- package/dist/client.test.d.ts.map +1 -0
- package/dist/command-runner.d.ts +74 -0
- package/dist/command-runner.d.ts.map +1 -0
- package/dist/command-runner.test.d.ts +2 -0
- package/dist/command-runner.test.d.ts.map +1 -0
- package/dist/completion-verifier.d.ts +147 -0
- package/dist/completion-verifier.d.ts.map +1 -0
- package/dist/completion-verifier.test.d.ts +9 -0
- package/dist/completion-verifier.test.d.ts.map +1 -0
- package/dist/concurrency.d.ts +23 -0
- package/dist/concurrency.d.ts.map +1 -0
- package/dist/contamination-guard.d.ts +81 -0
- package/dist/contamination-guard.d.ts.map +1 -0
- package/dist/{analyze-runs-DwCEkpO_.d.ts → contract/analyze-runs.d.ts} +9 -10
- package/dist/contract/analyze-runs.d.ts.map +1 -0
- package/dist/contract/define-agent-eval.d.ts +52 -0
- package/dist/contract/define-agent-eval.d.ts.map +1 -0
- package/dist/contract/diff.d.ts +114 -0
- package/dist/contract/diff.d.ts.map +1 -0
- package/dist/contract/index.d.ts +106 -640
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +131 -21
- package/dist/contract/index.js.map +1 -1
- package/dist/{insight-report-BBwvOh6x.d.ts → contract/insight-report.d.ts} +16 -19
- package/dist/contract/insight-report.d.ts.map +1 -0
- package/dist/contract/insight-types-fwd.d.ts +7 -0
- package/dist/contract/insight-types-fwd.d.ts.map +1 -0
- package/dist/contract/intake/agent-trace.d.ts +97 -0
- package/dist/contract/intake/agent-trace.d.ts.map +1 -0
- package/dist/{code-agent-session-BO8nCnv3.d.ts → contract/intake/code-agent-session.d.ts} +15 -17
- package/dist/contract/intake/code-agent-session.d.ts.map +1 -0
- package/dist/contract/intake/feedback-table.d.ts +87 -0
- package/dist/contract/intake/feedback-table.d.ts.map +1 -0
- package/dist/contract/intake/index.d.ts +22 -0
- package/dist/contract/intake/index.d.ts.map +1 -0
- package/dist/contract/intake/otel-spans.d.ts +33 -0
- package/dist/contract/intake/otel-spans.d.ts.map +1 -0
- package/dist/contract/self-improve.d.ts +284 -0
- package/dist/contract/self-improve.d.ts.map +1 -0
- package/dist/{control-runtime-DuFBYg7A.d.ts → control-runtime.d.ts} +23 -26
- package/dist/control-runtime.d.ts.map +1 -0
- package/dist/control-runtime.test.d.ts +2 -0
- package/dist/control-runtime.test.d.ts.map +1 -0
- package/dist/control.d.ts +11 -9
- package/dist/control.d.ts.map +1 -0
- package/dist/control.js +2 -2
- package/dist/convergence.d.ts +29 -0
- package/dist/convergence.d.ts.map +1 -0
- package/dist/{cost-ledger-DuSqlw5B.d.ts → cost-ledger.d.ts} +10 -11
- package/dist/cost-ledger.d.ts.map +1 -0
- package/dist/cost-ledger.test.d.ts +2 -0
- package/dist/cost-ledger.test.d.ts.map +1 -0
- package/dist/cost-report.d.ts +43 -0
- package/dist/cost-report.d.ts.map +1 -0
- package/dist/cost-report.test.d.ts +2 -0
- package/dist/cost-report.test.d.ts.map +1 -0
- package/dist/cost-tracker.d.ts +76 -0
- package/dist/cost-tracker.d.ts.map +1 -0
- package/dist/{counterfactual-Dwibr5IW.d.ts → counterfactual.d.ts} +12 -13
- package/dist/counterfactual.d.ts.map +1 -0
- package/dist/cross-trace-diff.d.ts +56 -0
- package/dist/cross-trace-diff.d.ts.map +1 -0
- package/dist/{dataset-BbGkaN2I.d.ts → dataset.d.ts} +11 -14
- package/dist/dataset.d.ts.map +1 -0
- package/dist/deploy-gate-layer.d.ts +125 -0
- package/dist/deploy-gate-layer.d.ts.map +1 -0
- package/dist/deploy-gate-layer.test.d.ts +2 -0
- package/dist/deploy-gate-layer.test.d.ts.map +1 -0
- package/dist/description-length-gate.d.ts +119 -0
- package/dist/description-length-gate.d.ts.map +1 -0
- package/dist/detectors/edge.test.d.ts +2 -0
- package/dist/detectors/edge.test.d.ts.map +1 -0
- package/dist/detectors/index.d.ts +81 -0
- package/dist/detectors/index.d.ts.map +1 -0
- package/dist/detectors/index.test.d.ts +2 -0
- package/dist/detectors/index.test.d.ts.map +1 -0
- package/dist/diagnose/causal-sweep.d.ts +100 -0
- package/dist/diagnose/causal-sweep.d.ts.map +1 -0
- package/dist/diagnose/index.d.ts +36 -0
- package/dist/diagnose/index.d.ts.map +1 -0
- package/dist/diagnose/remediation.d.ts +68 -0
- package/dist/diagnose/remediation.d.ts.map +1 -0
- package/dist/diagnose/repair.d.ts +77 -0
- package/dist/diagnose/repair.d.ts.map +1 -0
- package/dist/diagnose.d.ts +1 -251
- package/dist/diagnose.js +2 -2
- package/dist/discover-personas.d.ts +35 -0
- package/dist/discover-personas.d.ts.map +1 -0
- package/dist/driver.d.ts +95 -0
- package/dist/driver.d.ts.map +1 -0
- package/dist/driver.test.d.ts +8 -0
- package/dist/driver.test.d.ts.map +1 -0
- package/dist/dual-agent-bench.d.ts +81 -0
- package/dist/dual-agent-bench.d.ts.map +1 -0
- package/dist/error-count-extractor.d.ts +47 -0
- package/dist/error-count-extractor.d.ts.map +1 -0
- package/dist/error-count-extractor.test.d.ts +2 -0
- package/dist/error-count-extractor.test.d.ts.map +1 -0
- package/dist/{errors-CzMUYo7b.d.ts → errors.d.ts} +10 -11
- package/dist/errors.d.ts.map +1 -0
- package/dist/{researcher-DE6Gpnb4.d.ts → eval-campaign.d.ts} +34 -156
- package/dist/eval-campaign.d.ts.map +1 -0
- package/dist/eval-campaign.test.d.ts +2 -0
- package/dist/eval-campaign.test.d.ts.map +1 -0
- package/dist/eval-tools.d.ts +55 -0
- package/dist/eval-tools.d.ts.map +1 -0
- package/dist/eval-trace-store.d.ts +107 -0
- package/dist/eval-trace-store.d.ts.map +1 -0
- package/dist/eval-trace-store.test.d.ts +2 -0
- package/dist/eval-trace-store.test.d.ts.map +1 -0
- package/dist/executor.d.ts +38 -0
- package/dist/executor.d.ts.map +1 -0
- package/dist/executor.test.d.ts +10 -0
- package/dist/executor.test.d.ts.map +1 -0
- package/dist/experiment-tracker.d.ts +178 -0
- package/dist/experiment-tracker.d.ts.map +1 -0
- package/dist/experiment-tracker.test.d.ts +2 -0
- package/dist/experiment-tracker.test.d.ts.map +1 -0
- package/dist/failure-taxonomy.d.ts +38 -0
- package/dist/failure-taxonomy.d.ts.map +1 -0
- package/dist/{feedback-trajectory-D9OVLrg9.d.ts → feedback-trajectory.d.ts} +36 -38
- package/dist/feedback-trajectory.d.ts.map +1 -0
- package/dist/feedback-trajectory.test.d.ts +2 -0
- package/dist/feedback-trajectory.test.d.ts.map +1 -0
- package/dist/flow-layer.d.ts +90 -0
- package/dist/flow-layer.d.ts.map +1 -0
- package/dist/flow-layer.test.d.ts +2 -0
- package/dist/flow-layer.test.d.ts.map +1 -0
- package/dist/fuzz/capsule.d.ts +46 -0
- package/dist/fuzz/capsule.d.ts.map +1 -0
- package/dist/fuzz/cube.d.ts +36 -0
- package/dist/fuzz/cube.d.ts.map +1 -0
- package/dist/fuzz/explorer-cost.test.d.ts +2 -0
- package/dist/fuzz/explorer-cost.test.d.ts.map +1 -0
- package/dist/fuzz/explorer.d.ts +64 -0
- package/dist/fuzz/explorer.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.d.ts +16 -0
- package/dist/fuzz/fuzz-agent.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.test.d.ts +2 -0
- package/dist/fuzz/fuzz-agent.test.d.ts.map +1 -0
- package/dist/fuzz/gates.d.ts +33 -0
- package/dist/fuzz/gates.d.ts.map +1 -0
- package/dist/fuzz/index.d.ts +26 -0
- package/dist/fuzz/index.d.ts.map +1 -0
- package/dist/fuzz/policies.d.ts +28 -0
- package/dist/fuzz/policies.d.ts.map +1 -0
- package/dist/fuzz/tools.d.ts +20 -0
- package/dist/fuzz/tools.d.ts.map +1 -0
- package/dist/fuzz/types.d.ts +307 -0
- package/dist/fuzz/types.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -547
- package/dist/golden-matcher.d.ts +71 -0
- package/dist/golden-matcher.d.ts.map +1 -0
- package/dist/governance/eu-ai-act.d.ts +37 -0
- package/dist/governance/eu-ai-act.d.ts.map +1 -0
- package/dist/governance/index.d.ts +5 -135
- package/dist/governance/index.d.ts.map +1 -0
- package/dist/governance/nist-ai-rmf.d.ts +15 -0
- package/dist/governance/nist-ai-rmf.d.ts.map +1 -0
- package/dist/governance/soc2.d.ts +12 -0
- package/dist/governance/soc2.d.ts.map +1 -0
- package/dist/governance/types.d.ts +66 -0
- package/dist/governance/types.d.ts.map +1 -0
- package/dist/harness-optimizer.d.ts +82 -0
- package/dist/harness-optimizer.d.ts.map +1 -0
- package/dist/held-out-gate.d.ts +135 -0
- package/dist/held-out-gate.d.ts.map +1 -0
- package/dist/hosted/client.d.ts +73 -0
- package/dist/hosted/client.d.ts.map +1 -0
- package/dist/hosted/from-env.test.d.ts +8 -0
- package/dist/hosted/from-env.test.d.ts.map +1 -0
- package/dist/hosted/index.d.ts +10 -238
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/types.d.ts +159 -0
- package/dist/hosted/types.d.ts.map +1 -0
- package/dist/index.d.ts +277 -5643
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +317 -108
- package/dist/index.js.map +1 -1
- package/dist/integrity/backend-integrity.d.ts +71 -0
- package/dist/integrity/backend-integrity.d.ts.map +1 -0
- package/dist/integrity/preflight.d.ts +72 -0
- package/dist/integrity/preflight.d.ts.map +1 -0
- package/dist/integrity/preflight.test.d.ts +2 -0
- package/dist/integrity/preflight.test.d.ts.map +1 -0
- package/dist/integrity/single-backend.d.ts +67 -0
- package/dist/integrity/single-backend.d.ts.map +1 -0
- package/dist/intent-match-judge.d.ts +69 -0
- package/dist/intent-match-judge.d.ts.map +1 -0
- package/dist/intent-match-judge.test.d.ts +2 -0
- package/dist/intent-match-judge.test.d.ts.map +1 -0
- package/dist/{judge-calibration-0p2QcWNE.d.ts → judge-calibration.d.ts} +16 -17
- package/dist/judge-calibration.d.ts.map +1 -0
- package/dist/judge-ensemble.d.ts +66 -0
- package/dist/judge-ensemble.d.ts.map +1 -0
- package/dist/judge-ensemble.test.d.ts +8 -0
- package/dist/judge-ensemble.test.d.ts.map +1 -0
- package/dist/judge-families.d.ts +38 -0
- package/dist/judge-families.d.ts.map +1 -0
- package/dist/judge-panel.d.ts +65 -0
- package/dist/judge-panel.d.ts.map +1 -0
- package/dist/judge-retry.d.ts +70 -0
- package/dist/judge-retry.d.ts.map +1 -0
- package/dist/judge-runner.d.ts +36 -0
- package/dist/judge-runner.d.ts.map +1 -0
- package/dist/judge-runner.test.d.ts +2 -0
- package/dist/judge-runner.test.d.ts.map +1 -0
- package/dist/judges.d.ts +74 -0
- package/dist/judges.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.d.ts +89 -0
- package/dist/keyword-coverage-judge.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.test.d.ts +2 -0
- package/dist/keyword-coverage-judge.test.d.ts.map +1 -0
- package/dist/knowledge/index.d.ts +3 -103
- package/dist/knowledge/index.d.ts.map +1 -0
- package/dist/knowledge/readiness.d.ts +26 -0
- package/dist/knowledge/readiness.d.ts.map +1 -0
- package/dist/knowledge/types.d.ts +75 -0
- package/dist/knowledge/types.d.ts.map +1 -0
- package/dist/live-proof.d.ts +62 -0
- package/dist/live-proof.d.ts.map +1 -0
- package/dist/{llm-client-BeEcAokY.d.ts → llm-client.d.ts} +52 -23
- package/dist/llm-client.d.ts.map +1 -0
- package/dist/llm-client.test.d.ts +2 -0
- package/dist/llm-client.test.d.ts.map +1 -0
- package/dist/locked-jsonl-appender.d.ts +19 -0
- package/dist/locked-jsonl-appender.d.ts.map +1 -0
- package/dist/matrix/aggregation.d.ts +16 -0
- package/dist/matrix/aggregation.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +12 -30
- package/dist/matrix/index.d.ts.map +1 -0
- package/dist/matrix/runner.d.ts +15 -0
- package/dist/matrix/runner.d.ts.map +1 -0
- package/dist/{types-mn5Aqk7x.d.ts → matrix/types.d.ts} +11 -15
- package/dist/matrix/types.d.ts.map +1 -0
- package/dist/meta-eval/calibration.d.ts +47 -0
- package/dist/meta-eval/calibration.d.ts.map +1 -0
- package/dist/meta-eval/correlation-study.d.ts +53 -0
- package/dist/meta-eval/correlation-study.d.ts.map +1 -0
- package/dist/meta-eval/index.d.ts +6 -181
- package/dist/meta-eval/index.d.ts.map +1 -0
- package/dist/meta-eval/index.js +1 -1
- package/dist/{outcome-store-rnXLEqSn.d.ts → meta-eval/outcome-store.d.ts} +7 -8
- package/dist/meta-eval/outcome-store.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → meta-eval/rubric-predictive-validity.d.ts} +8 -11
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +1 -0
- package/dist/meta-eval/sentinel.d.ts +169 -0
- package/dist/meta-eval/sentinel.d.ts.map +1 -0
- package/dist/metrics.d.ts +63 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/model-seats.d.ts +71 -0
- package/dist/model-seats.d.ts.map +1 -0
- package/dist/model-seats.test.d.ts +2 -0
- package/dist/model-seats.test.d.ts.map +1 -0
- package/dist/muffled-gate-scanner.d.ts +102 -0
- package/dist/muffled-gate-scanner.d.ts.map +1 -0
- package/dist/{multi-layer-verifier-DUZXrPDA.d.ts → multi-layer-verifier.d.ts} +12 -15
- package/dist/multi-layer-verifier.d.ts.map +1 -0
- package/dist/multi-layer-verifier.test.d.ts +2 -0
- package/dist/multi-layer-verifier.test.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.d.ts +80 -0
- package/dist/multi-toolchain-layer.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.test.d.ts +2 -0
- package/dist/multi-toolchain-layer.test.d.ts.map +1 -0
- package/dist/multishot/default-tools.d.ts +34 -0
- package/dist/multishot/default-tools.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +7 -290
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js.map +1 -1
- package/dist/multishot/judges.d.ts +32 -0
- package/dist/multishot/judges.d.ts.map +1 -0
- package/dist/multishot/matrix.d.ts +107 -0
- package/dist/multishot/matrix.d.ts.map +1 -0
- package/dist/multishot/multishot.d.ts +23 -0
- package/dist/multishot/multishot.d.ts.map +1 -0
- package/dist/multishot/router.d.ts +37 -0
- package/dist/multishot/router.d.ts.map +1 -0
- package/dist/multishot/types.d.ts +60 -0
- package/dist/multishot/types.d.ts.map +1 -0
- package/dist/observability.d.ts +71 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/oracle.d.ts +55 -0
- package/dist/oracle.d.ts.map +1 -0
- package/dist/orthogonality.d.ts +35 -0
- package/dist/orthogonality.d.ts.map +1 -0
- package/dist/otel-pipeline.d.ts +31 -0
- package/dist/otel-pipeline.d.ts.map +1 -0
- package/dist/paraphrase.d.ts +107 -0
- package/dist/paraphrase.d.ts.map +1 -0
- package/dist/{pareto-E-pembql.d.ts → pareto.d.ts} +9 -10
- package/dist/pareto.d.ts.map +1 -0
- package/dist/partition-held-out.d.ts +70 -0
- package/dist/partition-held-out.d.ts.map +1 -0
- package/dist/partition-held-out.test.d.ts +2 -0
- package/dist/partition-held-out.test.d.ts.map +1 -0
- package/dist/perf/index.d.ts +13 -119
- package/dist/perf/index.d.ts.map +1 -0
- package/dist/perf/integrity.d.ts +30 -0
- package/dist/perf/integrity.d.ts.map +1 -0
- package/dist/perf/journey.d.ts +45 -0
- package/dist/perf/journey.d.ts.map +1 -0
- package/dist/perf/ratchet.d.ts +47 -0
- package/dist/perf/ratchet.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.d.ts +31 -0
- package/dist/pipelines/budget-breach.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.test.d.ts +2 -0
- package/dist/pipelines/budget-breach.test.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.d.ts +38 -0
- package/dist/pipelines/failure-cluster.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.test.d.ts +2 -0
- package/dist/pipelines/failure-cluster.test.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.d.ts +26 -0
- package/dist/pipelines/first-divergence.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.test.d.ts +2 -0
- package/dist/pipelines/first-divergence.test.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +8 -173
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +40 -15
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pipelines/judge-agreement.d.ts +26 -0
- package/dist/pipelines/judge-agreement.d.ts.map +1 -0
- package/dist/pipelines/judge-agreement.test.d.ts +2 -0
- package/dist/pipelines/judge-agreement.test.d.ts.map +1 -0
- package/dist/pipelines/regression.d.ts +23 -0
- package/dist/pipelines/regression.d.ts.map +1 -0
- package/dist/pipelines/regression.test.d.ts +2 -0
- package/dist/pipelines/regression.test.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.d.ts +32 -0
- package/dist/pipelines/stuck-loop.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.test.d.ts +2 -0
- package/dist/pipelines/stuck-loop.test.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.d.ts +34 -0
- package/dist/pipelines/tool-waste.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.test.d.ts +2 -0
- package/dist/pipelines/tool-waste.test.d.ts.map +1 -0
- package/dist/playbook.d.ts +16 -0
- package/dist/playbook.d.ts.map +1 -0
- package/dist/pr-review-benchmark.d.ts +88 -0
- package/dist/pr-review-benchmark.d.ts.map +1 -0
- package/dist/pr-review-benchmark.test.d.ts +2 -0
- package/dist/pr-review-benchmark.test.d.ts.map +1 -0
- package/dist/pre-registration.d.ts +125 -0
- package/dist/pre-registration.d.ts.map +1 -0
- package/dist/prm/builtin-rubrics.d.ts +33 -0
- package/dist/prm/builtin-rubrics.d.ts.map +1 -0
- package/dist/prm/index.d.ts +5 -100
- package/dist/prm/index.d.ts.map +1 -0
- package/dist/prm/index.js +55 -12
- package/dist/prm/index.js.map +1 -1
- package/dist/prm/inference.d.ts +29 -0
- package/dist/prm/inference.d.ts.map +1 -0
- package/dist/prm/inference.test.d.ts +2 -0
- package/dist/prm/inference.test.d.ts.map +1 -0
- package/dist/{rubric-BOfxn4ja.d.ts → prm/rubric.d.ts} +10 -13
- package/dist/prm/rubric.d.ts.map +1 -0
- package/dist/prm/training-export.d.ts +38 -0
- package/dist/prm/training-export.d.ts.map +1 -0
- package/dist/produced-state.d.ts +63 -0
- package/dist/produced-state.d.ts.map +1 -0
- package/dist/produced-state.test.d.ts +8 -0
- package/dist/produced-state.test.d.ts.map +1 -0
- package/dist/profile/baselines.d.ts +37 -0
- package/dist/profile/baselines.d.ts.map +1 -0
- package/dist/profile/index.d.ts +105 -0
- package/dist/profile/index.d.ts.map +1 -0
- package/dist/promotion-gate.d.ts +93 -0
- package/dist/promotion-gate.d.ts.map +1 -0
- package/dist/prompt-registry.d.ts +41 -0
- package/dist/prompt-registry.d.ts.map +1 -0
- package/dist/propose-review-control.d.ts +49 -0
- package/dist/propose-review-control.d.ts.map +1 -0
- package/dist/propose-review-control.test.d.ts +2 -0
- package/dist/propose-review-control.test.d.ts.map +1 -0
- package/dist/propose-review.d.ts +155 -0
- package/dist/propose-review.d.ts.map +1 -0
- package/dist/{red-team-BXHil6c8.d.ts → red-team.d.ts} +14 -16
- package/dist/red-team.d.ts.map +1 -0
- package/dist/reference-replay-steering.d.ts +11 -0
- package/dist/reference-replay-steering.d.ts.map +1 -0
- package/dist/reference-replay.d.ts +176 -0
- package/dist/reference-replay.d.ts.map +1 -0
- package/dist/reflective-mutation.d.ts +79 -0
- package/dist/reflective-mutation.d.ts.map +1 -0
- package/dist/registry.d.ts +31 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/release-confidence.d.ts +128 -0
- package/dist/release-confidence.d.ts.map +1 -0
- package/dist/release-report.d.ts +11 -0
- package/dist/release-report.d.ts.map +1 -0
- package/dist/replay.d.ts +120 -0
- package/dist/replay.d.ts.map +1 -0
- package/dist/reporter.d.ts +14 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporting.d.ts +15 -15
- package/dist/reporting.d.ts.map +1 -0
- package/dist/reporting.js +3 -3
- package/dist/researcher.d.ts +140 -0
- package/dist/researcher.d.ts.map +1 -0
- package/dist/reviewer.d.ts +118 -0
- package/dist/reviewer.d.ts.map +1 -0
- package/dist/reviewer.test.d.ts +2 -0
- package/dist/reviewer.test.d.ts.map +1 -0
- package/dist/reward-model-export.d.ts +60 -0
- package/dist/reward-model-export.d.ts.map +1 -0
- package/dist/rl/active-curriculum.d.ts +110 -0
- package/dist/rl/active-curriculum.d.ts.map +1 -0
- package/dist/rl/adaptation-eval.d.ts +109 -0
- package/dist/rl/adaptation-eval.d.ts.map +1 -0
- package/dist/{adversarial-DIVcDoI_.d.ts → rl/adversarial.d.ts} +6 -7
- package/dist/rl/adversarial.d.ts.map +1 -0
- package/dist/rl/compute-curves.d.ts +127 -0
- package/dist/rl/compute-curves.d.ts.map +1 -0
- package/dist/rl/contamination.d.ts +117 -0
- package/dist/rl/contamination.d.ts.map +1 -0
- package/dist/rl/corpus.d.ts +55 -0
- package/dist/rl/corpus.d.ts.map +1 -0
- package/dist/rl/corpus.test.d.ts +2 -0
- package/dist/rl/corpus.test.d.ts.map +1 -0
- package/dist/rl/dataset.d.ts +102 -0
- package/dist/rl/dataset.d.ts.map +1 -0
- package/dist/rl/dataset.test.d.ts +2 -0
- package/dist/rl/dataset.test.d.ts.map +1 -0
- package/dist/rl/exporters.d.ts +141 -0
- package/dist/rl/exporters.d.ts.map +1 -0
- package/dist/rl/index.d.ts +49 -0
- package/dist/rl/index.d.ts.map +1 -0
- package/dist/{off-policy-DiwuKKg7.d.ts → rl/off-policy.d.ts} +8 -9
- package/dist/rl/off-policy.d.ts.map +1 -0
- package/dist/rl/predictive-validity-researcher.d.ts +69 -0
- package/dist/rl/predictive-validity-researcher.d.ts.map +1 -0
- package/dist/rl/preferences.d.ts +141 -0
- package/dist/rl/preferences.d.ts.map +1 -0
- package/dist/rl/process-reward.d.ts +122 -0
- package/dist/rl/process-reward.d.ts.map +1 -0
- package/dist/rl/reward-hacking.d.ts +104 -0
- package/dist/rl/reward-hacking.d.ts.map +1 -0
- package/dist/rl/rl-campaign.d.ts +85 -0
- package/dist/rl/rl-campaign.d.ts.map +1 -0
- package/dist/rl/run-record-adapters.d.ts +56 -0
- package/dist/rl/run-record-adapters.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.d.ts +166 -0
- package/dist/rl/sim-fidelity.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.test.d.ts +2 -0
- package/dist/rl/sim-fidelity.test.d.ts.map +1 -0
- package/dist/rl/tournament.d.ts +115 -0
- package/dist/rl/tournament.d.ts.map +1 -0
- package/dist/rl/verifiable-reward.d.ts +124 -0
- package/dist/rl/verifiable-reward.d.ts.map +1 -0
- package/dist/rl.d.ts +1 -1192
- package/dist/rl.js +615 -615
- package/dist/rl.js.map +1 -1
- package/dist/{run-campaign-RDGAM5KJ.js → run-campaign-WXY7KI67.js} +3 -3
- package/dist/run-critic.d.ts +23 -0
- package/dist/run-critic.d.ts.map +1 -0
- package/dist/run-evidence.d.ts +32 -0
- package/dist/run-evidence.d.ts.map +1 -0
- package/dist/{run-record-e7vj1uZQ.d.ts → run-record.d.ts} +16 -122
- package/dist/run-record.d.ts.map +1 -0
- package/dist/run-record.test.d.ts +2 -0
- package/dist/run-record.test.d.ts.map +1 -0
- package/dist/run-score.d.ts +31 -0
- package/dist/run-score.d.ts.map +1 -0
- package/dist/runtime-trajectory.d.ts +47 -0
- package/dist/runtime-trajectory.d.ts.map +1 -0
- package/dist/{test-graded-scenario-BdVaPyHT.d.ts → sandbox-harness.d.ts} +34 -60
- package/dist/sandbox-harness.d.ts.map +1 -0
- package/dist/sandbox-harness.test.d.ts +2 -0
- package/dist/sandbox-harness.test.d.ts.map +1 -0
- package/dist/sandbox-pool.d.ts +74 -0
- package/dist/sandbox-pool.d.ts.map +1 -0
- package/dist/sandbox-pool.test.d.ts +2 -0
- package/dist/sandbox-pool.test.d.ts.map +1 -0
- package/dist/scorecard.d.ts +133 -0
- package/dist/scorecard.d.ts.map +1 -0
- package/dist/scorecard.test.d.ts +2 -0
- package/dist/scorecard.test.d.ts.map +1 -0
- package/dist/self-play.d.ts +69 -0
- package/dist/self-play.d.ts.map +1 -0
- package/dist/semantic-concept-judge.d.ts +135 -0
- package/dist/semantic-concept-judge.d.ts.map +1 -0
- package/dist/semantic-concept-judge.test.d.ts +2 -0
- package/dist/semantic-concept-judge.test.d.ts.map +1 -0
- package/dist/{sequential-5iSVfzl2.d.ts → sequential.d.ts} +9 -10
- package/dist/sequential.d.ts.map +1 -0
- package/dist/{series-convergence-D5OWMBg6.d.ts → series-convergence.d.ts} +4 -5
- package/dist/series-convergence.d.ts.map +1 -0
- package/dist/slo.d.ts +48 -0
- package/dist/slo.d.ts.map +1 -0
- package/dist/state-continuity.d.ts +47 -0
- package/dist/state-continuity.d.ts.map +1 -0
- package/dist/{statistics-C7PozGrZ.d.ts → statistics.d.ts} +152 -39
- package/dist/statistics.d.ts.map +1 -0
- package/dist/statistics.test.d.ts +2 -0
- package/dist/statistics.test.d.ts.map +1 -0
- package/dist/steering-optimizer.d.ts +58 -0
- package/dist/steering-optimizer.d.ts.map +1 -0
- package/dist/steering.d.ts +24 -0
- package/dist/steering.d.ts.map +1 -0
- package/dist/storyboard/code-edit.d.ts +64 -0
- package/dist/storyboard/code-edit.d.ts.map +1 -0
- package/dist/storyboard/code-edit.test.d.ts +2 -0
- package/dist/storyboard/code-edit.test.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +16 -81
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.test.d.ts +2 -0
- package/dist/storyboard/index.test.d.ts.map +1 -0
- package/dist/{summary-report-DGmUucwQ.d.ts → summary-report.d.ts} +23 -160
- package/dist/summary-report.d.ts.map +1 -0
- package/dist/telemetry/client.d.ts +35 -0
- package/dist/telemetry/client.d.ts.map +1 -0
- package/dist/telemetry/index.d.ts +17 -35
- package/dist/telemetry/index.d.ts.map +1 -0
- package/dist/telemetry/schema.d.ts +61 -0
- package/dist/telemetry/schema.d.ts.map +1 -0
- package/dist/telemetry/sink-fetch.d.ts +39 -0
- package/dist/telemetry/sink-fetch.d.ts.map +1 -0
- package/dist/telemetry/{file.d.ts → sink-file.d.ts} +5 -7
- package/dist/telemetry/sink-file.d.ts.map +1 -0
- package/dist/test-graded-scenario.d.ts +42 -0
- package/dist/test-graded-scenario.d.ts.map +1 -0
- package/dist/testing.d.ts +5 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +8 -0
- package/dist/testing.js.map +1 -0
- package/dist/tool-use-metrics.d.ts +35 -0
- package/dist/tool-use-metrics.d.ts.map +1 -0
- package/dist/trace/capture-fetch.d.ts +48 -0
- package/dist/trace/capture-fetch.d.ts.map +1 -0
- package/dist/trace/capture-fetch.test.d.ts +2 -0
- package/dist/trace/capture-fetch.test.d.ts.map +1 -0
- package/dist/{emitter-DEZwY14K.d.ts → trace/emitter.d.ts} +9 -12
- package/dist/trace/emitter.d.ts.map +1 -0
- package/dist/trace/extract-usage.d.ts +46 -0
- package/dist/trace/extract-usage.d.ts.map +1 -0
- package/dist/trace/extract-usage.test.d.ts +2 -0
- package/dist/trace/extract-usage.test.d.ts.map +1 -0
- package/dist/trace/index.d.ts +15 -0
- package/dist/trace/index.d.ts.map +1 -0
- package/dist/{integrity-VJ9A7aST.d.ts → trace/integrity.d.ts} +11 -14
- package/dist/trace/integrity.d.ts.map +1 -0
- package/dist/trace/otel-bridge.d.ts +29 -0
- package/dist/trace/otel-bridge.d.ts.map +1 -0
- package/dist/trace/otel-export.d.ts +52 -0
- package/dist/trace/otel-export.d.ts.map +1 -0
- package/dist/trace/otel.d.ts +57 -0
- package/dist/trace/otel.d.ts.map +1 -0
- package/dist/trace/otlp-attributes.d.ts +17 -0
- package/dist/trace/otlp-attributes.d.ts.map +1 -0
- package/dist/trace/query.d.ts +29 -0
- package/dist/trace/query.d.ts.map +1 -0
- package/dist/trace/query.test.d.ts +2 -0
- package/dist/trace/query.test.d.ts.map +1 -0
- package/dist/{raw-provider-sink-C46HDghv.d.ts → trace/raw-provider-sink.d.ts} +13 -14
- package/dist/trace/raw-provider-sink.d.ts.map +1 -0
- package/dist/{redact-B40YG2M_.d.ts → trace/redact.d.ts} +7 -8
- package/dist/trace/redact.d.ts.map +1 -0
- package/dist/{schema-m0gsnbt3.d.ts → trace/schema.d.ts} +29 -30
- package/dist/trace/schema.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.d.ts +72 -0
- package/dist/trace/store-to-otlp.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.test.d.ts +2 -0
- package/dist/trace/store-to-otlp.test.d.ts.map +1 -0
- package/dist/{store-CKUAgsJz.d.ts → trace/store.d.ts} +37 -13
- package/dist/trace/store.d.ts.map +1 -0
- package/dist/trace/store.test.d.ts +2 -0
- package/dist/trace/store.test.d.ts.map +1 -0
- package/dist/{analyst-C8HHvfJp.d.ts → trace-analyst/analyst.d.ts} +12 -14
- package/dist/trace-analyst/analyst.d.ts.map +1 -0
- package/dist/trace-analyst/analyst.test.d.ts +2 -0
- package/dist/trace-analyst/analyst.test.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts +40 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +2 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +1 -0
- package/dist/trace-analyst/hook.d.ts +55 -0
- package/dist/trace-analyst/hook.d.ts.map +1 -0
- package/dist/trace-analyst/index.d.ts +18 -0
- package/dist/trace-analyst/index.d.ts.map +1 -0
- package/dist/trace-analyst/insights.d.ts +71 -0
- package/dist/trace-analyst/insights.d.ts.map +1 -0
- package/dist/trace-analyst/insights.test.d.ts +2 -0
- package/dist/trace-analyst/insights.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-flatten.d.ts +42 -0
- package/dist/trace-analyst/otlp-flatten.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.d.ts +85 -0
- package/dist/trace-analyst/otlp-span.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-span.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts +115 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/prompts.d.ts +6 -0
- package/dist/trace-analyst/prompts.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.d.ts +126 -0
- package/dist/trace-analyst/store-otlp.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.test.d.ts +8 -0
- package/dist/trace-analyst/store-otlp.test.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/store.d.ts +63 -0
- package/dist/trace-analyst/store.d.ts.map +1 -0
- package/dist/trace-analyst/tools.d.ts +44 -0
- package/dist/trace-analyst/tools.d.ts.map +1 -0
- package/dist/trace-analyst/tools.test.d.ts +10 -0
- package/dist/trace-analyst/tools.test.d.ts.map +1 -0
- package/dist/{store-C1YxJDEK.d.ts → trace-analyst/types.d.ts} +18 -74
- package/dist/trace-analyst/types.d.ts.map +1 -0
- package/dist/trace-contracts.d.ts +180 -0
- package/dist/trace-contracts.d.ts.map +1 -0
- package/dist/traced-analyst.d.ts +26 -0
- package/dist/traced-analyst.d.ts.map +1 -0
- package/dist/traced-judges.d.ts +27 -0
- package/dist/traced-judges.d.ts.map +1 -0
- package/dist/traces.d.ts +4 -947
- package/dist/traces.d.ts.map +1 -0
- package/dist/traces.js +42 -12
- package/dist/{trajectory-GEdXJCL5.d.ts → trajectory.d.ts} +8 -9
- package/dist/trajectory.d.ts.map +1 -0
- package/dist/{types-Croy5h7V.d.ts → types.d.ts} +33 -33
- package/dist/types.d.ts.map +1 -0
- package/dist/ui-finding.d.ts +104 -0
- package/dist/ui-finding.d.ts.map +1 -0
- package/dist/verdict-cache.d.ts +78 -0
- package/dist/verdict-cache.d.ts.map +1 -0
- package/dist/verdict-cache.test.d.ts +2 -0
- package/dist/verdict-cache.test.d.ts.map +1 -0
- package/dist/{verdict-C9MlYujm.d.ts → verdict.d.ts} +2 -3
- package/dist/verdict.d.ts.map +1 -0
- package/dist/visual-diff.d.ts +32 -0
- package/dist/visual-diff.d.ts.map +1 -0
- package/dist/wire/handlers.d.ts +54 -0
- package/dist/wire/handlers.d.ts.map +1 -0
- package/dist/wire/index.d.ts +13 -570
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -2
- package/dist/wire/openapi.d.ts +3 -0
- package/dist/wire/openapi.d.ts.map +1 -0
- package/dist/wire/rpc.d.ts +21 -0
- package/dist/wire/rpc.d.ts.map +1 -0
- package/dist/wire/rubrics.d.ts +34 -0
- package/dist/wire/rubrics.d.ts.map +1 -0
- package/dist/wire/schemas.d.ts +410 -0
- package/dist/wire/schemas.d.ts.map +1 -0
- package/dist/wire/server.d.ts +60 -0
- package/dist/wire/server.d.ts.map +1 -0
- package/dist/workflow/event-schema.d.ts +5 -0
- package/dist/workflow/event-schema.d.ts.map +1 -0
- package/dist/workflow/feedback-pack.d.ts +99 -0
- package/dist/workflow/feedback-pack.d.ts.map +1 -0
- package/dist/workflow/index.d.ts +22 -495
- package/dist/workflow/index.d.ts.map +1 -0
- package/dist/workflow/index.js +2 -2
- package/dist/workflow/intelligence-export.d.ts +62 -0
- package/dist/workflow/intelligence-export.d.ts.map +1 -0
- package/dist/workflow/partner-report.d.ts +49 -0
- package/dist/workflow/partner-report.d.ts.map +1 -0
- package/dist/workflow/phase-graph.d.ts +43 -0
- package/dist/workflow/phase-graph.d.ts.map +1 -0
- package/dist/workflow/promotion-gate.d.ts +61 -0
- package/dist/workflow/promotion-gate.d.ts.map +1 -0
- package/dist/workflow/run-record.d.ts +12 -0
- package/dist/workflow/run-record.d.ts.map +1 -0
- package/dist/workflow/runtime-adapter.d.ts +20 -0
- package/dist/workflow/runtime-adapter.d.ts.map +1 -0
- package/dist/workflow/sanitize.d.ts +21 -0
- package/dist/workflow/sanitize.d.ts.map +1 -0
- package/dist/workflow/schema.d.ts +5 -0
- package/dist/workflow/schema.d.ts.map +1 -0
- package/dist/workflow/summary.d.ts +43 -0
- package/dist/workflow/summary.d.ts.map +1 -0
- package/dist/workflow/trace-event-fields.d.ts +6 -0
- package/dist/workflow/trace-event-fields.d.ts.map +1 -0
- package/dist/workflow/trajectory.d.ts +15 -0
- package/dist/workflow/trajectory.d.ts.map +1 -0
- package/dist/workflow/types.d.ts +68 -0
- package/dist/workflow/types.d.ts.map +1 -0
- package/dist/workspace-inspector.d.ts +67 -0
- package/dist/workspace-inspector.d.ts.map +1 -0
- package/dist/wrangler-deploy-runner.test.d.ts +2 -0
- package/dist/wrangler-deploy-runner.test.d.ts.map +1 -0
- package/docs/campaign-proposers.md +170 -0
- package/docs/concepts.md +8 -4
- package/docs/customer-journeys.md +15 -13
- package/docs/design/loop-taxonomy.md +34 -66
- package/docs/distributed-driver.md +14 -14
- package/docs/feature-guide.md +1 -1
- package/docs/hosted-ingest-spec.md +2 -3
- package/docs/multi-shot-optimization.md +8 -8
- package/docs/product-eval-adoption.md +1 -1
- package/docs/self-improvement-map.md +33 -29
- package/package.json +9 -15
- package/dist/calibration-Cpr3WaX3.d.ts +0 -101
- package/dist/chunk-47X6LRCE.js.map +0 -1
- package/dist/chunk-6SOJM3VR.js.map +0 -1
- package/dist/chunk-CVVHBFGN.js.map +0 -1
- package/dist/chunk-D3V5B42D.js.map +0 -1
- package/dist/chunk-E4GH6USR.js.map +0 -1
- package/dist/chunk-GSH6QNNS.js.map +0 -1
- package/dist/chunk-KWRRMR3J.js.map +0 -1
- package/dist/chunk-LMZQ2Z4U.js.map +0 -1
- package/dist/chunk-QAY5UIJO.js.map +0 -1
- package/dist/chunk-QG2OVF2D.js.map +0 -1
- package/dist/chunk-T375SUOZ.js.map +0 -1
- package/dist/chunk-TWS7AZEY.js.map +0 -1
- package/dist/chunk-UMMZHCPB.js.map +0 -1
- package/dist/chunk-XY4DDNEG.js.map +0 -1
- package/dist/chunk-ZFIBGEOL.js.map +0 -1
- package/dist/control-_Qb7skHX.d.ts +0 -259
- package/dist/corpus-BoR-041R.d.ts +0 -560
- package/dist/failure-cluster-CL7IVgkJ.d.ts +0 -76
- package/dist/harness-optimizer-EnEnQPsr.d.ts +0 -106
- package/dist/index-Bx3gZ8xl.d.ts +0 -159
- package/dist/pre-registration-mAnCugl9.d.ts +0 -523
- package/dist/provenance-LnqRT0sS.d.ts +0 -441
- package/dist/query-CqTxMwDw.d.ts +0 -31
- package/dist/release-report-euXIV_Sk.d.ts +0 -233
- package/dist/run-critic-BAIjX99r.d.ts +0 -56
- package/dist/run-improvement-loop-5z_l5zDz.d.ts +0 -427
- package/dist/runtime-trajectory-BDgfGZSr.d.ts +0 -49
- package/dist/semantic-concept-judge-Dn8Z6KEG.d.ts +0 -624
- package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
- package/docs/design/external-agent-wedge.md +0 -89
- package/docs/design/phase-d-rfc.md +0 -125
- package/docs/design/phase4-consumer-migration.md +0 -70
- package/docs/design/primitives-integration-spec.md +0 -393
- package/docs/design/product-self-improvement-loop.md +0 -146
- package/docs/design/self-improvement-engine.md +0 -140
- package/docs/design/self-improvement-protocol.md +0 -223
- package/docs/design/self-improvement-roadmap.md +0 -106
- package/docs/design/substrate-gaps.md +0 -118
- package/docs/phase-b-pairing-kit.md +0 -188
- package/docs/phase-b-runbook.md +0 -176
- package/docs/pilot/README.md +0 -62
- package/docs/pilot/customer-checklist.md +0 -90
- package/docs/pilot/integration-foreign-stack.md +0 -296
- package/docs/pilot/integration-tangle-stack.md +0 -248
- package/docs/pilot/one-pager.md +0 -161
- package/docs/pilot/sample-insight-report.json +0 -172
- package/docs/quickstart-external.md +0 -229
- package/docs/research/belief-state-agent-eval-roadmap.md +0 -593
- package/docs/research/research-roadmap.md +0 -205
- package/docs/specs/driver-honest-spec.md +0 -251
- package/docs/specs/hermes-self-improvement-audit.md +0 -93
- package/docs/specs/profile-versioning.md +0 -291
- package/docs/three-package-architecture.md +0 -168
- /package/dist/{chunk-QS3RBQPI.js.map → chunk-D5KBVULY.js.map} +0 -0
- /package/dist/{chunk-UHMJT4T7.js.map → chunk-IZCEK2HR.js.map} +0 -0
- /package/dist/{chunk-L3JOU6XM.js.map → chunk-NACM5RS7.js.map} +0 -0
- /package/dist/{chunk-4FBZZIYD.js.map → chunk-QTMYW64F.js.map} +0 -0
- /package/dist/{chunk-CY6U5S3X.js.map → chunk-URWVKRCS.js.map} +0 -0
- /package/dist/{chunk-Y47J2LJ3.js.map → chunk-Z6L6YSU6.js.map} +0 -0
- /package/dist/{run-campaign-RDGAM5KJ.js.map → run-campaign-WXY7KI67.js.map} +0 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* `distill` — run a teacher→student GEPA distillation against a gold JSONL.
|
|
4
|
+
*
|
|
5
|
+
* The TEACHER is an expensive workflow whose verdicts are frozen as gold; the
|
|
6
|
+
* STUDENT is a cheap single-shot analyst whose system prompt GEPA optimizes
|
|
7
|
+
* toward reproducing those gold verdicts. This is the LIVE, token-spending
|
|
8
|
+
* entry — invoked by hand, never in CI.
|
|
9
|
+
*
|
|
10
|
+
* Usage:
|
|
11
|
+
* distill \
|
|
12
|
+
* --gold <path/to/gold.jsonl> \
|
|
13
|
+
* --baseline <path/to/baseline-prompt.txt> \
|
|
14
|
+
* --model <cheap-student-model> \
|
|
15
|
+
* [--optimizer-model <reflection-model>] \
|
|
16
|
+
* [--generations N] [--population K] [--reps R] \
|
|
17
|
+
* [--test-every-nth 4] [--delta 0] \
|
|
18
|
+
* [--categorical f1,f2] [--array a1,a2] \
|
|
19
|
+
* [--run-dir <dir>]
|
|
20
|
+
*
|
|
21
|
+
* Auth: set LLM_API_KEY (+ optional LLM_BASE_URL) or TANGLE_API_KEY for the
|
|
22
|
+
* Tangle router. The same creds drive the student calls AND the GEPA
|
|
23
|
+
* reflection.
|
|
24
|
+
*
|
|
25
|
+
* Example (against the private skills-internal gold set, 53 records):
|
|
26
|
+
* TANGLE_API_KEY=$(cat /tmp/.tk) pnpm tsx src/campaign/distillation/cli.ts \
|
|
27
|
+
* --gold ~/code/skills-internal/audits/gold/skill-verdicts.gold.jsonl \
|
|
28
|
+
* --baseline ./baseline-skill-analyst.txt \
|
|
29
|
+
* --model gpt-4o-mini --optimizer-model gpt-4o \
|
|
30
|
+
* --generations 3 --population 4 \
|
|
31
|
+
* --categorical value_verdict,quality_score,generalization_rating,public_leak_risk,write_target_rating,subagent_recommended \
|
|
32
|
+
* --array top_actions
|
|
33
|
+
*/
|
|
34
|
+
export {};
|
|
35
|
+
//# sourceMappingURL=cli.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../../src/campaign/distillation/cli.ts"],"names":[],"mappings":";AACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"distillation.test.d.ts","sourceRoot":"","sources":["../../../src/campaign/distillation/distillation.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Gold scenarios for teacher→student distillation. The TEACHER is an
|
|
3
|
+
* expensive workflow (e.g. the 70-agent skill audit) whose verdicts are
|
|
4
|
+
* frozen as gold labels; the STUDENT is a cheap single-shot analyst whose
|
|
5
|
+
* prompt GEPA optimizes toward reproducing those labels.
|
|
6
|
+
*
|
|
7
|
+
* A `GoldScenario` is a `Scenario` (the substrate's input contract) carrying
|
|
8
|
+
* an OPAQUE `input` (what the student sees) and an OPAQUE `label` (the gold
|
|
9
|
+
* verdict the student's output is scored against). Both are typed `unknown`
|
|
10
|
+
* here: this module is domain-agnostic — it distills ANY analyst against ANY
|
|
11
|
+
* gold JSONL. The agreement comparator (see `agreement-judge.ts`) is what
|
|
12
|
+
* knows the label's shape.
|
|
13
|
+
*
|
|
14
|
+
* Loading + splitting are DETERMINISTIC and LLM-free: the gold set is the
|
|
15
|
+
* fixed ground truth, never regenerated here.
|
|
16
|
+
*/
|
|
17
|
+
import type { Scenario } from '../types';
|
|
18
|
+
/** A held gold record: opaque student-input + opaque gold-label, carried as a
|
|
19
|
+
* substrate `Scenario` so it flows through `runCampaign` unchanged. */
|
|
20
|
+
export interface GoldScenario<TInput = unknown, TLabel = unknown> extends Scenario {
|
|
21
|
+
kind: 'gold';
|
|
22
|
+
/** What the student analyst is shown (rendered into its user prompt). */
|
|
23
|
+
input: TInput;
|
|
24
|
+
/** The teacher's gold verdict — the target the student's output is scored
|
|
25
|
+
* against by the agreement judge. NEVER shown to the student. */
|
|
26
|
+
label: TLabel;
|
|
27
|
+
}
|
|
28
|
+
/** Read a gold JSONL (one `{scenarioId|id, input, label, split?}` per line) into
|
|
29
|
+
* `GoldScenario[]`. Deterministic, no LLM. Blank lines are skipped; a line
|
|
30
|
+
* missing an id, `input`, or `label` throws (a silent skip would corrupt the
|
|
31
|
+
* split silently — fail loud on a malformed gold set). */
|
|
32
|
+
export declare function loadGoldScenarios<TInput = unknown, TLabel = unknown>(jsonlPath: string): GoldScenario<TInput, TLabel>[];
|
|
33
|
+
/** Parse gold JSONL text directly (no fs). Exported so tests + in-memory
|
|
34
|
+
* callers exercise the same parse path as {@link loadGoldScenarios}. */
|
|
35
|
+
export declare function parseGoldJsonl<TInput = unknown, TLabel = unknown>(text: string, sourceLabel?: string): GoldScenario<TInput, TLabel>[];
|
|
36
|
+
export interface SplitGoldOptions {
|
|
37
|
+
/** Every Nth scenario (0-based index) goes to the TEST/holdout split; the
|
|
38
|
+
* rest train. Default 4 ⇒ a 25% holdout. Ignored for any scenario that
|
|
39
|
+
* carries an explicit `split:` tag (that is honored verbatim). */
|
|
40
|
+
testEveryNth?: number;
|
|
41
|
+
}
|
|
42
|
+
export interface GoldSplit<TInput, TLabel> {
|
|
43
|
+
/** Training scenarios — the optimization pool the proposer searches over. */
|
|
44
|
+
train: GoldScenario<TInput, TLabel>[];
|
|
45
|
+
/** Held-out scenarios — kept OUT of training; scored only at the gate. */
|
|
46
|
+
test: GoldScenario<TInput, TLabel>[];
|
|
47
|
+
}
|
|
48
|
+
/** Deterministic train/test split. A scenario tagged `split:train|test` is
|
|
49
|
+
* routed by that tag; the rest fall to a modulo split (`index % testEveryNth
|
|
50
|
+
* === 0 ⇒ test`). Pure — same input always yields the same split, so a gold
|
|
51
|
+
* set's holdout is stable across runs (a shuffled split would let a lucky
|
|
52
|
+
* seed flatter the gate). */
|
|
53
|
+
export declare function splitGold<TInput, TLabel>(scenarios: GoldScenario<TInput, TLabel>[], options?: SplitGoldOptions): GoldSplit<TInput, TLabel>;
|
|
54
|
+
//# sourceMappingURL=gold-scenarios.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"gold-scenarios.d.ts","sourceRoot":"","sources":["../../../src/campaign/distillation/gold-scenarios.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AAExC;wEACwE;AACxE,MAAM,WAAW,YAAY,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,CAAE,SAAQ,QAAQ;IAChF,IAAI,EAAE,MAAM,CAAA;IACZ,yEAAyE;IACzE,KAAK,EAAE,MAAM,CAAA;IACb;sEACkE;IAClE,KAAK,EAAE,MAAM,CAAA;CACd;AAYD;;;2DAG2D;AAC3D,wBAAgB,iBAAiB,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,EAClE,SAAS,EAAE,MAAM,GAChB,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAGhC;AAED;yEACyE;AACzE,wBAAgB,cAAc,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,GAAG,OAAO,EAC/D,IAAI,EAAE,MAAM,EACZ,WAAW,SAAa,GACvB,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAmDhC;AAED,MAAM,WAAW,gBAAgB;IAC/B;;uEAEmE;IACnE,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,SAAS,CAAC,MAAM,EAAE,MAAM;IACvC,6EAA6E;IAC7E,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;IACrC,0EAA0E;IAC1E,IAAI,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;CACrC;AAED;;;;8BAI8B;AAC9B,wBAAgB,SAAS,CAAC,MAAM,EAAE,MAAM,EACtC,SAAS,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,EACzC,OAAO,GAAE,gBAAqB,GAC7B,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAuB3B"}
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `runDistillation` — the teacher→student distillation loop. COMPOSES existing
|
|
3
|
+
* substrate primitives; reimplements none of them:
|
|
4
|
+
*
|
|
5
|
+
* - DRIVER = `gepaProposer` (reflective prompt optimizer)
|
|
6
|
+
* - LOOP = `runImprovementLoop` (outer: optimize → holdout re-score → gate)
|
|
7
|
+
* - MEASUREMENT = `runCampaign` (inside the loop) scoring the student
|
|
8
|
+
* - JUDGE = `buildAgreementJudge` — student label vs gold teacher label
|
|
9
|
+
* - GATE = caller-supplied (`heldOutGate` / `defaultProductionGate`)
|
|
10
|
+
* - STUDENT = a cheap single-shot analyst whose system prompt is the
|
|
11
|
+
* `MutableSurface` GEPA mutates; it calls the LLM through
|
|
12
|
+
* `createChatClient` and emits a JSON label.
|
|
13
|
+
*
|
|
14
|
+
* The surface IS the student's system prompt. Each generation GEPA rewrites it;
|
|
15
|
+
* `dispatchWithSurface` renders {surface + scenario.input} into a chat request,
|
|
16
|
+
* calls the (cheap) model, parses the produced JSON label, and returns it as
|
|
17
|
+
* the artifact. The agreement judge scores that label against the gold label.
|
|
18
|
+
*
|
|
19
|
+
* `autoOnPromote: 'none'` is FORCED — the loop never opens a PR; the caller
|
|
20
|
+
* (the `distill` CLI) decides what to do with the winning prompt.
|
|
21
|
+
*/
|
|
22
|
+
import { type ChatRequest, type CreateChatClientOpts } from '../../analyst/chat-client';
|
|
23
|
+
import type { LlmClientOptions } from '../../llm-client';
|
|
24
|
+
import { type RunImprovementLoopResult } from '../presets/run-improvement-loop';
|
|
25
|
+
import { type GepaProposerConstraints } from '../proposers/gepa';
|
|
26
|
+
import type { CampaignResult, Gate, JudgeConfig } from '../types';
|
|
27
|
+
import type { GoldScenario } from './gold-scenarios';
|
|
28
|
+
/** Render the student's prompt from {current surface, scenario input}. The
|
|
29
|
+
* surface is the system prompt; the scenario input is the user turn. Override
|
|
30
|
+
* to inject few-shot framing or a JSON-schema reminder. */
|
|
31
|
+
export type RenderStudentPrompt<TInput> = (args: {
|
|
32
|
+
surface: string;
|
|
33
|
+
input: TInput;
|
|
34
|
+
scenarioId: string;
|
|
35
|
+
}) => ChatRequest['messages'];
|
|
36
|
+
/** Parse the model's raw text into a typed produced label. Throws on
|
|
37
|
+
* unparseable output — a thrown dispatch is recorded as a failed cell (never
|
|
38
|
+
* silently scored 0), which is the honest signal that the prompt isn't
|
|
39
|
+
* emitting valid JSON yet. */
|
|
40
|
+
export type ParseStudentLabel<TProduced> = (rawContent: string, scenarioId: string) => TProduced;
|
|
41
|
+
export interface RunDistillationOptions<TProduced, TInput, TLabel> {
|
|
42
|
+
/** The student analyst's INITIAL system prompt — the baseline surface GEPA
|
|
43
|
+
* searches from. */
|
|
44
|
+
baselinePrompt: string;
|
|
45
|
+
/** Training scenarios (the optimization pool). */
|
|
46
|
+
train: GoldScenario<TInput, TLabel>[];
|
|
47
|
+
/** Held-out scenarios — kept OUT of training; scored only at the gate. */
|
|
48
|
+
holdout: GoldScenario<TInput, TLabel>[];
|
|
49
|
+
/** Transport for BOTH the student (cheap model) and the GEPA reflection
|
|
50
|
+
* (the optimizer model). The student calls it via `createChatClient`. */
|
|
51
|
+
llm: CreateChatClientOpts;
|
|
52
|
+
/** Router transport the GEPA proposer reflects through. `gepaProposer` uses the
|
|
53
|
+
* package `LlmClient` directly (`LlmClientOptions`), not the ChatClient —
|
|
54
|
+
* pass the router creds here. A test may inject `fetch` to stub the
|
|
55
|
+
* reflection HTTP and exercise the wiring without real tokens. */
|
|
56
|
+
reflectionLlm: LlmClientOptions;
|
|
57
|
+
/** Cheap model the student runs (e.g. a small/fast model). */
|
|
58
|
+
studentModel: string;
|
|
59
|
+
/** Model GEPA uses to propose prompt rewrites (typically a stronger model). */
|
|
60
|
+
optimizerModel: string;
|
|
61
|
+
/** Agreement judge — produced student label vs gold teacher label. */
|
|
62
|
+
judge: JudgeConfig<TProduced, GoldScenario<TInput, TLabel>>;
|
|
63
|
+
/** Promotion gate. Default: `heldOutGate` over the holdout. Pass
|
|
64
|
+
* `defaultProductionGate({ holdoutScenarios: holdout, ... })` for the full
|
|
65
|
+
* red-team / reward-hacking / canary stack. */
|
|
66
|
+
gate?: Gate<TProduced, GoldScenario<TInput, TLabel>>;
|
|
67
|
+
/** GEPA population size (candidates per generation). Default 4. */
|
|
68
|
+
populationSize?: number;
|
|
69
|
+
/** GEPA generations. Default 3. */
|
|
70
|
+
maxGenerations?: number;
|
|
71
|
+
/** Campaign reps per scenario. Default 1 — raise for CI bands on a flaky
|
|
72
|
+
* student. */
|
|
73
|
+
reps?: number;
|
|
74
|
+
/** Where campaign artifacts + traces land. Default a temp dir under cwd. */
|
|
75
|
+
runDir?: string;
|
|
76
|
+
/** Levers offered to the GEPA reflection prompt. */
|
|
77
|
+
mutationPrimitives?: string[];
|
|
78
|
+
/** GEPA structured-doc constraints (preserve sections, edit budget). */
|
|
79
|
+
constraints?: GepaProposerConstraints;
|
|
80
|
+
/** Gate's minimum holdout-agreement delta to ship. Default 0.0 — a
|
|
81
|
+
* distillation run reports the lift; the caller decides the bar. Only used
|
|
82
|
+
* when `gate` is omitted (the default `heldOutGate`). */
|
|
83
|
+
deltaThreshold?: number;
|
|
84
|
+
/** Render the student prompt. Default: surface as system, JSON-stringified
|
|
85
|
+
* input as the user turn with a JSON-only instruction. */
|
|
86
|
+
renderStudentPrompt?: RenderStudentPrompt<TInput>;
|
|
87
|
+
/** Parse the model's text into a produced label. Default: strict JSON parse
|
|
88
|
+
* with fenced-block stripping. */
|
|
89
|
+
parseStudentLabel?: ParseStudentLabel<TProduced>;
|
|
90
|
+
/** Per-student-call sampling temperature. Default 0 (deterministic student;
|
|
91
|
+
* the optimization signal must come from the PROMPT, not sampling noise). */
|
|
92
|
+
studentTemperature?: number;
|
|
93
|
+
/** Per-student-call max tokens. Default 1024. */
|
|
94
|
+
studentMaxTokens?: number;
|
|
95
|
+
}
|
|
96
|
+
export interface RunDistillationResult<TProduced, TInput, TLabel> extends RunImprovementLoopResult<TProduced, GoldScenario<TInput, TLabel>> {
|
|
97
|
+
/** The winning student prompt (a string surface). */
|
|
98
|
+
winnerPrompt: string;
|
|
99
|
+
/** Mean agreement on the HOLDOUT — baseline vs winner. The headline number:
|
|
100
|
+
* did distillation move the student closer to the teacher on UNSEEN gold? */
|
|
101
|
+
holdoutAgreement: {
|
|
102
|
+
baseline: number;
|
|
103
|
+
winner: number;
|
|
104
|
+
delta: number;
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
export declare function runDistillation<TProduced, TInput, TLabel>(opts: RunDistillationOptions<TProduced, TInput, TLabel>): Promise<RunDistillationResult<TProduced, TInput, TLabel>>;
|
|
108
|
+
/** Default student prompt render: surface as system, JSON input as the user
|
|
109
|
+
* turn, with a JSON-only output instruction. */
|
|
110
|
+
export declare function defaultRenderStudentPrompt<TInput>(args: {
|
|
111
|
+
surface: string;
|
|
112
|
+
input: TInput;
|
|
113
|
+
scenarioId: string;
|
|
114
|
+
}): ChatRequest['messages'];
|
|
115
|
+
/** Default label parse: strip a ```json fence if present, then `JSON.parse`.
|
|
116
|
+
* Throws on failure so the cell is recorded as failed, not silently zeroed. */
|
|
117
|
+
export declare function defaultParseStudentLabel<TProduced>(rawContent: string, scenarioId: string): TProduced;
|
|
118
|
+
export type { CampaignResult };
|
|
119
|
+
//# sourceMappingURL=run-distillation.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"run-distillation.d.ts","sourceRoot":"","sources":["../../../src/campaign/distillation/run-distillation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EACL,KAAK,WAAW,EAEhB,KAAK,oBAAoB,EAE1B,MAAM,2BAA2B,CAAA;AAClC,OAAO,KAAK,EAAiB,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAEvE,OAAO,EAAE,KAAK,wBAAwB,EAAsB,MAAM,iCAAiC,CAAA;AACnG,OAAO,EAAE,KAAK,uBAAuB,EAAgB,MAAM,mBAAmB,CAAA;AAE9E,OAAO,KAAK,EAAE,cAAc,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,UAAU,CAAA;AACjE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAEpD;;4DAE4D;AAC5D,MAAM,MAAM,mBAAmB,CAAC,MAAM,IAAI,CAAC,IAAI,EAAE;IAC/C,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;CACnB,KAAK,WAAW,CAAC,UAAU,CAAC,CAAA;AAE7B;;;+BAG+B;AAC/B,MAAM,MAAM,iBAAiB,CAAC,SAAS,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,SAAS,CAAA;AAEhG,MAAM,WAAW,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM;IAC/D;yBACqB;IACrB,cAAc,EAAE,MAAM,CAAA;IACtB,kDAAkD;IAClD,KAAK,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;IACrC,0EAA0E;IAC1E,OAAO,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAA;IACvC;8EAC0E;IAC1E,GAAG,EAAE,oBAAoB,CAAA;IACzB;;;uEAGmE;IACnE,aAAa,EAAE,gBAAgB,CAAA;IAC/B,8DAA8D;IAC9D,YAAY,EAAE,MAAM,CAAA;IACpB,+EAA+E;IAC/E,cAAc,EAAE,MAAM,CAAA;IACtB,sEAAsE;IACtE,KAAK,EAAE,WAAW,CAAC,SAAS,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAC3D;;oDAEgD;IAChD,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IACpD,mEAAmE;IACnE,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,mCAAmC;IACnC,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB;mBACe;IACf,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,4EAA4E;IAC5E,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC7B,wEAAwE;IACxE,WAAW,CAAC,EAAE,uBAAuB,CAAA;IACrC;;8DAE0D;IAC1D,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB;+DAC2D;IAC3D,mBAAmB,CAAC,EAAE,mBAAmB,CAAC,MAAM,CAAC,CAAA;IACjD;uCACmC;IACnC,iBAAiB,CAAC,EAAE,iBAAiB,CAAC,SAAS,CAAC,CAAA;IAChD;kFAC8E;IAC9E,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,iDAAiD;IACjD,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,WAAW,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAC9D,SAAQ,wBAAwB,CAAC,SAAS,EAAE,YAAY,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzE,qDAAqD;IACrD,YAAY,EAAE,MAAM,CAAA;IACpB;kFAC8E;IAC9E,gBAAgB,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CACtE;AAED,wBAAsB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,EAC7D,IAAI,EAAE,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,GACtD,OAAO,CAAC,qBAAqB,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAyE3D;AA6BD;iDACiD;AACjD,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,IAAI,EAAE;IACvD,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;CACnB,GAAG,WAAW,CAAC,UAAU,CAAC,CAU1B;AAED;gFACgF;AAChF,wBAAgB,wBAAwB,CAAC,SAAS,EAChD,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,GACjB,SAAS,CAcX;AA0BD,YAAY,EAAE,cAAc,EAAE,CAAA"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Compose multiple `Gate` implementations — every gate must pass for the
|
|
3
|
+
* composite to ship. Closes the alignment reviewer's "default-only
|
|
4
|
+
* heldOutGate + costGate would happily promote a reward-hacked prompt"
|
|
5
|
+
* concern by making safety gates first-class composable defaults.
|
|
6
|
+
*/
|
|
7
|
+
import type { Gate, Scenario } from '../types';
|
|
8
|
+
/** Compose gates — all must `ship` for the composite to `ship`. First
|
|
9
|
+
* non-ship verdict short-circuits the composite verdict, but ALL gates run
|
|
10
|
+
* (so the result records every gate's reason — useful for diagnostics). */
|
|
11
|
+
export declare function composeGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(...gates: Array<Gate<TArtifact, TScenario>>): Gate<TArtifact, TScenario>;
|
|
12
|
+
//# sourceMappingURL=compose.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"compose.d.ts","sourceRoot":"","sources":["../../../src/campaign/gates/compose.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAyC,QAAQ,EAAE,MAAM,UAAU,CAAA;AAErF;;4EAE4E;AAC5E,wBAAgB,WAAW,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,SAAS,QAAQ,GAAG,QAAQ,EACpF,GAAG,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,GAC1C,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAgD5B"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `defaultProductionGate` — composes the substrate's existing safety
|
|
3
|
+
* primitives (red-team / reward-hacking / canary / heldout) into a single
|
|
4
|
+
* Gate.decide shape. Closes the alignment + Anthropic-SI reviewers' "safety
|
|
5
|
+
* primitives are off the critical path" blocker.
|
|
6
|
+
*
|
|
7
|
+
* The composition is opinionated — when consumers wire `runImprovementLoop`,
|
|
8
|
+
* THIS gate is the default. Consumers can still pass a custom gate to
|
|
9
|
+
* override; the recommended pattern is to compose THIS gate with whatever
|
|
10
|
+
* extra domain-specific gates they need (`composeGate(defaultProductionGate(...), customGate)`).
|
|
11
|
+
*/
|
|
12
|
+
import type { RedTeamCase } from '../../red-team';
|
|
13
|
+
import type { RunRecord } from '../../run-record';
|
|
14
|
+
import type { Gate, Scenario } from '../types';
|
|
15
|
+
export interface DefaultProductionGateOptions {
|
|
16
|
+
/** Required: scenarios held out from training; substrate compares
|
|
17
|
+
* candidate-on-holdout vs baseline-on-holdout. */
|
|
18
|
+
holdoutScenarios: Scenario[];
|
|
19
|
+
/** Minimum held-out lift the **paired-bootstrap CI lower bound** must clear
|
|
20
|
+
* to ship — NOT a point estimate. Default 0 ⇒ "confidently positive at the
|
|
21
|
+
* confidence level". Interpreted in the judge's native composite scale (set
|
|
22
|
+
* e.g. 2 for a 0-100 rubric to require a ≥2-point significant gain). */
|
|
23
|
+
deltaThreshold?: number;
|
|
24
|
+
/** Confidence level for the held-out + dimension bootstraps. Default 0.95. */
|
|
25
|
+
confidence?: number;
|
|
26
|
+
/** Bootstrap resamples. Default 2000. */
|
|
27
|
+
bootstrapResamples?: number;
|
|
28
|
+
/** Fixed bootstrap seed for a deterministic verdict. Default 1337. */
|
|
29
|
+
bootstrapSeed?: number;
|
|
30
|
+
/** Minimum paired holdout observations (scenarios × reps) before a
|
|
31
|
+
* significance claim is allowed; below it the gate HOLDS with `few_runs`
|
|
32
|
+
* rather than reading a degenerate CI. Default 3. */
|
|
33
|
+
minProductiveRuns?: number;
|
|
34
|
+
/** Critical judge dimensions that must NOT significantly regress even when
|
|
35
|
+
* the net composite rises (anti-Goodhart). The gate HOLDS if any listed
|
|
36
|
+
* dimension's paired-delta CI lower bound < −`regressionTolerance`. E.g.
|
|
37
|
+
* `['hallucination_free']` for a legal agent. */
|
|
38
|
+
criticalDimensions?: string[];
|
|
39
|
+
/** Tolerance for the per-dimension regression guard, in the dimension's
|
|
40
|
+
* native scale. When omitted it auto-scales off observed magnitudes:
|
|
41
|
+
* 0.05 on [0,1], 5 on 0-100. */
|
|
42
|
+
regressionTolerance?: number;
|
|
43
|
+
/** Total $ budget for ALL cells in this campaign — including baseline + candidate.
|
|
44
|
+
* Composite verdict refuses to ship when spend exceeded budget. */
|
|
45
|
+
budgetUsd?: number;
|
|
46
|
+
/** Red-team cases to probe candidate outputs against. When omitted the
|
|
47
|
+
* substrate uses `DEFAULT_RED_TEAM_CORPUS`. Provide a domain-specific
|
|
48
|
+
* battery for tighter coverage. */
|
|
49
|
+
redTeamBattery?: RedTeamCase[];
|
|
50
|
+
/** Run records (oldest-first) needed for the reward-hacking detector.
|
|
51
|
+
* Substrate populates from prior production-loop generations. */
|
|
52
|
+
recentRuns?: RunRecord[];
|
|
53
|
+
/** When true, the gate refuses to ship if the reward-hacking detector
|
|
54
|
+
* fires at the `gaming` severity. Default true. */
|
|
55
|
+
blockOnRewardHackingGaming?: boolean;
|
|
56
|
+
}
|
|
57
|
+
export declare function defaultProductionGate<TArtifact, TScenario extends Scenario>(options: DefaultProductionGateOptions): Gate<TArtifact, TScenario>;
|
|
58
|
+
//# sourceMappingURL=default-production-gate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"default-production-gate.d.ts","sourceRoot":"","sources":["../../../src/campaign/gates/default-production-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAIjD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,KAAK,EAAE,IAAI,EAA2B,QAAQ,EAAE,MAAM,UAAU,CAAA;AAGvE,MAAM,WAAW,4BAA4B;IAC3C;uDACmD;IACnD,gBAAgB,EAAE,QAAQ,EAAE,CAAA;IAC5B;;;6EAGyE;IACzE,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,8EAA8E;IAC9E,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,yCAAyC;IACzC,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,sEAAsE;IACtE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;0DAEsD;IACtD,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;;;sDAGkD;IAClD,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC7B;;qCAEiC;IACjC,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;wEACoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;wCAEoC;IACpC,cAAc,CAAC,EAAE,WAAW,EAAE,CAAA;IAC9B;sEACkE;IAClE,UAAU,CAAC,EAAE,SAAS,EAAE,CAAA;IACxB;wDACoD;IACpD,0BAA0B,CAAC,EAAE,OAAO,CAAA;CACrC;AAED,wBAAgB,qBAAqB,CAAC,SAAS,EAAE,SAAS,SAAS,QAAQ,EACzE,OAAO,EAAE,4BAA4B,GACpC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAuL5B"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Thin Gate adapter — exposes delta-threshold-on-holdout as a composable
|
|
3
|
+
* `Gate`. Use when you want held-out as one of N composed gates instead of
|
|
4
|
+
* the full `defaultProductionGate` stack.
|
|
5
|
+
*/
|
|
6
|
+
import type { Gate, Scenario } from '../types';
|
|
7
|
+
export interface HeldOutGateOptions<TScenario extends Scenario = Scenario> {
|
|
8
|
+
scenarios: TScenario[];
|
|
9
|
+
deltaThreshold?: number;
|
|
10
|
+
}
|
|
11
|
+
export declare function heldOutGate<TArtifact, TScenario extends Scenario>(options: HeldOutGateOptions<TScenario>): Gate<TArtifact, TScenario>;
|
|
12
|
+
//# sourceMappingURL=heldout-gate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"heldout-gate.d.ts","sourceRoot":"","sources":["../../../src/campaign/gates/heldout-gate.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,IAAI,EAA2B,QAAQ,EAAE,MAAM,UAAU,CAAA;AAEvE,MAAM,WAAW,kBAAkB,CAAC,SAAS,SAAS,QAAQ,GAAG,QAAQ;IACvE,SAAS,EAAE,SAAS,EAAE,CAAA;IACtB,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED,wBAAgB,WAAW,CAAC,SAAS,EAAE,SAAS,SAAS,QAAQ,EAC/D,OAAO,EAAE,kBAAkB,CAAC,SAAS,CAAC,GACrC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAwB5B"}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Promotion policy over the evidence VECTOR — the substrate's answer to "never
|
|
3
|
+
* collapse the multi-objective promotion decision into one scalar." A
|
|
4
|
+
* `defaultProductionGate` is one opinionated composition; this module factors
|
|
5
|
+
* the decision into two reusable pieces so MANY policies can compete over the
|
|
6
|
+
* SAME evidence (the quant-desk pattern: one evidence bus, plural strategies):
|
|
7
|
+
*
|
|
8
|
+
* buildEvidenceVector(ctx, objectives, opts) -> EvidenceVector // the bus
|
|
9
|
+
* PromotionPolicy = (ev: EvidenceVector) => GateResult // a strategy
|
|
10
|
+
* paretoPolicy(ev) // the default strategy
|
|
11
|
+
* paretoSignificanceGate(options): Gate // bus + policy as a Gate
|
|
12
|
+
*
|
|
13
|
+
* The Pareto policy is SYMMETRIC multi-objective: every objective is BOTH a
|
|
14
|
+
* potential gain source AND a safety floor (unlike `defaultProductionGate`,
|
|
15
|
+
* where only `composite` can win and `criticalDimensions` are pure floors). A
|
|
16
|
+
* candidate ships iff it weakly DOMINATES the baseline at the confidence level —
|
|
17
|
+
* no objective credibly worse (CI floor breach) AND at least one objective
|
|
18
|
+
* credibly better (CI gain). Insufficient evidence on ANY axis -> need_more_work
|
|
19
|
+
* (NOT folded into hold: "gather more reps" and "reject" are different actions).
|
|
20
|
+
*
|
|
21
|
+
* Cost/latency are NOT CI axes here — `GateContext` carries only an aggregate
|
|
22
|
+
* per-side cost, no per-cell observation vector to bootstrap. Treat them as hard
|
|
23
|
+
* constraints (compose with a budget gate via `composeGate`), not faked CIs.
|
|
24
|
+
*/
|
|
25
|
+
import type { Direction } from '../../pareto';
|
|
26
|
+
import { type PairedBootstrapResult } from '../../statistics';
|
|
27
|
+
import type { Gate, GateContext, GateResult, Scenario } from '../types';
|
|
28
|
+
/** Where an objective's per-cell scalar comes from. `composite` reads the
|
|
29
|
+
* judge's composite; `dimension` reads a named per-dimension score. */
|
|
30
|
+
export type ObjectiveSource = {
|
|
31
|
+
kind: 'composite';
|
|
32
|
+
} | {
|
|
33
|
+
kind: 'dimension';
|
|
34
|
+
dimension: string;
|
|
35
|
+
};
|
|
36
|
+
export interface PromotionObjective {
|
|
37
|
+
/** Stable label used in reports + `contributingGates`. */
|
|
38
|
+
name: string;
|
|
39
|
+
source: ObjectiveSource;
|
|
40
|
+
/** 'maximize' (quality dims) or 'minimize' (error/risk/length dims). Orients
|
|
41
|
+
* the paired delta so a positive bootstrap always means "candidate better". */
|
|
42
|
+
direction: Direction;
|
|
43
|
+
/** The good-direction paired-delta CI lower bound must EXCEED this to count
|
|
44
|
+
* as a significant gain on this axis. Interpreted in the judge's native
|
|
45
|
+
* scale. Default 0 (⇒ "confidently better"). */
|
|
46
|
+
gainThreshold?: number;
|
|
47
|
+
/** A floor breach (regression) is declared when the good-direction CI lower
|
|
48
|
+
* bound is below −floorTolerance. When omitted it auto-scales off observed
|
|
49
|
+
* magnitudes (0.05 on [0,1], 5 on 0-100), matching `dimensionRegressions`. */
|
|
50
|
+
floorTolerance?: number;
|
|
51
|
+
}
|
|
52
|
+
/** Per-axis verdict from the good-direction paired bootstrap. */
|
|
53
|
+
export type AxisVerdict = 'improved' | 'regressed' | 'flat' | 'few_runs';
|
|
54
|
+
export interface AxisEvidence {
|
|
55
|
+
name: string;
|
|
56
|
+
source: ObjectiveSource;
|
|
57
|
+
direction: Direction;
|
|
58
|
+
/** Paired bootstrap on the GOOD-DIRECTION delta (oriented by `direction`):
|
|
59
|
+
* a positive value means the candidate is better on this axis. */
|
|
60
|
+
bootstrap: PairedBootstrapResult;
|
|
61
|
+
/** Paired observations contributing to this axis. */
|
|
62
|
+
n: number;
|
|
63
|
+
gainThreshold: number;
|
|
64
|
+
floorTolerance: number;
|
|
65
|
+
verdict: AxisVerdict;
|
|
66
|
+
}
|
|
67
|
+
export interface EvidenceVector {
|
|
68
|
+
/** One entry per objective — NOTHING averaged across axes. */
|
|
69
|
+
axes: AxisEvidence[];
|
|
70
|
+
/** Smallest paired n across axes that produced observations — the binding
|
|
71
|
+
* evidence-sufficiency constraint. 0 when no axis produced observations. */
|
|
72
|
+
minN: number;
|
|
73
|
+
/** Aggregate per-side cost from the gate context (a constraint input, not a
|
|
74
|
+
* CI axis — see the module header). */
|
|
75
|
+
cost: {
|
|
76
|
+
candidate: number;
|
|
77
|
+
baseline: number;
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
/** A promotion strategy: a pure function from the evidence vector to a verdict.
|
|
81
|
+
* Many policies can run over the same `EvidenceVector` and disagree — that's
|
|
82
|
+
* the point (competing strategies, shared evidence). */
|
|
83
|
+
export type PromotionPolicy = (ev: EvidenceVector) => GateResult;
|
|
84
|
+
export interface BuildEvidenceVectorOptions {
|
|
85
|
+
/** Minimum paired observations before an axis can claim significance; below
|
|
86
|
+
* it the axis is `few_runs`. Default 3. */
|
|
87
|
+
minProductiveRuns?: number;
|
|
88
|
+
/** Confidence level for every axis bootstrap. Default 0.95. */
|
|
89
|
+
confidence?: number;
|
|
90
|
+
/** Bootstrap resamples. Default 2000. */
|
|
91
|
+
resamples?: number;
|
|
92
|
+
/** Fixed bootstrap seed for a deterministic, reproducible verdict. Default 1337. */
|
|
93
|
+
seed?: number;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* The Evidence Bus. For each objective, pair candidate vs baseline by full
|
|
97
|
+
* cellId and bootstrap a CI on the good-direction paired delta. Reuses the
|
|
98
|
+
* exact `pairHoldout` + `pairedBootstrap` machinery the held-out gate uses, so
|
|
99
|
+
* a single source of truth governs pairing granularity + scale handling.
|
|
100
|
+
*/
|
|
101
|
+
export declare function buildEvidenceVector<TArtifact, TScenario extends Scenario>(ctx: GateContext<TArtifact, TScenario>, objectives: PromotionObjective[], opts?: BuildEvidenceVectorOptions): EvidenceVector;
|
|
102
|
+
/**
|
|
103
|
+
* The default strategy: symmetric multi-objective Pareto significance. Ship iff
|
|
104
|
+
* the candidate weakly dominates the baseline at the confidence level — no axis
|
|
105
|
+
* credibly worse AND ≥1 axis credibly better. Floor breach on any axis → hold
|
|
106
|
+
* (anti-Goodhart, dominates everything). Insufficient evidence on any axis →
|
|
107
|
+
* need_more_work. Statistically equivalent → hold (never ship noise).
|
|
108
|
+
*/
|
|
109
|
+
export declare const paretoPolicy: PromotionPolicy;
|
|
110
|
+
export interface ParetoSignificanceGateOptions extends BuildEvidenceVectorOptions {
|
|
111
|
+
/** The objective vector. Every axis is both a gain source and a safety floor. */
|
|
112
|
+
objectives: PromotionObjective[];
|
|
113
|
+
/** Strategy applied to the evidence vector. Default `paretoPolicy`. Override
|
|
114
|
+
* to run a stricter/looser strategy over the SAME bus (competing policies). */
|
|
115
|
+
policy?: PromotionPolicy;
|
|
116
|
+
/** Override the gate name in reports. */
|
|
117
|
+
name?: string;
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Wrap the bus + a policy as a `Gate`. Plugs into the existing
|
|
121
|
+
* `runImprovementLoop({ gate })` slot and composes via `composeGate`; default
|
|
122
|
+
* loop behavior is unchanged because consumers opt in by passing this gate.
|
|
123
|
+
*/
|
|
124
|
+
export declare function paretoSignificanceGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(options: ParetoSignificanceGateOptions): Gate<TArtifact, TScenario>;
|
|
125
|
+
//# sourceMappingURL=promotion-policy.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promotion-policy.d.ts","sourceRoot":"","sources":["../../../src/campaign/gates/promotion-policy.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAC7C,OAAO,EAAE,KAAK,qBAAqB,EAAmB,MAAM,kBAAkB,CAAA;AAC9E,OAAO,KAAK,EAAE,IAAI,EAAE,WAAW,EAAgB,UAAU,EAAc,QAAQ,EAAE,MAAM,UAAU,CAAA;AAGjG;wEACwE;AACxE,MAAM,MAAM,eAAe,GAAG;IAAE,IAAI,EAAE,WAAW,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,CAAA;AAE9F,MAAM,WAAW,kBAAkB;IACjC,0DAA0D;IAC1D,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,eAAe,CAAA;IACvB;oFACgF;IAChF,SAAS,EAAE,SAAS,CAAA;IACpB;;qDAEiD;IACjD,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;mFAE+E;IAC/E,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED,iEAAiE;AACjE,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,WAAW,GAAG,MAAM,GAAG,UAAU,CAAA;AAExE,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,eAAe,CAAA;IACvB,SAAS,EAAE,SAAS,CAAA;IACpB;uEACmE;IACnE,SAAS,EAAE,qBAAqB,CAAA;IAChC,qDAAqD;IACrD,CAAC,EAAE,MAAM,CAAA;IACT,aAAa,EAAE,MAAM,CAAA;IACrB,cAAc,EAAE,MAAM,CAAA;IACtB,OAAO,EAAE,WAAW,CAAA;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,8DAA8D;IAC9D,IAAI,EAAE,YAAY,EAAE,CAAA;IACpB;iFAC6E;IAC7E,IAAI,EAAE,MAAM,CAAA;IACZ;4CACwC;IACxC,IAAI,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAA;CAC9C;AAED;;yDAEyD;AACzD,MAAM,MAAM,eAAe,GAAG,CAAC,EAAE,EAAE,cAAc,KAAK,UAAU,CAAA;AAEhE,MAAM,WAAW,0BAA0B;IACzC;gDAC4C;IAC5C,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,+DAA+D;IAC/D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,yCAAyC;IACzC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,oFAAoF;IACpF,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,SAAS,SAAS,QAAQ,EACvE,GAAG,EAAE,WAAW,CAAC,SAAS,EAAE,SAAS,CAAC,EACtC,UAAU,EAAE,kBAAkB,EAAE,EAChC,IAAI,GAAE,0BAA+B,GACpC,cAAc,CA+DhB;AAED;;;;;;GAMG;AACH,eAAO,MAAM,YAAY,EAAE,eAoE1B,CAAA;AAED,MAAM,WAAW,6BAA8B,SAAQ,0BAA0B;IAC/E,iFAAiF;IACjF,UAAU,EAAE,kBAAkB,EAAE,CAAA;IAChC;oFACgF;IAChF,MAAM,CAAC,EAAE,eAAe,CAAA;IACxB,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,SAAS,QAAQ,GAAG,QAAQ,EAC/F,OAAO,EAAE,6BAA6B,GACrC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAY5B"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"promotion-policy.test.d.ts","sourceRoot":"","sources":["../../../src/campaign/gates/promotion-policy.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Anytime-valid sequential promotion gate — an e-process (betting
|
|
3
|
+
* test-martingale, see `eProcess` in `statistics.ts`) over paired
|
|
4
|
+
* per-scenario deltas, so a campaign stops the MOMENT evidence decides
|
|
5
|
+
* instead of burning a fixed-n budget. Decisions remain valid at any
|
|
6
|
+
* data-dependent stopping time (Ville's inequality), which is exactly what
|
|
7
|
+
* fixed-n machinery cannot offer: peeking at a bootstrap CI after every
|
|
8
|
+
* observation and stopping on the first significant peek inflates type-I
|
|
9
|
+
* error far beyond alpha.
|
|
10
|
+
*
|
|
11
|
+
* REPLACES, never layers on, a fixed-n gate. Running `heldoutSignificance`
|
|
12
|
+
* or `paretoSignificanceGate` repeatedly on a growing sample and stopping
|
|
13
|
+
* early is optional stopping no matter how it is dressed up; this gate is
|
|
14
|
+
* the valid way to stop early. Use one or the other per evidence stream.
|
|
15
|
+
*
|
|
16
|
+
* Pre-registration binding: anytime validity holds only for the
|
|
17
|
+
* PRE-REGISTERED statistic. When a `SignedManifest` is bound, the gate takes
|
|
18
|
+
* alpha from `manifest.alpha`, the observation budget from
|
|
19
|
+
* `manifest.preRegisteredN`, orients deltas by `manifest.direction`, and
|
|
20
|
+
* shifts the null boundary by `manifest.minEffect` — re-deciding the same
|
|
21
|
+
* stream under different parameters after seeing data would reopen optional
|
|
22
|
+
* stopping under a fancier name. The manifest's content hash is verified at
|
|
23
|
+
* construction (sync, same `sha256-content` scheme as `signManifest`).
|
|
24
|
+
*
|
|
25
|
+
* Non-iid caveat (stated honestly): the supermartingale guarantee needs each
|
|
26
|
+
* delta's conditional mean under H0 to stay ≤ the null boundary given the
|
|
27
|
+
* past — exchangeable scenario deltas suffice. Scenario streams ordered by
|
|
28
|
+
* difficulty or by scenario family violate this; `decide(ctx)` therefore
|
|
29
|
+
* shuffles the paired deltas with a SEEDED permutation by default (the
|
|
30
|
+
* permutation is data-independent, so bet predictability is preserved).
|
|
31
|
+
* Stratified betting (per-stratum λ) is future work, not implemented here.
|
|
32
|
+
*/
|
|
33
|
+
import { type SignedManifest } from '../../pre-registration';
|
|
34
|
+
import { type EProcessState } from '../../statistics';
|
|
35
|
+
import type { Gate, GenerationRecord, Scenario } from '../types';
|
|
36
|
+
export type SequentialDecision = 'promote' | 'continue' | 'undecided-at-maxN';
|
|
37
|
+
export interface SequentialObservation {
|
|
38
|
+
decision: SequentialDecision;
|
|
39
|
+
/** Current e-value (the betting wealth) against H0. */
|
|
40
|
+
eValue: number;
|
|
41
|
+
/** Paired deltas consumed so far. */
|
|
42
|
+
n: number;
|
|
43
|
+
/** Names the decision basis. For 'undecided-at-maxN' it states explicitly
|
|
44
|
+
* that exhausting the budget is NOT evidence of no effect. */
|
|
45
|
+
reason: string;
|
|
46
|
+
}
|
|
47
|
+
export interface SequentialPairedGateOptions {
|
|
48
|
+
/** Type-I budget. With `preRegistration` bound this MUST match
|
|
49
|
+
* `manifest.alpha` (conflict throws). Default 0.05. */
|
|
50
|
+
alpha?: number;
|
|
51
|
+
/** Minimum paired deltas before a promote may fire. The stopping rule is
|
|
52
|
+
* "first n ≥ minN with e-value ≥ 1/alpha" — still a valid stopping time.
|
|
53
|
+
* Default 5. */
|
|
54
|
+
minN?: number;
|
|
55
|
+
/** Pre-registered observation budget. Required unless `preRegistration`
|
|
56
|
+
* supplies it via `preRegisteredN` (conflict throws). */
|
|
57
|
+
maxN?: number;
|
|
58
|
+
/** Bet truncation forwarded to `eProcess`. Default 0.5. */
|
|
59
|
+
maxBet?: number;
|
|
60
|
+
/** Bound on |delta| in the judge's native scale; deltas are mapped to
|
|
61
|
+
* x = (d/scale + 1)/2 ∈ [0,1]. A delta outside ±scale throws (use
|
|
62
|
+
* `detectScale` to pick 1 vs 100 BEFORE streaming). Default 1. */
|
|
63
|
+
scale?: number;
|
|
64
|
+
/** Seed for the data-independent shuffle of paired deltas in `decide(ctx)`
|
|
65
|
+
* (exchangeability guard). Default 1337. */
|
|
66
|
+
shuffleSeed?: number;
|
|
67
|
+
/** Bind the pre-registered hypothesis. Verified (content hash) at
|
|
68
|
+
* construction; alpha/maxN/direction/minEffect come FROM the manifest. */
|
|
69
|
+
preRegistration?: SignedManifest;
|
|
70
|
+
/** Override the gate name in reports. */
|
|
71
|
+
name?: string;
|
|
72
|
+
}
|
|
73
|
+
export interface SequentialPairedGate<TArtifact = unknown, TScenario extends Scenario = Scenario> extends Gate<TArtifact, TScenario> {
|
|
74
|
+
/** Streaming entry point: feed one paired per-scenario delta
|
|
75
|
+
* (candidate − baseline, native scale). Each gate instance carries ONE
|
|
76
|
+
* observe-stream; `decide(ctx)` runs on its own fresh stream and never
|
|
77
|
+
* consumes or advances this one. 'promote' is sticky; observing past the
|
|
78
|
+
* pre-registered maxN throws (extending a finished stream after seeing
|
|
79
|
+
* the result reopens optional stopping — start a NEW pre-registered
|
|
80
|
+
* test). */
|
|
81
|
+
observe(delta: number): SequentialObservation;
|
|
82
|
+
/** Read-only snapshot of the observe-stream. */
|
|
83
|
+
state(): EProcessState & {
|
|
84
|
+
decision: SequentialDecision;
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Anytime-valid sequential paired gate. Conforms to the existing `Gate`
|
|
89
|
+
* contract (`decide(ctx)` consumes candidate vs baseline judge scores via
|
|
90
|
+
* `pairHoldout` — same pairing granularity as the fixed-n gates: full cellId,
|
|
91
|
+
* never scenarioId) and adds a streaming `observe(delta)` entry for campaigns
|
|
92
|
+
* that score cells incrementally and want to stop mid-stream.
|
|
93
|
+
*
|
|
94
|
+
* Decision mapping onto the substrate's five-valued `GateDecision`:
|
|
95
|
+
* - 'promote' → 'ship'
|
|
96
|
+
* - 'continue' → 'need_more_work' (stream ended before maxN with
|
|
97
|
+
* the e-value undecided — more reps could decide)
|
|
98
|
+
* - 'undecided-at-maxN' → 'hold', with the reason stating it is NOT
|
|
99
|
+
* evidence of no effect (never a silent default)
|
|
100
|
+
*/
|
|
101
|
+
export declare function sequentialPairedGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(options: SequentialPairedGateOptions): SequentialPairedGate<TArtifact, TScenario>;
|
|
102
|
+
export interface SequentialDecideOptions {
|
|
103
|
+
/** Type-I budget for the early-stop evidence. Default 0.05. */
|
|
104
|
+
alpha?: number;
|
|
105
|
+
/** Minimum paired deltas before a stop may fire. Default 5. */
|
|
106
|
+
minN?: number;
|
|
107
|
+
/** Bet truncation forwarded to `eProcess`. Default 0.5. */
|
|
108
|
+
maxBet?: number;
|
|
109
|
+
/** Bound on |per-scenario composite delta|. Default 1. */
|
|
110
|
+
scale?: number;
|
|
111
|
+
}
|
|
112
|
+
export interface SequentialDecideFn {
|
|
113
|
+
(args: {
|
|
114
|
+
history: GenerationRecord[];
|
|
115
|
+
}): {
|
|
116
|
+
stop: boolean;
|
|
117
|
+
reason?: string;
|
|
118
|
+
};
|
|
119
|
+
/** Read-only snapshot of the accumulated e-process (observability + tests). */
|
|
120
|
+
state(): EProcessState;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* `SurfaceProposer.decide` adapter — stops the optimization loop the moment
|
|
124
|
+
* the e-process decides the loop has produced a real improvement, instead of
|
|
125
|
+
* always running `maxGenerations`.
|
|
126
|
+
*
|
|
127
|
+
* Stream: for each generation g ≥ 1, the per-scenario composite deltas of
|
|
128
|
+
* generation g's top candidate vs the generation-0 top candidate (the
|
|
129
|
+
* incumbent the loop set out to beat), paired by scenarioId. H0: no proposed
|
|
130
|
+
* surface improves any scenario's expected composite over the incumbent —
|
|
131
|
+
* under it every delta has conditional mean ≤ 0 and the e-process is valid.
|
|
132
|
+
* Once wealth ≥ 1/alpha the loop stops and hands the winner to the promotion
|
|
133
|
+
* gate (which re-scores on HELD-OUT data — this adapter only spends the
|
|
134
|
+
* exploration budget, it never promotes).
|
|
135
|
+
*
|
|
136
|
+
* Honesty caveats: (1) the incumbent's scores are measured once and shared
|
|
137
|
+
* across all generations' deltas, so type-I control is exact only insofar as
|
|
138
|
+
* those scores approximate the incumbent's true per-scenario means (more reps
|
|
139
|
+
* → tighter); (2) an UNDECIDED process never stops the loop — absence of a
|
|
140
|
+
* crossing is NOT evidence of no effect, so the loop simply runs its normal
|
|
141
|
+
* course. Calling the adapter repeatedly with a growing history consumes each
|
|
142
|
+
* generation exactly once (re-feeding an already-seen record would double-count
|
|
143
|
+
* evidence).
|
|
144
|
+
*/
|
|
145
|
+
export declare function sequentialDecide(options?: SequentialDecideOptions): SequentialDecideFn;
|
|
146
|
+
//# sourceMappingURL=sequential.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sequential.d.ts","sourceRoot":"","sources":["../../../src/campaign/gates/sequential.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAGH,OAAO,EAAgB,KAAK,cAAc,EAAE,MAAM,wBAAwB,CAAA;AAC1E,OAAO,EAAE,KAAK,aAAa,EAAwB,MAAM,kBAAkB,CAAA;AAC3E,OAAO,KAAK,EAAE,IAAI,EAA2B,gBAAgB,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAA;AAGzF,MAAM,MAAM,kBAAkB,GAAG,SAAS,GAAG,UAAU,GAAG,mBAAmB,CAAA;AAE7E,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,uDAAuD;IACvD,MAAM,EAAE,MAAM,CAAA;IACd,qCAAqC;IACrC,CAAC,EAAE,MAAM,CAAA;IACT;mEAC+D;IAC/D,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,2BAA2B;IAC1C;4DACwD;IACxD,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;qBAEiB;IACjB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;8DAC0D;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;uEAEmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;iDAC6C;IAC7C,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;+EAC2E;IAC3E,eAAe,CAAC,EAAE,cAAc,CAAA;IAChC,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,oBAAoB,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,SAAS,QAAQ,GAAG,QAAQ,CAC9F,SAAQ,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC;IAClC;;;;;;iBAMa;IACb,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,qBAAqB,CAAA;IAC7C,gDAAgD;IAChD,KAAK,IAAI,aAAa,GAAG;QAAE,QAAQ,EAAE,kBAAkB,CAAA;KAAE,CAAA;CAC1D;AA2LD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,SAAS,QAAQ,GAAG,QAAQ,EAC7F,OAAO,EAAE,2BAA2B,GACnC,oBAAoB,CAAC,SAAS,EAAE,SAAS,CAAC,CA0E5C;AAED,MAAM,WAAW,uBAAuB;IACtC,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,+DAA+D;IAC/D,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,0DAA0D;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,kBAAkB;IACjC,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,gBAAgB,EAAE,CAAA;KAAE,GAAG;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAC3E,+EAA+E;IAC/E,KAAK,IAAI,aAAa,CAAA;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,GAAE,uBAA4B,GAAG,kBAAkB,CAuE1F"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sequential.test.d.ts","sourceRoot":"","sources":["../../../src/campaign/gates/sequential.test.ts"],"names":[],"mappings":""}
|