@tangle-network/agent-eval 0.93.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +44 -30
- package/dist/action-policy.d.ts +24 -0
- package/dist/action-policy.d.ts.map +1 -0
- package/dist/action-policy.test.d.ts +2 -0
- package/dist/action-policy.test.d.ts.map +1 -0
- package/dist/active-learning.d.ts +41 -0
- package/dist/active-learning.d.ts.map +1 -0
- package/dist/adapters/http.d.ts +15 -21
- package/dist/adapters/http.d.ts.map +1 -0
- package/dist/adapters/http.js.map +1 -1
- package/dist/adapters/langchain.d.ts +7 -13
- package/dist/adapters/langchain.d.ts.map +1 -0
- package/dist/adapters/otel.d.ts +13 -24
- package/dist/adapters/otel.d.ts.map +1 -0
- package/dist/agent-profile-cell.d.ts +101 -0
- package/dist/agent-profile-cell.d.ts.map +1 -0
- package/dist/agent-profile.d.ts +27 -0
- package/dist/agent-profile.d.ts.map +1 -0
- package/dist/agent-profile.test.d.ts +2 -0
- package/dist/agent-profile.test.d.ts.map +1 -0
- package/dist/analyst/adapters.d.ts +62 -0
- package/dist/analyst/adapters.d.ts.map +1 -0
- package/dist/analyst/analyst.test.d.ts +2 -0
- package/dist/analyst/analyst.test.d.ts.map +1 -0
- package/dist/analyst/ax-service.d.ts +27 -0
- package/dist/analyst/ax-service.d.ts.map +1 -0
- package/dist/analyst/behavioral-analyst.d.ts +28 -0
- package/dist/analyst/behavioral-analyst.d.ts.map +1 -0
- package/dist/analyst/chat-client.d.ts +91 -0
- package/dist/analyst/chat-client.d.ts.map +1 -0
- package/dist/analyst/default-registry.d.ts +27 -0
- package/dist/analyst/default-registry.d.ts.map +1 -0
- package/dist/analyst/default-registry.test.d.ts +2 -0
- package/dist/analyst/default-registry.test.d.ts.map +1 -0
- package/dist/analyst/finding-signature.d.ts +48 -0
- package/dist/analyst/finding-signature.d.ts.map +1 -0
- package/dist/analyst/finding-subject.d.ts +146 -0
- package/dist/analyst/finding-subject.d.ts.map +1 -0
- package/dist/analyst/finding-subject.test.d.ts +2 -0
- package/dist/analyst/finding-subject.test.d.ts.map +1 -0
- package/dist/analyst/findings-store.d.ts +75 -0
- package/dist/analyst/findings-store.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +28 -256
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +6 -66
- package/dist/analyst/index.js.map +1 -1
- package/dist/{kind-factory-5b7xXXOr.d.ts → analyst/kind-factory.d.ts} +11 -63
- package/dist/analyst/kind-factory.d.ts.map +1 -0
- package/dist/analyst/kinds/failure-mode.d.ts +19 -0
- package/dist/analyst/kinds/failure-mode.d.ts.map +1 -0
- package/dist/analyst/kinds/improvement.d.ts +23 -0
- package/dist/analyst/kinds/improvement.d.ts.map +1 -0
- package/dist/analyst/kinds/index.d.ts +22 -0
- package/dist/analyst/kinds/index.d.ts.map +1 -0
- package/dist/analyst/kinds/kinds.test.d.ts +2 -0
- package/dist/analyst/kinds/kinds.test.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts +28 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +22 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.d.ts +84 -0
- package/dist/analyst/kinds/skill-usage.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts +2 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.d.ts +26 -0
- package/dist/analyst/parse-tolerant.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.test.d.ts +2 -0
- package/dist/analyst/parse-tolerant.test.d.ts.map +1 -0
- package/dist/analyst/registry.budget.test.d.ts +2 -0
- package/dist/analyst/registry.budget.test.d.ts.map +1 -0
- package/dist/{default-registry-zoGHUQEH.d.ts → analyst/registry.d.ts} +8 -37
- package/dist/analyst/registry.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.d.ts +35 -0
- package/dist/analyst/steer-firewall.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.test.d.ts +2 -0
- package/dist/analyst/steer-firewall.test.d.ts.map +1 -0
- package/dist/analyst/structure-findings.d.ts +37 -0
- package/dist/analyst/structure-findings.d.ts.map +1 -0
- package/dist/analyst/structure-findings.test.d.ts +2 -0
- package/dist/analyst/structure-findings.test.d.ts.map +1 -0
- package/dist/analyst/tool-groups.d.ts +34 -0
- package/dist/analyst/tool-groups.d.ts.map +1 -0
- package/dist/{types-2VVIL04s.d.ts → analyst/types.d.ts} +19 -112
- package/dist/analyst/types.d.ts.map +1 -0
- package/dist/anti-slop.d.ts +59 -0
- package/dist/anti-slop.d.ts.map +1 -0
- package/dist/artifact-validator.d.ts +74 -0
- package/dist/artifact-validator.d.ts.map +1 -0
- package/dist/attestation.d.ts +63 -0
- package/dist/attestation.d.ts.map +1 -0
- package/dist/attestation.test.d.ts +2 -0
- package/dist/attestation.test.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +15 -16
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.test.d.ts +2 -0
- package/dist/authenticity/index.test.d.ts.map +1 -0
- package/dist/auto-pr.d.ts +120 -0
- package/dist/auto-pr.d.ts.map +1 -0
- package/dist/{baseline-DE36-Np7.d.ts → baseline.d.ts} +8 -44
- package/dist/baseline.d.ts.map +1 -0
- package/dist/behavior-dsl.d.ts +73 -0
- package/dist/behavior-dsl.d.ts.map +1 -0
- package/dist/belief-state/calibration.d.ts +10 -0
- package/dist/belief-state/calibration.d.ts.map +1 -0
- package/dist/belief-state/calibration.test.d.ts +2 -0
- package/dist/belief-state/calibration.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.d.ts +66 -0
- package/dist/belief-state/code-agent-corpus.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts +2 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.d.ts +22 -0
- package/dist/belief-state/code-agent-evidence.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts +2 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/extract.d.ts +7 -0
- package/dist/belief-state/extract.d.ts.map +1 -0
- package/dist/belief-state/extract.test.d.ts +2 -0
- package/dist/belief-state/extract.test.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +14 -604
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1 -1
- package/dist/belief-state/ope.d.ts +17 -0
- package/dist/belief-state/ope.d.ts.map +1 -0
- package/dist/belief-state/ope.test.d.ts +2 -0
- package/dist/belief-state/ope.test.d.ts.map +1 -0
- package/dist/belief-state/phase0-measurement.d.ts +55 -0
- package/dist/belief-state/phase0-measurement.d.ts.map +1 -0
- package/dist/belief-state/report.d.ts +17 -0
- package/dist/belief-state/report.d.ts.map +1 -0
- package/dist/belief-state/report.test.d.ts +2 -0
- package/dist/belief-state/report.test.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.d.ts +23 -0
- package/dist/belief-state/research-evidence.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.test.d.ts +2 -0
- package/dist/belief-state/research-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +32 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.d.ts +87 -0
- package/dist/belief-state/runtime-hooks.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.test.d.ts +2 -0
- package/dist/belief-state/runtime-hooks.test.d.ts.map +1 -0
- package/dist/belief-state/selective.d.ts +15 -0
- package/dist/belief-state/selective.d.ts.map +1 -0
- package/dist/belief-state/selective.test.d.ts +2 -0
- package/dist/belief-state/selective.test.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.d.ts +80 -0
- package/dist/belief-state/shadow-probe.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.test.d.ts +2 -0
- package/dist/belief-state/shadow-probe.test.d.ts.map +1 -0
- package/dist/belief-state/types.d.ts +195 -0
- package/dist/belief-state/types.d.ts.map +1 -0
- package/dist/belief-state/types.test.d.ts +2 -0
- package/dist/belief-state/types.test.d.ts.map +1 -0
- package/dist/benchmark.d.ts +14 -0
- package/dist/benchmark.d.ts.map +1 -0
- package/dist/benchmarks/index.d.ts +23 -4
- package/dist/benchmarks/index.d.ts.map +1 -0
- package/dist/benchmarks/routing/dataset.d.ts +34 -0
- package/dist/benchmarks/routing/dataset.d.ts.map +1 -0
- package/dist/benchmarks/routing/index.d.ts +34 -0
- package/dist/benchmarks/routing/index.d.ts.map +1 -0
- package/dist/benchmarks/types.d.ts +49 -0
- package/dist/benchmarks/types.d.ts.map +1 -0
- package/dist/bisector.d.ts +81 -0
- package/dist/bisector.d.ts.map +1 -0
- package/dist/budget-guard.d.ts +31 -0
- package/dist/budget-guard.d.ts.map +1 -0
- package/dist/builder-eval/builder-session.d.ts +111 -0
- package/dist/builder-eval/builder-session.d.ts.map +1 -0
- package/dist/builder-eval/correlation.d.ts +32 -0
- package/dist/builder-eval/correlation.d.ts.map +1 -0
- package/dist/builder-eval/index.d.ts +5 -250
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/project-registry.d.ts +51 -0
- package/dist/builder-eval/project-registry.d.ts.map +1 -0
- package/dist/builder-eval/three-layer-eval.d.ts +55 -0
- package/dist/builder-eval/three-layer-eval.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.d.ts +108 -0
- package/dist/campaign/analyst-surface.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.test.d.ts +2 -0
- package/dist/campaign/analyst-surface.test.d.ts.map +1 -0
- package/dist/campaign/auto-pr.d.ts +46 -0
- package/dist/campaign/auto-pr.d.ts.map +1 -0
- package/dist/campaign/distillation/agreement-judge.d.ts +69 -0
- package/dist/campaign/distillation/agreement-judge.d.ts.map +1 -0
- package/dist/campaign/distillation/cli.d.ts +35 -0
- package/dist/campaign/distillation/cli.d.ts.map +1 -0
- package/dist/campaign/distillation/distillation.test.d.ts +2 -0
- package/dist/campaign/distillation/distillation.test.d.ts.map +1 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts +54 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +1 -0
- package/dist/campaign/distillation/run-distillation.d.ts +119 -0
- package/dist/campaign/distillation/run-distillation.d.ts.map +1 -0
- package/dist/campaign/gates/compose.d.ts +12 -0
- package/dist/campaign/gates/compose.d.ts.map +1 -0
- package/dist/campaign/gates/default-production-gate.d.ts +58 -0
- package/dist/campaign/gates/default-production-gate.d.ts.map +1 -0
- package/dist/campaign/gates/heldout-gate.d.ts +12 -0
- package/dist/campaign/gates/heldout-gate.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.d.ts +125 -0
- package/dist/campaign/gates/promotion-policy.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts +2 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.d.ts +146 -0
- package/dist/campaign/gates/sequential.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.test.d.ts +2 -0
- package/dist/campaign/gates/sequential.test.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.d.ts +99 -0
- package/dist/campaign/gates/statistical-heldout.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts +2 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +1 -0
- package/dist/campaign/index.d.ts +38 -1341
- package/dist/campaign/index.d.ts.map +1 -0
- package/dist/campaign/index.js +1865 -1318
- package/dist/campaign/index.js.map +1 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +59 -0
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +1 -0
- package/dist/campaign/presets/compare-proposers.d.ts +146 -0
- package/dist/campaign/presets/compare-proposers.d.ts.map +1 -0
- package/dist/campaign/presets/playback.d.ts +120 -0
- package/dist/campaign/presets/playback.d.ts.map +1 -0
- package/dist/campaign/presets/playback.test.d.ts +2 -0
- package/dist/campaign/presets/playback.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-eval.d.ts +14 -0
- package/dist/campaign/presets/run-eval.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts +63 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +2 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-optimization.d.ts +92 -0
- package/dist/campaign/presets/run-optimization.d.ts.map +1 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts +151 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +1 -0
- package/dist/campaign/presets/run-skill-opt.d.ts +96 -0
- package/dist/campaign/presets/run-skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/_findings-text.d.ts +22 -0
- package/dist/campaign/proposers/_findings-text.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.d.ts +33 -0
- package/dist/campaign/proposers/ace.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.test.d.ts +2 -0
- package/dist/campaign/proposers/ace.test.d.ts.map +1 -0
- package/dist/campaign/proposers/analysis-edit.d.ts +32 -0
- package/dist/campaign/proposers/analysis-edit.d.ts.map +1 -0
- package/dist/campaign/proposers/evolutionary.d.ts +20 -0
- package/dist/campaign/proposers/evolutionary.d.ts.map +1 -0
- package/dist/campaign/proposers/fapo.d.ts +120 -0
- package/dist/campaign/proposers/fapo.d.ts.map +1 -0
- package/dist/campaign/proposers/gepa.d.ts +86 -0
- package/dist/campaign/proposers/gepa.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.d.ts +44 -0
- package/dist/campaign/proposers/halo.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.test.d.ts +2 -0
- package/dist/campaign/proposers/halo.test.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.d.ts +47 -0
- package/dist/campaign/proposers/memory.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.test.d.ts +2 -0
- package/dist/campaign/proposers/memory.test.d.ts.map +1 -0
- package/dist/campaign/proposers/skill-opt.d.ts +88 -0
- package/dist/campaign/proposers/skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.d.ts +48 -0
- package/dist/campaign/proposers/trace-analyst.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts +2 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +1 -0
- package/dist/campaign/provenance.d.ts +185 -0
- package/dist/campaign/provenance.d.ts.map +1 -0
- package/dist/campaign/run-campaign.d.ts +90 -0
- package/dist/campaign/run-campaign.d.ts.map +1 -0
- package/dist/campaign/score-utils.d.ts +26 -0
- package/dist/campaign/score-utils.d.ts.map +1 -0
- package/dist/campaign/skill-patch.d.ts +62 -0
- package/dist/campaign/skill-patch.d.ts.map +1 -0
- package/dist/campaign/storage.d.ts +38 -0
- package/dist/campaign/storage.d.ts.map +1 -0
- package/dist/{types-BU-7W85F.d.ts → campaign/types.d.ts} +98 -99
- package/dist/campaign/types.d.ts.map +1 -0
- package/dist/campaign/worktree/index.d.ts +53 -0
- package/dist/campaign/worktree/index.d.ts.map +1 -0
- package/dist/canary.d.ts +101 -0
- package/dist/canary.d.ts.map +1 -0
- package/dist/causal-attribution.d.ts +45 -0
- package/dist/causal-attribution.d.ts.map +1 -0
- package/dist/{chunk-TWS7AZEY.js → chunk-2T4EZACH.js} +2 -2
- package/dist/chunk-2T4EZACH.js.map +1 -0
- package/dist/{chunk-LMZQ2Z4U.js → chunk-56GC6VYO.js} +126 -1
- package/dist/chunk-56GC6VYO.js.map +1 -0
- package/dist/{chunk-QG2OVF2D.js → chunk-77T4STFI.js} +154 -112
- package/dist/chunk-77T4STFI.js.map +1 -0
- package/dist/{chunk-ZFIBGEOL.js → chunk-7QTQKIDD.js} +179 -178
- package/dist/chunk-7QTQKIDD.js.map +1 -0
- package/dist/{chunk-UMMZHCPB.js → chunk-AQ5WQAIV.js} +26 -9
- package/dist/chunk-AQ5WQAIV.js.map +1 -0
- package/dist/{chunk-CVVHBFGN.js → chunk-CWNP4DV4.js} +53 -5
- package/dist/chunk-CWNP4DV4.js.map +1 -0
- package/dist/{chunk-QS3RBQPI.js → chunk-D5KBVULY.js} +2 -2
- package/dist/chunk-DJWX3GVS.js +81 -0
- package/dist/chunk-DJWX3GVS.js.map +1 -0
- package/dist/{chunk-S6OZEZQK.js → chunk-HMA63UEO.js} +37 -9
- package/dist/{chunk-S6OZEZQK.js.map → chunk-HMA63UEO.js.map} +1 -1
- package/dist/{chunk-UHMJT4T7.js → chunk-IZCEK2HR.js} +2 -2
- package/dist/{chunk-6SOJM3VR.js → chunk-KKWJD5E6.js} +21 -21
- package/dist/chunk-KKWJD5E6.js.map +1 -0
- package/dist/{chunk-KWRRMR3J.js → chunk-LO6IOIJ2.js} +10 -10
- package/dist/chunk-LO6IOIJ2.js.map +1 -0
- package/dist/{chunk-L3JOU6XM.js → chunk-NACM5RS7.js} +3 -3
- package/dist/{chunk-E4GH6USR.js → chunk-NZEQVRH5.js} +2 -2
- package/dist/chunk-NZEQVRH5.js.map +1 -0
- package/dist/{chunk-XY4DDNEG.js → chunk-PSWWQXHF.js} +14 -89
- package/dist/chunk-PSWWQXHF.js.map +1 -0
- package/dist/{chunk-4FBZZIYD.js → chunk-QTMYW64F.js} +2 -2
- package/dist/{chunk-D3V5B42D.js → chunk-S4SYLDFX.js} +7 -4
- package/dist/chunk-S4SYLDFX.js.map +1 -0
- package/dist/{chunk-47X6LRCE.js → chunk-UFSG7ACU.js} +10 -7
- package/dist/chunk-UFSG7ACU.js.map +1 -0
- package/dist/{chunk-CY6U5S3X.js → chunk-URWVKRCS.js} +2 -2
- package/dist/{chunk-GSH6QNNS.js → chunk-X74V6ESX.js} +102 -79
- package/dist/chunk-X74V6ESX.js.map +1 -0
- package/dist/{chunk-QAY5UIJO.js → chunk-YBIGNSCZ.js} +178 -59
- package/dist/chunk-YBIGNSCZ.js.map +1 -0
- package/dist/{chunk-Y47J2LJ3.js → chunk-Z6L6YSU6.js} +5 -5
- package/dist/{chunk-T375SUOZ.js → chunk-ZOPLCJSY.js} +86 -31
- package/dist/chunk-ZOPLCJSY.js.map +1 -0
- package/dist/ci-gate.d.ts +44 -0
- package/dist/ci-gate.d.ts.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client.d.ts +77 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.test.d.ts +2 -0
- package/dist/client.test.d.ts.map +1 -0
- package/dist/command-runner.d.ts +74 -0
- package/dist/command-runner.d.ts.map +1 -0
- package/dist/command-runner.test.d.ts +2 -0
- package/dist/command-runner.test.d.ts.map +1 -0
- package/dist/completion-verifier.d.ts +147 -0
- package/dist/completion-verifier.d.ts.map +1 -0
- package/dist/completion-verifier.test.d.ts +9 -0
- package/dist/completion-verifier.test.d.ts.map +1 -0
- package/dist/concurrency.d.ts +23 -0
- package/dist/concurrency.d.ts.map +1 -0
- package/dist/contamination-guard.d.ts +81 -0
- package/dist/contamination-guard.d.ts.map +1 -0
- package/dist/{analyze-runs-DwCEkpO_.d.ts → contract/analyze-runs.d.ts} +9 -10
- package/dist/contract/analyze-runs.d.ts.map +1 -0
- package/dist/contract/define-agent-eval.d.ts +52 -0
- package/dist/contract/define-agent-eval.d.ts.map +1 -0
- package/dist/contract/diff.d.ts +114 -0
- package/dist/contract/diff.d.ts.map +1 -0
- package/dist/contract/index.d.ts +106 -640
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +131 -21
- package/dist/contract/index.js.map +1 -1
- package/dist/{insight-report-BBwvOh6x.d.ts → contract/insight-report.d.ts} +16 -19
- package/dist/contract/insight-report.d.ts.map +1 -0
- package/dist/contract/insight-types-fwd.d.ts +7 -0
- package/dist/contract/insight-types-fwd.d.ts.map +1 -0
- package/dist/contract/intake/agent-trace.d.ts +97 -0
- package/dist/contract/intake/agent-trace.d.ts.map +1 -0
- package/dist/{code-agent-session-BO8nCnv3.d.ts → contract/intake/code-agent-session.d.ts} +15 -17
- package/dist/contract/intake/code-agent-session.d.ts.map +1 -0
- package/dist/contract/intake/feedback-table.d.ts +87 -0
- package/dist/contract/intake/feedback-table.d.ts.map +1 -0
- package/dist/contract/intake/index.d.ts +22 -0
- package/dist/contract/intake/index.d.ts.map +1 -0
- package/dist/contract/intake/otel-spans.d.ts +33 -0
- package/dist/contract/intake/otel-spans.d.ts.map +1 -0
- package/dist/contract/self-improve.d.ts +284 -0
- package/dist/contract/self-improve.d.ts.map +1 -0
- package/dist/{control-runtime-DuFBYg7A.d.ts → control-runtime.d.ts} +23 -26
- package/dist/control-runtime.d.ts.map +1 -0
- package/dist/control-runtime.test.d.ts +2 -0
- package/dist/control-runtime.test.d.ts.map +1 -0
- package/dist/control.d.ts +11 -9
- package/dist/control.d.ts.map +1 -0
- package/dist/control.js +2 -2
- package/dist/convergence.d.ts +29 -0
- package/dist/convergence.d.ts.map +1 -0
- package/dist/{cost-ledger-DuSqlw5B.d.ts → cost-ledger.d.ts} +10 -11
- package/dist/cost-ledger.d.ts.map +1 -0
- package/dist/cost-ledger.test.d.ts +2 -0
- package/dist/cost-ledger.test.d.ts.map +1 -0
- package/dist/cost-report.d.ts +43 -0
- package/dist/cost-report.d.ts.map +1 -0
- package/dist/cost-report.test.d.ts +2 -0
- package/dist/cost-report.test.d.ts.map +1 -0
- package/dist/cost-tracker.d.ts +76 -0
- package/dist/cost-tracker.d.ts.map +1 -0
- package/dist/{counterfactual-Dwibr5IW.d.ts → counterfactual.d.ts} +12 -13
- package/dist/counterfactual.d.ts.map +1 -0
- package/dist/cross-trace-diff.d.ts +56 -0
- package/dist/cross-trace-diff.d.ts.map +1 -0
- package/dist/{dataset-BbGkaN2I.d.ts → dataset.d.ts} +11 -14
- package/dist/dataset.d.ts.map +1 -0
- package/dist/deploy-gate-layer.d.ts +125 -0
- package/dist/deploy-gate-layer.d.ts.map +1 -0
- package/dist/deploy-gate-layer.test.d.ts +2 -0
- package/dist/deploy-gate-layer.test.d.ts.map +1 -0
- package/dist/description-length-gate.d.ts +119 -0
- package/dist/description-length-gate.d.ts.map +1 -0
- package/dist/detectors/edge.test.d.ts +2 -0
- package/dist/detectors/edge.test.d.ts.map +1 -0
- package/dist/detectors/index.d.ts +81 -0
- package/dist/detectors/index.d.ts.map +1 -0
- package/dist/detectors/index.test.d.ts +2 -0
- package/dist/detectors/index.test.d.ts.map +1 -0
- package/dist/diagnose/causal-sweep.d.ts +100 -0
- package/dist/diagnose/causal-sweep.d.ts.map +1 -0
- package/dist/diagnose/index.d.ts +36 -0
- package/dist/diagnose/index.d.ts.map +1 -0
- package/dist/diagnose/remediation.d.ts +68 -0
- package/dist/diagnose/remediation.d.ts.map +1 -0
- package/dist/diagnose/repair.d.ts +77 -0
- package/dist/diagnose/repair.d.ts.map +1 -0
- package/dist/diagnose.d.ts +1 -251
- package/dist/diagnose.js +2 -2
- package/dist/discover-personas.d.ts +35 -0
- package/dist/discover-personas.d.ts.map +1 -0
- package/dist/driver.d.ts +95 -0
- package/dist/driver.d.ts.map +1 -0
- package/dist/driver.test.d.ts +8 -0
- package/dist/driver.test.d.ts.map +1 -0
- package/dist/dual-agent-bench.d.ts +81 -0
- package/dist/dual-agent-bench.d.ts.map +1 -0
- package/dist/error-count-extractor.d.ts +47 -0
- package/dist/error-count-extractor.d.ts.map +1 -0
- package/dist/error-count-extractor.test.d.ts +2 -0
- package/dist/error-count-extractor.test.d.ts.map +1 -0
- package/dist/{errors-CzMUYo7b.d.ts → errors.d.ts} +10 -11
- package/dist/errors.d.ts.map +1 -0
- package/dist/{researcher-DE6Gpnb4.d.ts → eval-campaign.d.ts} +34 -156
- package/dist/eval-campaign.d.ts.map +1 -0
- package/dist/eval-campaign.test.d.ts +2 -0
- package/dist/eval-campaign.test.d.ts.map +1 -0
- package/dist/eval-tools.d.ts +55 -0
- package/dist/eval-tools.d.ts.map +1 -0
- package/dist/eval-trace-store.d.ts +107 -0
- package/dist/eval-trace-store.d.ts.map +1 -0
- package/dist/eval-trace-store.test.d.ts +2 -0
- package/dist/eval-trace-store.test.d.ts.map +1 -0
- package/dist/executor.d.ts +38 -0
- package/dist/executor.d.ts.map +1 -0
- package/dist/executor.test.d.ts +10 -0
- package/dist/executor.test.d.ts.map +1 -0
- package/dist/experiment-tracker.d.ts +178 -0
- package/dist/experiment-tracker.d.ts.map +1 -0
- package/dist/experiment-tracker.test.d.ts +2 -0
- package/dist/experiment-tracker.test.d.ts.map +1 -0
- package/dist/failure-taxonomy.d.ts +38 -0
- package/dist/failure-taxonomy.d.ts.map +1 -0
- package/dist/{feedback-trajectory-D9OVLrg9.d.ts → feedback-trajectory.d.ts} +36 -38
- package/dist/feedback-trajectory.d.ts.map +1 -0
- package/dist/feedback-trajectory.test.d.ts +2 -0
- package/dist/feedback-trajectory.test.d.ts.map +1 -0
- package/dist/flow-layer.d.ts +90 -0
- package/dist/flow-layer.d.ts.map +1 -0
- package/dist/flow-layer.test.d.ts +2 -0
- package/dist/flow-layer.test.d.ts.map +1 -0
- package/dist/fuzz/capsule.d.ts +46 -0
- package/dist/fuzz/capsule.d.ts.map +1 -0
- package/dist/fuzz/cube.d.ts +36 -0
- package/dist/fuzz/cube.d.ts.map +1 -0
- package/dist/fuzz/explorer-cost.test.d.ts +2 -0
- package/dist/fuzz/explorer-cost.test.d.ts.map +1 -0
- package/dist/fuzz/explorer.d.ts +64 -0
- package/dist/fuzz/explorer.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.d.ts +16 -0
- package/dist/fuzz/fuzz-agent.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.test.d.ts +2 -0
- package/dist/fuzz/fuzz-agent.test.d.ts.map +1 -0
- package/dist/fuzz/gates.d.ts +33 -0
- package/dist/fuzz/gates.d.ts.map +1 -0
- package/dist/fuzz/index.d.ts +26 -0
- package/dist/fuzz/index.d.ts.map +1 -0
- package/dist/fuzz/policies.d.ts +28 -0
- package/dist/fuzz/policies.d.ts.map +1 -0
- package/dist/fuzz/tools.d.ts +20 -0
- package/dist/fuzz/tools.d.ts.map +1 -0
- package/dist/fuzz/types.d.ts +307 -0
- package/dist/fuzz/types.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -547
- package/dist/golden-matcher.d.ts +71 -0
- package/dist/golden-matcher.d.ts.map +1 -0
- package/dist/governance/eu-ai-act.d.ts +37 -0
- package/dist/governance/eu-ai-act.d.ts.map +1 -0
- package/dist/governance/index.d.ts +5 -135
- package/dist/governance/index.d.ts.map +1 -0
- package/dist/governance/nist-ai-rmf.d.ts +15 -0
- package/dist/governance/nist-ai-rmf.d.ts.map +1 -0
- package/dist/governance/soc2.d.ts +12 -0
- package/dist/governance/soc2.d.ts.map +1 -0
- package/dist/governance/types.d.ts +66 -0
- package/dist/governance/types.d.ts.map +1 -0
- package/dist/harness-optimizer.d.ts +82 -0
- package/dist/harness-optimizer.d.ts.map +1 -0
- package/dist/held-out-gate.d.ts +135 -0
- package/dist/held-out-gate.d.ts.map +1 -0
- package/dist/hosted/client.d.ts +73 -0
- package/dist/hosted/client.d.ts.map +1 -0
- package/dist/hosted/from-env.test.d.ts +8 -0
- package/dist/hosted/from-env.test.d.ts.map +1 -0
- package/dist/hosted/index.d.ts +10 -238
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/types.d.ts +159 -0
- package/dist/hosted/types.d.ts.map +1 -0
- package/dist/index.d.ts +277 -5643
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +317 -108
- package/dist/index.js.map +1 -1
- package/dist/integrity/backend-integrity.d.ts +71 -0
- package/dist/integrity/backend-integrity.d.ts.map +1 -0
- package/dist/integrity/preflight.d.ts +72 -0
- package/dist/integrity/preflight.d.ts.map +1 -0
- package/dist/integrity/preflight.test.d.ts +2 -0
- package/dist/integrity/preflight.test.d.ts.map +1 -0
- package/dist/integrity/single-backend.d.ts +67 -0
- package/dist/integrity/single-backend.d.ts.map +1 -0
- package/dist/intent-match-judge.d.ts +69 -0
- package/dist/intent-match-judge.d.ts.map +1 -0
- package/dist/intent-match-judge.test.d.ts +2 -0
- package/dist/intent-match-judge.test.d.ts.map +1 -0
- package/dist/{judge-calibration-0p2QcWNE.d.ts → judge-calibration.d.ts} +16 -17
- package/dist/judge-calibration.d.ts.map +1 -0
- package/dist/judge-ensemble.d.ts +66 -0
- package/dist/judge-ensemble.d.ts.map +1 -0
- package/dist/judge-ensemble.test.d.ts +8 -0
- package/dist/judge-ensemble.test.d.ts.map +1 -0
- package/dist/judge-families.d.ts +38 -0
- package/dist/judge-families.d.ts.map +1 -0
- package/dist/judge-panel.d.ts +65 -0
- package/dist/judge-panel.d.ts.map +1 -0
- package/dist/judge-retry.d.ts +70 -0
- package/dist/judge-retry.d.ts.map +1 -0
- package/dist/judge-runner.d.ts +36 -0
- package/dist/judge-runner.d.ts.map +1 -0
- package/dist/judge-runner.test.d.ts +2 -0
- package/dist/judge-runner.test.d.ts.map +1 -0
- package/dist/judges.d.ts +74 -0
- package/dist/judges.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.d.ts +89 -0
- package/dist/keyword-coverage-judge.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.test.d.ts +2 -0
- package/dist/keyword-coverage-judge.test.d.ts.map +1 -0
- package/dist/knowledge/index.d.ts +3 -103
- package/dist/knowledge/index.d.ts.map +1 -0
- package/dist/knowledge/readiness.d.ts +26 -0
- package/dist/knowledge/readiness.d.ts.map +1 -0
- package/dist/knowledge/types.d.ts +75 -0
- package/dist/knowledge/types.d.ts.map +1 -0
- package/dist/live-proof.d.ts +62 -0
- package/dist/live-proof.d.ts.map +1 -0
- package/dist/{llm-client-BeEcAokY.d.ts → llm-client.d.ts} +52 -23
- package/dist/llm-client.d.ts.map +1 -0
- package/dist/llm-client.test.d.ts +2 -0
- package/dist/llm-client.test.d.ts.map +1 -0
- package/dist/locked-jsonl-appender.d.ts +19 -0
- package/dist/locked-jsonl-appender.d.ts.map +1 -0
- package/dist/matrix/aggregation.d.ts +16 -0
- package/dist/matrix/aggregation.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +12 -30
- package/dist/matrix/index.d.ts.map +1 -0
- package/dist/matrix/runner.d.ts +15 -0
- package/dist/matrix/runner.d.ts.map +1 -0
- package/dist/{types-mn5Aqk7x.d.ts → matrix/types.d.ts} +11 -15
- package/dist/matrix/types.d.ts.map +1 -0
- package/dist/meta-eval/calibration.d.ts +47 -0
- package/dist/meta-eval/calibration.d.ts.map +1 -0
- package/dist/meta-eval/correlation-study.d.ts +53 -0
- package/dist/meta-eval/correlation-study.d.ts.map +1 -0
- package/dist/meta-eval/index.d.ts +6 -181
- package/dist/meta-eval/index.d.ts.map +1 -0
- package/dist/meta-eval/index.js +1 -1
- package/dist/{outcome-store-rnXLEqSn.d.ts → meta-eval/outcome-store.d.ts} +7 -8
- package/dist/meta-eval/outcome-store.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → meta-eval/rubric-predictive-validity.d.ts} +8 -11
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +1 -0
- package/dist/meta-eval/sentinel.d.ts +169 -0
- package/dist/meta-eval/sentinel.d.ts.map +1 -0
- package/dist/metrics.d.ts +63 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/model-seats.d.ts +71 -0
- package/dist/model-seats.d.ts.map +1 -0
- package/dist/model-seats.test.d.ts +2 -0
- package/dist/model-seats.test.d.ts.map +1 -0
- package/dist/muffled-gate-scanner.d.ts +102 -0
- package/dist/muffled-gate-scanner.d.ts.map +1 -0
- package/dist/{multi-layer-verifier-DUZXrPDA.d.ts → multi-layer-verifier.d.ts} +12 -15
- package/dist/multi-layer-verifier.d.ts.map +1 -0
- package/dist/multi-layer-verifier.test.d.ts +2 -0
- package/dist/multi-layer-verifier.test.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.d.ts +80 -0
- package/dist/multi-toolchain-layer.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.test.d.ts +2 -0
- package/dist/multi-toolchain-layer.test.d.ts.map +1 -0
- package/dist/multishot/default-tools.d.ts +34 -0
- package/dist/multishot/default-tools.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +7 -290
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js.map +1 -1
- package/dist/multishot/judges.d.ts +32 -0
- package/dist/multishot/judges.d.ts.map +1 -0
- package/dist/multishot/matrix.d.ts +107 -0
- package/dist/multishot/matrix.d.ts.map +1 -0
- package/dist/multishot/multishot.d.ts +23 -0
- package/dist/multishot/multishot.d.ts.map +1 -0
- package/dist/multishot/router.d.ts +37 -0
- package/dist/multishot/router.d.ts.map +1 -0
- package/dist/multishot/types.d.ts +60 -0
- package/dist/multishot/types.d.ts.map +1 -0
- package/dist/observability.d.ts +71 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/oracle.d.ts +55 -0
- package/dist/oracle.d.ts.map +1 -0
- package/dist/orthogonality.d.ts +35 -0
- package/dist/orthogonality.d.ts.map +1 -0
- package/dist/otel-pipeline.d.ts +31 -0
- package/dist/otel-pipeline.d.ts.map +1 -0
- package/dist/paraphrase.d.ts +107 -0
- package/dist/paraphrase.d.ts.map +1 -0
- package/dist/{pareto-E-pembql.d.ts → pareto.d.ts} +9 -10
- package/dist/pareto.d.ts.map +1 -0
- package/dist/partition-held-out.d.ts +70 -0
- package/dist/partition-held-out.d.ts.map +1 -0
- package/dist/partition-held-out.test.d.ts +2 -0
- package/dist/partition-held-out.test.d.ts.map +1 -0
- package/dist/perf/index.d.ts +13 -119
- package/dist/perf/index.d.ts.map +1 -0
- package/dist/perf/integrity.d.ts +30 -0
- package/dist/perf/integrity.d.ts.map +1 -0
- package/dist/perf/journey.d.ts +45 -0
- package/dist/perf/journey.d.ts.map +1 -0
- package/dist/perf/ratchet.d.ts +47 -0
- package/dist/perf/ratchet.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.d.ts +31 -0
- package/dist/pipelines/budget-breach.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.test.d.ts +2 -0
- package/dist/pipelines/budget-breach.test.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.d.ts +38 -0
- package/dist/pipelines/failure-cluster.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.test.d.ts +2 -0
- package/dist/pipelines/failure-cluster.test.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.d.ts +26 -0
- package/dist/pipelines/first-divergence.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.test.d.ts +2 -0
- package/dist/pipelines/first-divergence.test.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +8 -173
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +40 -15
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pipelines/judge-agreement.d.ts +26 -0
- package/dist/pipelines/judge-agreement.d.ts.map +1 -0
- package/dist/pipelines/judge-agreement.test.d.ts +2 -0
- package/dist/pipelines/judge-agreement.test.d.ts.map +1 -0
- package/dist/pipelines/regression.d.ts +23 -0
- package/dist/pipelines/regression.d.ts.map +1 -0
- package/dist/pipelines/regression.test.d.ts +2 -0
- package/dist/pipelines/regression.test.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.d.ts +32 -0
- package/dist/pipelines/stuck-loop.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.test.d.ts +2 -0
- package/dist/pipelines/stuck-loop.test.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.d.ts +34 -0
- package/dist/pipelines/tool-waste.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.test.d.ts +2 -0
- package/dist/pipelines/tool-waste.test.d.ts.map +1 -0
- package/dist/playbook.d.ts +16 -0
- package/dist/playbook.d.ts.map +1 -0
- package/dist/pr-review-benchmark.d.ts +88 -0
- package/dist/pr-review-benchmark.d.ts.map +1 -0
- package/dist/pr-review-benchmark.test.d.ts +2 -0
- package/dist/pr-review-benchmark.test.d.ts.map +1 -0
- package/dist/pre-registration.d.ts +125 -0
- package/dist/pre-registration.d.ts.map +1 -0
- package/dist/prm/builtin-rubrics.d.ts +33 -0
- package/dist/prm/builtin-rubrics.d.ts.map +1 -0
- package/dist/prm/index.d.ts +5 -100
- package/dist/prm/index.d.ts.map +1 -0
- package/dist/prm/index.js +55 -12
- package/dist/prm/index.js.map +1 -1
- package/dist/prm/inference.d.ts +29 -0
- package/dist/prm/inference.d.ts.map +1 -0
- package/dist/prm/inference.test.d.ts +2 -0
- package/dist/prm/inference.test.d.ts.map +1 -0
- package/dist/{rubric-BOfxn4ja.d.ts → prm/rubric.d.ts} +10 -13
- package/dist/prm/rubric.d.ts.map +1 -0
- package/dist/prm/training-export.d.ts +38 -0
- package/dist/prm/training-export.d.ts.map +1 -0
- package/dist/produced-state.d.ts +63 -0
- package/dist/produced-state.d.ts.map +1 -0
- package/dist/produced-state.test.d.ts +8 -0
- package/dist/produced-state.test.d.ts.map +1 -0
- package/dist/profile/baselines.d.ts +37 -0
- package/dist/profile/baselines.d.ts.map +1 -0
- package/dist/profile/index.d.ts +105 -0
- package/dist/profile/index.d.ts.map +1 -0
- package/dist/promotion-gate.d.ts +93 -0
- package/dist/promotion-gate.d.ts.map +1 -0
- package/dist/prompt-registry.d.ts +41 -0
- package/dist/prompt-registry.d.ts.map +1 -0
- package/dist/propose-review-control.d.ts +49 -0
- package/dist/propose-review-control.d.ts.map +1 -0
- package/dist/propose-review-control.test.d.ts +2 -0
- package/dist/propose-review-control.test.d.ts.map +1 -0
- package/dist/propose-review.d.ts +155 -0
- package/dist/propose-review.d.ts.map +1 -0
- package/dist/{red-team-BXHil6c8.d.ts → red-team.d.ts} +14 -16
- package/dist/red-team.d.ts.map +1 -0
- package/dist/reference-replay-steering.d.ts +11 -0
- package/dist/reference-replay-steering.d.ts.map +1 -0
- package/dist/reference-replay.d.ts +176 -0
- package/dist/reference-replay.d.ts.map +1 -0
- package/dist/reflective-mutation.d.ts +79 -0
- package/dist/reflective-mutation.d.ts.map +1 -0
- package/dist/registry.d.ts +31 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/release-confidence.d.ts +128 -0
- package/dist/release-confidence.d.ts.map +1 -0
- package/dist/release-report.d.ts +11 -0
- package/dist/release-report.d.ts.map +1 -0
- package/dist/replay.d.ts +120 -0
- package/dist/replay.d.ts.map +1 -0
- package/dist/reporter.d.ts +14 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporting.d.ts +15 -15
- package/dist/reporting.d.ts.map +1 -0
- package/dist/reporting.js +3 -3
- package/dist/researcher.d.ts +140 -0
- package/dist/researcher.d.ts.map +1 -0
- package/dist/reviewer.d.ts +118 -0
- package/dist/reviewer.d.ts.map +1 -0
- package/dist/reviewer.test.d.ts +2 -0
- package/dist/reviewer.test.d.ts.map +1 -0
- package/dist/reward-model-export.d.ts +60 -0
- package/dist/reward-model-export.d.ts.map +1 -0
- package/dist/rl/active-curriculum.d.ts +110 -0
- package/dist/rl/active-curriculum.d.ts.map +1 -0
- package/dist/rl/adaptation-eval.d.ts +109 -0
- package/dist/rl/adaptation-eval.d.ts.map +1 -0
- package/dist/{adversarial-DIVcDoI_.d.ts → rl/adversarial.d.ts} +6 -7
- package/dist/rl/adversarial.d.ts.map +1 -0
- package/dist/rl/compute-curves.d.ts +127 -0
- package/dist/rl/compute-curves.d.ts.map +1 -0
- package/dist/rl/contamination.d.ts +117 -0
- package/dist/rl/contamination.d.ts.map +1 -0
- package/dist/rl/corpus.d.ts +55 -0
- package/dist/rl/corpus.d.ts.map +1 -0
- package/dist/rl/corpus.test.d.ts +2 -0
- package/dist/rl/corpus.test.d.ts.map +1 -0
- package/dist/rl/dataset.d.ts +102 -0
- package/dist/rl/dataset.d.ts.map +1 -0
- package/dist/rl/dataset.test.d.ts +2 -0
- package/dist/rl/dataset.test.d.ts.map +1 -0
- package/dist/rl/exporters.d.ts +141 -0
- package/dist/rl/exporters.d.ts.map +1 -0
- package/dist/rl/index.d.ts +49 -0
- package/dist/rl/index.d.ts.map +1 -0
- package/dist/{off-policy-DiwuKKg7.d.ts → rl/off-policy.d.ts} +8 -9
- package/dist/rl/off-policy.d.ts.map +1 -0
- package/dist/rl/predictive-validity-researcher.d.ts +69 -0
- package/dist/rl/predictive-validity-researcher.d.ts.map +1 -0
- package/dist/rl/preferences.d.ts +141 -0
- package/dist/rl/preferences.d.ts.map +1 -0
- package/dist/rl/process-reward.d.ts +122 -0
- package/dist/rl/process-reward.d.ts.map +1 -0
- package/dist/rl/reward-hacking.d.ts +104 -0
- package/dist/rl/reward-hacking.d.ts.map +1 -0
- package/dist/rl/rl-campaign.d.ts +85 -0
- package/dist/rl/rl-campaign.d.ts.map +1 -0
- package/dist/rl/run-record-adapters.d.ts +56 -0
- package/dist/rl/run-record-adapters.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.d.ts +166 -0
- package/dist/rl/sim-fidelity.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.test.d.ts +2 -0
- package/dist/rl/sim-fidelity.test.d.ts.map +1 -0
- package/dist/rl/tournament.d.ts +115 -0
- package/dist/rl/tournament.d.ts.map +1 -0
- package/dist/rl/verifiable-reward.d.ts +124 -0
- package/dist/rl/verifiable-reward.d.ts.map +1 -0
- package/dist/rl.d.ts +1 -1192
- package/dist/rl.js +615 -615
- package/dist/rl.js.map +1 -1
- package/dist/{run-campaign-RDGAM5KJ.js → run-campaign-WXY7KI67.js} +3 -3
- package/dist/run-critic.d.ts +23 -0
- package/dist/run-critic.d.ts.map +1 -0
- package/dist/run-evidence.d.ts +32 -0
- package/dist/run-evidence.d.ts.map +1 -0
- package/dist/{run-record-e7vj1uZQ.d.ts → run-record.d.ts} +16 -122
- package/dist/run-record.d.ts.map +1 -0
- package/dist/run-record.test.d.ts +2 -0
- package/dist/run-record.test.d.ts.map +1 -0
- package/dist/run-score.d.ts +31 -0
- package/dist/run-score.d.ts.map +1 -0
- package/dist/runtime-trajectory.d.ts +47 -0
- package/dist/runtime-trajectory.d.ts.map +1 -0
- package/dist/{test-graded-scenario-BdVaPyHT.d.ts → sandbox-harness.d.ts} +34 -60
- package/dist/sandbox-harness.d.ts.map +1 -0
- package/dist/sandbox-harness.test.d.ts +2 -0
- package/dist/sandbox-harness.test.d.ts.map +1 -0
- package/dist/sandbox-pool.d.ts +74 -0
- package/dist/sandbox-pool.d.ts.map +1 -0
- package/dist/sandbox-pool.test.d.ts +2 -0
- package/dist/sandbox-pool.test.d.ts.map +1 -0
- package/dist/scorecard.d.ts +133 -0
- package/dist/scorecard.d.ts.map +1 -0
- package/dist/scorecard.test.d.ts +2 -0
- package/dist/scorecard.test.d.ts.map +1 -0
- package/dist/self-play.d.ts +69 -0
- package/dist/self-play.d.ts.map +1 -0
- package/dist/semantic-concept-judge.d.ts +135 -0
- package/dist/semantic-concept-judge.d.ts.map +1 -0
- package/dist/semantic-concept-judge.test.d.ts +2 -0
- package/dist/semantic-concept-judge.test.d.ts.map +1 -0
- package/dist/{sequential-5iSVfzl2.d.ts → sequential.d.ts} +9 -10
- package/dist/sequential.d.ts.map +1 -0
- package/dist/{series-convergence-D5OWMBg6.d.ts → series-convergence.d.ts} +4 -5
- package/dist/series-convergence.d.ts.map +1 -0
- package/dist/slo.d.ts +48 -0
- package/dist/slo.d.ts.map +1 -0
- package/dist/state-continuity.d.ts +47 -0
- package/dist/state-continuity.d.ts.map +1 -0
- package/dist/{statistics-C7PozGrZ.d.ts → statistics.d.ts} +152 -39
- package/dist/statistics.d.ts.map +1 -0
- package/dist/statistics.test.d.ts +2 -0
- package/dist/statistics.test.d.ts.map +1 -0
- package/dist/steering-optimizer.d.ts +58 -0
- package/dist/steering-optimizer.d.ts.map +1 -0
- package/dist/steering.d.ts +24 -0
- package/dist/steering.d.ts.map +1 -0
- package/dist/storyboard/code-edit.d.ts +64 -0
- package/dist/storyboard/code-edit.d.ts.map +1 -0
- package/dist/storyboard/code-edit.test.d.ts +2 -0
- package/dist/storyboard/code-edit.test.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +16 -81
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.test.d.ts +2 -0
- package/dist/storyboard/index.test.d.ts.map +1 -0
- package/dist/{summary-report-DGmUucwQ.d.ts → summary-report.d.ts} +23 -160
- package/dist/summary-report.d.ts.map +1 -0
- package/dist/telemetry/client.d.ts +35 -0
- package/dist/telemetry/client.d.ts.map +1 -0
- package/dist/telemetry/index.d.ts +17 -35
- package/dist/telemetry/index.d.ts.map +1 -0
- package/dist/telemetry/schema.d.ts +61 -0
- package/dist/telemetry/schema.d.ts.map +1 -0
- package/dist/telemetry/sink-fetch.d.ts +39 -0
- package/dist/telemetry/sink-fetch.d.ts.map +1 -0
- package/dist/telemetry/{file.d.ts → sink-file.d.ts} +5 -7
- package/dist/telemetry/sink-file.d.ts.map +1 -0
- package/dist/test-graded-scenario.d.ts +42 -0
- package/dist/test-graded-scenario.d.ts.map +1 -0
- package/dist/testing.d.ts +5 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +8 -0
- package/dist/testing.js.map +1 -0
- package/dist/tool-use-metrics.d.ts +35 -0
- package/dist/tool-use-metrics.d.ts.map +1 -0
- package/dist/trace/capture-fetch.d.ts +48 -0
- package/dist/trace/capture-fetch.d.ts.map +1 -0
- package/dist/trace/capture-fetch.test.d.ts +2 -0
- package/dist/trace/capture-fetch.test.d.ts.map +1 -0
- package/dist/{emitter-DEZwY14K.d.ts → trace/emitter.d.ts} +9 -12
- package/dist/trace/emitter.d.ts.map +1 -0
- package/dist/trace/extract-usage.d.ts +46 -0
- package/dist/trace/extract-usage.d.ts.map +1 -0
- package/dist/trace/extract-usage.test.d.ts +2 -0
- package/dist/trace/extract-usage.test.d.ts.map +1 -0
- package/dist/trace/index.d.ts +15 -0
- package/dist/trace/index.d.ts.map +1 -0
- package/dist/{integrity-VJ9A7aST.d.ts → trace/integrity.d.ts} +11 -14
- package/dist/trace/integrity.d.ts.map +1 -0
- package/dist/trace/otel-bridge.d.ts +29 -0
- package/dist/trace/otel-bridge.d.ts.map +1 -0
- package/dist/trace/otel-export.d.ts +52 -0
- package/dist/trace/otel-export.d.ts.map +1 -0
- package/dist/trace/otel.d.ts +57 -0
- package/dist/trace/otel.d.ts.map +1 -0
- package/dist/trace/otlp-attributes.d.ts +17 -0
- package/dist/trace/otlp-attributes.d.ts.map +1 -0
- package/dist/trace/query.d.ts +29 -0
- package/dist/trace/query.d.ts.map +1 -0
- package/dist/trace/query.test.d.ts +2 -0
- package/dist/trace/query.test.d.ts.map +1 -0
- package/dist/{raw-provider-sink-C46HDghv.d.ts → trace/raw-provider-sink.d.ts} +13 -14
- package/dist/trace/raw-provider-sink.d.ts.map +1 -0
- package/dist/{redact-B40YG2M_.d.ts → trace/redact.d.ts} +7 -8
- package/dist/trace/redact.d.ts.map +1 -0
- package/dist/{schema-m0gsnbt3.d.ts → trace/schema.d.ts} +29 -30
- package/dist/trace/schema.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.d.ts +72 -0
- package/dist/trace/store-to-otlp.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.test.d.ts +2 -0
- package/dist/trace/store-to-otlp.test.d.ts.map +1 -0
- package/dist/{store-CKUAgsJz.d.ts → trace/store.d.ts} +37 -13
- package/dist/trace/store.d.ts.map +1 -0
- package/dist/trace/store.test.d.ts +2 -0
- package/dist/trace/store.test.d.ts.map +1 -0
- package/dist/{analyst-C8HHvfJp.d.ts → trace-analyst/analyst.d.ts} +12 -14
- package/dist/trace-analyst/analyst.d.ts.map +1 -0
- package/dist/trace-analyst/analyst.test.d.ts +2 -0
- package/dist/trace-analyst/analyst.test.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts +40 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +2 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +1 -0
- package/dist/trace-analyst/hook.d.ts +55 -0
- package/dist/trace-analyst/hook.d.ts.map +1 -0
- package/dist/trace-analyst/index.d.ts +18 -0
- package/dist/trace-analyst/index.d.ts.map +1 -0
- package/dist/trace-analyst/insights.d.ts +71 -0
- package/dist/trace-analyst/insights.d.ts.map +1 -0
- package/dist/trace-analyst/insights.test.d.ts +2 -0
- package/dist/trace-analyst/insights.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-flatten.d.ts +42 -0
- package/dist/trace-analyst/otlp-flatten.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.d.ts +85 -0
- package/dist/trace-analyst/otlp-span.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-span.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts +115 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/prompts.d.ts +6 -0
- package/dist/trace-analyst/prompts.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.d.ts +126 -0
- package/dist/trace-analyst/store-otlp.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.test.d.ts +8 -0
- package/dist/trace-analyst/store-otlp.test.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/store.d.ts +63 -0
- package/dist/trace-analyst/store.d.ts.map +1 -0
- package/dist/trace-analyst/tools.d.ts +44 -0
- package/dist/trace-analyst/tools.d.ts.map +1 -0
- package/dist/trace-analyst/tools.test.d.ts +10 -0
- package/dist/trace-analyst/tools.test.d.ts.map +1 -0
- package/dist/{store-C1YxJDEK.d.ts → trace-analyst/types.d.ts} +18 -74
- package/dist/trace-analyst/types.d.ts.map +1 -0
- package/dist/trace-contracts.d.ts +180 -0
- package/dist/trace-contracts.d.ts.map +1 -0
- package/dist/traced-analyst.d.ts +26 -0
- package/dist/traced-analyst.d.ts.map +1 -0
- package/dist/traced-judges.d.ts +27 -0
- package/dist/traced-judges.d.ts.map +1 -0
- package/dist/traces.d.ts +4 -947
- package/dist/traces.d.ts.map +1 -0
- package/dist/traces.js +42 -12
- package/dist/{trajectory-GEdXJCL5.d.ts → trajectory.d.ts} +8 -9
- package/dist/trajectory.d.ts.map +1 -0
- package/dist/{types-Croy5h7V.d.ts → types.d.ts} +33 -33
- package/dist/types.d.ts.map +1 -0
- package/dist/ui-finding.d.ts +104 -0
- package/dist/ui-finding.d.ts.map +1 -0
- package/dist/verdict-cache.d.ts +78 -0
- package/dist/verdict-cache.d.ts.map +1 -0
- package/dist/verdict-cache.test.d.ts +2 -0
- package/dist/verdict-cache.test.d.ts.map +1 -0
- package/dist/{verdict-C9MlYujm.d.ts → verdict.d.ts} +2 -3
- package/dist/verdict.d.ts.map +1 -0
- package/dist/visual-diff.d.ts +32 -0
- package/dist/visual-diff.d.ts.map +1 -0
- package/dist/wire/handlers.d.ts +54 -0
- package/dist/wire/handlers.d.ts.map +1 -0
- package/dist/wire/index.d.ts +13 -570
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -2
- package/dist/wire/openapi.d.ts +3 -0
- package/dist/wire/openapi.d.ts.map +1 -0
- package/dist/wire/rpc.d.ts +21 -0
- package/dist/wire/rpc.d.ts.map +1 -0
- package/dist/wire/rubrics.d.ts +34 -0
- package/dist/wire/rubrics.d.ts.map +1 -0
- package/dist/wire/schemas.d.ts +410 -0
- package/dist/wire/schemas.d.ts.map +1 -0
- package/dist/wire/server.d.ts +60 -0
- package/dist/wire/server.d.ts.map +1 -0
- package/dist/workflow/event-schema.d.ts +5 -0
- package/dist/workflow/event-schema.d.ts.map +1 -0
- package/dist/workflow/feedback-pack.d.ts +99 -0
- package/dist/workflow/feedback-pack.d.ts.map +1 -0
- package/dist/workflow/index.d.ts +22 -495
- package/dist/workflow/index.d.ts.map +1 -0
- package/dist/workflow/index.js +2 -2
- package/dist/workflow/intelligence-export.d.ts +62 -0
- package/dist/workflow/intelligence-export.d.ts.map +1 -0
- package/dist/workflow/partner-report.d.ts +49 -0
- package/dist/workflow/partner-report.d.ts.map +1 -0
- package/dist/workflow/phase-graph.d.ts +43 -0
- package/dist/workflow/phase-graph.d.ts.map +1 -0
- package/dist/workflow/promotion-gate.d.ts +61 -0
- package/dist/workflow/promotion-gate.d.ts.map +1 -0
- package/dist/workflow/run-record.d.ts +12 -0
- package/dist/workflow/run-record.d.ts.map +1 -0
- package/dist/workflow/runtime-adapter.d.ts +20 -0
- package/dist/workflow/runtime-adapter.d.ts.map +1 -0
- package/dist/workflow/sanitize.d.ts +21 -0
- package/dist/workflow/sanitize.d.ts.map +1 -0
- package/dist/workflow/schema.d.ts +5 -0
- package/dist/workflow/schema.d.ts.map +1 -0
- package/dist/workflow/summary.d.ts +43 -0
- package/dist/workflow/summary.d.ts.map +1 -0
- package/dist/workflow/trace-event-fields.d.ts +6 -0
- package/dist/workflow/trace-event-fields.d.ts.map +1 -0
- package/dist/workflow/trajectory.d.ts +15 -0
- package/dist/workflow/trajectory.d.ts.map +1 -0
- package/dist/workflow/types.d.ts +68 -0
- package/dist/workflow/types.d.ts.map +1 -0
- package/dist/workspace-inspector.d.ts +67 -0
- package/dist/workspace-inspector.d.ts.map +1 -0
- package/dist/wrangler-deploy-runner.test.d.ts +2 -0
- package/dist/wrangler-deploy-runner.test.d.ts.map +1 -0
- package/docs/campaign-proposers.md +170 -0
- package/docs/concepts.md +8 -4
- package/docs/customer-journeys.md +15 -13
- package/docs/design/loop-taxonomy.md +34 -66
- package/docs/distributed-driver.md +14 -14
- package/docs/feature-guide.md +1 -1
- package/docs/hosted-ingest-spec.md +2 -3
- package/docs/multi-shot-optimization.md +8 -8
- package/docs/product-eval-adoption.md +1 -1
- package/docs/self-improvement-map.md +33 -29
- package/package.json +9 -15
- package/dist/calibration-Cpr3WaX3.d.ts +0 -101
- package/dist/chunk-47X6LRCE.js.map +0 -1
- package/dist/chunk-6SOJM3VR.js.map +0 -1
- package/dist/chunk-CVVHBFGN.js.map +0 -1
- package/dist/chunk-D3V5B42D.js.map +0 -1
- package/dist/chunk-E4GH6USR.js.map +0 -1
- package/dist/chunk-GSH6QNNS.js.map +0 -1
- package/dist/chunk-KWRRMR3J.js.map +0 -1
- package/dist/chunk-LMZQ2Z4U.js.map +0 -1
- package/dist/chunk-QAY5UIJO.js.map +0 -1
- package/dist/chunk-QG2OVF2D.js.map +0 -1
- package/dist/chunk-T375SUOZ.js.map +0 -1
- package/dist/chunk-TWS7AZEY.js.map +0 -1
- package/dist/chunk-UMMZHCPB.js.map +0 -1
- package/dist/chunk-XY4DDNEG.js.map +0 -1
- package/dist/chunk-ZFIBGEOL.js.map +0 -1
- package/dist/control-_Qb7skHX.d.ts +0 -259
- package/dist/corpus-BoR-041R.d.ts +0 -560
- package/dist/failure-cluster-CL7IVgkJ.d.ts +0 -76
- package/dist/harness-optimizer-EnEnQPsr.d.ts +0 -106
- package/dist/index-Bx3gZ8xl.d.ts +0 -159
- package/dist/pre-registration-mAnCugl9.d.ts +0 -523
- package/dist/provenance-LnqRT0sS.d.ts +0 -441
- package/dist/query-CqTxMwDw.d.ts +0 -31
- package/dist/release-report-euXIV_Sk.d.ts +0 -233
- package/dist/run-critic-BAIjX99r.d.ts +0 -56
- package/dist/run-improvement-loop-5z_l5zDz.d.ts +0 -427
- package/dist/runtime-trajectory-BDgfGZSr.d.ts +0 -49
- package/dist/semantic-concept-judge-Dn8Z6KEG.d.ts +0 -624
- package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
- package/docs/design/external-agent-wedge.md +0 -89
- package/docs/design/phase-d-rfc.md +0 -125
- package/docs/design/phase4-consumer-migration.md +0 -70
- package/docs/design/primitives-integration-spec.md +0 -393
- package/docs/design/product-self-improvement-loop.md +0 -146
- package/docs/design/self-improvement-engine.md +0 -140
- package/docs/design/self-improvement-protocol.md +0 -223
- package/docs/design/self-improvement-roadmap.md +0 -106
- package/docs/design/substrate-gaps.md +0 -118
- package/docs/phase-b-pairing-kit.md +0 -188
- package/docs/phase-b-runbook.md +0 -176
- package/docs/pilot/README.md +0 -62
- package/docs/pilot/customer-checklist.md +0 -90
- package/docs/pilot/integration-foreign-stack.md +0 -296
- package/docs/pilot/integration-tangle-stack.md +0 -248
- package/docs/pilot/one-pager.md +0 -161
- package/docs/pilot/sample-insight-report.json +0 -172
- package/docs/quickstart-external.md +0 -229
- package/docs/research/belief-state-agent-eval-roadmap.md +0 -593
- package/docs/research/research-roadmap.md +0 -205
- package/docs/specs/driver-honest-spec.md +0 -251
- package/docs/specs/hermes-self-improvement-audit.md +0 -93
- package/docs/specs/profile-versioning.md +0 -291
- package/docs/three-package-architecture.md +0 -168
- /package/dist/{chunk-QS3RBQPI.js.map → chunk-D5KBVULY.js.map} +0 -0
- /package/dist/{chunk-UHMJT4T7.js.map → chunk-IZCEK2HR.js.map} +0 -0
- /package/dist/{chunk-L3JOU6XM.js.map → chunk-NACM5RS7.js.map} +0 -0
- /package/dist/{chunk-4FBZZIYD.js.map → chunk-QTMYW64F.js.map} +0 -0
- /package/dist/{chunk-CY6U5S3X.js.map → chunk-URWVKRCS.js.map} +0 -0
- /package/dist/{chunk-Y47J2LJ3.js.map → chunk-Z6L6YSU6.js.map} +0 -0
- /package/dist/{run-campaign-RDGAM5KJ.js.map → run-campaign-WXY7KI67.js.map} +0 -0
|
@@ -1,16 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
|
|
1
|
+
import { type ContinuousAgreement, type ContinuousAgreementOptions } from './judge-calibration';
|
|
2
|
+
import type { JudgeScore } from './types';
|
|
4
3
|
/** Identity: dimensions already follow "higher = better" by prompt convention
|
|
5
4
|
* (inverted dims like hallucination are scored 10 = best at the source). */
|
|
6
|
-
declare const normalizeScores: (scores: JudgeScore[]) => JudgeScore[];
|
|
5
|
+
export declare const normalizeScores: (scores: JudgeScore[]) => JudgeScore[];
|
|
7
6
|
/** Weighted mean — falls back to uniform weights when omitted */
|
|
8
|
-
declare function weightedMean(scores: {
|
|
7
|
+
export declare function weightedMean(scores: {
|
|
9
8
|
score: number;
|
|
10
9
|
weight?: number;
|
|
11
10
|
}[]): number;
|
|
12
11
|
/** Bootstrap confidence interval */
|
|
13
|
-
declare function confidenceInterval(scores: number[], confidence?: number, opts?: {
|
|
12
|
+
export declare function confidenceInterval(scores: number[], confidence?: number, opts?: {
|
|
14
13
|
seed?: number;
|
|
15
14
|
resamples?: number;
|
|
16
15
|
}): {
|
|
@@ -24,24 +23,24 @@ declare function confidenceInterval(scores: number[], confidence?: number, opts?
|
|
|
24
23
|
* Each inner array is one judge's scores for all items.
|
|
25
24
|
* All arrays must have the same length (same items scored).
|
|
26
25
|
*/
|
|
27
|
-
declare function interRaterReliability(judgeScores: JudgeScore[][]): number;
|
|
26
|
+
export declare function interRaterReliability(judgeScores: JudgeScore[][]): number;
|
|
28
27
|
/**
|
|
29
28
|
* Mann-Whitney U test for comparing two independent groups.
|
|
30
29
|
* Returns U statistic and approximate p-value (normal approximation).
|
|
31
30
|
*/
|
|
32
|
-
declare function mannWhitneyU(a: number[], b: number[]): {
|
|
31
|
+
export declare function mannWhitneyU(a: number[], b: number[]): {
|
|
33
32
|
u: number;
|
|
34
33
|
p: number;
|
|
35
34
|
};
|
|
36
35
|
/** Partial credit: returns 0-1 ratio of current toward target */
|
|
37
|
-
declare function partialCredit(current: number, target: number): number;
|
|
36
|
+
export declare function partialCredit(current: number, target: number): number;
|
|
38
37
|
/**
|
|
39
38
|
* Paired t-test — before/after measurements on the SAME items.
|
|
40
39
|
* Pairing removes inter-item variance, giving tighter significance than
|
|
41
40
|
* an unpaired test when comparing prompt v1 vs prompt v2 on identical
|
|
42
41
|
* scenarios.
|
|
43
42
|
*/
|
|
44
|
-
declare function pairedTTest(before: number[], after: number[]): {
|
|
43
|
+
export declare function pairedTTest(before: number[], after: number[]): {
|
|
45
44
|
t: number;
|
|
46
45
|
df: number;
|
|
47
46
|
p: number;
|
|
@@ -50,7 +49,7 @@ declare function pairedTTest(before: number[], after: number[]): {
|
|
|
50
49
|
* Wilcoxon signed-rank test — paired non-parametric alternative.
|
|
51
50
|
* Use when the differences aren't normally distributed.
|
|
52
51
|
*/
|
|
53
|
-
declare function wilcoxonSignedRank(before: number[], after: number[]): {
|
|
52
|
+
export declare function wilcoxonSignedRank(before: number[], after: number[]): {
|
|
54
53
|
w: number;
|
|
55
54
|
p: number;
|
|
56
55
|
};
|
|
@@ -59,8 +58,8 @@ declare function wilcoxonSignedRank(before: number[], after: number[]): {
|
|
|
59
58
|
* Positive d means group b has higher mean than group a.
|
|
60
59
|
* Rule of thumb: |d| < 0.2 negligible, 0.2–0.5 small, 0.5–0.8 medium, > 0.8 large.
|
|
61
60
|
*/
|
|
62
|
-
declare function cohensD(a: number[], b: number[]): number;
|
|
63
|
-
type CliffsMagnitude = 'negligible' | 'small' | 'medium' | 'large';
|
|
61
|
+
export declare function cohensD(a: number[], b: number[]): number;
|
|
62
|
+
export type CliffsMagnitude = 'negligible' | 'small' | 'medium' | 'large';
|
|
64
63
|
/**
|
|
65
64
|
* Cliff's delta — a non-parametric effect size for two independent samples.
|
|
66
65
|
* `δ = (#(after > before) − #(after < before)) / (n_before · n_after)`,
|
|
@@ -71,14 +70,14 @@ type CliffsMagnitude = 'negligible' | 'small' | 'medium' | 'large';
|
|
|
71
70
|
* `pairedBootstrap` / `wilcoxonSignedRank` for the non-parametric reporting
|
|
72
71
|
* path. Returns 0 when either sample is empty.
|
|
73
72
|
*/
|
|
74
|
-
declare function cliffsDelta(before: number[], after: number[]): number;
|
|
73
|
+
export declare function cliffsDelta(before: number[], after: number[]): number;
|
|
75
74
|
/**
|
|
76
75
|
* Map a Cliff's delta to a qualitative magnitude using the standard
|
|
77
76
|
* Romano et al. thresholds (|δ|): <0.147 negligible, <0.33 small,
|
|
78
77
|
* <0.474 medium, else large.
|
|
79
78
|
*/
|
|
80
|
-
declare function interpretCliffs(delta: number): CliffsMagnitude;
|
|
81
|
-
interface WeightedCompositeInput {
|
|
79
|
+
export declare function interpretCliffs(delta: number): CliffsMagnitude;
|
|
80
|
+
export interface WeightedCompositeInput {
|
|
82
81
|
/** Per-dimension scores (typically 0..1). */
|
|
83
82
|
dims: Record<string, number>;
|
|
84
83
|
/** Weight per dimension. Every weighted dimension MUST be present in
|
|
@@ -89,7 +88,7 @@ interface WeightedCompositeInput {
|
|
|
89
88
|
/** Optional pass threshold; when set, the result reports `pass`. */
|
|
90
89
|
threshold?: number;
|
|
91
90
|
}
|
|
92
|
-
interface WeightedCompositeResult {
|
|
91
|
+
export interface WeightedCompositeResult {
|
|
93
92
|
composite: number;
|
|
94
93
|
pass?: boolean;
|
|
95
94
|
}
|
|
@@ -102,8 +101,8 @@ interface WeightedCompositeResult {
|
|
|
102
101
|
* weight is negative, or if the weights sum to 0 — none of which can produce
|
|
103
102
|
* a meaningful composite.
|
|
104
103
|
*/
|
|
105
|
-
declare function weightedComposite(input: WeightedCompositeInput): WeightedCompositeResult;
|
|
106
|
-
interface CorpusScoreRecord {
|
|
104
|
+
export declare function weightedComposite(input: WeightedCompositeInput): WeightedCompositeResult;
|
|
105
|
+
export interface CorpusScoreRecord {
|
|
107
106
|
/** Stable identifier for the rated item (scenario, span, turn, …). */
|
|
108
107
|
itemId: string;
|
|
109
108
|
/** Identifier for the judge that produced this score. */
|
|
@@ -113,14 +112,14 @@ interface CorpusScoreRecord {
|
|
|
113
112
|
/** Numeric score; must be finite. */
|
|
114
113
|
score: number;
|
|
115
114
|
}
|
|
116
|
-
interface CorpusAgreementPerDimension extends ContinuousAgreement {
|
|
115
|
+
export interface CorpusAgreementPerDimension extends ContinuousAgreement {
|
|
117
116
|
dimension: string;
|
|
118
117
|
/** Item IDs that contributed to this dimension's matrix (every judge scored them). */
|
|
119
118
|
itemIds: string[];
|
|
120
119
|
/** Judge IDs that contributed to this dimension's matrix. */
|
|
121
120
|
judgeIds: string[];
|
|
122
121
|
}
|
|
123
|
-
interface CorpusAgreementReport {
|
|
122
|
+
export interface CorpusAgreementReport {
|
|
124
123
|
/** Per-dimension ICC(2,1) + κ_w + Pearson + Spearman + bootstrap CIs. */
|
|
125
124
|
perDimension: CorpusAgreementPerDimension[];
|
|
126
125
|
/** Mean ICC across dimensions (NaN if no dimension yielded a finite ICC). */
|
|
@@ -132,7 +131,7 @@ interface CorpusAgreementReport {
|
|
|
132
131
|
/** Judges seen across the corpus (sorted). */
|
|
133
132
|
judgeIds: string[];
|
|
134
133
|
}
|
|
135
|
-
interface CorpusAgreementOptions extends ContinuousAgreementOptions {
|
|
134
|
+
export interface CorpusAgreementOptions extends ContinuousAgreementOptions {
|
|
136
135
|
/**
|
|
137
136
|
* Restrict the audit to these dimensions. Default = every dimension
|
|
138
137
|
* that appears in the input. A dimension named here but absent from
|
|
@@ -162,7 +161,7 @@ interface CorpusAgreementOptions extends ContinuousAgreementOptions {
|
|
|
162
161
|
* another dimension throws (would silently shrink the matrix).
|
|
163
162
|
* - Duplicate (itemId, judgeName, dimension) records throw.
|
|
164
163
|
*/
|
|
165
|
-
declare function corpusInterRaterAgreement(records: CorpusScoreRecord[], opts?: CorpusAgreementOptions): CorpusAgreementReport;
|
|
164
|
+
export declare function corpusInterRaterAgreement(records: CorpusScoreRecord[], opts?: CorpusAgreementOptions): CorpusAgreementReport;
|
|
166
165
|
/**
|
|
167
166
|
* Convenience adapter for `JudgeScore[]` data keyed externally by item.
|
|
168
167
|
*
|
|
@@ -171,7 +170,7 @@ declare function corpusInterRaterAgreement(records: CorpusScoreRecord[], opts?:
|
|
|
171
170
|
* agreement without manually flattening. `itemId` must be unique per
|
|
172
171
|
* row of `itemsScores`.
|
|
173
172
|
*/
|
|
174
|
-
declare function corpusInterRaterAgreementFromJudgeScores(itemsScores: Array<{
|
|
173
|
+
export declare function corpusInterRaterAgreementFromJudgeScores(itemsScores: Array<{
|
|
175
174
|
itemId: string;
|
|
176
175
|
scores: JudgeScore[];
|
|
177
176
|
}>, opts?: CorpusAgreementOptions): CorpusAgreementReport;
|
|
@@ -181,7 +180,7 @@ declare function corpusInterRaterAgreementFromJudgeScores(itemsScores: Array<{
|
|
|
181
180
|
* n = 2 * ( (z_{1-α/2} + z_{1-β}) / d )^2
|
|
182
181
|
* where d is Cohen's d. Returns Infinity for effect ≤ 0.
|
|
183
182
|
*/
|
|
184
|
-
declare function requiredSampleSize(opts: {
|
|
183
|
+
export declare function requiredSampleSize(opts: {
|
|
185
184
|
effect: number;
|
|
186
185
|
alpha?: number;
|
|
187
186
|
power?: number;
|
|
@@ -193,14 +192,47 @@ declare function requiredSampleSize(opts: {
|
|
|
193
192
|
* sd(deltas) for score units; treat as a lower bound — Wilcoxon and bootstrap
|
|
194
193
|
* have asymptotic relative efficiency below 1 vs the t-test on heavy tails.
|
|
195
194
|
*/
|
|
196
|
-
declare function pairedMde(opts: {
|
|
195
|
+
export declare function pairedMde(opts: {
|
|
197
196
|
nPaired: number;
|
|
198
197
|
alpha?: number;
|
|
199
198
|
power?: number;
|
|
200
199
|
twoSided?: boolean;
|
|
201
200
|
}): number;
|
|
201
|
+
/**
|
|
202
|
+
* Number of paired observations needed for a McNemar test to reach a target
|
|
203
|
+
* power — the pre-registration companion to {@link mcnemar}. Parametrised by the
|
|
204
|
+
* expected discordant-cell probabilities `p10` (P[treatment wins on a pair]) and
|
|
205
|
+
* `p01` (P[control wins]); concordant pairs carry no information, so the count
|
|
206
|
+
* is driven entirely by the discordant rate. Lachin's (1992) asymptotic normal
|
|
207
|
+
* approximation: with discordant rate `pDisc = p10 + p01` and marginal effect
|
|
208
|
+
* `δ = p10 − p01`,
|
|
209
|
+
* n = ( z_{1-α/2}·√pDisc + z_{1-β}·√(pDisc − δ²) )² / δ².
|
|
210
|
+
* Returns Infinity when there is no effect (p10 === p01). Asymptotic — at the
|
|
211
|
+
* tiny discordant counts where the exact {@link mcnemar} differs from the normal
|
|
212
|
+
* approximation, treat the result as a lower bound and prefer the discordant-pair
|
|
213
|
+
* floor.
|
|
214
|
+
*/
|
|
215
|
+
export declare function mcnemarRequiredN(opts: {
|
|
216
|
+
p10: number;
|
|
217
|
+
p01: number;
|
|
218
|
+
alpha?: number;
|
|
219
|
+
power?: number;
|
|
220
|
+
twoSided?: boolean;
|
|
221
|
+
}): number;
|
|
222
|
+
/**
|
|
223
|
+
* Power of a McNemar test at a given number of paired observations, the inverse
|
|
224
|
+
* of {@link mcnemarRequiredN} (same Lachin asymptotic model, same parameters).
|
|
225
|
+
* Returns a value in [0, 1]; equals `alpha` when there is no effect.
|
|
226
|
+
*/
|
|
227
|
+
export declare function mcnemarPower(opts: {
|
|
228
|
+
p10: number;
|
|
229
|
+
p01: number;
|
|
230
|
+
nPairs: number;
|
|
231
|
+
alpha?: number;
|
|
232
|
+
twoSided?: boolean;
|
|
233
|
+
}): number;
|
|
202
234
|
/** Bonferroni adjustment: multiply every p-value by the test count, clamp at 1. */
|
|
203
|
-
declare function bonferroni(pValues: number[], alpha?: number): {
|
|
235
|
+
export declare function bonferroni(pValues: number[], alpha?: number): {
|
|
204
236
|
adjusted: number[];
|
|
205
237
|
significant: boolean[];
|
|
206
238
|
};
|
|
@@ -208,11 +240,11 @@ declare function bonferroni(pValues: number[], alpha?: number): {
|
|
|
208
240
|
* Benjamini–Hochberg false discovery rate. Returns adjusted q-values and
|
|
209
241
|
* significance at the target FDR; handles ties and preserves q monotonicity.
|
|
210
242
|
*/
|
|
211
|
-
declare function benjaminiHochberg(pValues: number[], fdr?: number): {
|
|
243
|
+
export declare function benjaminiHochberg(pValues: number[], fdr?: number): {
|
|
212
244
|
qValues: number[];
|
|
213
245
|
significant: boolean[];
|
|
214
246
|
};
|
|
215
|
-
interface PairedBootstrapResult {
|
|
247
|
+
export interface PairedBootstrapResult {
|
|
216
248
|
/** Number of paired observations. */
|
|
217
249
|
n: number;
|
|
218
250
|
/** Median of paired deltas (after − before). */
|
|
@@ -228,7 +260,7 @@ interface PairedBootstrapResult {
|
|
|
228
260
|
/** Number of bootstrap resamples used. */
|
|
229
261
|
resamples: number;
|
|
230
262
|
}
|
|
231
|
-
interface PairedBootstrapOptions {
|
|
263
|
+
export interface PairedBootstrapOptions {
|
|
232
264
|
/** Confidence level. Default 0.95. */
|
|
233
265
|
confidence?: number;
|
|
234
266
|
/** Bootstrap resample count. Default 2000. */
|
|
@@ -244,8 +276,90 @@ interface PairedBootstrapOptions {
|
|
|
244
276
|
* lower bound is what the promotion gate checks — `low > threshold` means the
|
|
245
277
|
* gain is real at the confidence level. Throws on unequal sample sizes.
|
|
246
278
|
*/
|
|
247
|
-
declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
|
|
248
|
-
|
|
279
|
+
export declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
|
|
280
|
+
/** A binomial proportion estimate with a confidence interval. */
|
|
281
|
+
export interface ProportionInterval {
|
|
282
|
+
/** Point estimate successes / n (0 when n = 0). */
|
|
283
|
+
estimate: number;
|
|
284
|
+
/** Lower bound, clamped to [0, 1]. */
|
|
285
|
+
lower: number;
|
|
286
|
+
/** Upper bound, clamped to [0, 1]. */
|
|
287
|
+
upper: number;
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Wilson score interval for a binomial proportion. Correct at small n and near
|
|
291
|
+
* 0/1, where the normal (Wald) approximation produces bounds outside [0, 1] and
|
|
292
|
+
* understates coverage. Use this for any pass-rate / hit-rate / realness-rate
|
|
293
|
+
* CI — the continuous `confidenceInterval` assumes the wrong distribution for a
|
|
294
|
+
* proportion. `n = 0 ⇒ {0, 0, 0}`.
|
|
295
|
+
*/
|
|
296
|
+
export declare function wilson(successes: number, n: number, confidence?: number): ProportionInterval;
|
|
297
|
+
/** Result of a McNemar paired-binary significance test. */
|
|
298
|
+
export interface McNemarResult {
|
|
299
|
+
/** Total paired observations. */
|
|
300
|
+
n: number;
|
|
301
|
+
/** Discordant pairs (b + c) — the only ones that carry signal. */
|
|
302
|
+
nDiscordant: number;
|
|
303
|
+
/** Pairs where treatment succeeded and control failed ("newly correct"). */
|
|
304
|
+
b: number;
|
|
305
|
+
/** Pairs where control succeeded and treatment failed ("newly wrong"). */
|
|
306
|
+
c: number;
|
|
307
|
+
/** Continuity-corrected chi-square statistic (reference; exact p drives the call). */
|
|
308
|
+
statistic: number;
|
|
309
|
+
/** Two-sided p-value. Exact (binomial sign test on discordant pairs). */
|
|
310
|
+
pValue: number;
|
|
311
|
+
}
|
|
312
|
+
/**
|
|
313
|
+
* McNemar's test for paired binary outcomes — the correct significance test for
|
|
314
|
+
* "does treatment change the success rate vs control on the SAME items". Only
|
|
315
|
+
* discordant pairs (one arm right, the other wrong) carry information; concordant
|
|
316
|
+
* pairs are uninformative, so a paired t-test / two-proportion z-test on the raw
|
|
317
|
+
* rates is wrong here. The p-value is exact: under H0 the b "treatment-wins" are
|
|
318
|
+
* Binomial(b + c, 0.5), so the two-sided p is the doubled binomial tail — correct
|
|
319
|
+
* at the small discordant counts typical of eval runs (no continuity-corrected
|
|
320
|
+
* chi-square approximation needed, though it is returned as `statistic` for
|
|
321
|
+
* reference). Inputs are paired 0/1 (or boolean) arrays, control first to match
|
|
322
|
+
* the module's (before, after) convention. Throws on unequal lengths.
|
|
323
|
+
*/
|
|
324
|
+
export declare function mcnemar(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>): McNemarResult;
|
|
325
|
+
/** A paired binary effect size (treatment rate − control rate) with a CI. */
|
|
326
|
+
export interface RiskDifferenceResult {
|
|
327
|
+
/** Total paired observations. */
|
|
328
|
+
n: number;
|
|
329
|
+
/** Discordant pairs: treatment-win count. */
|
|
330
|
+
b: number;
|
|
331
|
+
/** Discordant pairs: control-win count. */
|
|
332
|
+
c: number;
|
|
333
|
+
/** Paired risk difference p(treatment) − p(control) = (b − c) / n. */
|
|
334
|
+
riskDifference: number;
|
|
335
|
+
/** Lower bound of the CI, clamped to [-1, 1]. */
|
|
336
|
+
lower: number;
|
|
337
|
+
/** Upper bound of the CI, clamped to [-1, 1]. */
|
|
338
|
+
upper: number;
|
|
339
|
+
/** Confidence level used. */
|
|
340
|
+
confidence: number;
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Paired risk difference (the effect-size companion to {@link mcnemar}): the
|
|
344
|
+
* change in success rate p(treatment) − p(control) on matched items, which for
|
|
345
|
+
* paired binary data equals (b − c) / n. The CI uses the paired variance from
|
|
346
|
+
* the discordant counts, not the independent-samples formula (which overstates
|
|
347
|
+
* the interval by ignoring the pairing). Inputs are paired 0/1 (or boolean)
|
|
348
|
+
* arrays, control first. Throws on unequal lengths.
|
|
349
|
+
*/
|
|
350
|
+
export declare function pairedRiskDifference(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>, confidence?: number): RiskDifferenceResult;
|
|
351
|
+
/**
|
|
352
|
+
* Unbiased pass@k for code generation (Chen et al. 2021, "Evaluating Large
|
|
353
|
+
* Language Models Trained on Code"). Given `n` independent samples for one
|
|
354
|
+
* problem of which `c` pass, the probability that at least one of a random k of
|
|
355
|
+
* them passes is 1 − C(n−c, k) / C(n, k). Estimating pass@k as "did any of the
|
|
356
|
+
* first k pass" is biased high at small n; this is the variance-reduced estimator
|
|
357
|
+
* averaged implicitly over all k-subsets. Average the per-problem values across
|
|
358
|
+
* the suite for the corpus pass@k. Computed in the numerically stable product
|
|
359
|
+
* form. Requires 1 ≤ k ≤ n and 0 ≤ c ≤ n.
|
|
360
|
+
*/
|
|
361
|
+
export declare function passAtK(n: number, c: number, k: number): number;
|
|
362
|
+
export interface EProcessOptions {
|
|
249
363
|
/** Type-I error budget. The process decides when wealth ≥ 1/alpha
|
|
250
364
|
* (Ville's inequality). Default 0.05. */
|
|
251
365
|
alpha?: number;
|
|
@@ -258,7 +372,7 @@ interface EProcessOptions {
|
|
|
258
372
|
* A pre-registered minEffect shifts this — see `sequentialPairedGate`. */
|
|
259
373
|
nullMean?: number;
|
|
260
374
|
}
|
|
261
|
-
interface EProcessStep {
|
|
375
|
+
export interface EProcessStep {
|
|
262
376
|
/** Current wealth W_n — the e-value against H0 after n observations. */
|
|
263
377
|
wealth: number;
|
|
264
378
|
/** Observations consumed so far. */
|
|
@@ -266,7 +380,7 @@ interface EProcessStep {
|
|
|
266
380
|
/** True from the first n where W_n ≥ 1/alpha onward (sticky). */
|
|
267
381
|
decided: boolean;
|
|
268
382
|
}
|
|
269
|
-
interface EProcessState extends EProcessStep {
|
|
383
|
+
export interface EProcessState extends EProcessStep {
|
|
270
384
|
alpha: number;
|
|
271
385
|
maxBet: number;
|
|
272
386
|
nullMean: number;
|
|
@@ -275,7 +389,7 @@ interface EProcessState extends EProcessStep {
|
|
|
275
389
|
/** Observation count at the first threshold crossing; undefined until decided. */
|
|
276
390
|
decidedAtN?: number;
|
|
277
391
|
}
|
|
278
|
-
interface EProcess {
|
|
392
|
+
export interface EProcess {
|
|
279
393
|
/** Consume one observation x ∈ [0,1]. Throws on non-finite / out-of-range
|
|
280
394
|
* input — a silent clamp would corrupt the type-I guarantee. */
|
|
281
395
|
update(x: number): EProcessStep;
|
|
@@ -308,11 +422,10 @@ interface EProcess {
|
|
|
308
422
|
* wealth keeps updating after the crossing (the e-process remains valid), but
|
|
309
423
|
* the decision time is the first crossing.
|
|
310
424
|
*/
|
|
311
|
-
declare function eProcess(opts?: EProcessOptions): EProcess;
|
|
425
|
+
export declare function eProcess(opts?: EProcessOptions): EProcess;
|
|
312
426
|
/** Tiny seedable PRNG (mulberry32) — deterministic resampling/shuffling, not
|
|
313
427
|
* cryptographic. Exported so e-process shuffles and bootstrap resampling
|
|
314
428
|
* share ONE PRNG implementation; a seed is REQUIRED (unseeded randomness in
|
|
315
429
|
* gate verdicts is non-reproducible by construction). */
|
|
316
|
-
declare function mulberry32(seed: number): () => number;
|
|
317
|
-
|
|
318
|
-
export { partialCredit as A, requiredSampleSize as B, type CorpusAgreementReport as C, weightedComposite as D, type EProcessState as E, weightedMean as F, type PairedBootstrapOptions as P, type WeightedCompositeInput as W, type PairedBootstrapResult as a, benjaminiHochberg as b, type CliffsMagnitude as c, type CorpusAgreementOptions as d, type CorpusAgreementPerDimension as e, type CorpusScoreRecord as f, type EProcess as g, type EProcessOptions as h, type EProcessStep as i, type WeightedCompositeResult as j, bonferroni as k, cliffsDelta as l, cohensD as m, confidenceInterval as n, corpusInterRaterAgreement as o, pairedBootstrap as p, corpusInterRaterAgreementFromJudgeScores as q, eProcess as r, interRaterReliability as s, interpretCliffs as t, mannWhitneyU as u, mulberry32 as v, wilcoxonSignedRank as w, normalizeScores as x, pairedMde as y, pairedTTest as z };
|
|
430
|
+
export declare function mulberry32(seed: number): () => number;
|
|
431
|
+
//# sourceMappingURL=statistics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"statistics.d.ts","sourceRoot":"","sources":["../src/statistics.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,mBAAmB,EACxB,KAAK,0BAA0B,EAEhC,MAAM,qBAAqB,CAAA;AAC5B,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAEzC;6EAC6E;AAC7E,eAAO,MAAM,eAAe,GAAI,QAAQ,UAAU,EAAE,KAAG,UAAU,EAAY,CAAA;AAE7E,iEAAiE;AACjE,wBAAgB,YAAY,CAAC,MAAM,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,GAAG,MAAM,CAUjF;AAED,oCAAoC;AACpC,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,MAAM,EAAE,EAChB,UAAU,SAAO,EACjB,IAAI,GAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAO,GAC/C;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CA8BhD;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,WAAW,EAAE,UAAU,EAAE,EAAE,GAAG,MAAM,CAkDzE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CA4C/E;AAED,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAGrE;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CACzB,MAAM,EAAE,MAAM,EAAE,EAChB,KAAK,EAAE,MAAM,EAAE,GACd;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CAmBtC;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,CAAC,EAAE,MAAM,CAAA;CAAE,CA8B9F;AAED;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAWxD;AAED,MAAM,MAAM,eAAe,GAAG,YAAY,GAAG,OAAO,GAAG,QAAQ,GAAG,OAAO,CAAA;AAEzE;;;;;;;;;GASG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAWrE;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,eAAe,CAM9D;AAED,MAAM,WAAW,sBAAsB;IACrC,6CAA6C;IAC7C,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC5B;;;+CAG2C;IAC3C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC/B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAA;IACjB,IAAI,CAAC,EAAE,OAAO,CAAA;CACf;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,sBAAsB,GAAG,uBAAuB,CA2BxF;AAiBD,MAAM,WAAW,iBAAiB;IAChC,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAA;IACd,yDAAyD;IACzD,SAAS,EAAE,MAAM,CAAA;IACjB,uDAAuD;IACvD,SAAS,EAAE,MAAM,CAAA;IACjB,qCAAqC;IACrC,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,2BAA4B,SAAQ,mBAAmB;IACtE,SAAS,EAAE,MAAM,CAAA;IACjB,sFAAsF;IACtF,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,6DAA6D;IAC7D,QAAQ,EAAE,MAAM,EAAE,CAAA;CACnB;AAED,MAAM,WAAW,qBAAqB;IACpC,yEAAyE;IACzE,YAAY,EAAE,2BAA2B,EAAE,CAAA;IAC3C,6EAA6E;IAC7E,UAAU,EAAE,MAAM,CAAA;IAClB,8DAA8D;IAC9D,oBAAoB,EAAE,MAAM,CAAA;IAC5B,qCAAqC;IACrC,UAAU,EAAE,MAAM,EAAE,CAAA;IACpB,8CAA8C;IAC9C,QAAQ,EAAE,MAAM,EAAE,CAAA;CACnB;AAED,MAAM,WAAW,sBAAuB,SAAQ,0BAA0B;IACxE;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,EAAE,CAAA;IACrB;;;;OAIG;IACH,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;CAClB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,iBAAiB,EAAE,EAC5B,IAAI,GAAE,sBAA2B,GAChC,qBAAqB,CAgHvB;AAED;;;;;;;GAOG;AACH,wBAAgB,wCAAwC,CACtD,WAAW,EAAE,KAAK,CAAC;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,UAAU,EAAE,CAAA;CAAE,CAAC,EAC5D,IAAI,GAAE,sBAA2B,GAChC,qBAAqB,CAoBvB;AAqFD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB,GAAG,MAAM,CAUT;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB,GAAG,MAAM,CAQT;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE;IACrC,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB,GAAG,MAAM,CAiBT;AAED;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE;IACjC,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB,GAAG,MAAM,CAcT;AAED,mFAAmF;AACnF,wBAAgB,UAAU,CACxB,OAAO,EAAE,MAAM,EAAE,EACjB,KAAK,SAAO,GACX;IAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,EAAE,CAAA;CAAE,CAKhD;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,MAAM,EAAE,EACjB,GAAG,SAAO,GACT;IAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,WAAW,EAAE,OAAO,EAAE,CAAA;CAAE,CAgB/C;AAID,MAAM,WAAW,qBAAqB;IACpC,qCAAqC;IACrC,CAAC,EAAE,MAAM,CAAA;IACT,gDAAgD;IAChD,MAAM,EAAE,MAAM,CAAA;IACd,6BAA6B;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,+DAA+D;IAC/D,GAAG,EAAE,MAAM,CAAA;IACX,+DAA+D;IAC/D,IAAI,EAAE,MAAM,CAAA;IACZ,yCAAyC;IACzC,UAAU,EAAE,MAAM,CAAA;IAClB,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,sBAAsB;IACrC,sCAAsC;IACtC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,gDAAgD;IAChD,SAAS,CAAC,EAAE,QAAQ,GAAG,MAAM,CAAA;IAC7B,0DAA0D;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,MAAM,EAAE,EAChB,KAAK,EAAE,MAAM,EAAE,EACf,IAAI,GAAE,sBAA2B,GAChC,qBAAqB,CAqDvB;AAcD,iEAAiE;AACjE,MAAM,WAAW,kBAAkB;IACjC,mDAAmD;IACnD,QAAQ,EAAE,MAAM,CAAA;IAChB,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAA;IACb,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAA;CACd;AAED;;;;;;GAMG;AACH,wBAAgB,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,SAAO,GAAG,kBAAkB,CAgB1F;AAED,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC5B,iCAAiC;IACjC,CAAC,EAAE,MAAM,CAAA;IACT,kEAAkE;IAClE,WAAW,EAAE,MAAM,CAAA;IACnB,4EAA4E;IAC5E,CAAC,EAAE,MAAM,CAAA;IACT,0EAA0E;IAC1E,CAAC,EAAE,MAAM,CAAA;IACT,sFAAsF;IACtF,SAAS,EAAE,MAAM,CAAA;IACjB,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,OAAO,CACrB,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,OAAO,CAAC,EACpC,SAAS,EAAE,SAAS,CAAC,MAAM,GAAG,OAAO,CAAC,GACrC,aAAa,CAgBf;AAED,6EAA6E;AAC7E,MAAM,WAAW,oBAAoB;IACnC,iCAAiC;IACjC,CAAC,EAAE,MAAM,CAAA;IACT,6CAA6C;IAC7C,CAAC,EAAE,MAAM,CAAA;IACT,2CAA2C;IAC3C,CAAC,EAAE,MAAM,CAAA;IACT,sEAAsE;IACtE,cAAc,EAAE,MAAM,CAAA;IACtB,iDAAiD;IACjD,KAAK,EAAE,MAAM,CAAA;IACb,iDAAiD;IACjD,KAAK,EAAE,MAAM,CAAA;IACb,6BAA6B;IAC7B,UAAU,EAAE,MAAM,CAAA;CACnB;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,SAAS,CAAC,MAAM,GAAG,OAAO,CAAC,EACpC,SAAS,EAAE,SAAS,CAAC,MAAM,GAAG,OAAO,CAAC,EACtC,UAAU,SAAO,GAChB,oBAAoB,CA6BtB;AAED;;;;;;;;;GASG;AACH,wBAAgB,OAAO,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAW/D;AAuBD,MAAM,WAAW,eAAe;IAC9B;8CAC0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;uBAEmB;IACnB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;+EAE2E;IAC3E,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,wEAAwE;IACxE,MAAM,EAAE,MAAM,CAAA;IACd,oCAAoC;IACpC,CAAC,EAAE,MAAM,CAAA;IACT,iEAAiE;IACjE,OAAO,EAAE,OAAO,CAAA;CACjB;AAED,MAAM,WAAW,aAAc,SAAQ,YAAY;IACjD,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,qCAAqC;IACrC,SAAS,EAAE,MAAM,CAAA;IACjB,kFAAkF;IAClF,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,QAAQ;IACvB;qEACiE;IACjE,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,YAAY,CAAA;IAC/B,KAAK,IAAI,aAAa,CAAA;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAgB,QAAQ,CAAC,IAAI,GAAE,eAAoB,GAAG,QAAQ,CAoD7D;AA8DD;;;0DAG0D;AAC1D,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CASrD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"statistics.test.d.ts","sourceRoot":"","sources":["../src/statistics.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { type RunScore, type RunScoreWeights } from './run-score';
|
|
2
|
+
import type { SteeringBundle } from './steering';
|
|
3
|
+
export type SteeringOptimizerBackend = 'pairwise' | 'ax-gepa';
|
|
4
|
+
export interface SteeringOptimizationRow {
|
|
5
|
+
variantId: string;
|
|
6
|
+
scenarioId: string;
|
|
7
|
+
bundle: SteeringBundle;
|
|
8
|
+
score: RunScore;
|
|
9
|
+
metadata?: Record<string, unknown>;
|
|
10
|
+
}
|
|
11
|
+
export interface SteeringOptimizationSelector {
|
|
12
|
+
backend: SteeringOptimizerBackend;
|
|
13
|
+
signature?: string;
|
|
14
|
+
labels?: string[];
|
|
15
|
+
rationale?: string;
|
|
16
|
+
}
|
|
17
|
+
export interface SteeringOptimizationResult {
|
|
18
|
+
backend: SteeringOptimizerBackend;
|
|
19
|
+
recommendedVariantId: string;
|
|
20
|
+
rationale: string;
|
|
21
|
+
rankings: Array<{
|
|
22
|
+
variantId: string;
|
|
23
|
+
mean: number;
|
|
24
|
+
runs: number;
|
|
25
|
+
}>;
|
|
26
|
+
selector?: SteeringOptimizationSelector;
|
|
27
|
+
/** Runnable handle on the trained classifier. Present only when the
|
|
28
|
+
* ax-gepa backend completed training; calls the optimized selector
|
|
29
|
+
* program via ax's `forward`. */
|
|
30
|
+
selectVariant?: (row: {
|
|
31
|
+
task: string;
|
|
32
|
+
split: string;
|
|
33
|
+
seedPreview: string;
|
|
34
|
+
}) => Promise<{
|
|
35
|
+
variantId: string;
|
|
36
|
+
rationale: string;
|
|
37
|
+
}>;
|
|
38
|
+
skipped?: boolean;
|
|
39
|
+
}
|
|
40
|
+
export interface SteeringOptimizerConfig {
|
|
41
|
+
weights?: Partial<RunScoreWeights>;
|
|
42
|
+
}
|
|
43
|
+
export interface AxSteeringOptimizerConfig extends SteeringOptimizerConfig {
|
|
44
|
+
provider: 'openai' | 'anthropic';
|
|
45
|
+
apiKey: string;
|
|
46
|
+
model: string;
|
|
47
|
+
teacherModel?: string;
|
|
48
|
+
minScenarioWinners?: number;
|
|
49
|
+
}
|
|
50
|
+
export declare class PairwiseSteeringOptimizer {
|
|
51
|
+
optimize(rows: SteeringOptimizationRow[], config?: SteeringOptimizerConfig): SteeringOptimizationResult;
|
|
52
|
+
}
|
|
53
|
+
export declare class AxGepaSteeringOptimizer {
|
|
54
|
+
private readonly config;
|
|
55
|
+
constructor(config: AxSteeringOptimizerConfig);
|
|
56
|
+
optimize(rows: SteeringOptimizationRow[]): Promise<SteeringOptimizationResult>;
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=steering-optimizer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"steering-optimizer.d.ts","sourceRoot":"","sources":["../src/steering-optimizer.ts"],"names":[],"mappings":"AACA,OAAO,EAAqB,KAAK,QAAQ,EAAE,KAAK,eAAe,EAAE,MAAM,aAAa,CAAA;AACpF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD,MAAM,MAAM,wBAAwB,GAAG,UAAU,GAAG,SAAS,CAAA;AAE7D,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,MAAM,EAAE,cAAc,CAAA;IACtB,KAAK,EAAE,QAAQ,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,4BAA4B;IAC3C,OAAO,EAAE,wBAAwB,CAAA;IACjC,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,0BAA0B;IACzC,OAAO,EAAE,wBAAwB,CAAA;IACjC,oBAAoB,EAAE,MAAM,CAAA;IAC5B,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IAClE,QAAQ,CAAC,EAAE,4BAA4B,CAAA;IACvC;;sCAEkC;IAClC,aAAa,CAAC,EAAE,CAAC,GAAG,EAAE;QACpB,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,EAAE,MAAM,CAAA;QACb,WAAW,EAAE,MAAM,CAAA;KACpB,KAAK,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACvD,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,yBAA0B,SAAQ,uBAAuB;IACxE,QAAQ,EAAE,QAAQ,GAAG,WAAW,CAAA;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AASD,qBAAa,yBAAyB;IACpC,QAAQ,CACN,IAAI,EAAE,uBAAuB,EAAE,EAC/B,MAAM,GAAE,uBAA4B,GACnC,0BAA0B;CAU9B;AAED,qBAAa,uBAAuB;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM;gBAAN,MAAM,EAAE,yBAAyB;IAExD,QAAQ,CAAC,IAAI,EAAE,uBAAuB,EAAE,GAAG,OAAO,CAAC,0BAA0B,CAAC;CAuFrF"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export interface SteeringRolePrompt {
|
|
2
|
+
system?: string;
|
|
3
|
+
append?: string;
|
|
4
|
+
}
|
|
5
|
+
export interface SteeringBundle {
|
|
6
|
+
id: string;
|
|
7
|
+
coderPrompt?: string;
|
|
8
|
+
continuePrompt?: string;
|
|
9
|
+
reviewerPrompts?: Record<string, string>;
|
|
10
|
+
skills?: string[];
|
|
11
|
+
rolePrompts?: Record<string, SteeringRolePrompt>;
|
|
12
|
+
metadata?: Record<string, unknown>;
|
|
13
|
+
}
|
|
14
|
+
export interface SteeringDelta {
|
|
15
|
+
coderPrompt?: string;
|
|
16
|
+
continuePrompt?: string;
|
|
17
|
+
reviewerPrompts?: Record<string, string>;
|
|
18
|
+
skills?: string[];
|
|
19
|
+
rolePrompts?: Record<string, SteeringRolePrompt>;
|
|
20
|
+
metadata?: Record<string, unknown>;
|
|
21
|
+
}
|
|
22
|
+
export declare function mergeSteeringBundle(base: SteeringBundle, delta: SteeringDelta): SteeringBundle;
|
|
23
|
+
export declare function renderSteeringText(bundle: SteeringBundle): string;
|
|
24
|
+
//# sourceMappingURL=steering.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"steering.d.ts","sourceRoot":"","sources":["../src/steering.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAA;IACV,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACxC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,aAAa;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACxC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAA;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,GAAG,cAAc,CAmB9F;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,cAAc,GAAG,MAAM,CAejE"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Code-edit extraction + the "watch the agent write code" animation.
|
|
3
|
+
*
|
|
4
|
+
* The base storyboard (./index) compresses a run into scenes but keeps scenes
|
|
5
|
+
* payload-free (title + narration + evidence span ids). To actually SHOW the
|
|
6
|
+
* code an agent wrote, this module pulls the concrete edit (path, diff,
|
|
7
|
+
* before/after, language) out of the raw edit-tool spans and renders a
|
|
8
|
+
* self-contained HTML replay that types each file out, character by character,
|
|
9
|
+
* like an editor recording.
|
|
10
|
+
*
|
|
11
|
+
* Additive on purpose: it reuses the canonical `Span` (trace/schema) and the
|
|
12
|
+
* `Storyboard` IR (./index) without modifying the reducer/compiler — so it
|
|
13
|
+
* composes with the existing pipeline and a Remotion consumer can read the same
|
|
14
|
+
* `CodeEdit[]` to render an MP4.
|
|
15
|
+
*/
|
|
16
|
+
import type { Span } from '../trace/schema';
|
|
17
|
+
import type { Scene, Storyboard } from './index';
|
|
18
|
+
/** A concrete code change lifted from a tool span. */
|
|
19
|
+
export interface CodeEdit {
|
|
20
|
+
/** Span the edit came from — back-reference into the trace. */
|
|
21
|
+
spanId: string;
|
|
22
|
+
path: string;
|
|
23
|
+
language?: string;
|
|
24
|
+
/** Unified diff, when the tool carried one. */
|
|
25
|
+
diff?: string;
|
|
26
|
+
/** Full file body before / after, when present — enables true keystroke
|
|
27
|
+
* animation rather than only flashing a diff. */
|
|
28
|
+
before?: string;
|
|
29
|
+
after?: string;
|
|
30
|
+
additions: number;
|
|
31
|
+
deletions: number;
|
|
32
|
+
startedAt: number;
|
|
33
|
+
}
|
|
34
|
+
/** Extract a CodeEdit from a single span, or undefined if it is not an edit. */
|
|
35
|
+
export declare function codeEditFromSpan(span: Span): CodeEdit | undefined;
|
|
36
|
+
/** All code edits in a run, in chronological order. */
|
|
37
|
+
export declare function extractCodeEdits(spans: readonly Span[]): CodeEdit[];
|
|
38
|
+
/** Pair each code-edit (`diff`) scene with its concrete edit, matched via the
|
|
39
|
+
* scene's evidence span ids. Scenes without a recoverable edit are dropped. */
|
|
40
|
+
export declare function codeEditsForStoryboard(storyboard: Storyboard, spans: readonly Span[]): Array<{
|
|
41
|
+
scene: Scene;
|
|
42
|
+
edit: CodeEdit;
|
|
43
|
+
}>;
|
|
44
|
+
/** The text the animation types for an edit: prefer the full new file body,
|
|
45
|
+
* then the added lines of a diff, then a minimal placeholder so the scene is
|
|
46
|
+
* never blank. */
|
|
47
|
+
export declare function editAnimationText(edit: CodeEdit): string;
|
|
48
|
+
export interface CodeAnimationOptions {
|
|
49
|
+
/** Page title. Default "Agent writing code". */
|
|
50
|
+
title?: string;
|
|
51
|
+
/** Characters typed per animation tick. Higher = faster. Default 4. */
|
|
52
|
+
charsPerTick?: number;
|
|
53
|
+
/** Pause between files, ms. Default 900. */
|
|
54
|
+
pauseBetweenMs?: number;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Render a self-contained HTML page that animates the agent writing each file,
|
|
58
|
+
* character by character, with a filename tab, line numbers, and a blinking
|
|
59
|
+
* cursor — the shareable "watch it build" clip. No build step, no assets, no
|
|
60
|
+
* deps. The same `CodeEdit[]` feeds a Remotion MP4 renderer in a consumer
|
|
61
|
+
* package; this is the dependency-free baseline.
|
|
62
|
+
*/
|
|
63
|
+
export declare function renderCodeAnimationHtml(source: readonly Span[] | readonly CodeEdit[], opts?: CodeAnimationOptions): string;
|
|
64
|
+
//# sourceMappingURL=code-edit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"code-edit.d.ts","sourceRoot":"","sources":["../../src/storyboard/code-edit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAKhD,sDAAsD;AACtD,MAAM,WAAW,QAAQ;IACvB,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,+CAA+C;IAC/C,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;sDACkD;IAClD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;CAClB;AA2DD,gFAAgF;AAChF,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,IAAI,GAAG,QAAQ,GAAG,SAAS,CAqBjE;AAED,uDAAuD;AACvD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,IAAI,EAAE,GAAG,QAAQ,EAAE,CAKnE;AAED;gFACgF;AAChF,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,UAAU,EACtB,KAAK,EAAE,SAAS,IAAI,EAAE,GACrB,KAAK,CAAC;IAAE,KAAK,EAAE,KAAK,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAA;CAAE,CAAC,CAWzC;AAED;;mBAEmB;AACnB,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,QAAQ,GAAG,MAAM,CAWxD;AAMD,MAAM,WAAW,oBAAoB;IACnC,gDAAgD;IAChD,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,uEAAuE;IACvE,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,SAAS,IAAI,EAAE,GAAG,SAAS,QAAQ,EAAE,EAC7C,IAAI,GAAE,oBAAyB,GAC9B,MAAM,CA+FR"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"code-edit.test.d.ts","sourceRoot":"","sources":["../../src/storyboard/code-edit.test.ts"],"names":[],"mappings":""}
|