@tangle-network/agent-eval 0.93.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +44 -30
- package/dist/action-policy.d.ts +24 -0
- package/dist/action-policy.d.ts.map +1 -0
- package/dist/action-policy.test.d.ts +2 -0
- package/dist/action-policy.test.d.ts.map +1 -0
- package/dist/active-learning.d.ts +41 -0
- package/dist/active-learning.d.ts.map +1 -0
- package/dist/adapters/http.d.ts +15 -21
- package/dist/adapters/http.d.ts.map +1 -0
- package/dist/adapters/http.js.map +1 -1
- package/dist/adapters/langchain.d.ts +7 -13
- package/dist/adapters/langchain.d.ts.map +1 -0
- package/dist/adapters/otel.d.ts +13 -24
- package/dist/adapters/otel.d.ts.map +1 -0
- package/dist/agent-profile-cell.d.ts +101 -0
- package/dist/agent-profile-cell.d.ts.map +1 -0
- package/dist/agent-profile.d.ts +27 -0
- package/dist/agent-profile.d.ts.map +1 -0
- package/dist/agent-profile.test.d.ts +2 -0
- package/dist/agent-profile.test.d.ts.map +1 -0
- package/dist/analyst/adapters.d.ts +62 -0
- package/dist/analyst/adapters.d.ts.map +1 -0
- package/dist/analyst/analyst.test.d.ts +2 -0
- package/dist/analyst/analyst.test.d.ts.map +1 -0
- package/dist/analyst/ax-service.d.ts +27 -0
- package/dist/analyst/ax-service.d.ts.map +1 -0
- package/dist/analyst/behavioral-analyst.d.ts +28 -0
- package/dist/analyst/behavioral-analyst.d.ts.map +1 -0
- package/dist/analyst/chat-client.d.ts +91 -0
- package/dist/analyst/chat-client.d.ts.map +1 -0
- package/dist/analyst/default-registry.d.ts +27 -0
- package/dist/analyst/default-registry.d.ts.map +1 -0
- package/dist/analyst/default-registry.test.d.ts +2 -0
- package/dist/analyst/default-registry.test.d.ts.map +1 -0
- package/dist/analyst/finding-signature.d.ts +48 -0
- package/dist/analyst/finding-signature.d.ts.map +1 -0
- package/dist/analyst/finding-subject.d.ts +146 -0
- package/dist/analyst/finding-subject.d.ts.map +1 -0
- package/dist/analyst/finding-subject.test.d.ts +2 -0
- package/dist/analyst/finding-subject.test.d.ts.map +1 -0
- package/dist/analyst/findings-store.d.ts +75 -0
- package/dist/analyst/findings-store.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +28 -256
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +6 -66
- package/dist/analyst/index.js.map +1 -1
- package/dist/{kind-factory-5b7xXXOr.d.ts → analyst/kind-factory.d.ts} +11 -63
- package/dist/analyst/kind-factory.d.ts.map +1 -0
- package/dist/analyst/kinds/failure-mode.d.ts +19 -0
- package/dist/analyst/kinds/failure-mode.d.ts.map +1 -0
- package/dist/analyst/kinds/improvement.d.ts +23 -0
- package/dist/analyst/kinds/improvement.d.ts.map +1 -0
- package/dist/analyst/kinds/index.d.ts +22 -0
- package/dist/analyst/kinds/index.d.ts.map +1 -0
- package/dist/analyst/kinds/kinds.test.d.ts +2 -0
- package/dist/analyst/kinds/kinds.test.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts +28 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +22 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.d.ts +84 -0
- package/dist/analyst/kinds/skill-usage.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts +2 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.d.ts +26 -0
- package/dist/analyst/parse-tolerant.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.test.d.ts +2 -0
- package/dist/analyst/parse-tolerant.test.d.ts.map +1 -0
- package/dist/analyst/registry.budget.test.d.ts +2 -0
- package/dist/analyst/registry.budget.test.d.ts.map +1 -0
- package/dist/{default-registry-zoGHUQEH.d.ts → analyst/registry.d.ts} +8 -37
- package/dist/analyst/registry.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.d.ts +35 -0
- package/dist/analyst/steer-firewall.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.test.d.ts +2 -0
- package/dist/analyst/steer-firewall.test.d.ts.map +1 -0
- package/dist/analyst/structure-findings.d.ts +37 -0
- package/dist/analyst/structure-findings.d.ts.map +1 -0
- package/dist/analyst/structure-findings.test.d.ts +2 -0
- package/dist/analyst/structure-findings.test.d.ts.map +1 -0
- package/dist/analyst/tool-groups.d.ts +34 -0
- package/dist/analyst/tool-groups.d.ts.map +1 -0
- package/dist/{types-2VVIL04s.d.ts → analyst/types.d.ts} +19 -112
- package/dist/analyst/types.d.ts.map +1 -0
- package/dist/anti-slop.d.ts +59 -0
- package/dist/anti-slop.d.ts.map +1 -0
- package/dist/artifact-validator.d.ts +74 -0
- package/dist/artifact-validator.d.ts.map +1 -0
- package/dist/attestation.d.ts +63 -0
- package/dist/attestation.d.ts.map +1 -0
- package/dist/attestation.test.d.ts +2 -0
- package/dist/attestation.test.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +15 -16
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.test.d.ts +2 -0
- package/dist/authenticity/index.test.d.ts.map +1 -0
- package/dist/auto-pr.d.ts +120 -0
- package/dist/auto-pr.d.ts.map +1 -0
- package/dist/{baseline-DE36-Np7.d.ts → baseline.d.ts} +8 -44
- package/dist/baseline.d.ts.map +1 -0
- package/dist/behavior-dsl.d.ts +73 -0
- package/dist/behavior-dsl.d.ts.map +1 -0
- package/dist/belief-state/calibration.d.ts +10 -0
- package/dist/belief-state/calibration.d.ts.map +1 -0
- package/dist/belief-state/calibration.test.d.ts +2 -0
- package/dist/belief-state/calibration.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.d.ts +66 -0
- package/dist/belief-state/code-agent-corpus.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts +2 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.d.ts +22 -0
- package/dist/belief-state/code-agent-evidence.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts +2 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/extract.d.ts +7 -0
- package/dist/belief-state/extract.d.ts.map +1 -0
- package/dist/belief-state/extract.test.d.ts +2 -0
- package/dist/belief-state/extract.test.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +14 -604
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/index.js +1 -1
- package/dist/belief-state/ope.d.ts +17 -0
- package/dist/belief-state/ope.d.ts.map +1 -0
- package/dist/belief-state/ope.test.d.ts +2 -0
- package/dist/belief-state/ope.test.d.ts.map +1 -0
- package/dist/belief-state/phase0-measurement.d.ts +55 -0
- package/dist/belief-state/phase0-measurement.d.ts.map +1 -0
- package/dist/belief-state/report.d.ts +17 -0
- package/dist/belief-state/report.d.ts.map +1 -0
- package/dist/belief-state/report.test.d.ts +2 -0
- package/dist/belief-state/report.test.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.d.ts +23 -0
- package/dist/belief-state/research-evidence.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.test.d.ts +2 -0
- package/dist/belief-state/research-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +32 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.d.ts +87 -0
- package/dist/belief-state/runtime-hooks.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.test.d.ts +2 -0
- package/dist/belief-state/runtime-hooks.test.d.ts.map +1 -0
- package/dist/belief-state/selective.d.ts +15 -0
- package/dist/belief-state/selective.d.ts.map +1 -0
- package/dist/belief-state/selective.test.d.ts +2 -0
- package/dist/belief-state/selective.test.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.d.ts +80 -0
- package/dist/belief-state/shadow-probe.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.test.d.ts +2 -0
- package/dist/belief-state/shadow-probe.test.d.ts.map +1 -0
- package/dist/belief-state/types.d.ts +195 -0
- package/dist/belief-state/types.d.ts.map +1 -0
- package/dist/belief-state/types.test.d.ts +2 -0
- package/dist/belief-state/types.test.d.ts.map +1 -0
- package/dist/benchmark.d.ts +14 -0
- package/dist/benchmark.d.ts.map +1 -0
- package/dist/benchmarks/index.d.ts +23 -4
- package/dist/benchmarks/index.d.ts.map +1 -0
- package/dist/benchmarks/routing/dataset.d.ts +34 -0
- package/dist/benchmarks/routing/dataset.d.ts.map +1 -0
- package/dist/benchmarks/routing/index.d.ts +34 -0
- package/dist/benchmarks/routing/index.d.ts.map +1 -0
- package/dist/benchmarks/types.d.ts +49 -0
- package/dist/benchmarks/types.d.ts.map +1 -0
- package/dist/bisector.d.ts +81 -0
- package/dist/bisector.d.ts.map +1 -0
- package/dist/budget-guard.d.ts +31 -0
- package/dist/budget-guard.d.ts.map +1 -0
- package/dist/builder-eval/builder-session.d.ts +111 -0
- package/dist/builder-eval/builder-session.d.ts.map +1 -0
- package/dist/builder-eval/correlation.d.ts +32 -0
- package/dist/builder-eval/correlation.d.ts.map +1 -0
- package/dist/builder-eval/index.d.ts +5 -250
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/index.js +2 -2
- package/dist/builder-eval/project-registry.d.ts +51 -0
- package/dist/builder-eval/project-registry.d.ts.map +1 -0
- package/dist/builder-eval/three-layer-eval.d.ts +55 -0
- package/dist/builder-eval/three-layer-eval.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.d.ts +108 -0
- package/dist/campaign/analyst-surface.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.test.d.ts +2 -0
- package/dist/campaign/analyst-surface.test.d.ts.map +1 -0
- package/dist/campaign/auto-pr.d.ts +46 -0
- package/dist/campaign/auto-pr.d.ts.map +1 -0
- package/dist/campaign/distillation/agreement-judge.d.ts +69 -0
- package/dist/campaign/distillation/agreement-judge.d.ts.map +1 -0
- package/dist/campaign/distillation/cli.d.ts +35 -0
- package/dist/campaign/distillation/cli.d.ts.map +1 -0
- package/dist/campaign/distillation/distillation.test.d.ts +2 -0
- package/dist/campaign/distillation/distillation.test.d.ts.map +1 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts +54 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +1 -0
- package/dist/campaign/distillation/run-distillation.d.ts +119 -0
- package/dist/campaign/distillation/run-distillation.d.ts.map +1 -0
- package/dist/campaign/gates/compose.d.ts +12 -0
- package/dist/campaign/gates/compose.d.ts.map +1 -0
- package/dist/campaign/gates/default-production-gate.d.ts +58 -0
- package/dist/campaign/gates/default-production-gate.d.ts.map +1 -0
- package/dist/campaign/gates/heldout-gate.d.ts +12 -0
- package/dist/campaign/gates/heldout-gate.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.d.ts +125 -0
- package/dist/campaign/gates/promotion-policy.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts +2 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.d.ts +146 -0
- package/dist/campaign/gates/sequential.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.test.d.ts +2 -0
- package/dist/campaign/gates/sequential.test.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.d.ts +99 -0
- package/dist/campaign/gates/statistical-heldout.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts +2 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +1 -0
- package/dist/campaign/index.d.ts +38 -1341
- package/dist/campaign/index.d.ts.map +1 -0
- package/dist/campaign/index.js +1865 -1318
- package/dist/campaign/index.js.map +1 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +59 -0
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +1 -0
- package/dist/campaign/presets/compare-proposers.d.ts +146 -0
- package/dist/campaign/presets/compare-proposers.d.ts.map +1 -0
- package/dist/campaign/presets/playback.d.ts +120 -0
- package/dist/campaign/presets/playback.d.ts.map +1 -0
- package/dist/campaign/presets/playback.test.d.ts +2 -0
- package/dist/campaign/presets/playback.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-eval.d.ts +14 -0
- package/dist/campaign/presets/run-eval.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts +63 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +2 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-optimization.d.ts +92 -0
- package/dist/campaign/presets/run-optimization.d.ts.map +1 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts +151 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +1 -0
- package/dist/campaign/presets/run-skill-opt.d.ts +96 -0
- package/dist/campaign/presets/run-skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/_findings-text.d.ts +22 -0
- package/dist/campaign/proposers/_findings-text.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.d.ts +33 -0
- package/dist/campaign/proposers/ace.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.test.d.ts +2 -0
- package/dist/campaign/proposers/ace.test.d.ts.map +1 -0
- package/dist/campaign/proposers/analysis-edit.d.ts +32 -0
- package/dist/campaign/proposers/analysis-edit.d.ts.map +1 -0
- package/dist/campaign/proposers/evolutionary.d.ts +20 -0
- package/dist/campaign/proposers/evolutionary.d.ts.map +1 -0
- package/dist/campaign/proposers/fapo.d.ts +120 -0
- package/dist/campaign/proposers/fapo.d.ts.map +1 -0
- package/dist/campaign/proposers/gepa.d.ts +86 -0
- package/dist/campaign/proposers/gepa.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.d.ts +44 -0
- package/dist/campaign/proposers/halo.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.test.d.ts +2 -0
- package/dist/campaign/proposers/halo.test.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.d.ts +47 -0
- package/dist/campaign/proposers/memory.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.test.d.ts +2 -0
- package/dist/campaign/proposers/memory.test.d.ts.map +1 -0
- package/dist/campaign/proposers/skill-opt.d.ts +88 -0
- package/dist/campaign/proposers/skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.d.ts +48 -0
- package/dist/campaign/proposers/trace-analyst.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts +2 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +1 -0
- package/dist/campaign/provenance.d.ts +185 -0
- package/dist/campaign/provenance.d.ts.map +1 -0
- package/dist/campaign/run-campaign.d.ts +90 -0
- package/dist/campaign/run-campaign.d.ts.map +1 -0
- package/dist/campaign/score-utils.d.ts +26 -0
- package/dist/campaign/score-utils.d.ts.map +1 -0
- package/dist/campaign/skill-patch.d.ts +62 -0
- package/dist/campaign/skill-patch.d.ts.map +1 -0
- package/dist/campaign/storage.d.ts +38 -0
- package/dist/campaign/storage.d.ts.map +1 -0
- package/dist/{types-BU-7W85F.d.ts → campaign/types.d.ts} +98 -99
- package/dist/campaign/types.d.ts.map +1 -0
- package/dist/campaign/worktree/index.d.ts +53 -0
- package/dist/campaign/worktree/index.d.ts.map +1 -0
- package/dist/canary.d.ts +101 -0
- package/dist/canary.d.ts.map +1 -0
- package/dist/causal-attribution.d.ts +45 -0
- package/dist/causal-attribution.d.ts.map +1 -0
- package/dist/{chunk-TWS7AZEY.js → chunk-2T4EZACH.js} +2 -2
- package/dist/chunk-2T4EZACH.js.map +1 -0
- package/dist/{chunk-LMZQ2Z4U.js → chunk-56GC6VYO.js} +126 -1
- package/dist/chunk-56GC6VYO.js.map +1 -0
- package/dist/{chunk-QG2OVF2D.js → chunk-77T4STFI.js} +154 -112
- package/dist/chunk-77T4STFI.js.map +1 -0
- package/dist/{chunk-ZFIBGEOL.js → chunk-7QTQKIDD.js} +179 -178
- package/dist/chunk-7QTQKIDD.js.map +1 -0
- package/dist/{chunk-UMMZHCPB.js → chunk-AQ5WQAIV.js} +26 -9
- package/dist/chunk-AQ5WQAIV.js.map +1 -0
- package/dist/{chunk-CVVHBFGN.js → chunk-CWNP4DV4.js} +53 -5
- package/dist/chunk-CWNP4DV4.js.map +1 -0
- package/dist/{chunk-QS3RBQPI.js → chunk-D5KBVULY.js} +2 -2
- package/dist/chunk-DJWX3GVS.js +81 -0
- package/dist/chunk-DJWX3GVS.js.map +1 -0
- package/dist/{chunk-S6OZEZQK.js → chunk-HMA63UEO.js} +37 -9
- package/dist/{chunk-S6OZEZQK.js.map → chunk-HMA63UEO.js.map} +1 -1
- package/dist/{chunk-UHMJT4T7.js → chunk-IZCEK2HR.js} +2 -2
- package/dist/{chunk-6SOJM3VR.js → chunk-KKWJD5E6.js} +21 -21
- package/dist/chunk-KKWJD5E6.js.map +1 -0
- package/dist/{chunk-KWRRMR3J.js → chunk-LO6IOIJ2.js} +10 -10
- package/dist/chunk-LO6IOIJ2.js.map +1 -0
- package/dist/{chunk-L3JOU6XM.js → chunk-NACM5RS7.js} +3 -3
- package/dist/{chunk-E4GH6USR.js → chunk-NZEQVRH5.js} +2 -2
- package/dist/chunk-NZEQVRH5.js.map +1 -0
- package/dist/{chunk-XY4DDNEG.js → chunk-PSWWQXHF.js} +14 -89
- package/dist/chunk-PSWWQXHF.js.map +1 -0
- package/dist/{chunk-4FBZZIYD.js → chunk-QTMYW64F.js} +2 -2
- package/dist/{chunk-D3V5B42D.js → chunk-S4SYLDFX.js} +7 -4
- package/dist/chunk-S4SYLDFX.js.map +1 -0
- package/dist/{chunk-47X6LRCE.js → chunk-UFSG7ACU.js} +10 -7
- package/dist/chunk-UFSG7ACU.js.map +1 -0
- package/dist/{chunk-CY6U5S3X.js → chunk-URWVKRCS.js} +2 -2
- package/dist/{chunk-GSH6QNNS.js → chunk-X74V6ESX.js} +102 -79
- package/dist/chunk-X74V6ESX.js.map +1 -0
- package/dist/{chunk-QAY5UIJO.js → chunk-YBIGNSCZ.js} +178 -59
- package/dist/chunk-YBIGNSCZ.js.map +1 -0
- package/dist/{chunk-Y47J2LJ3.js → chunk-Z6L6YSU6.js} +5 -5
- package/dist/{chunk-T375SUOZ.js → chunk-ZOPLCJSY.js} +86 -31
- package/dist/chunk-ZOPLCJSY.js.map +1 -0
- package/dist/ci-gate.d.ts +44 -0
- package/dist/ci-gate.d.ts.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +2 -2
- package/dist/client.d.ts +77 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.test.d.ts +2 -0
- package/dist/client.test.d.ts.map +1 -0
- package/dist/command-runner.d.ts +74 -0
- package/dist/command-runner.d.ts.map +1 -0
- package/dist/command-runner.test.d.ts +2 -0
- package/dist/command-runner.test.d.ts.map +1 -0
- package/dist/completion-verifier.d.ts +147 -0
- package/dist/completion-verifier.d.ts.map +1 -0
- package/dist/completion-verifier.test.d.ts +9 -0
- package/dist/completion-verifier.test.d.ts.map +1 -0
- package/dist/concurrency.d.ts +23 -0
- package/dist/concurrency.d.ts.map +1 -0
- package/dist/contamination-guard.d.ts +81 -0
- package/dist/contamination-guard.d.ts.map +1 -0
- package/dist/{analyze-runs-DwCEkpO_.d.ts → contract/analyze-runs.d.ts} +9 -10
- package/dist/contract/analyze-runs.d.ts.map +1 -0
- package/dist/contract/define-agent-eval.d.ts +52 -0
- package/dist/contract/define-agent-eval.d.ts.map +1 -0
- package/dist/contract/diff.d.ts +114 -0
- package/dist/contract/diff.d.ts.map +1 -0
- package/dist/contract/index.d.ts +106 -640
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +131 -21
- package/dist/contract/index.js.map +1 -1
- package/dist/{insight-report-BBwvOh6x.d.ts → contract/insight-report.d.ts} +16 -19
- package/dist/contract/insight-report.d.ts.map +1 -0
- package/dist/contract/insight-types-fwd.d.ts +7 -0
- package/dist/contract/insight-types-fwd.d.ts.map +1 -0
- package/dist/contract/intake/agent-trace.d.ts +97 -0
- package/dist/contract/intake/agent-trace.d.ts.map +1 -0
- package/dist/{code-agent-session-BO8nCnv3.d.ts → contract/intake/code-agent-session.d.ts} +15 -17
- package/dist/contract/intake/code-agent-session.d.ts.map +1 -0
- package/dist/contract/intake/feedback-table.d.ts +87 -0
- package/dist/contract/intake/feedback-table.d.ts.map +1 -0
- package/dist/contract/intake/index.d.ts +22 -0
- package/dist/contract/intake/index.d.ts.map +1 -0
- package/dist/contract/intake/otel-spans.d.ts +33 -0
- package/dist/contract/intake/otel-spans.d.ts.map +1 -0
- package/dist/contract/self-improve.d.ts +284 -0
- package/dist/contract/self-improve.d.ts.map +1 -0
- package/dist/{control-runtime-DuFBYg7A.d.ts → control-runtime.d.ts} +23 -26
- package/dist/control-runtime.d.ts.map +1 -0
- package/dist/control-runtime.test.d.ts +2 -0
- package/dist/control-runtime.test.d.ts.map +1 -0
- package/dist/control.d.ts +11 -9
- package/dist/control.d.ts.map +1 -0
- package/dist/control.js +2 -2
- package/dist/convergence.d.ts +29 -0
- package/dist/convergence.d.ts.map +1 -0
- package/dist/{cost-ledger-DuSqlw5B.d.ts → cost-ledger.d.ts} +10 -11
- package/dist/cost-ledger.d.ts.map +1 -0
- package/dist/cost-ledger.test.d.ts +2 -0
- package/dist/cost-ledger.test.d.ts.map +1 -0
- package/dist/cost-report.d.ts +43 -0
- package/dist/cost-report.d.ts.map +1 -0
- package/dist/cost-report.test.d.ts +2 -0
- package/dist/cost-report.test.d.ts.map +1 -0
- package/dist/cost-tracker.d.ts +76 -0
- package/dist/cost-tracker.d.ts.map +1 -0
- package/dist/{counterfactual-Dwibr5IW.d.ts → counterfactual.d.ts} +12 -13
- package/dist/counterfactual.d.ts.map +1 -0
- package/dist/cross-trace-diff.d.ts +56 -0
- package/dist/cross-trace-diff.d.ts.map +1 -0
- package/dist/{dataset-BbGkaN2I.d.ts → dataset.d.ts} +11 -14
- package/dist/dataset.d.ts.map +1 -0
- package/dist/deploy-gate-layer.d.ts +125 -0
- package/dist/deploy-gate-layer.d.ts.map +1 -0
- package/dist/deploy-gate-layer.test.d.ts +2 -0
- package/dist/deploy-gate-layer.test.d.ts.map +1 -0
- package/dist/description-length-gate.d.ts +119 -0
- package/dist/description-length-gate.d.ts.map +1 -0
- package/dist/detectors/edge.test.d.ts +2 -0
- package/dist/detectors/edge.test.d.ts.map +1 -0
- package/dist/detectors/index.d.ts +81 -0
- package/dist/detectors/index.d.ts.map +1 -0
- package/dist/detectors/index.test.d.ts +2 -0
- package/dist/detectors/index.test.d.ts.map +1 -0
- package/dist/diagnose/causal-sweep.d.ts +100 -0
- package/dist/diagnose/causal-sweep.d.ts.map +1 -0
- package/dist/diagnose/index.d.ts +36 -0
- package/dist/diagnose/index.d.ts.map +1 -0
- package/dist/diagnose/remediation.d.ts +68 -0
- package/dist/diagnose/remediation.d.ts.map +1 -0
- package/dist/diagnose/repair.d.ts +77 -0
- package/dist/diagnose/repair.d.ts.map +1 -0
- package/dist/diagnose.d.ts +1 -251
- package/dist/diagnose.js +2 -2
- package/dist/discover-personas.d.ts +35 -0
- package/dist/discover-personas.d.ts.map +1 -0
- package/dist/driver.d.ts +95 -0
- package/dist/driver.d.ts.map +1 -0
- package/dist/driver.test.d.ts +8 -0
- package/dist/driver.test.d.ts.map +1 -0
- package/dist/dual-agent-bench.d.ts +81 -0
- package/dist/dual-agent-bench.d.ts.map +1 -0
- package/dist/error-count-extractor.d.ts +47 -0
- package/dist/error-count-extractor.d.ts.map +1 -0
- package/dist/error-count-extractor.test.d.ts +2 -0
- package/dist/error-count-extractor.test.d.ts.map +1 -0
- package/dist/{errors-CzMUYo7b.d.ts → errors.d.ts} +10 -11
- package/dist/errors.d.ts.map +1 -0
- package/dist/{researcher-DE6Gpnb4.d.ts → eval-campaign.d.ts} +34 -156
- package/dist/eval-campaign.d.ts.map +1 -0
- package/dist/eval-campaign.test.d.ts +2 -0
- package/dist/eval-campaign.test.d.ts.map +1 -0
- package/dist/eval-tools.d.ts +55 -0
- package/dist/eval-tools.d.ts.map +1 -0
- package/dist/eval-trace-store.d.ts +107 -0
- package/dist/eval-trace-store.d.ts.map +1 -0
- package/dist/eval-trace-store.test.d.ts +2 -0
- package/dist/eval-trace-store.test.d.ts.map +1 -0
- package/dist/executor.d.ts +38 -0
- package/dist/executor.d.ts.map +1 -0
- package/dist/executor.test.d.ts +10 -0
- package/dist/executor.test.d.ts.map +1 -0
- package/dist/experiment-tracker.d.ts +178 -0
- package/dist/experiment-tracker.d.ts.map +1 -0
- package/dist/experiment-tracker.test.d.ts +2 -0
- package/dist/experiment-tracker.test.d.ts.map +1 -0
- package/dist/failure-taxonomy.d.ts +38 -0
- package/dist/failure-taxonomy.d.ts.map +1 -0
- package/dist/{feedback-trajectory-D9OVLrg9.d.ts → feedback-trajectory.d.ts} +36 -38
- package/dist/feedback-trajectory.d.ts.map +1 -0
- package/dist/feedback-trajectory.test.d.ts +2 -0
- package/dist/feedback-trajectory.test.d.ts.map +1 -0
- package/dist/flow-layer.d.ts +90 -0
- package/dist/flow-layer.d.ts.map +1 -0
- package/dist/flow-layer.test.d.ts +2 -0
- package/dist/flow-layer.test.d.ts.map +1 -0
- package/dist/fuzz/capsule.d.ts +46 -0
- package/dist/fuzz/capsule.d.ts.map +1 -0
- package/dist/fuzz/cube.d.ts +36 -0
- package/dist/fuzz/cube.d.ts.map +1 -0
- package/dist/fuzz/explorer-cost.test.d.ts +2 -0
- package/dist/fuzz/explorer-cost.test.d.ts.map +1 -0
- package/dist/fuzz/explorer.d.ts +64 -0
- package/dist/fuzz/explorer.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.d.ts +16 -0
- package/dist/fuzz/fuzz-agent.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.test.d.ts +2 -0
- package/dist/fuzz/fuzz-agent.test.d.ts.map +1 -0
- package/dist/fuzz/gates.d.ts +33 -0
- package/dist/fuzz/gates.d.ts.map +1 -0
- package/dist/fuzz/index.d.ts +26 -0
- package/dist/fuzz/index.d.ts.map +1 -0
- package/dist/fuzz/policies.d.ts +28 -0
- package/dist/fuzz/policies.d.ts.map +1 -0
- package/dist/fuzz/tools.d.ts +20 -0
- package/dist/fuzz/tools.d.ts.map +1 -0
- package/dist/fuzz/types.d.ts +307 -0
- package/dist/fuzz/types.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -547
- package/dist/golden-matcher.d.ts +71 -0
- package/dist/golden-matcher.d.ts.map +1 -0
- package/dist/governance/eu-ai-act.d.ts +37 -0
- package/dist/governance/eu-ai-act.d.ts.map +1 -0
- package/dist/governance/index.d.ts +5 -135
- package/dist/governance/index.d.ts.map +1 -0
- package/dist/governance/nist-ai-rmf.d.ts +15 -0
- package/dist/governance/nist-ai-rmf.d.ts.map +1 -0
- package/dist/governance/soc2.d.ts +12 -0
- package/dist/governance/soc2.d.ts.map +1 -0
- package/dist/governance/types.d.ts +66 -0
- package/dist/governance/types.d.ts.map +1 -0
- package/dist/harness-optimizer.d.ts +82 -0
- package/dist/harness-optimizer.d.ts.map +1 -0
- package/dist/held-out-gate.d.ts +135 -0
- package/dist/held-out-gate.d.ts.map +1 -0
- package/dist/hosted/client.d.ts +73 -0
- package/dist/hosted/client.d.ts.map +1 -0
- package/dist/hosted/from-env.test.d.ts +8 -0
- package/dist/hosted/from-env.test.d.ts.map +1 -0
- package/dist/hosted/index.d.ts +10 -238
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/types.d.ts +159 -0
- package/dist/hosted/types.d.ts.map +1 -0
- package/dist/index.d.ts +277 -5643
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +317 -108
- package/dist/index.js.map +1 -1
- package/dist/integrity/backend-integrity.d.ts +71 -0
- package/dist/integrity/backend-integrity.d.ts.map +1 -0
- package/dist/integrity/preflight.d.ts +72 -0
- package/dist/integrity/preflight.d.ts.map +1 -0
- package/dist/integrity/preflight.test.d.ts +2 -0
- package/dist/integrity/preflight.test.d.ts.map +1 -0
- package/dist/integrity/single-backend.d.ts +67 -0
- package/dist/integrity/single-backend.d.ts.map +1 -0
- package/dist/intent-match-judge.d.ts +69 -0
- package/dist/intent-match-judge.d.ts.map +1 -0
- package/dist/intent-match-judge.test.d.ts +2 -0
- package/dist/intent-match-judge.test.d.ts.map +1 -0
- package/dist/{judge-calibration-0p2QcWNE.d.ts → judge-calibration.d.ts} +16 -17
- package/dist/judge-calibration.d.ts.map +1 -0
- package/dist/judge-ensemble.d.ts +66 -0
- package/dist/judge-ensemble.d.ts.map +1 -0
- package/dist/judge-ensemble.test.d.ts +8 -0
- package/dist/judge-ensemble.test.d.ts.map +1 -0
- package/dist/judge-families.d.ts +38 -0
- package/dist/judge-families.d.ts.map +1 -0
- package/dist/judge-panel.d.ts +65 -0
- package/dist/judge-panel.d.ts.map +1 -0
- package/dist/judge-retry.d.ts +70 -0
- package/dist/judge-retry.d.ts.map +1 -0
- package/dist/judge-runner.d.ts +36 -0
- package/dist/judge-runner.d.ts.map +1 -0
- package/dist/judge-runner.test.d.ts +2 -0
- package/dist/judge-runner.test.d.ts.map +1 -0
- package/dist/judges.d.ts +74 -0
- package/dist/judges.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.d.ts +89 -0
- package/dist/keyword-coverage-judge.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.test.d.ts +2 -0
- package/dist/keyword-coverage-judge.test.d.ts.map +1 -0
- package/dist/knowledge/index.d.ts +3 -103
- package/dist/knowledge/index.d.ts.map +1 -0
- package/dist/knowledge/readiness.d.ts +26 -0
- package/dist/knowledge/readiness.d.ts.map +1 -0
- package/dist/knowledge/types.d.ts +75 -0
- package/dist/knowledge/types.d.ts.map +1 -0
- package/dist/live-proof.d.ts +62 -0
- package/dist/live-proof.d.ts.map +1 -0
- package/dist/{llm-client-BeEcAokY.d.ts → llm-client.d.ts} +52 -23
- package/dist/llm-client.d.ts.map +1 -0
- package/dist/llm-client.test.d.ts +2 -0
- package/dist/llm-client.test.d.ts.map +1 -0
- package/dist/locked-jsonl-appender.d.ts +19 -0
- package/dist/locked-jsonl-appender.d.ts.map +1 -0
- package/dist/matrix/aggregation.d.ts +16 -0
- package/dist/matrix/aggregation.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +12 -30
- package/dist/matrix/index.d.ts.map +1 -0
- package/dist/matrix/runner.d.ts +15 -0
- package/dist/matrix/runner.d.ts.map +1 -0
- package/dist/{types-mn5Aqk7x.d.ts → matrix/types.d.ts} +11 -15
- package/dist/matrix/types.d.ts.map +1 -0
- package/dist/meta-eval/calibration.d.ts +47 -0
- package/dist/meta-eval/calibration.d.ts.map +1 -0
- package/dist/meta-eval/correlation-study.d.ts +53 -0
- package/dist/meta-eval/correlation-study.d.ts.map +1 -0
- package/dist/meta-eval/index.d.ts +6 -181
- package/dist/meta-eval/index.d.ts.map +1 -0
- package/dist/meta-eval/index.js +1 -1
- package/dist/{outcome-store-rnXLEqSn.d.ts → meta-eval/outcome-store.d.ts} +7 -8
- package/dist/meta-eval/outcome-store.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → meta-eval/rubric-predictive-validity.d.ts} +8 -11
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +1 -0
- package/dist/meta-eval/sentinel.d.ts +169 -0
- package/dist/meta-eval/sentinel.d.ts.map +1 -0
- package/dist/metrics.d.ts +63 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/model-seats.d.ts +71 -0
- package/dist/model-seats.d.ts.map +1 -0
- package/dist/model-seats.test.d.ts +2 -0
- package/dist/model-seats.test.d.ts.map +1 -0
- package/dist/muffled-gate-scanner.d.ts +102 -0
- package/dist/muffled-gate-scanner.d.ts.map +1 -0
- package/dist/{multi-layer-verifier-DUZXrPDA.d.ts → multi-layer-verifier.d.ts} +12 -15
- package/dist/multi-layer-verifier.d.ts.map +1 -0
- package/dist/multi-layer-verifier.test.d.ts +2 -0
- package/dist/multi-layer-verifier.test.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.d.ts +80 -0
- package/dist/multi-toolchain-layer.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.test.d.ts +2 -0
- package/dist/multi-toolchain-layer.test.d.ts.map +1 -0
- package/dist/multishot/default-tools.d.ts +34 -0
- package/dist/multishot/default-tools.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +7 -290
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js.map +1 -1
- package/dist/multishot/judges.d.ts +32 -0
- package/dist/multishot/judges.d.ts.map +1 -0
- package/dist/multishot/matrix.d.ts +107 -0
- package/dist/multishot/matrix.d.ts.map +1 -0
- package/dist/multishot/multishot.d.ts +23 -0
- package/dist/multishot/multishot.d.ts.map +1 -0
- package/dist/multishot/router.d.ts +37 -0
- package/dist/multishot/router.d.ts.map +1 -0
- package/dist/multishot/types.d.ts +60 -0
- package/dist/multishot/types.d.ts.map +1 -0
- package/dist/observability.d.ts +71 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/oracle.d.ts +55 -0
- package/dist/oracle.d.ts.map +1 -0
- package/dist/orthogonality.d.ts +35 -0
- package/dist/orthogonality.d.ts.map +1 -0
- package/dist/otel-pipeline.d.ts +31 -0
- package/dist/otel-pipeline.d.ts.map +1 -0
- package/dist/paraphrase.d.ts +107 -0
- package/dist/paraphrase.d.ts.map +1 -0
- package/dist/{pareto-E-pembql.d.ts → pareto.d.ts} +9 -10
- package/dist/pareto.d.ts.map +1 -0
- package/dist/partition-held-out.d.ts +70 -0
- package/dist/partition-held-out.d.ts.map +1 -0
- package/dist/partition-held-out.test.d.ts +2 -0
- package/dist/partition-held-out.test.d.ts.map +1 -0
- package/dist/perf/index.d.ts +13 -119
- package/dist/perf/index.d.ts.map +1 -0
- package/dist/perf/integrity.d.ts +30 -0
- package/dist/perf/integrity.d.ts.map +1 -0
- package/dist/perf/journey.d.ts +45 -0
- package/dist/perf/journey.d.ts.map +1 -0
- package/dist/perf/ratchet.d.ts +47 -0
- package/dist/perf/ratchet.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.d.ts +31 -0
- package/dist/pipelines/budget-breach.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.test.d.ts +2 -0
- package/dist/pipelines/budget-breach.test.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.d.ts +38 -0
- package/dist/pipelines/failure-cluster.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.test.d.ts +2 -0
- package/dist/pipelines/failure-cluster.test.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.d.ts +26 -0
- package/dist/pipelines/first-divergence.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.test.d.ts +2 -0
- package/dist/pipelines/first-divergence.test.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +8 -173
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/index.js +40 -15
- package/dist/pipelines/index.js.map +1 -1
- package/dist/pipelines/judge-agreement.d.ts +26 -0
- package/dist/pipelines/judge-agreement.d.ts.map +1 -0
- package/dist/pipelines/judge-agreement.test.d.ts +2 -0
- package/dist/pipelines/judge-agreement.test.d.ts.map +1 -0
- package/dist/pipelines/regression.d.ts +23 -0
- package/dist/pipelines/regression.d.ts.map +1 -0
- package/dist/pipelines/regression.test.d.ts +2 -0
- package/dist/pipelines/regression.test.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.d.ts +32 -0
- package/dist/pipelines/stuck-loop.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.test.d.ts +2 -0
- package/dist/pipelines/stuck-loop.test.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.d.ts +34 -0
- package/dist/pipelines/tool-waste.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.test.d.ts +2 -0
- package/dist/pipelines/tool-waste.test.d.ts.map +1 -0
- package/dist/playbook.d.ts +16 -0
- package/dist/playbook.d.ts.map +1 -0
- package/dist/pr-review-benchmark.d.ts +88 -0
- package/dist/pr-review-benchmark.d.ts.map +1 -0
- package/dist/pr-review-benchmark.test.d.ts +2 -0
- package/dist/pr-review-benchmark.test.d.ts.map +1 -0
- package/dist/pre-registration.d.ts +125 -0
- package/dist/pre-registration.d.ts.map +1 -0
- package/dist/prm/builtin-rubrics.d.ts +33 -0
- package/dist/prm/builtin-rubrics.d.ts.map +1 -0
- package/dist/prm/index.d.ts +5 -100
- package/dist/prm/index.d.ts.map +1 -0
- package/dist/prm/index.js +55 -12
- package/dist/prm/index.js.map +1 -1
- package/dist/prm/inference.d.ts +29 -0
- package/dist/prm/inference.d.ts.map +1 -0
- package/dist/prm/inference.test.d.ts +2 -0
- package/dist/prm/inference.test.d.ts.map +1 -0
- package/dist/{rubric-BOfxn4ja.d.ts → prm/rubric.d.ts} +10 -13
- package/dist/prm/rubric.d.ts.map +1 -0
- package/dist/prm/training-export.d.ts +38 -0
- package/dist/prm/training-export.d.ts.map +1 -0
- package/dist/produced-state.d.ts +63 -0
- package/dist/produced-state.d.ts.map +1 -0
- package/dist/produced-state.test.d.ts +8 -0
- package/dist/produced-state.test.d.ts.map +1 -0
- package/dist/profile/baselines.d.ts +37 -0
- package/dist/profile/baselines.d.ts.map +1 -0
- package/dist/profile/index.d.ts +105 -0
- package/dist/profile/index.d.ts.map +1 -0
- package/dist/promotion-gate.d.ts +93 -0
- package/dist/promotion-gate.d.ts.map +1 -0
- package/dist/prompt-registry.d.ts +41 -0
- package/dist/prompt-registry.d.ts.map +1 -0
- package/dist/propose-review-control.d.ts +49 -0
- package/dist/propose-review-control.d.ts.map +1 -0
- package/dist/propose-review-control.test.d.ts +2 -0
- package/dist/propose-review-control.test.d.ts.map +1 -0
- package/dist/propose-review.d.ts +155 -0
- package/dist/propose-review.d.ts.map +1 -0
- package/dist/{red-team-BXHil6c8.d.ts → red-team.d.ts} +14 -16
- package/dist/red-team.d.ts.map +1 -0
- package/dist/reference-replay-steering.d.ts +11 -0
- package/dist/reference-replay-steering.d.ts.map +1 -0
- package/dist/reference-replay.d.ts +176 -0
- package/dist/reference-replay.d.ts.map +1 -0
- package/dist/reflective-mutation.d.ts +79 -0
- package/dist/reflective-mutation.d.ts.map +1 -0
- package/dist/registry.d.ts +31 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/release-confidence.d.ts +128 -0
- package/dist/release-confidence.d.ts.map +1 -0
- package/dist/release-report.d.ts +11 -0
- package/dist/release-report.d.ts.map +1 -0
- package/dist/replay.d.ts +120 -0
- package/dist/replay.d.ts.map +1 -0
- package/dist/reporter.d.ts +14 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporting.d.ts +15 -15
- package/dist/reporting.d.ts.map +1 -0
- package/dist/reporting.js +3 -3
- package/dist/researcher.d.ts +140 -0
- package/dist/researcher.d.ts.map +1 -0
- package/dist/reviewer.d.ts +118 -0
- package/dist/reviewer.d.ts.map +1 -0
- package/dist/reviewer.test.d.ts +2 -0
- package/dist/reviewer.test.d.ts.map +1 -0
- package/dist/reward-model-export.d.ts +60 -0
- package/dist/reward-model-export.d.ts.map +1 -0
- package/dist/rl/active-curriculum.d.ts +110 -0
- package/dist/rl/active-curriculum.d.ts.map +1 -0
- package/dist/rl/adaptation-eval.d.ts +109 -0
- package/dist/rl/adaptation-eval.d.ts.map +1 -0
- package/dist/{adversarial-DIVcDoI_.d.ts → rl/adversarial.d.ts} +6 -7
- package/dist/rl/adversarial.d.ts.map +1 -0
- package/dist/rl/compute-curves.d.ts +127 -0
- package/dist/rl/compute-curves.d.ts.map +1 -0
- package/dist/rl/contamination.d.ts +117 -0
- package/dist/rl/contamination.d.ts.map +1 -0
- package/dist/rl/corpus.d.ts +55 -0
- package/dist/rl/corpus.d.ts.map +1 -0
- package/dist/rl/corpus.test.d.ts +2 -0
- package/dist/rl/corpus.test.d.ts.map +1 -0
- package/dist/rl/dataset.d.ts +102 -0
- package/dist/rl/dataset.d.ts.map +1 -0
- package/dist/rl/dataset.test.d.ts +2 -0
- package/dist/rl/dataset.test.d.ts.map +1 -0
- package/dist/rl/exporters.d.ts +141 -0
- package/dist/rl/exporters.d.ts.map +1 -0
- package/dist/rl/index.d.ts +49 -0
- package/dist/rl/index.d.ts.map +1 -0
- package/dist/{off-policy-DiwuKKg7.d.ts → rl/off-policy.d.ts} +8 -9
- package/dist/rl/off-policy.d.ts.map +1 -0
- package/dist/rl/predictive-validity-researcher.d.ts +69 -0
- package/dist/rl/predictive-validity-researcher.d.ts.map +1 -0
- package/dist/rl/preferences.d.ts +141 -0
- package/dist/rl/preferences.d.ts.map +1 -0
- package/dist/rl/process-reward.d.ts +122 -0
- package/dist/rl/process-reward.d.ts.map +1 -0
- package/dist/rl/reward-hacking.d.ts +104 -0
- package/dist/rl/reward-hacking.d.ts.map +1 -0
- package/dist/rl/rl-campaign.d.ts +85 -0
- package/dist/rl/rl-campaign.d.ts.map +1 -0
- package/dist/rl/run-record-adapters.d.ts +56 -0
- package/dist/rl/run-record-adapters.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.d.ts +166 -0
- package/dist/rl/sim-fidelity.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.test.d.ts +2 -0
- package/dist/rl/sim-fidelity.test.d.ts.map +1 -0
- package/dist/rl/tournament.d.ts +115 -0
- package/dist/rl/tournament.d.ts.map +1 -0
- package/dist/rl/verifiable-reward.d.ts +124 -0
- package/dist/rl/verifiable-reward.d.ts.map +1 -0
- package/dist/rl.d.ts +1 -1192
- package/dist/rl.js +615 -615
- package/dist/rl.js.map +1 -1
- package/dist/{run-campaign-RDGAM5KJ.js → run-campaign-WXY7KI67.js} +3 -3
- package/dist/run-critic.d.ts +23 -0
- package/dist/run-critic.d.ts.map +1 -0
- package/dist/run-evidence.d.ts +32 -0
- package/dist/run-evidence.d.ts.map +1 -0
- package/dist/{run-record-e7vj1uZQ.d.ts → run-record.d.ts} +16 -122
- package/dist/run-record.d.ts.map +1 -0
- package/dist/run-record.test.d.ts +2 -0
- package/dist/run-record.test.d.ts.map +1 -0
- package/dist/run-score.d.ts +31 -0
- package/dist/run-score.d.ts.map +1 -0
- package/dist/runtime-trajectory.d.ts +47 -0
- package/dist/runtime-trajectory.d.ts.map +1 -0
- package/dist/{test-graded-scenario-BdVaPyHT.d.ts → sandbox-harness.d.ts} +34 -60
- package/dist/sandbox-harness.d.ts.map +1 -0
- package/dist/sandbox-harness.test.d.ts +2 -0
- package/dist/sandbox-harness.test.d.ts.map +1 -0
- package/dist/sandbox-pool.d.ts +74 -0
- package/dist/sandbox-pool.d.ts.map +1 -0
- package/dist/sandbox-pool.test.d.ts +2 -0
- package/dist/sandbox-pool.test.d.ts.map +1 -0
- package/dist/scorecard.d.ts +133 -0
- package/dist/scorecard.d.ts.map +1 -0
- package/dist/scorecard.test.d.ts +2 -0
- package/dist/scorecard.test.d.ts.map +1 -0
- package/dist/self-play.d.ts +69 -0
- package/dist/self-play.d.ts.map +1 -0
- package/dist/semantic-concept-judge.d.ts +135 -0
- package/dist/semantic-concept-judge.d.ts.map +1 -0
- package/dist/semantic-concept-judge.test.d.ts +2 -0
- package/dist/semantic-concept-judge.test.d.ts.map +1 -0
- package/dist/{sequential-5iSVfzl2.d.ts → sequential.d.ts} +9 -10
- package/dist/sequential.d.ts.map +1 -0
- package/dist/{series-convergence-D5OWMBg6.d.ts → series-convergence.d.ts} +4 -5
- package/dist/series-convergence.d.ts.map +1 -0
- package/dist/slo.d.ts +48 -0
- package/dist/slo.d.ts.map +1 -0
- package/dist/state-continuity.d.ts +47 -0
- package/dist/state-continuity.d.ts.map +1 -0
- package/dist/{statistics-C7PozGrZ.d.ts → statistics.d.ts} +152 -39
- package/dist/statistics.d.ts.map +1 -0
- package/dist/statistics.test.d.ts +2 -0
- package/dist/statistics.test.d.ts.map +1 -0
- package/dist/steering-optimizer.d.ts +58 -0
- package/dist/steering-optimizer.d.ts.map +1 -0
- package/dist/steering.d.ts +24 -0
- package/dist/steering.d.ts.map +1 -0
- package/dist/storyboard/code-edit.d.ts +64 -0
- package/dist/storyboard/code-edit.d.ts.map +1 -0
- package/dist/storyboard/code-edit.test.d.ts +2 -0
- package/dist/storyboard/code-edit.test.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +16 -81
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.test.d.ts +2 -0
- package/dist/storyboard/index.test.d.ts.map +1 -0
- package/dist/{summary-report-DGmUucwQ.d.ts → summary-report.d.ts} +23 -160
- package/dist/summary-report.d.ts.map +1 -0
- package/dist/telemetry/client.d.ts +35 -0
- package/dist/telemetry/client.d.ts.map +1 -0
- package/dist/telemetry/index.d.ts +17 -35
- package/dist/telemetry/index.d.ts.map +1 -0
- package/dist/telemetry/schema.d.ts +61 -0
- package/dist/telemetry/schema.d.ts.map +1 -0
- package/dist/telemetry/sink-fetch.d.ts +39 -0
- package/dist/telemetry/sink-fetch.d.ts.map +1 -0
- package/dist/telemetry/{file.d.ts → sink-file.d.ts} +5 -7
- package/dist/telemetry/sink-file.d.ts.map +1 -0
- package/dist/test-graded-scenario.d.ts +42 -0
- package/dist/test-graded-scenario.d.ts.map +1 -0
- package/dist/testing.d.ts +5 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +8 -0
- package/dist/testing.js.map +1 -0
- package/dist/tool-use-metrics.d.ts +35 -0
- package/dist/tool-use-metrics.d.ts.map +1 -0
- package/dist/trace/capture-fetch.d.ts +48 -0
- package/dist/trace/capture-fetch.d.ts.map +1 -0
- package/dist/trace/capture-fetch.test.d.ts +2 -0
- package/dist/trace/capture-fetch.test.d.ts.map +1 -0
- package/dist/{emitter-DEZwY14K.d.ts → trace/emitter.d.ts} +9 -12
- package/dist/trace/emitter.d.ts.map +1 -0
- package/dist/trace/extract-usage.d.ts +46 -0
- package/dist/trace/extract-usage.d.ts.map +1 -0
- package/dist/trace/extract-usage.test.d.ts +2 -0
- package/dist/trace/extract-usage.test.d.ts.map +1 -0
- package/dist/trace/index.d.ts +15 -0
- package/dist/trace/index.d.ts.map +1 -0
- package/dist/{integrity-VJ9A7aST.d.ts → trace/integrity.d.ts} +11 -14
- package/dist/trace/integrity.d.ts.map +1 -0
- package/dist/trace/otel-bridge.d.ts +29 -0
- package/dist/trace/otel-bridge.d.ts.map +1 -0
- package/dist/trace/otel-export.d.ts +52 -0
- package/dist/trace/otel-export.d.ts.map +1 -0
- package/dist/trace/otel.d.ts +57 -0
- package/dist/trace/otel.d.ts.map +1 -0
- package/dist/trace/otlp-attributes.d.ts +17 -0
- package/dist/trace/otlp-attributes.d.ts.map +1 -0
- package/dist/trace/query.d.ts +29 -0
- package/dist/trace/query.d.ts.map +1 -0
- package/dist/trace/query.test.d.ts +2 -0
- package/dist/trace/query.test.d.ts.map +1 -0
- package/dist/{raw-provider-sink-C46HDghv.d.ts → trace/raw-provider-sink.d.ts} +13 -14
- package/dist/trace/raw-provider-sink.d.ts.map +1 -0
- package/dist/{redact-B40YG2M_.d.ts → trace/redact.d.ts} +7 -8
- package/dist/trace/redact.d.ts.map +1 -0
- package/dist/{schema-m0gsnbt3.d.ts → trace/schema.d.ts} +29 -30
- package/dist/trace/schema.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.d.ts +72 -0
- package/dist/trace/store-to-otlp.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.test.d.ts +2 -0
- package/dist/trace/store-to-otlp.test.d.ts.map +1 -0
- package/dist/{store-CKUAgsJz.d.ts → trace/store.d.ts} +37 -13
- package/dist/trace/store.d.ts.map +1 -0
- package/dist/trace/store.test.d.ts +2 -0
- package/dist/trace/store.test.d.ts.map +1 -0
- package/dist/{analyst-C8HHvfJp.d.ts → trace-analyst/analyst.d.ts} +12 -14
- package/dist/trace-analyst/analyst.d.ts.map +1 -0
- package/dist/trace-analyst/analyst.test.d.ts +2 -0
- package/dist/trace-analyst/analyst.test.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts +40 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +2 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +1 -0
- package/dist/trace-analyst/hook.d.ts +55 -0
- package/dist/trace-analyst/hook.d.ts.map +1 -0
- package/dist/trace-analyst/index.d.ts +18 -0
- package/dist/trace-analyst/index.d.ts.map +1 -0
- package/dist/trace-analyst/insights.d.ts +71 -0
- package/dist/trace-analyst/insights.d.ts.map +1 -0
- package/dist/trace-analyst/insights.test.d.ts +2 -0
- package/dist/trace-analyst/insights.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-flatten.d.ts +42 -0
- package/dist/trace-analyst/otlp-flatten.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.d.ts +85 -0
- package/dist/trace-analyst/otlp-span.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-span.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts +115 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/prompts.d.ts +6 -0
- package/dist/trace-analyst/prompts.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.d.ts +126 -0
- package/dist/trace-analyst/store-otlp.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.test.d.ts +8 -0
- package/dist/trace-analyst/store-otlp.test.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/store.d.ts +63 -0
- package/dist/trace-analyst/store.d.ts.map +1 -0
- package/dist/trace-analyst/tools.d.ts +44 -0
- package/dist/trace-analyst/tools.d.ts.map +1 -0
- package/dist/trace-analyst/tools.test.d.ts +10 -0
- package/dist/trace-analyst/tools.test.d.ts.map +1 -0
- package/dist/{store-C1YxJDEK.d.ts → trace-analyst/types.d.ts} +18 -74
- package/dist/trace-analyst/types.d.ts.map +1 -0
- package/dist/trace-contracts.d.ts +180 -0
- package/dist/trace-contracts.d.ts.map +1 -0
- package/dist/traced-analyst.d.ts +26 -0
- package/dist/traced-analyst.d.ts.map +1 -0
- package/dist/traced-judges.d.ts +27 -0
- package/dist/traced-judges.d.ts.map +1 -0
- package/dist/traces.d.ts +4 -947
- package/dist/traces.d.ts.map +1 -0
- package/dist/traces.js +42 -12
- package/dist/{trajectory-GEdXJCL5.d.ts → trajectory.d.ts} +8 -9
- package/dist/trajectory.d.ts.map +1 -0
- package/dist/{types-Croy5h7V.d.ts → types.d.ts} +33 -33
- package/dist/types.d.ts.map +1 -0
- package/dist/ui-finding.d.ts +104 -0
- package/dist/ui-finding.d.ts.map +1 -0
- package/dist/verdict-cache.d.ts +78 -0
- package/dist/verdict-cache.d.ts.map +1 -0
- package/dist/verdict-cache.test.d.ts +2 -0
- package/dist/verdict-cache.test.d.ts.map +1 -0
- package/dist/{verdict-C9MlYujm.d.ts → verdict.d.ts} +2 -3
- package/dist/verdict.d.ts.map +1 -0
- package/dist/visual-diff.d.ts +32 -0
- package/dist/visual-diff.d.ts.map +1 -0
- package/dist/wire/handlers.d.ts +54 -0
- package/dist/wire/handlers.d.ts.map +1 -0
- package/dist/wire/index.d.ts +13 -570
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/index.js +2 -2
- package/dist/wire/openapi.d.ts +3 -0
- package/dist/wire/openapi.d.ts.map +1 -0
- package/dist/wire/rpc.d.ts +21 -0
- package/dist/wire/rpc.d.ts.map +1 -0
- package/dist/wire/rubrics.d.ts +34 -0
- package/dist/wire/rubrics.d.ts.map +1 -0
- package/dist/wire/schemas.d.ts +410 -0
- package/dist/wire/schemas.d.ts.map +1 -0
- package/dist/wire/server.d.ts +60 -0
- package/dist/wire/server.d.ts.map +1 -0
- package/dist/workflow/event-schema.d.ts +5 -0
- package/dist/workflow/event-schema.d.ts.map +1 -0
- package/dist/workflow/feedback-pack.d.ts +99 -0
- package/dist/workflow/feedback-pack.d.ts.map +1 -0
- package/dist/workflow/index.d.ts +22 -495
- package/dist/workflow/index.d.ts.map +1 -0
- package/dist/workflow/index.js +2 -2
- package/dist/workflow/intelligence-export.d.ts +62 -0
- package/dist/workflow/intelligence-export.d.ts.map +1 -0
- package/dist/workflow/partner-report.d.ts +49 -0
- package/dist/workflow/partner-report.d.ts.map +1 -0
- package/dist/workflow/phase-graph.d.ts +43 -0
- package/dist/workflow/phase-graph.d.ts.map +1 -0
- package/dist/workflow/promotion-gate.d.ts +61 -0
- package/dist/workflow/promotion-gate.d.ts.map +1 -0
- package/dist/workflow/run-record.d.ts +12 -0
- package/dist/workflow/run-record.d.ts.map +1 -0
- package/dist/workflow/runtime-adapter.d.ts +20 -0
- package/dist/workflow/runtime-adapter.d.ts.map +1 -0
- package/dist/workflow/sanitize.d.ts +21 -0
- package/dist/workflow/sanitize.d.ts.map +1 -0
- package/dist/workflow/schema.d.ts +5 -0
- package/dist/workflow/schema.d.ts.map +1 -0
- package/dist/workflow/summary.d.ts +43 -0
- package/dist/workflow/summary.d.ts.map +1 -0
- package/dist/workflow/trace-event-fields.d.ts +6 -0
- package/dist/workflow/trace-event-fields.d.ts.map +1 -0
- package/dist/workflow/trajectory.d.ts +15 -0
- package/dist/workflow/trajectory.d.ts.map +1 -0
- package/dist/workflow/types.d.ts +68 -0
- package/dist/workflow/types.d.ts.map +1 -0
- package/dist/workspace-inspector.d.ts +67 -0
- package/dist/workspace-inspector.d.ts.map +1 -0
- package/dist/wrangler-deploy-runner.test.d.ts +2 -0
- package/dist/wrangler-deploy-runner.test.d.ts.map +1 -0
- package/docs/campaign-proposers.md +170 -0
- package/docs/concepts.md +8 -4
- package/docs/customer-journeys.md +15 -13
- package/docs/design/loop-taxonomy.md +34 -66
- package/docs/distributed-driver.md +14 -14
- package/docs/feature-guide.md +1 -1
- package/docs/hosted-ingest-spec.md +2 -3
- package/docs/multi-shot-optimization.md +8 -8
- package/docs/product-eval-adoption.md +1 -1
- package/docs/self-improvement-map.md +33 -29
- package/package.json +9 -15
- package/dist/calibration-Cpr3WaX3.d.ts +0 -101
- package/dist/chunk-47X6LRCE.js.map +0 -1
- package/dist/chunk-6SOJM3VR.js.map +0 -1
- package/dist/chunk-CVVHBFGN.js.map +0 -1
- package/dist/chunk-D3V5B42D.js.map +0 -1
- package/dist/chunk-E4GH6USR.js.map +0 -1
- package/dist/chunk-GSH6QNNS.js.map +0 -1
- package/dist/chunk-KWRRMR3J.js.map +0 -1
- package/dist/chunk-LMZQ2Z4U.js.map +0 -1
- package/dist/chunk-QAY5UIJO.js.map +0 -1
- package/dist/chunk-QG2OVF2D.js.map +0 -1
- package/dist/chunk-T375SUOZ.js.map +0 -1
- package/dist/chunk-TWS7AZEY.js.map +0 -1
- package/dist/chunk-UMMZHCPB.js.map +0 -1
- package/dist/chunk-XY4DDNEG.js.map +0 -1
- package/dist/chunk-ZFIBGEOL.js.map +0 -1
- package/dist/control-_Qb7skHX.d.ts +0 -259
- package/dist/corpus-BoR-041R.d.ts +0 -560
- package/dist/failure-cluster-CL7IVgkJ.d.ts +0 -76
- package/dist/harness-optimizer-EnEnQPsr.d.ts +0 -106
- package/dist/index-Bx3gZ8xl.d.ts +0 -159
- package/dist/pre-registration-mAnCugl9.d.ts +0 -523
- package/dist/provenance-LnqRT0sS.d.ts +0 -441
- package/dist/query-CqTxMwDw.d.ts +0 -31
- package/dist/release-report-euXIV_Sk.d.ts +0 -233
- package/dist/run-critic-BAIjX99r.d.ts +0 -56
- package/dist/run-improvement-loop-5z_l5zDz.d.ts +0 -427
- package/dist/runtime-trajectory-BDgfGZSr.d.ts +0 -49
- package/dist/semantic-concept-judge-Dn8Z6KEG.d.ts +0 -624
- package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
- package/docs/design/external-agent-wedge.md +0 -89
- package/docs/design/phase-d-rfc.md +0 -125
- package/docs/design/phase4-consumer-migration.md +0 -70
- package/docs/design/primitives-integration-spec.md +0 -393
- package/docs/design/product-self-improvement-loop.md +0 -146
- package/docs/design/self-improvement-engine.md +0 -140
- package/docs/design/self-improvement-protocol.md +0 -223
- package/docs/design/self-improvement-roadmap.md +0 -106
- package/docs/design/substrate-gaps.md +0 -118
- package/docs/phase-b-pairing-kit.md +0 -188
- package/docs/phase-b-runbook.md +0 -176
- package/docs/pilot/README.md +0 -62
- package/docs/pilot/customer-checklist.md +0 -90
- package/docs/pilot/integration-foreign-stack.md +0 -296
- package/docs/pilot/integration-tangle-stack.md +0 -248
- package/docs/pilot/one-pager.md +0 -161
- package/docs/pilot/sample-insight-report.json +0 -172
- package/docs/quickstart-external.md +0 -229
- package/docs/research/belief-state-agent-eval-roadmap.md +0 -593
- package/docs/research/research-roadmap.md +0 -205
- package/docs/specs/driver-honest-spec.md +0 -251
- package/docs/specs/hermes-self-improvement-audit.md +0 -93
- package/docs/specs/profile-versioning.md +0 -291
- package/docs/three-package-architecture.md +0 -168
- /package/dist/{chunk-QS3RBQPI.js.map → chunk-D5KBVULY.js.map} +0 -0
- /package/dist/{chunk-UHMJT4T7.js.map → chunk-IZCEK2HR.js.map} +0 -0
- /package/dist/{chunk-L3JOU6XM.js.map → chunk-NACM5RS7.js.map} +0 -0
- /package/dist/{chunk-4FBZZIYD.js.map → chunk-QTMYW64F.js.map} +0 -0
- /package/dist/{chunk-CY6U5S3X.js.map → chunk-URWVKRCS.js.map} +0 -0
- /package/dist/{chunk-Y47J2LJ3.js.map → chunk-Z6L6YSU6.js.map} +0 -0
- /package/dist/{run-campaign-RDGAM5KJ.js.map → run-campaign-WXY7KI67.js.map} +0 -0
|
@@ -1,181 +1,6 @@
|
|
|
1
|
-
export
|
|
2
|
-
export
|
|
3
|
-
export
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
import '../store-CKUAgsJz.js';
|
|
8
|
-
import '../schema-m0gsnbt3.js';
|
|
9
|
-
import '../run-record-e7vj1uZQ.js';
|
|
10
|
-
import '../errors-CzMUYo7b.js';
|
|
11
|
-
import '../types-Croy5h7V.js';
|
|
12
|
-
import '@tangle-network/tcloud';
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Judge sentinel — eval trustworthiness as a continuously measured,
|
|
16
|
-
* alarmed trend.
|
|
17
|
-
*
|
|
18
|
-
* Judges are models; models change underneath us; calibration decays.
|
|
19
|
-
* This module composes three existing instruments into one loop:
|
|
20
|
-
*
|
|
21
|
-
* snapshot — adapters turn real instrument outputs (`calibrateJudge` /
|
|
22
|
-
* `calibrateJudgeContinuous`, `corpusInterRaterAgreement` /
|
|
23
|
-
* `continuousAgreement`, a rerun sentinel golden set) into
|
|
24
|
-
* `SentinelSnapshot`s
|
|
25
|
-
* series — `analyzeSeries` (src/series-convergence) classifies each
|
|
26
|
-
* judge×metric history as stabilized / drifting / noisy
|
|
27
|
-
* alarm — `judgeSentinelReport` turns trends + thresholds into named
|
|
28
|
-
* alarms; `evalHealthStamp` is the tiny object a campaign
|
|
29
|
-
* attaches to its verdicts
|
|
30
|
-
*
|
|
31
|
-
* Alarm conditions:
|
|
32
|
-
* - convergence state `drifting-down` on any tracked metric
|
|
33
|
-
* - irr below `minIrr`
|
|
34
|
-
* - calibrationKappa / sentinelPassRate dropped vs the series baseline
|
|
35
|
-
* beyond `maxKappaDrop` / `maxSentinelDrop`
|
|
36
|
-
* - judge model changed with no post-change golden-grounded snapshot
|
|
37
|
-
* (the silent-upgrade trap: judges agreeing with each other after a
|
|
38
|
-
* model swap proves nothing — only re-measuring against gold does)
|
|
39
|
-
* - newest snapshot older than `staleAfterDays` relative to `asOf`
|
|
40
|
-
*
|
|
41
|
-
* Wire-in: run the snapshot adapters from a post-campaign hook or a
|
|
42
|
-
* nightly job, append to a `SentinelStore`, then compute
|
|
43
|
-
* `judgeSentinelReport(await store.history(), { asOf })` with `asOf` =
|
|
44
|
-
* the campaign timestamp. Attach `evalHealthStamp(report)` to campaign
|
|
45
|
-
* verdicts: an alarmed sentinel means every downstream verdict carries
|
|
46
|
-
* an integrity warning until the judge is recalibrated.
|
|
47
|
-
*
|
|
48
|
-
* The module is clock-free — every timestamp (`at`, `asOf`) is
|
|
49
|
-
* caller-supplied ISO-8601, so reports are reproducible.
|
|
50
|
-
*/
|
|
51
|
-
|
|
52
|
-
declare const SENTINEL_METRIC_NAMES: readonly ["irr", "calibrationKappa", "sentinelPassRate"];
|
|
53
|
-
type SentinelMetricName = (typeof SENTINEL_METRIC_NAMES)[number];
|
|
54
|
-
interface SentinelMetrics {
|
|
55
|
-
/** Inter-rater reliability — ICC(2,1) from agreement instruments. */
|
|
56
|
-
irr?: number;
|
|
57
|
-
/** Weighted κ vs the human golden set (`calibrateJudge*`). */
|
|
58
|
-
calibrationKappa?: number;
|
|
59
|
-
/** Fraction of a fixed sentinel golden set the judge still scores within tolerance. */
|
|
60
|
-
sentinelPassRate?: number;
|
|
61
|
-
}
|
|
62
|
-
interface SentinelSnapshot {
|
|
63
|
-
/** Caller-supplied ISO-8601 timestamp — the module never reads a clock. */
|
|
64
|
-
at: string;
|
|
65
|
-
judgeId: string;
|
|
66
|
-
/** Exact model identity behind the judge (pin the full version string). */
|
|
67
|
-
judgeModel: string;
|
|
68
|
-
/**
|
|
69
|
-
* Metrics measured at `at`. May be empty: an empty-metrics snapshot is a
|
|
70
|
-
* model-change marker — it records the new `judgeModel` immediately and
|
|
71
|
-
* the silent-upgrade alarm stays raised until a golden-grounded snapshot
|
|
72
|
-
* (calibrationKappa or sentinelPassRate) follows.
|
|
73
|
-
*/
|
|
74
|
-
metrics: SentinelMetrics;
|
|
75
|
-
}
|
|
76
|
-
/** Identity + timestamp the caller supplies alongside an instrument output. */
|
|
77
|
-
interface SnapshotMeta {
|
|
78
|
-
at: string;
|
|
79
|
-
judgeId: string;
|
|
80
|
-
judgeModel: string;
|
|
81
|
-
}
|
|
82
|
-
/** Shape guard for snapshots crossing a boundary (store append, JSONL load, report input). */
|
|
83
|
-
declare function validateSentinelSnapshot(snapshot: SentinelSnapshot, source?: string): void;
|
|
84
|
-
/**
|
|
85
|
-
* From `calibrateJudge` / `calibrateJudgeContinuous` output. Prefers the
|
|
86
|
-
* un-rounded continuous κ_w when the report carries one (integer κ
|
|
87
|
-
* discards information for [0,1] judges).
|
|
88
|
-
*/
|
|
89
|
-
declare function snapshotFromCalibration(report: CalibrationResult | ContinuousCalibrationResult, meta: SnapshotMeta): SentinelSnapshot;
|
|
90
|
-
/**
|
|
91
|
-
* From `corpusInterRaterAgreement` (statistics.ts) or `continuousAgreement`
|
|
92
|
-
* (judge-calibration.ts) output. IRR = ICC(2,1) — the reliability
|
|
93
|
-
* coefficient both instruments compute. For ensemble-level agreement,
|
|
94
|
-
* `meta.judgeId` names the ensemble and `meta.judgeModel` pins its
|
|
95
|
-
* composition (e.g. a joined list of member model strings).
|
|
96
|
-
*/
|
|
97
|
-
declare function snapshotFromAgreement(report: CorpusAgreementReport | ContinuousAgreement, meta: SnapshotMeta): SentinelSnapshot;
|
|
98
|
-
interface SentinelSetOptions {
|
|
99
|
-
/** Max |judge − human| for an item to count as passed. Default 0.1 (tuned for [0,1] scores). */
|
|
100
|
-
tolerance?: number;
|
|
101
|
-
}
|
|
102
|
-
/**
|
|
103
|
-
* From a rerun of a fixed sentinel golden set: the same `GoldenItem`s the
|
|
104
|
-
* judge was originally calibrated on, re-scored periodically. Pass rate =
|
|
105
|
-
* fraction of joined items where the judge stays within `tolerance` of the
|
|
106
|
-
* human score. Items the judge didn't score are excluded from the
|
|
107
|
-
* denominator; zero joined items is an error, not a 0% pass rate.
|
|
108
|
-
*/
|
|
109
|
-
declare function snapshotFromSentinelSet(scores: CandidateScore[], golden: GoldenItem[], meta: SnapshotMeta, options?: SentinelSetOptions): SentinelSnapshot;
|
|
110
|
-
interface SentinelStore {
|
|
111
|
-
append(snapshot: SentinelSnapshot): Promise<void>;
|
|
112
|
-
/** All snapshots (optionally for one judge), in append order. */
|
|
113
|
-
history(judgeId?: string): Promise<SentinelSnapshot[]>;
|
|
114
|
-
}
|
|
115
|
-
declare function inMemorySentinelStore(initial?: SentinelSnapshot[]): SentinelStore;
|
|
116
|
-
/**
|
|
117
|
-
* JSONL store — one snapshot per line, appended atomically per call.
|
|
118
|
-
* A corrupt or shape-invalid line is a loud error naming the file and
|
|
119
|
-
* line number: a sentinel history that silently drops records would
|
|
120
|
-
* defeat the drift detection it exists to provide.
|
|
121
|
-
*/
|
|
122
|
-
declare function fileSentinelStore(path: string): SentinelStore;
|
|
123
|
-
interface SentinelThresholds {
|
|
124
|
-
/** Floor for irr — below it the judge ensemble is unreliable regardless of trend. Default 0.6. */
|
|
125
|
-
minIrr?: number;
|
|
126
|
-
/** Max tolerated calibrationKappa drop vs the series baseline. Default 0.15. */
|
|
127
|
-
maxKappaDrop?: number;
|
|
128
|
-
/** Max tolerated sentinelPassRate drop vs the series baseline. Default 0.1. */
|
|
129
|
-
maxSentinelDrop?: number;
|
|
130
|
-
/** Newest snapshot older than this (days, vs asOf) = the sentinel itself went blind. Default 30. */
|
|
131
|
-
staleAfterDays?: number;
|
|
132
|
-
}
|
|
133
|
-
interface JudgeSentinelOptions {
|
|
134
|
-
/** Caller-supplied ISO timestamp staleness is measured against. */
|
|
135
|
-
asOf: string;
|
|
136
|
-
thresholds?: SentinelThresholds;
|
|
137
|
-
/** Forwarded to `analyzeSeries` (window, stableCv, driftRun). */
|
|
138
|
-
convergence?: SeriesConvergenceOptions;
|
|
139
|
-
}
|
|
140
|
-
interface SentinelTrend {
|
|
141
|
-
judgeId: string;
|
|
142
|
-
metric: SentinelMetricName;
|
|
143
|
-
/** Verbatim `analyzeSeries` state for this judge×metric history. */
|
|
144
|
-
state: SeriesConvergenceResult['state'];
|
|
145
|
-
/** Newest value in the series. */
|
|
146
|
-
current: number;
|
|
147
|
-
/** Oldest value in the series — the reference the drop thresholds compare against. */
|
|
148
|
-
baseline: number;
|
|
149
|
-
/** current − baseline (signed; negative = decayed). */
|
|
150
|
-
drift: number;
|
|
151
|
-
alarmed: boolean;
|
|
152
|
-
reason?: string;
|
|
153
|
-
}
|
|
154
|
-
interface SentinelReport {
|
|
155
|
-
perJudge: SentinelTrend[];
|
|
156
|
-
alarms: string[];
|
|
157
|
-
healthy: boolean;
|
|
158
|
-
/**
|
|
159
|
-
* `judgeId:metric` pairs with too few snapshots for the convergence
|
|
160
|
-
* machine. Named, not counted — a blind spot you can't see is a blind
|
|
161
|
-
* spot you won't fix. Floor/drop checks still apply to thin series, so
|
|
162
|
-
* insufficient history alone never masks an alarm.
|
|
163
|
-
*/
|
|
164
|
-
insufficientHistory: string[];
|
|
165
|
-
}
|
|
166
|
-
declare function judgeSentinelReport(history: SentinelSnapshot[], opts: JudgeSentinelOptions): SentinelReport;
|
|
167
|
-
interface EvalHealthStamp {
|
|
168
|
-
healthy: boolean;
|
|
169
|
-
alarms: string[];
|
|
170
|
-
}
|
|
171
|
-
/**
|
|
172
|
-
* The object a campaign attaches to its verdicts. Compute it from the
|
|
173
|
-
* sentinel report in a post-campaign hook (or a nightly job feeding the
|
|
174
|
-
* next day's campaigns) and store it alongside the verdict payload.
|
|
175
|
-
* `healthy: false` means the judges that produced those verdicts have an
|
|
176
|
-
* unresolved drift / decay / silent-upgrade / staleness alarm — treat the
|
|
177
|
-
* verdicts as carrying an integrity warning until recalibration clears it.
|
|
178
|
-
*/
|
|
179
|
-
declare function evalHealthStamp(report: SentinelReport): EvalHealthStamp;
|
|
180
|
-
|
|
181
|
-
export { type EvalHealthStamp, type JudgeSentinelOptions, SENTINEL_METRIC_NAMES, type SentinelMetricName, type SentinelMetrics, type SentinelReport, type SentinelSetOptions, type SentinelSnapshot, type SentinelStore, type SentinelThresholds, type SentinelTrend, type SnapshotMeta, evalHealthStamp, fileSentinelStore, inMemorySentinelStore, judgeSentinelReport, snapshotFromAgreement, snapshotFromCalibration, snapshotFromSentinelSet, validateSentinelSnapshot };
|
|
1
|
+
export * from './calibration';
|
|
2
|
+
export * from './correlation-study';
|
|
3
|
+
export * from './outcome-store';
|
|
4
|
+
export * from './rubric-predictive-validity';
|
|
5
|
+
export * from './sentinel';
|
|
6
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/meta-eval/index.ts"],"names":[],"mappings":"AAAA,cAAc,eAAe,CAAA;AAC7B,cAAc,qBAAqB,CAAA;AACnC,cAAc,iBAAiB,CAAA;AAC/B,cAAc,8BAA8B,CAAA;AAC5C,cAAc,YAAY,CAAA"}
|
package/dist/meta-eval/index.js
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
* The whole point of this module is to make the meta-eval correlation
|
|
11
11
|
* question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.
|
|
12
12
|
*/
|
|
13
|
-
interface DeploymentOutcome {
|
|
13
|
+
export interface DeploymentOutcome {
|
|
14
14
|
runId: string;
|
|
15
15
|
capturedAt: number;
|
|
16
16
|
/** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */
|
|
@@ -20,7 +20,7 @@ interface DeploymentOutcome {
|
|
|
20
20
|
/** Free-form provenance (source system, pipeline version). */
|
|
21
21
|
source?: string;
|
|
22
22
|
}
|
|
23
|
-
interface OutcomeFilter {
|
|
23
|
+
export interface OutcomeFilter {
|
|
24
24
|
runIds?: string[];
|
|
25
25
|
since?: number;
|
|
26
26
|
until?: number;
|
|
@@ -30,24 +30,24 @@ interface OutcomeFilter {
|
|
|
30
30
|
};
|
|
31
31
|
source?: string;
|
|
32
32
|
}
|
|
33
|
-
interface OutcomeStore {
|
|
33
|
+
export interface OutcomeStore {
|
|
34
34
|
append(outcome: DeploymentOutcome): Promise<void>;
|
|
35
35
|
/** All outcomes attached to this run (a single run can have many — multiple
|
|
36
36
|
* capture windows over deployment time). */
|
|
37
37
|
forRun(runId: string): Promise<DeploymentOutcome[]>;
|
|
38
38
|
list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
|
|
39
39
|
}
|
|
40
|
-
declare class InMemoryOutcomeStore implements OutcomeStore {
|
|
40
|
+
export declare class InMemoryOutcomeStore implements OutcomeStore {
|
|
41
41
|
private items;
|
|
42
42
|
append(outcome: DeploymentOutcome): Promise<void>;
|
|
43
43
|
forRun(runId: string): Promise<DeploymentOutcome[]>;
|
|
44
44
|
list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
|
|
45
45
|
}
|
|
46
|
-
interface FileSystemOutcomeStoreOptions {
|
|
46
|
+
export interface FileSystemOutcomeStoreOptions {
|
|
47
47
|
dir: string;
|
|
48
48
|
maxBytes?: number;
|
|
49
49
|
}
|
|
50
|
-
declare class FileSystemOutcomeStore implements OutcomeStore {
|
|
50
|
+
export declare class FileSystemOutcomeStore implements OutcomeStore {
|
|
51
51
|
private dir;
|
|
52
52
|
private maxBytes;
|
|
53
53
|
private memo?;
|
|
@@ -59,5 +59,4 @@ declare class FileSystemOutcomeStore implements OutcomeStore {
|
|
|
59
59
|
forRun(runId: string): Promise<DeploymentOutcome[]>;
|
|
60
60
|
list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>;
|
|
61
61
|
}
|
|
62
|
-
|
|
63
|
-
export { type DeploymentOutcome as D, FileSystemOutcomeStore as F, InMemoryOutcomeStore as I, type OutcomeFilter as O, type FileSystemOutcomeStoreOptions as a, type OutcomeStore as b };
|
|
62
|
+
//# sourceMappingURL=outcome-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"outcome-store.d.ts","sourceRoot":"","sources":["../../src/meta-eval/outcome-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;IAClB,6EAA6E;IAC7E,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC/B,yEAAyE;IACzE,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC/B,8DAA8D;IAC9D,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;IACtC,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjD;iDAC6C;IAC7C,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAA;IACnD,IAAI,CAAC,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAA;CAC3D;AAED,qBAAa,oBAAqB,YAAW,YAAY;IACvD,OAAO,CAAC,KAAK,CAA0B;IAEjC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;IAIjD,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAInD,IAAI,CAAC,MAAM,GAAE,aAAkB,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;CAGrE;AAED,MAAM,WAAW,6BAA6B;IAC5C,GAAG,EAAE,MAAM,CAAA;IACX,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,qBAAa,sBAAuB,YAAW,YAAY;IACzD,OAAO,CAAC,GAAG,CAAQ;IACnB,OAAO,CAAC,QAAQ,CAAQ;IACxB,OAAO,CAAC,IAAI,CAAC,CAAsB;IACnC,OAAO,CAAC,MAAM,CAAQ;gBAEV,OAAO,EAAE,6BAA6B;YAKpC,SAAS;IAKjB,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC;YAiBzC,IAAI;IAuBZ,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAInD,IAAI,CAAC,MAAM,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;CAGjE"}
|
package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → meta-eval/rubric-predictive-validity.d.ts}
RENAMED
|
@@ -1,6 +1,3 @@
|
|
|
1
|
-
import { R as RunRecord } from './run-record-e7vj1uZQ.js';
|
|
2
|
-
import { b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
|
|
3
|
-
|
|
4
1
|
/**
|
|
5
2
|
* Rubric predictive validity — does our eval rubric predict deployment
|
|
6
3
|
* outcomes?
|
|
@@ -32,8 +29,9 @@ import { b as OutcomeStore } from './outcome-store-rnXLEqSn.js';
|
|
|
32
29
|
* decorative rubrics, re-weight composite scores, or trigger a
|
|
33
30
|
* recalibration sweep when predictive validity drops below a threshold.
|
|
34
31
|
*/
|
|
35
|
-
|
|
36
|
-
|
|
32
|
+
import type { RunRecord } from '../run-record';
|
|
33
|
+
import type { OutcomeStore } from './outcome-store';
|
|
34
|
+
export interface RubricPredictiveValidityInput {
|
|
37
35
|
/**
|
|
38
36
|
* Canonical campaign output. Each record's `outcome.raw[<rubricId>]`
|
|
39
37
|
* provides the eval score; missing keys are silently skipped per pair.
|
|
@@ -64,7 +62,7 @@ interface RubricPredictiveValidityInput {
|
|
|
64
62
|
*/
|
|
65
63
|
reduction?: 'latest' | 'mean' | 'max';
|
|
66
64
|
}
|
|
67
|
-
interface RubricOutcomePair {
|
|
65
|
+
export interface RubricOutcomePair {
|
|
68
66
|
rubric: string;
|
|
69
67
|
outcome: string;
|
|
70
68
|
n: number;
|
|
@@ -82,7 +80,7 @@ interface RubricOutcomePair {
|
|
|
82
80
|
*/
|
|
83
81
|
verdict: 'load_bearing' | 'informative' | 'decorative';
|
|
84
82
|
}
|
|
85
|
-
interface RubricRanking {
|
|
83
|
+
export interface RubricRanking {
|
|
86
84
|
rubric: string;
|
|
87
85
|
/** Outcome metric this rubric correlated best with. */
|
|
88
86
|
bestOutcome: string;
|
|
@@ -91,7 +89,7 @@ interface RubricRanking {
|
|
|
91
89
|
n: number;
|
|
92
90
|
verdict: RubricOutcomePair['verdict'];
|
|
93
91
|
}
|
|
94
|
-
interface RubricPredictiveValidityReport {
|
|
92
|
+
export interface RubricPredictiveValidityReport {
|
|
95
93
|
pairs: RubricOutcomePair[];
|
|
96
94
|
/** Per-rubric best pair, sorted descending by |spearman|. */
|
|
97
95
|
ranked: RubricRanking[];
|
|
@@ -100,6 +98,5 @@ interface RubricPredictiveValidityReport {
|
|
|
100
98
|
/** Rubrics that were declared but never produced a usable score. */
|
|
101
99
|
rubricsWithoutData: string[];
|
|
102
100
|
}
|
|
103
|
-
declare function rubricPredictiveValidity(input: RubricPredictiveValidityInput): Promise<RubricPredictiveValidityReport>;
|
|
104
|
-
|
|
105
|
-
export { type RubricOutcomePair as R, type RubricPredictiveValidityInput as a, type RubricPredictiveValidityReport as b, type RubricRanking as c, rubricPredictiveValidity as r };
|
|
101
|
+
export declare function rubricPredictiveValidity(input: RubricPredictiveValidityInput): Promise<RubricPredictiveValidityReport>;
|
|
102
|
+
//# sourceMappingURL=rubric-predictive-validity.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rubric-predictive-validity.d.ts","sourceRoot":"","sources":["../../src/meta-eval/rubric-predictive-validity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,KAAK,EAAqB,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAEtE,MAAM,WAAW,6BAA6B;IAC5C;;;OAGG;IACH,IAAI,EAAE,SAAS,EAAE,CAAA;IACjB,QAAQ,EAAE,YAAY,CAAA;IACtB;;;;OAIG;IACH,cAAc,EAAE,MAAM,EAAE,CAAA;IACxB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;IAClB,wEAAwE;IACxE,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,+CAA+C;IAC/C,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,+EAA+E;IAC/E,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;OAGG;IACH,SAAS,CAAC,EAAE,QAAQ,GAAG,MAAM,GAAG,KAAK,CAAA;CACtC;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,CAAC,EAAE,MAAM,CAAA;IACT,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;IACnC;;;;;OAKG;IACH,OAAO,EAAE,cAAc,GAAG,aAAa,GAAG,YAAY,CAAA;CACvD;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,uDAAuD;IACvD,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;IACf,CAAC,EAAE,MAAM,CAAA;IACT,OAAO,EAAE,iBAAiB,CAAC,SAAS,CAAC,CAAA;CACtC;AAED,MAAM,WAAW,8BAA8B;IAC7C,KAAK,EAAE,iBAAiB,EAAE,CAAA;IAC1B,6DAA6D;IAC7D,MAAM,EAAE,aAAa,EAAE,CAAA;IACvB,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAA;IACnB,oEAAoE;IACpE,kBAAkB,EAAE,MAAM,EAAE,CAAA;CAC7B;AAED,wBAAsB,wBAAwB,CAC5C,KAAK,EAAE,6BAA6B,GACnC,OAAO,CAAC,8BAA8B,CAAC,CAyGzC"}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Judge sentinel — eval trustworthiness as a continuously measured,
|
|
3
|
+
* alarmed trend.
|
|
4
|
+
*
|
|
5
|
+
* Judges are models; models change underneath us; calibration decays.
|
|
6
|
+
* This module composes three existing instruments into one loop:
|
|
7
|
+
*
|
|
8
|
+
* snapshot — adapters turn real instrument outputs (`calibrateJudge` /
|
|
9
|
+
* `calibrateJudgeContinuous`, `corpusInterRaterAgreement` /
|
|
10
|
+
* `continuousAgreement`, a rerun sentinel golden set) into
|
|
11
|
+
* `SentinelSnapshot`s
|
|
12
|
+
* series — `analyzeSeries` (src/series-convergence) classifies each
|
|
13
|
+
* judge×metric history as stabilized / drifting / noisy
|
|
14
|
+
* alarm — `judgeSentinelReport` turns trends + thresholds into named
|
|
15
|
+
* alarms; `evalHealthStamp` is the tiny object a campaign
|
|
16
|
+
* attaches to its verdicts
|
|
17
|
+
*
|
|
18
|
+
* Alarm conditions:
|
|
19
|
+
* - convergence state `drifting-down` on any tracked metric
|
|
20
|
+
* - irr below `minIrr`
|
|
21
|
+
* - calibrationKappa / sentinelPassRate dropped vs the series baseline
|
|
22
|
+
* beyond `maxKappaDrop` / `maxSentinelDrop`
|
|
23
|
+
* - judge model changed with no post-change golden-grounded snapshot
|
|
24
|
+
* (the silent-upgrade trap: judges agreeing with each other after a
|
|
25
|
+
* model swap proves nothing — only re-measuring against gold does)
|
|
26
|
+
* - newest snapshot older than `staleAfterDays` relative to `asOf`
|
|
27
|
+
*
|
|
28
|
+
* Wire-in: run the snapshot adapters from a post-campaign hook or a
|
|
29
|
+
* nightly job, append to a `SentinelStore`, then compute
|
|
30
|
+
* `judgeSentinelReport(await store.history(), { asOf })` with `asOf` =
|
|
31
|
+
* the campaign timestamp. Attach `evalHealthStamp(report)` to campaign
|
|
32
|
+
* verdicts: an alarmed sentinel means every downstream verdict carries
|
|
33
|
+
* an integrity warning until the judge is recalibrated.
|
|
34
|
+
*
|
|
35
|
+
* The module is clock-free — every timestamp (`at`, `asOf`) is
|
|
36
|
+
* caller-supplied ISO-8601, so reports are reproducible.
|
|
37
|
+
*/
|
|
38
|
+
import type { CalibrationResult, CandidateScore, ContinuousAgreement, ContinuousCalibrationResult, GoldenItem } from '../judge-calibration';
|
|
39
|
+
import { type SeriesConvergenceOptions, type SeriesConvergenceResult } from '../series-convergence';
|
|
40
|
+
import type { CorpusAgreementReport } from '../statistics';
|
|
41
|
+
export declare const SENTINEL_METRIC_NAMES: readonly ["irr", "calibrationKappa", "sentinelPassRate"];
|
|
42
|
+
export type SentinelMetricName = (typeof SENTINEL_METRIC_NAMES)[number];
|
|
43
|
+
export interface SentinelMetrics {
|
|
44
|
+
/** Inter-rater reliability — ICC(2,1) from agreement instruments. */
|
|
45
|
+
irr?: number;
|
|
46
|
+
/** Weighted κ vs the human golden set (`calibrateJudge*`). */
|
|
47
|
+
calibrationKappa?: number;
|
|
48
|
+
/** Fraction of a fixed sentinel golden set the judge still scores within tolerance. */
|
|
49
|
+
sentinelPassRate?: number;
|
|
50
|
+
}
|
|
51
|
+
export interface SentinelSnapshot {
|
|
52
|
+
/** Caller-supplied ISO-8601 timestamp — the module never reads a clock. */
|
|
53
|
+
at: string;
|
|
54
|
+
judgeId: string;
|
|
55
|
+
/** Exact model identity behind the judge (pin the full version string). */
|
|
56
|
+
judgeModel: string;
|
|
57
|
+
/**
|
|
58
|
+
* Metrics measured at `at`. May be empty: an empty-metrics snapshot is a
|
|
59
|
+
* model-change marker — it records the new `judgeModel` immediately and
|
|
60
|
+
* the silent-upgrade alarm stays raised until a golden-grounded snapshot
|
|
61
|
+
* (calibrationKappa or sentinelPassRate) follows.
|
|
62
|
+
*/
|
|
63
|
+
metrics: SentinelMetrics;
|
|
64
|
+
}
|
|
65
|
+
/** Identity + timestamp the caller supplies alongside an instrument output. */
|
|
66
|
+
export interface SnapshotMeta {
|
|
67
|
+
at: string;
|
|
68
|
+
judgeId: string;
|
|
69
|
+
judgeModel: string;
|
|
70
|
+
}
|
|
71
|
+
/** Shape guard for snapshots crossing a boundary (store append, JSONL load, report input). */
|
|
72
|
+
export declare function validateSentinelSnapshot(snapshot: SentinelSnapshot, source?: string): void;
|
|
73
|
+
/**
|
|
74
|
+
* From `calibrateJudge` / `calibrateJudgeContinuous` output. Prefers the
|
|
75
|
+
* un-rounded continuous κ_w when the report carries one (integer κ
|
|
76
|
+
* discards information for [0,1] judges).
|
|
77
|
+
*/
|
|
78
|
+
export declare function snapshotFromCalibration(report: CalibrationResult | ContinuousCalibrationResult, meta: SnapshotMeta): SentinelSnapshot;
|
|
79
|
+
/**
|
|
80
|
+
* From `corpusInterRaterAgreement` (statistics.ts) or `continuousAgreement`
|
|
81
|
+
* (judge-calibration.ts) output. IRR = ICC(2,1) — the reliability
|
|
82
|
+
* coefficient both instruments compute. For ensemble-level agreement,
|
|
83
|
+
* `meta.judgeId` names the ensemble and `meta.judgeModel` pins its
|
|
84
|
+
* composition (e.g. a joined list of member model strings).
|
|
85
|
+
*/
|
|
86
|
+
export declare function snapshotFromAgreement(report: CorpusAgreementReport | ContinuousAgreement, meta: SnapshotMeta): SentinelSnapshot;
|
|
87
|
+
export interface SentinelSetOptions {
|
|
88
|
+
/** Max |judge − human| for an item to count as passed. Default 0.1 (tuned for [0,1] scores). */
|
|
89
|
+
tolerance?: number;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* From a rerun of a fixed sentinel golden set: the same `GoldenItem`s the
|
|
93
|
+
* judge was originally calibrated on, re-scored periodically. Pass rate =
|
|
94
|
+
* fraction of joined items where the judge stays within `tolerance` of the
|
|
95
|
+
* human score. Items the judge didn't score are excluded from the
|
|
96
|
+
* denominator; zero joined items is an error, not a 0% pass rate.
|
|
97
|
+
*/
|
|
98
|
+
export declare function snapshotFromSentinelSet(scores: CandidateScore[], golden: GoldenItem[], meta: SnapshotMeta, options?: SentinelSetOptions): SentinelSnapshot;
|
|
99
|
+
export interface SentinelStore {
|
|
100
|
+
append(snapshot: SentinelSnapshot): Promise<void>;
|
|
101
|
+
/** All snapshots (optionally for one judge), in append order. */
|
|
102
|
+
history(judgeId?: string): Promise<SentinelSnapshot[]>;
|
|
103
|
+
}
|
|
104
|
+
export declare function inMemorySentinelStore(initial?: SentinelSnapshot[]): SentinelStore;
|
|
105
|
+
/**
|
|
106
|
+
* JSONL store — one snapshot per line, appended atomically per call.
|
|
107
|
+
* A corrupt or shape-invalid line is a loud error naming the file and
|
|
108
|
+
* line number: a sentinel history that silently drops records would
|
|
109
|
+
* defeat the drift detection it exists to provide.
|
|
110
|
+
*/
|
|
111
|
+
export declare function fileSentinelStore(path: string): SentinelStore;
|
|
112
|
+
export interface SentinelThresholds {
|
|
113
|
+
/** Floor for irr — below it the judge ensemble is unreliable regardless of trend. Default 0.6. */
|
|
114
|
+
minIrr?: number;
|
|
115
|
+
/** Max tolerated calibrationKappa drop vs the series baseline. Default 0.15. */
|
|
116
|
+
maxKappaDrop?: number;
|
|
117
|
+
/** Max tolerated sentinelPassRate drop vs the series baseline. Default 0.1. */
|
|
118
|
+
maxSentinelDrop?: number;
|
|
119
|
+
/** Newest snapshot older than this (days, vs asOf) = the sentinel itself went blind. Default 30. */
|
|
120
|
+
staleAfterDays?: number;
|
|
121
|
+
}
|
|
122
|
+
export interface JudgeSentinelOptions {
|
|
123
|
+
/** Caller-supplied ISO timestamp staleness is measured against. */
|
|
124
|
+
asOf: string;
|
|
125
|
+
thresholds?: SentinelThresholds;
|
|
126
|
+
/** Forwarded to `analyzeSeries` (window, stableCv, driftRun). */
|
|
127
|
+
convergence?: SeriesConvergenceOptions;
|
|
128
|
+
}
|
|
129
|
+
export interface SentinelTrend {
|
|
130
|
+
judgeId: string;
|
|
131
|
+
metric: SentinelMetricName;
|
|
132
|
+
/** Verbatim `analyzeSeries` state for this judge×metric history. */
|
|
133
|
+
state: SeriesConvergenceResult['state'];
|
|
134
|
+
/** Newest value in the series. */
|
|
135
|
+
current: number;
|
|
136
|
+
/** Oldest value in the series — the reference the drop thresholds compare against. */
|
|
137
|
+
baseline: number;
|
|
138
|
+
/** current − baseline (signed; negative = decayed). */
|
|
139
|
+
drift: number;
|
|
140
|
+
alarmed: boolean;
|
|
141
|
+
reason?: string;
|
|
142
|
+
}
|
|
143
|
+
export interface SentinelReport {
|
|
144
|
+
perJudge: SentinelTrend[];
|
|
145
|
+
alarms: string[];
|
|
146
|
+
healthy: boolean;
|
|
147
|
+
/**
|
|
148
|
+
* `judgeId:metric` pairs with too few snapshots for the convergence
|
|
149
|
+
* machine. Named, not counted — a blind spot you can't see is a blind
|
|
150
|
+
* spot you won't fix. Floor/drop checks still apply to thin series, so
|
|
151
|
+
* insufficient history alone never masks an alarm.
|
|
152
|
+
*/
|
|
153
|
+
insufficientHistory: string[];
|
|
154
|
+
}
|
|
155
|
+
export declare function judgeSentinelReport(history: SentinelSnapshot[], opts: JudgeSentinelOptions): SentinelReport;
|
|
156
|
+
export interface EvalHealthStamp {
|
|
157
|
+
healthy: boolean;
|
|
158
|
+
alarms: string[];
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* The object a campaign attaches to its verdicts. Compute it from the
|
|
162
|
+
* sentinel report in a post-campaign hook (or a nightly job feeding the
|
|
163
|
+
* next day's campaigns) and store it alongside the verdict payload.
|
|
164
|
+
* `healthy: false` means the judges that produced those verdicts have an
|
|
165
|
+
* unresolved drift / decay / silent-upgrade / staleness alarm — treat the
|
|
166
|
+
* verdicts as carrying an integrity warning until recalibration clears it.
|
|
167
|
+
*/
|
|
168
|
+
export declare function evalHealthStamp(report: SentinelReport): EvalHealthStamp;
|
|
169
|
+
//# sourceMappingURL=sentinel.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sentinel.d.ts","sourceRoot":"","sources":["../../src/meta-eval/sentinel.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAGH,OAAO,KAAK,EACV,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACnB,2BAA2B,EAC3B,UAAU,EACX,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,EAC7B,MAAM,uBAAuB,CAAA;AAC9B,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAA;AAE1D,eAAO,MAAM,qBAAqB,0DAA2D,CAAA;AAE7F,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,qBAAqB,CAAC,CAAC,MAAM,CAAC,CAAA;AAEvE,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,8DAA8D;IAC9D,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,uFAAuF;IACvF,gBAAgB,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B,2EAA2E;IAC3E,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAA;IAClB;;;;;OAKG;IACH,OAAO,EAAE,eAAe,CAAA;CACzB;AAED,+EAA+E;AAC/E,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;CACnB;AAYD,8FAA8F;AAC9F,wBAAgB,wBAAwB,CAAC,QAAQ,EAAE,gBAAgB,EAAE,MAAM,SAAa,GAAG,IAAI,CAwB9F;AAID;;;;GAIG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,iBAAiB,GAAG,2BAA2B,EACvD,IAAI,EAAE,YAAY,GACjB,gBAAgB,CAUlB;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,qBAAqB,GAAG,mBAAmB,EACnD,IAAI,EAAE,YAAY,GACjB,gBAAgB,CAUlB;AAED,MAAM,WAAW,kBAAkB;IACjC,gGAAgG;IAChG,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAED;;;;;;GAMG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,cAAc,EAAE,EACxB,MAAM,EAAE,UAAU,EAAE,EACpB,IAAI,EAAE,YAAY,EAClB,OAAO,GAAE,kBAAuB,GAC/B,gBAAgB,CAwClB;AAID,MAAM,WAAW,aAAa;IAC5B,MAAM,CAAC,QAAQ,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjD,iEAAiE;IACjE,OAAO,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAAA;CACvD;AAED,wBAAgB,qBAAqB,CAAC,OAAO,GAAE,gBAAgB,EAAO,GAAG,aAAa,CAarF;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,CAsC7D;AAID,MAAM,WAAW,kBAAkB;IACjC,kGAAkG;IAClG,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,gFAAgF;IAChF,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,+EAA+E;IAC/E,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,oGAAoG;IACpG,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,oBAAoB;IACnC,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,CAAC,EAAE,kBAAkB,CAAA;IAC/B,iEAAiE;IACjE,WAAW,CAAC,EAAE,wBAAwB,CAAA;CACvC;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,kBAAkB,CAAA;IAC1B,oEAAoE;IACpE,KAAK,EAAE,uBAAuB,CAAC,OAAO,CAAC,CAAA;IACvC,kCAAkC;IAClC,OAAO,EAAE,MAAM,CAAA;IACf,sFAAsF;IACtF,QAAQ,EAAE,MAAM,CAAA;IAChB,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,OAAO,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,aAAa,EAAE,CAAA;IACzB,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,OAAO,EAAE,OAAO,CAAA;IAChB;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,EAAE,CAAA;CAC9B;AAID,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,gBAAgB,EAAE,EAC3B,IAAI,EAAE,oBAAoB,GACzB,cAAc,CAyGhB;AAID,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,OAAO,CAAA;IAChB,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,cAAc,GAAG,eAAe,CAEvE"}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import type { ProductClient } from './client';
|
|
2
|
+
import type { DriverState, TurnMetrics } from './types';
|
|
3
|
+
interface TokenPrice {
|
|
4
|
+
input: number;
|
|
5
|
+
output: number;
|
|
6
|
+
}
|
|
7
|
+
/** Per-1K token pricing for exact model ids. */
|
|
8
|
+
export declare const MODEL_PRICING: Record<string, TokenPrice>;
|
|
9
|
+
/** Resolve pricing for a model id: exact table, then family fallback.
|
|
10
|
+
* Returns null when the id matches nothing (caller decides — never a
|
|
11
|
+
* silent-zero masquerading as a real $0 cost). */
|
|
12
|
+
export declare function resolveModelPricing(model: string): TokenPrice | null;
|
|
13
|
+
/** True when `model` has known pricing (exact or family). Lets cost-aware
|
|
14
|
+
* callers distinguish a real $0 from an unpriced model. */
|
|
15
|
+
export declare function isModelPriced(model: string): boolean;
|
|
16
|
+
/** Estimate token count from string length (chars / 4 approximation) */
|
|
17
|
+
export declare function estimateTokens(text: string): number;
|
|
18
|
+
/** Calculate cost in USD from token counts and model. Unknown models warn
|
|
19
|
+
* once (not a silent zero) and return 0 so callers that ignore pricing keep
|
|
20
|
+
* working; cost-sensitive callers should gate on {@link isModelPriced}. */
|
|
21
|
+
export declare function estimateCost(inputTokens: number, outputTokens: number, model: string): number;
|
|
22
|
+
/**
|
|
23
|
+
* TokenCounter — accumulates token usage and cost across turns.
|
|
24
|
+
*/
|
|
25
|
+
export declare class TokenCounter {
|
|
26
|
+
private totalInput;
|
|
27
|
+
private totalOutput;
|
|
28
|
+
private totalCost;
|
|
29
|
+
private model;
|
|
30
|
+
constructor(model?: string);
|
|
31
|
+
/** Record tokens for a turn, returns per-turn cost */
|
|
32
|
+
record(inputTokens: number, outputTokens: number): number;
|
|
33
|
+
/** Estimate and record from raw text */
|
|
34
|
+
recordFromText(inputText: string, outputText: string): {
|
|
35
|
+
inputTokens: number;
|
|
36
|
+
outputTokens: number;
|
|
37
|
+
cost: number;
|
|
38
|
+
};
|
|
39
|
+
getTotalInput(): number;
|
|
40
|
+
getTotalOutput(): number;
|
|
41
|
+
getTotalCost(): number;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* MetricsCollector — collects per-turn metrics from the product.
|
|
45
|
+
*
|
|
46
|
+
* After each turn, queries the product's APIs to measure state changes.
|
|
47
|
+
*/
|
|
48
|
+
export declare class MetricsCollector {
|
|
49
|
+
private client;
|
|
50
|
+
private workspaceId;
|
|
51
|
+
private metrics;
|
|
52
|
+
constructor(client: ProductClient, workspaceId: string);
|
|
53
|
+
/** Collect metrics after a turn completes */
|
|
54
|
+
collect(turn: number, responseLatencyMs: number, responseChars: number, codeBlocksProduced: number, blocksExtracted: number, completionCriteriaMet: number, completionCriteriaTotal: number, qualityScore?: number, inputTokens?: number, outputTokens?: number, estimatedCostUsd?: number): Promise<TurnMetrics>;
|
|
55
|
+
/** Get current product state */
|
|
56
|
+
getState(): Promise<DriverState>;
|
|
57
|
+
/** Get all collected metrics */
|
|
58
|
+
getMetrics(): TurnMetrics[];
|
|
59
|
+
/** Get convergence curve (completion% over turns) */
|
|
60
|
+
getConvergenceCurve(): number[];
|
|
61
|
+
}
|
|
62
|
+
export {};
|
|
63
|
+
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../src/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AAC7C,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAEvD,UAAU,UAAU;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;CACf;AAED,gDAAgD;AAChD,eAAO,MAAM,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAOpD,CAAA;AAgCD;;mDAEmD;AACnD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAQpE;AAED;4DAC4D;AAC5D,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAEpD;AAID,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD;AAED;;4EAE4E;AAC5E,wBAAgB,YAAY,CAAC,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAa7F;AAED;;GAEG;AACH,qBAAa,YAAY;IACvB,OAAO,CAAC,UAAU,CAAI;IACtB,OAAO,CAAC,WAAW,CAAI;IACvB,OAAO,CAAC,SAAS,CAAI;IACrB,OAAO,CAAC,KAAK,CAAQ;gBAET,KAAK,SAAW;IAI5B,sDAAsD;IACtD,MAAM,CAAC,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,GAAG,MAAM;IAQzD,wCAAwC;IACxC,cAAc,CACZ,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB;QAAE,WAAW,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAO9D,aAAa,IAAI,MAAM;IAGvB,cAAc,IAAI,MAAM;IAGxB,YAAY,IAAI,MAAM;CAGvB;AAED;;;;GAIG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,MAAM,CAAe;IAC7B,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,OAAO,CAAoB;gBACvB,MAAM,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM;IAKtD,6CAA6C;IACvC,OAAO,CACX,IAAI,EAAE,MAAM,EACZ,iBAAiB,EAAE,MAAM,EACzB,aAAa,EAAE,MAAM,EACrB,kBAAkB,EAAE,MAAM,EAC1B,eAAe,EAAE,MAAM,EACvB,qBAAqB,EAAE,MAAM,EAC7B,uBAAuB,EAAE,MAAM,EAC/B,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,SAAI,EACf,YAAY,SAAI,EAChB,gBAAgB,SAAI,GACnB,OAAO,CAAC,WAAW,CAAC;IA2BvB,gCAAgC;IAC1B,QAAQ,IAAI,OAAO,CAAC,WAAW,CAAC;IAsBtC,gCAAgC;IAChC,UAAU,IAAI,WAAW,EAAE;IAI3B,qDAAqD;IACrD,mBAAmB,IAAI,MAAM,EAAE;CAGhC"}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ModelSeats — the program's model seating chart.
|
|
3
|
+
*
|
|
4
|
+
* One object names which model fills each role in an eval program: the worker
|
|
5
|
+
* under evaluation, the judge panel, the analyst, the reflection/driver model,
|
|
6
|
+
* and the verifier. Re-tiering an entire program (economy ↔ frontier) is one
|
|
7
|
+
* swapped object instead of a hunt through call sites.
|
|
8
|
+
*
|
|
9
|
+
* Wiring points — consumers thread seats; this module implements none of them
|
|
10
|
+
* (those files belong to other surfaces):
|
|
11
|
+
* - `judges` → `ensembleJudge({ models: seats.judges, … })` (src/judge-panel.ts)
|
|
12
|
+
* and the `JudgeConfig`s handed to `makeEvalTools({ judges })`
|
|
13
|
+
* (src/eval-tools.ts).
|
|
14
|
+
* - `reflection` → `selfImprove({ llm: { model: seats.reflection } })` — the
|
|
15
|
+
* `gepaProposer` reflection model (src/contract/self-improve.ts);
|
|
16
|
+
* same seat for any custom `SurfaceProposer`'s LLM.
|
|
17
|
+
* - `worker` → the dispatch model the agent itself calls — the model an
|
|
18
|
+
* `AgentProfile` declares.
|
|
19
|
+
* - `analyst` → the LLM behind `analyzeRuns` / analyst-registry kinds.
|
|
20
|
+
* - `verifier` → completion-verifier / objective-checker model.
|
|
21
|
+
* - campaign cells thread `judges` + driver models the same way; that wiring
|
|
22
|
+
* lands with the campaign surface, not here.
|
|
23
|
+
*
|
|
24
|
+
* `resolveSeat` is the only read path: an unset seat with no explicit fallback
|
|
25
|
+
* throws — a model id is a budget decision, never a silent default.
|
|
26
|
+
*/
|
|
27
|
+
import { ConfigError } from './errors';
|
|
28
|
+
export interface ModelSeats {
|
|
29
|
+
/** The model under evaluation — what the agent itself dispatches with. */
|
|
30
|
+
worker?: string;
|
|
31
|
+
/** Judge-panel model ids — thread into `ensembleJudge({ models })`. */
|
|
32
|
+
judges?: string[];
|
|
33
|
+
/** Analyst model — `analyzeRuns` / analyst-registry LLM calls. */
|
|
34
|
+
analyst?: string;
|
|
35
|
+
/** Reflection/proposer model — `gepaProposer` mutation proposals. */
|
|
36
|
+
reflection?: string;
|
|
37
|
+
/** Verifier model — completion/objective checking. */
|
|
38
|
+
verifier?: string;
|
|
39
|
+
}
|
|
40
|
+
export type SeatName = keyof ModelSeats;
|
|
41
|
+
export type SeatPresetName = keyof typeof seatPresets;
|
|
42
|
+
/**
|
|
43
|
+
* Tier presets — plain data, swap or spread freely.
|
|
44
|
+
*
|
|
45
|
+
* `economy` uses the fleet-policy ids: every id resolves through the
|
|
46
|
+
* substrate's family pricing (no costUnknown axis) and the judge trio spans
|
|
47
|
+
* three provider families (moonshot / deepseek / openai), so it passes
|
|
48
|
+
* `assertCrossFamily` as-is.
|
|
49
|
+
*
|
|
50
|
+
* `frontier` is deliberately EMPTY: entitled frontier ids vary per router
|
|
51
|
+
* account, and a hardcoded claude/gpt-5 id 401s on keys that lack it. Supply
|
|
52
|
+
* your own: `{ ...seatPresets.frontier, worker: '<your-frontier-id>', … }` —
|
|
53
|
+
* `resolveSeat` throws on every seat you haven't filled.
|
|
54
|
+
*/
|
|
55
|
+
export declare const seatPresets: Record<'economy' | 'frontier', ModelSeats>;
|
|
56
|
+
/** Thrown by `resolveSeat` when a seat is unset and no fallback was given. */
|
|
57
|
+
export declare class SeatUnsetError extends ConfigError {
|
|
58
|
+
readonly seat: SeatName;
|
|
59
|
+
constructor(seat: SeatName);
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* Read one seat. Blank strings and empty arrays count as unset (env-var
|
|
63
|
+
* plumbing produces them); malformed values (non-string seat, non-array or
|
|
64
|
+
* blank-entry `judges`) throw `ValidationError`. When the seat is unset, an
|
|
65
|
+
* explicit `fallback` is returned (`[fallback]` for `judges` — a one-model
|
|
66
|
+
* panel); without one, `SeatUnsetError`.
|
|
67
|
+
*/
|
|
68
|
+
export declare function resolveSeat(seats: ModelSeats, seat: 'judges', fallback?: string): string[];
|
|
69
|
+
export declare function resolveSeat(seats: ModelSeats, seat: Exclude<SeatName, 'judges'>, fallback?: string): string;
|
|
70
|
+
export declare function resolveSeat(seats: ModelSeats, seat: SeatName, fallback?: string): string | string[];
|
|
71
|
+
//# sourceMappingURL=model-seats.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-seats.d.ts","sourceRoot":"","sources":["../src/model-seats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,WAAW,EAAmB,MAAM,UAAU,CAAA;AAEvD,MAAM,WAAW,UAAU;IACzB,0EAA0E;IAC1E,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,uEAAuE;IACvE,MAAM,CAAC,EAAE,MAAM,EAAE,CAAA;IACjB,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,qEAAqE;IACrE,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,sDAAsD;IACtD,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAA;AAEvC,MAAM,MAAM,cAAc,GAAG,MAAM,OAAO,WAAW,CAAA;AAErD;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,SAAS,GAAG,UAAU,EAAE,UAAU,CASlE,CAAA;AAED,8EAA8E;AAC9E,qBAAa,cAAe,SAAQ,WAAW;aACjB,IAAI,EAAE,QAAQ;gBAAd,IAAI,EAAE,QAAQ;CAM3C;AAED;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAA;AAC3F,wBAAgB,WAAW,CACzB,KAAK,EAAE,UAAU,EACjB,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,EACjC,QAAQ,CAAC,EAAE,MAAM,GAChB,MAAM,CAAA;AACT,wBAAgB,WAAW,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,EAAE,CAAA"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"model-seats.test.d.ts","sourceRoot":"","sources":["../src/model-seats.test.ts"],"names":[],"mappings":""}
|