@tangle-network/agent-eval 0.95.0 → 0.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/adapters/http.d.ts +17 -10
- package/dist/adapters/langchain.d.ts +14 -7
- package/dist/adapters/otel.d.ts +25 -13
- package/dist/{rl/adversarial.d.ts → adversarial-DIVcDoI_.d.ts} +7 -6
- package/dist/analyst/index.d.ts +236 -28
- package/dist/analyst/index.js +1 -1
- package/dist/{trace-analyst/analyst.d.ts → analyst-C8HHvfJp.d.ts} +14 -12
- package/dist/{contract/analyze-runs.d.ts → analyze-runs-DtT6F_6T.d.ts} +10 -9
- package/dist/authenticity/index.d.ts +16 -15
- package/dist/{baseline.d.ts → baseline-Bbid3WoO.d.ts} +44 -8
- package/dist/belief-state/index.d.ts +605 -14
- package/dist/benchmarks/index.d.ts +5 -23
- package/dist/builder-eval/index.d.ts +250 -5
- package/dist/calibration-BPmzuVPk.d.ts +101 -0
- package/dist/campaign/index.d.ts +1452 -38
- package/dist/{chunk-AQ5WQAIV.js → chunk-3NHEO6ZC.js} +2 -2
- package/dist/chunk-3NHEO6ZC.js.map +1 -0
- package/dist/cli.d.ts +0 -2
- package/dist/{contract/intake/code-agent-session.d.ts → code-agent-session-CPHRCb4-.d.ts} +17 -15
- package/dist/contract/index.d.ts +824 -107
- package/dist/contract/index.js +145 -1
- package/dist/contract/index.js.map +1 -1
- package/dist/control-Doncu-B_.d.ts +259 -0
- package/dist/{control-runtime.d.ts → control-runtime-Acf9CGhw.d.ts} +26 -23
- package/dist/control.d.ts +10 -11
- package/dist/corpus-D4YW9UoJ.d.ts +560 -0
- package/dist/{cost-ledger.d.ts → cost-ledger-DuSqlw5B.d.ts} +11 -10
- package/dist/{counterfactual.d.ts → counterfactual-DlOz8PBx.d.ts} +13 -12
- package/dist/{dataset.d.ts → dataset-BbGkaN2I.d.ts} +14 -11
- package/dist/{analyst/registry.d.ts → default-registry-GyE8X5SP.d.ts} +37 -8
- package/dist/diagnose.d.ts +252 -1
- package/dist/{trace/emitter.d.ts → emitter-C2rqGH_l.d.ts} +12 -9
- package/dist/{errors.d.ts → errors-CzMUYo7b.d.ts} +11 -10
- package/dist/failure-cluster-DH9Flgcf.d.ts +76 -0
- package/dist/{feedback-trajectory.d.ts → feedback-trajectory-BxY0cKfs.d.ts} +38 -36
- package/dist/fuzz.d.ts +547 -1
- package/dist/gepa-Dprxvz8r.d.ts +414 -0
- package/dist/governance/index.d.ts +135 -5
- package/dist/harness-optimizer-mOl9XX_O.d.ts +106 -0
- package/dist/hosted/index.d.ts +239 -10
- package/dist/index-_Y4oNOOb.d.ts +159 -0
- package/dist/index.d.ts +5660 -277
- package/dist/index.js +1 -1
- package/dist/{contract/insight-report.d.ts → insight-report-BnRjTibG.d.ts} +19 -16
- package/dist/{trace/integrity.d.ts → integrity-D2t12mMw.d.ts} +14 -11
- package/dist/{judge-calibration.d.ts → judge-calibration-0p2QcWNE.d.ts} +17 -16
- package/dist/{analyst/kind-factory.d.ts → kind-factory-X3eDYbKn.d.ts} +61 -10
- package/dist/knowledge/index.d.ts +103 -3
- package/dist/{llm-client.d.ts → llm-client-Bj7g0rqu.d.ts} +23 -21
- package/dist/matrix/index.d.ts +30 -12
- package/dist/meta-eval/index.d.ts +182 -6
- package/dist/{multi-layer-verifier.d.ts → multi-layer-verifier-DUZXrPDA.d.ts} +15 -12
- package/dist/multishot/index.d.ts +290 -7
- package/dist/{rl/off-policy.d.ts → off-policy-DiwuKKg7.d.ts} +9 -8
- package/dist/openapi.json +1 -1
- package/dist/{meta-eval/outcome-store.d.ts → outcome-store-rnXLEqSn.d.ts} +8 -7
- package/dist/{pareto.d.ts → pareto-E-pembql.d.ts} +10 -9
- package/dist/perf/index.d.ts +119 -13
- package/dist/pipelines/index.d.ts +173 -8
- package/dist/pre-registration-nfUdc9EQ.d.ts +483 -0
- package/dist/prm/index.d.ts +104 -5
- package/dist/provenance-CIxfBnkl.d.ts +426 -0
- package/dist/query-B7GGjRox.d.ts +32 -0
- package/dist/{trace/raw-provider-sink.d.ts → raw-provider-sink-C46HDghv.d.ts} +14 -13
- package/dist/{red-team.d.ts → red-team-BWdoyleI.d.ts} +16 -14
- package/dist/{trace/redact.d.ts → redact-B40YG2M_.d.ts} +8 -7
- package/dist/release-report-pidWUMZ2.d.ts +233 -0
- package/dist/reporting.d.ts +16 -15
- package/dist/{eval-campaign.d.ts → researcher-Jr8ME1dZ.d.ts} +156 -34
- package/dist/rl.d.ts +1193 -1
- package/dist/{prm/rubric.d.ts → rubric-Cc6UHvUb.d.ts} +13 -10
- package/dist/{meta-eval/rubric-predictive-validity.d.ts → rubric-predictive-validity-C2hDKM8Z.d.ts} +11 -8
- package/dist/run-critic-CmMf05uV.d.ts +56 -0
- package/dist/{run-record.d.ts → run-record-CP2ObebC.d.ts} +117 -15
- package/dist/runtime-trajectory-BOUUjI0y.d.ts +49 -0
- package/dist/{trace/schema.d.ts → schema-m0gsnbt3.d.ts} +30 -29
- package/dist/semantic-concept-judge-DSBB2Cfp.d.ts +624 -0
- package/dist/{sequential.d.ts → sequential-5iSVfzl2.d.ts} +10 -9
- package/dist/{series-convergence.d.ts → series-convergence-D5OWMBg6.d.ts} +5 -4
- package/dist/sink-fetch-B1Yg4Til.d.ts +101 -0
- package/dist/{statistics.d.ts → statistics-CCJpTGOS.d.ts} +48 -46
- package/dist/{trace/store.d.ts → store-BcFXE6LG.d.ts} +12 -21
- package/dist/{trace-analyst/types.d.ts → store-C1YxJDEK.d.ts} +74 -18
- package/dist/storyboard/index.d.ts +81 -16
- package/dist/{summary-report.d.ts → summary-report-CInXwsza.d.ts} +160 -23
- package/dist/telemetry/{sink-file.d.ts → file.d.ts} +7 -5
- package/dist/telemetry/index.d.ts +35 -17
- package/dist/{sandbox-harness.d.ts → test-graded-scenario-DeODGLra.d.ts} +60 -15
- package/dist/{locked-jsonl-appender.d.ts → testing-C21CHsq2.d.ts} +4 -3
- package/dist/testing.d.ts +1 -5
- package/dist/traces.d.ts +976 -4
- package/dist/{trajectory.d.ts → trajectory-2TkpSEVh.d.ts} +9 -8
- package/dist/{analyst/types.d.ts → types-B5x54y6n.d.ts} +112 -19
- package/dist/{campaign/types.d.ts → types-BMahhhio.d.ts} +47 -44
- package/dist/{matrix/types.d.ts → types-BUxNaJ8c.d.ts} +11 -9
- package/dist/{types.d.ts → types-C7DGg5ex.d.ts} +33 -31
- package/dist/{verdict.d.ts → verdict-C9MlYujm.d.ts} +3 -2
- package/dist/wire/index.d.ts +570 -13
- package/dist/workflow/index.d.ts +496 -22
- package/package.json +2 -2
- package/dist/action-policy.d.ts +0 -24
- package/dist/action-policy.d.ts.map +0 -1
- package/dist/action-policy.test.d.ts +0 -2
- package/dist/action-policy.test.d.ts.map +0 -1
- package/dist/active-learning.d.ts +0 -41
- package/dist/active-learning.d.ts.map +0 -1
- package/dist/adapters/http.d.ts.map +0 -1
- package/dist/adapters/langchain.d.ts.map +0 -1
- package/dist/adapters/otel.d.ts.map +0 -1
- package/dist/agent-profile-cell.d.ts +0 -101
- package/dist/agent-profile-cell.d.ts.map +0 -1
- package/dist/agent-profile.d.ts +0 -27
- package/dist/agent-profile.d.ts.map +0 -1
- package/dist/agent-profile.test.d.ts +0 -2
- package/dist/agent-profile.test.d.ts.map +0 -1
- package/dist/analyst/adapters.d.ts +0 -62
- package/dist/analyst/adapters.d.ts.map +0 -1
- package/dist/analyst/analyst.test.d.ts +0 -2
- package/dist/analyst/analyst.test.d.ts.map +0 -1
- package/dist/analyst/ax-service.d.ts +0 -27
- package/dist/analyst/ax-service.d.ts.map +0 -1
- package/dist/analyst/behavioral-analyst.d.ts +0 -28
- package/dist/analyst/behavioral-analyst.d.ts.map +0 -1
- package/dist/analyst/chat-client.d.ts +0 -91
- package/dist/analyst/chat-client.d.ts.map +0 -1
- package/dist/analyst/default-registry.d.ts +0 -27
- package/dist/analyst/default-registry.d.ts.map +0 -1
- package/dist/analyst/default-registry.test.d.ts +0 -2
- package/dist/analyst/default-registry.test.d.ts.map +0 -1
- package/dist/analyst/finding-signature.d.ts +0 -48
- package/dist/analyst/finding-signature.d.ts.map +0 -1
- package/dist/analyst/finding-subject.d.ts +0 -146
- package/dist/analyst/finding-subject.d.ts.map +0 -1
- package/dist/analyst/finding-subject.test.d.ts +0 -2
- package/dist/analyst/finding-subject.test.d.ts.map +0 -1
- package/dist/analyst/findings-store.d.ts +0 -75
- package/dist/analyst/findings-store.d.ts.map +0 -1
- package/dist/analyst/index.d.ts.map +0 -1
- package/dist/analyst/kind-factory.d.ts.map +0 -1
- package/dist/analyst/kinds/failure-mode.d.ts +0 -19
- package/dist/analyst/kinds/failure-mode.d.ts.map +0 -1
- package/dist/analyst/kinds/improvement.d.ts +0 -23
- package/dist/analyst/kinds/improvement.d.ts.map +0 -1
- package/dist/analyst/kinds/index.d.ts +0 -22
- package/dist/analyst/kinds/index.d.ts.map +0 -1
- package/dist/analyst/kinds/kinds.test.d.ts +0 -2
- package/dist/analyst/kinds/kinds.test.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-gap.d.ts +0 -28
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +0 -22
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.d.ts +0 -84
- package/dist/analyst/kinds/skill-usage.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.test.d.ts +0 -2
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.d.ts +0 -26
- package/dist/analyst/parse-tolerant.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.test.d.ts +0 -2
- package/dist/analyst/parse-tolerant.test.d.ts.map +0 -1
- package/dist/analyst/registry.budget.test.d.ts +0 -2
- package/dist/analyst/registry.budget.test.d.ts.map +0 -1
- package/dist/analyst/registry.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.d.ts +0 -35
- package/dist/analyst/steer-firewall.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.test.d.ts +0 -2
- package/dist/analyst/steer-firewall.test.d.ts.map +0 -1
- package/dist/analyst/structure-findings.d.ts +0 -37
- package/dist/analyst/structure-findings.d.ts.map +0 -1
- package/dist/analyst/structure-findings.test.d.ts +0 -2
- package/dist/analyst/structure-findings.test.d.ts.map +0 -1
- package/dist/analyst/tool-groups.d.ts +0 -34
- package/dist/analyst/tool-groups.d.ts.map +0 -1
- package/dist/analyst/types.d.ts.map +0 -1
- package/dist/anti-slop.d.ts +0 -59
- package/dist/anti-slop.d.ts.map +0 -1
- package/dist/artifact-validator.d.ts +0 -74
- package/dist/artifact-validator.d.ts.map +0 -1
- package/dist/attestation.d.ts +0 -63
- package/dist/attestation.d.ts.map +0 -1
- package/dist/attestation.test.d.ts +0 -2
- package/dist/attestation.test.d.ts.map +0 -1
- package/dist/authenticity/index.d.ts.map +0 -1
- package/dist/authenticity/index.test.d.ts +0 -2
- package/dist/authenticity/index.test.d.ts.map +0 -1
- package/dist/auto-pr.d.ts +0 -120
- package/dist/auto-pr.d.ts.map +0 -1
- package/dist/baseline.d.ts.map +0 -1
- package/dist/behavior-dsl.d.ts +0 -73
- package/dist/behavior-dsl.d.ts.map +0 -1
- package/dist/belief-state/calibration.d.ts +0 -10
- package/dist/belief-state/calibration.d.ts.map +0 -1
- package/dist/belief-state/calibration.test.d.ts +0 -2
- package/dist/belief-state/calibration.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.d.ts +0 -66
- package/dist/belief-state/code-agent-corpus.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.test.d.ts +0 -2
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.d.ts +0 -22
- package/dist/belief-state/code-agent-evidence.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.test.d.ts +0 -2
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/extract.d.ts +0 -7
- package/dist/belief-state/extract.d.ts.map +0 -1
- package/dist/belief-state/extract.test.d.ts +0 -2
- package/dist/belief-state/extract.test.d.ts.map +0 -1
- package/dist/belief-state/index.d.ts.map +0 -1
- package/dist/belief-state/ope.d.ts +0 -17
- package/dist/belief-state/ope.d.ts.map +0 -1
- package/dist/belief-state/ope.test.d.ts +0 -2
- package/dist/belief-state/ope.test.d.ts.map +0 -1
- package/dist/belief-state/phase0-measurement.d.ts +0 -55
- package/dist/belief-state/phase0-measurement.d.ts.map +0 -1
- package/dist/belief-state/report.d.ts +0 -17
- package/dist/belief-state/report.d.ts.map +0 -1
- package/dist/belief-state/report.test.d.ts +0 -2
- package/dist/belief-state/report.test.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.d.ts +0 -23
- package/dist/belief-state/research-evidence.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.test.d.ts +0 -2
- package/dist/belief-state/research-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +0 -32
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.d.ts +0 -87
- package/dist/belief-state/runtime-hooks.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.test.d.ts +0 -2
- package/dist/belief-state/runtime-hooks.test.d.ts.map +0 -1
- package/dist/belief-state/selective.d.ts +0 -15
- package/dist/belief-state/selective.d.ts.map +0 -1
- package/dist/belief-state/selective.test.d.ts +0 -2
- package/dist/belief-state/selective.test.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.d.ts +0 -80
- package/dist/belief-state/shadow-probe.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.test.d.ts +0 -2
- package/dist/belief-state/shadow-probe.test.d.ts.map +0 -1
- package/dist/belief-state/types.d.ts +0 -195
- package/dist/belief-state/types.d.ts.map +0 -1
- package/dist/belief-state/types.test.d.ts +0 -2
- package/dist/belief-state/types.test.d.ts.map +0 -1
- package/dist/benchmark.d.ts +0 -14
- package/dist/benchmark.d.ts.map +0 -1
- package/dist/benchmarks/index.d.ts.map +0 -1
- package/dist/benchmarks/routing/dataset.d.ts +0 -34
- package/dist/benchmarks/routing/dataset.d.ts.map +0 -1
- package/dist/benchmarks/routing/index.d.ts +0 -34
- package/dist/benchmarks/routing/index.d.ts.map +0 -1
- package/dist/benchmarks/types.d.ts +0 -49
- package/dist/benchmarks/types.d.ts.map +0 -1
- package/dist/bisector.d.ts +0 -81
- package/dist/bisector.d.ts.map +0 -1
- package/dist/budget-guard.d.ts +0 -31
- package/dist/budget-guard.d.ts.map +0 -1
- package/dist/builder-eval/builder-session.d.ts +0 -111
- package/dist/builder-eval/builder-session.d.ts.map +0 -1
- package/dist/builder-eval/correlation.d.ts +0 -32
- package/dist/builder-eval/correlation.d.ts.map +0 -1
- package/dist/builder-eval/index.d.ts.map +0 -1
- package/dist/builder-eval/project-registry.d.ts +0 -51
- package/dist/builder-eval/project-registry.d.ts.map +0 -1
- package/dist/builder-eval/three-layer-eval.d.ts +0 -55
- package/dist/builder-eval/three-layer-eval.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.d.ts +0 -108
- package/dist/campaign/analyst-surface.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.test.d.ts +0 -2
- package/dist/campaign/analyst-surface.test.d.ts.map +0 -1
- package/dist/campaign/auto-pr.d.ts +0 -46
- package/dist/campaign/auto-pr.d.ts.map +0 -1
- package/dist/campaign/distillation/agreement-judge.d.ts +0 -69
- package/dist/campaign/distillation/agreement-judge.d.ts.map +0 -1
- package/dist/campaign/distillation/cli.d.ts +0 -35
- package/dist/campaign/distillation/cli.d.ts.map +0 -1
- package/dist/campaign/distillation/distillation.test.d.ts +0 -2
- package/dist/campaign/distillation/distillation.test.d.ts.map +0 -1
- package/dist/campaign/distillation/gold-scenarios.d.ts +0 -54
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +0 -1
- package/dist/campaign/distillation/run-distillation.d.ts +0 -119
- package/dist/campaign/distillation/run-distillation.d.ts.map +0 -1
- package/dist/campaign/gates/compose.d.ts +0 -12
- package/dist/campaign/gates/compose.d.ts.map +0 -1
- package/dist/campaign/gates/default-production-gate.d.ts +0 -58
- package/dist/campaign/gates/default-production-gate.d.ts.map +0 -1
- package/dist/campaign/gates/heldout-gate.d.ts +0 -12
- package/dist/campaign/gates/heldout-gate.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.d.ts +0 -125
- package/dist/campaign/gates/promotion-policy.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.test.d.ts +0 -2
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.d.ts +0 -146
- package/dist/campaign/gates/sequential.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.test.d.ts +0 -2
- package/dist/campaign/gates/sequential.test.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.d.ts +0 -99
- package/dist/campaign/gates/statistical-heldout.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.test.d.ts +0 -2
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +0 -1
- package/dist/campaign/index.d.ts.map +0 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +0 -59
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +0 -1
- package/dist/campaign/presets/compare-proposers.d.ts +0 -146
- package/dist/campaign/presets/compare-proposers.d.ts.map +0 -1
- package/dist/campaign/presets/playback.d.ts +0 -120
- package/dist/campaign/presets/playback.d.ts.map +0 -1
- package/dist/campaign/presets/playback.test.d.ts +0 -2
- package/dist/campaign/presets/playback.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-eval.d.ts +0 -14
- package/dist/campaign/presets/run-eval.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.d.ts +0 -63
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +0 -2
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-optimization.d.ts +0 -92
- package/dist/campaign/presets/run-optimization.d.ts.map +0 -1
- package/dist/campaign/presets/run-profile-matrix.d.ts +0 -151
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +0 -1
- package/dist/campaign/presets/run-skill-opt.d.ts +0 -96
- package/dist/campaign/presets/run-skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/_findings-text.d.ts +0 -22
- package/dist/campaign/proposers/_findings-text.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.d.ts +0 -33
- package/dist/campaign/proposers/ace.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.test.d.ts +0 -2
- package/dist/campaign/proposers/ace.test.d.ts.map +0 -1
- package/dist/campaign/proposers/analysis-edit.d.ts +0 -32
- package/dist/campaign/proposers/analysis-edit.d.ts.map +0 -1
- package/dist/campaign/proposers/evolutionary.d.ts +0 -20
- package/dist/campaign/proposers/evolutionary.d.ts.map +0 -1
- package/dist/campaign/proposers/fapo.d.ts +0 -120
- package/dist/campaign/proposers/fapo.d.ts.map +0 -1
- package/dist/campaign/proposers/gepa.d.ts +0 -86
- package/dist/campaign/proposers/gepa.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.d.ts +0 -44
- package/dist/campaign/proposers/halo.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.test.d.ts +0 -2
- package/dist/campaign/proposers/halo.test.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.d.ts +0 -47
- package/dist/campaign/proposers/memory.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.test.d.ts +0 -2
- package/dist/campaign/proposers/memory.test.d.ts.map +0 -1
- package/dist/campaign/proposers/skill-opt.d.ts +0 -88
- package/dist/campaign/proposers/skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.d.ts +0 -48
- package/dist/campaign/proposers/trace-analyst.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.test.d.ts +0 -2
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +0 -1
- package/dist/campaign/provenance.d.ts +0 -185
- package/dist/campaign/provenance.d.ts.map +0 -1
- package/dist/campaign/run-campaign.d.ts +0 -90
- package/dist/campaign/run-campaign.d.ts.map +0 -1
- package/dist/campaign/score-utils.d.ts +0 -26
- package/dist/campaign/score-utils.d.ts.map +0 -1
- package/dist/campaign/skill-patch.d.ts +0 -62
- package/dist/campaign/skill-patch.d.ts.map +0 -1
- package/dist/campaign/storage.d.ts +0 -38
- package/dist/campaign/storage.d.ts.map +0 -1
- package/dist/campaign/types.d.ts.map +0 -1
- package/dist/campaign/worktree/index.d.ts +0 -53
- package/dist/campaign/worktree/index.d.ts.map +0 -1
- package/dist/canary.d.ts +0 -101
- package/dist/canary.d.ts.map +0 -1
- package/dist/causal-attribution.d.ts +0 -45
- package/dist/causal-attribution.d.ts.map +0 -1
- package/dist/chunk-AQ5WQAIV.js.map +0 -1
- package/dist/ci-gate.d.ts +0 -44
- package/dist/ci-gate.d.ts.map +0 -1
- package/dist/cli.d.ts.map +0 -1
- package/dist/client.d.ts +0 -77
- package/dist/client.d.ts.map +0 -1
- package/dist/client.test.d.ts +0 -2
- package/dist/client.test.d.ts.map +0 -1
- package/dist/command-runner.d.ts +0 -74
- package/dist/command-runner.d.ts.map +0 -1
- package/dist/command-runner.test.d.ts +0 -2
- package/dist/command-runner.test.d.ts.map +0 -1
- package/dist/completion-verifier.d.ts +0 -147
- package/dist/completion-verifier.d.ts.map +0 -1
- package/dist/completion-verifier.test.d.ts +0 -9
- package/dist/completion-verifier.test.d.ts.map +0 -1
- package/dist/concurrency.d.ts +0 -23
- package/dist/concurrency.d.ts.map +0 -1
- package/dist/contamination-guard.d.ts +0 -81
- package/dist/contamination-guard.d.ts.map +0 -1
- package/dist/contract/analyze-runs.d.ts.map +0 -1
- package/dist/contract/define-agent-eval.d.ts +0 -52
- package/dist/contract/define-agent-eval.d.ts.map +0 -1
- package/dist/contract/diff.d.ts +0 -114
- package/dist/contract/diff.d.ts.map +0 -1
- package/dist/contract/index.d.ts.map +0 -1
- package/dist/contract/insight-report.d.ts.map +0 -1
- package/dist/contract/insight-types-fwd.d.ts +0 -7
- package/dist/contract/insight-types-fwd.d.ts.map +0 -1
- package/dist/contract/intake/agent-trace.d.ts +0 -97
- package/dist/contract/intake/agent-trace.d.ts.map +0 -1
- package/dist/contract/intake/code-agent-session.d.ts.map +0 -1
- package/dist/contract/intake/feedback-table.d.ts +0 -87
- package/dist/contract/intake/feedback-table.d.ts.map +0 -1
- package/dist/contract/intake/index.d.ts +0 -22
- package/dist/contract/intake/index.d.ts.map +0 -1
- package/dist/contract/intake/otel-spans.d.ts +0 -33
- package/dist/contract/intake/otel-spans.d.ts.map +0 -1
- package/dist/contract/self-improve.d.ts +0 -284
- package/dist/contract/self-improve.d.ts.map +0 -1
- package/dist/control-runtime.d.ts.map +0 -1
- package/dist/control-runtime.test.d.ts +0 -2
- package/dist/control-runtime.test.d.ts.map +0 -1
- package/dist/control.d.ts.map +0 -1
- package/dist/convergence.d.ts +0 -29
- package/dist/convergence.d.ts.map +0 -1
- package/dist/cost-ledger.d.ts.map +0 -1
- package/dist/cost-ledger.test.d.ts +0 -2
- package/dist/cost-ledger.test.d.ts.map +0 -1
- package/dist/cost-report.d.ts +0 -43
- package/dist/cost-report.d.ts.map +0 -1
- package/dist/cost-report.test.d.ts +0 -2
- package/dist/cost-report.test.d.ts.map +0 -1
- package/dist/cost-tracker.d.ts +0 -76
- package/dist/cost-tracker.d.ts.map +0 -1
- package/dist/counterfactual.d.ts.map +0 -1
- package/dist/cross-trace-diff.d.ts +0 -56
- package/dist/cross-trace-diff.d.ts.map +0 -1
- package/dist/dataset.d.ts.map +0 -1
- package/dist/deploy-gate-layer.d.ts +0 -125
- package/dist/deploy-gate-layer.d.ts.map +0 -1
- package/dist/deploy-gate-layer.test.d.ts +0 -2
- package/dist/deploy-gate-layer.test.d.ts.map +0 -1
- package/dist/description-length-gate.d.ts +0 -119
- package/dist/description-length-gate.d.ts.map +0 -1
- package/dist/detectors/edge.test.d.ts +0 -2
- package/dist/detectors/edge.test.d.ts.map +0 -1
- package/dist/detectors/index.d.ts +0 -81
- package/dist/detectors/index.d.ts.map +0 -1
- package/dist/detectors/index.test.d.ts +0 -2
- package/dist/detectors/index.test.d.ts.map +0 -1
- package/dist/diagnose/causal-sweep.d.ts +0 -100
- package/dist/diagnose/causal-sweep.d.ts.map +0 -1
- package/dist/diagnose/index.d.ts +0 -36
- package/dist/diagnose/index.d.ts.map +0 -1
- package/dist/diagnose/remediation.d.ts +0 -68
- package/dist/diagnose/remediation.d.ts.map +0 -1
- package/dist/diagnose/repair.d.ts +0 -77
- package/dist/diagnose/repair.d.ts.map +0 -1
- package/dist/discover-personas.d.ts +0 -35
- package/dist/discover-personas.d.ts.map +0 -1
- package/dist/driver.d.ts +0 -95
- package/dist/driver.d.ts.map +0 -1
- package/dist/driver.test.d.ts +0 -8
- package/dist/driver.test.d.ts.map +0 -1
- package/dist/dual-agent-bench.d.ts +0 -81
- package/dist/dual-agent-bench.d.ts.map +0 -1
- package/dist/error-count-extractor.d.ts +0 -47
- package/dist/error-count-extractor.d.ts.map +0 -1
- package/dist/error-count-extractor.test.d.ts +0 -2
- package/dist/error-count-extractor.test.d.ts.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/eval-campaign.d.ts.map +0 -1
- package/dist/eval-campaign.test.d.ts +0 -2
- package/dist/eval-campaign.test.d.ts.map +0 -1
- package/dist/eval-tools.d.ts +0 -55
- package/dist/eval-tools.d.ts.map +0 -1
- package/dist/eval-trace-store.d.ts +0 -107
- package/dist/eval-trace-store.d.ts.map +0 -1
- package/dist/eval-trace-store.test.d.ts +0 -2
- package/dist/eval-trace-store.test.d.ts.map +0 -1
- package/dist/executor.d.ts +0 -38
- package/dist/executor.d.ts.map +0 -1
- package/dist/executor.test.d.ts +0 -10
- package/dist/executor.test.d.ts.map +0 -1
- package/dist/experiment-tracker.d.ts +0 -178
- package/dist/experiment-tracker.d.ts.map +0 -1
- package/dist/experiment-tracker.test.d.ts +0 -2
- package/dist/experiment-tracker.test.d.ts.map +0 -1
- package/dist/failure-taxonomy.d.ts +0 -38
- package/dist/failure-taxonomy.d.ts.map +0 -1
- package/dist/feedback-trajectory.d.ts.map +0 -1
- package/dist/feedback-trajectory.test.d.ts +0 -2
- package/dist/feedback-trajectory.test.d.ts.map +0 -1
- package/dist/flow-layer.d.ts +0 -90
- package/dist/flow-layer.d.ts.map +0 -1
- package/dist/flow-layer.test.d.ts +0 -2
- package/dist/flow-layer.test.d.ts.map +0 -1
- package/dist/fuzz/capsule.d.ts +0 -46
- package/dist/fuzz/capsule.d.ts.map +0 -1
- package/dist/fuzz/cube.d.ts +0 -36
- package/dist/fuzz/cube.d.ts.map +0 -1
- package/dist/fuzz/explorer-cost.test.d.ts +0 -2
- package/dist/fuzz/explorer-cost.test.d.ts.map +0 -1
- package/dist/fuzz/explorer.d.ts +0 -64
- package/dist/fuzz/explorer.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.d.ts +0 -16
- package/dist/fuzz/fuzz-agent.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.test.d.ts +0 -2
- package/dist/fuzz/fuzz-agent.test.d.ts.map +0 -1
- package/dist/fuzz/gates.d.ts +0 -33
- package/dist/fuzz/gates.d.ts.map +0 -1
- package/dist/fuzz/index.d.ts +0 -26
- package/dist/fuzz/index.d.ts.map +0 -1
- package/dist/fuzz/policies.d.ts +0 -28
- package/dist/fuzz/policies.d.ts.map +0 -1
- package/dist/fuzz/tools.d.ts +0 -20
- package/dist/fuzz/tools.d.ts.map +0 -1
- package/dist/fuzz/types.d.ts +0 -307
- package/dist/fuzz/types.d.ts.map +0 -1
- package/dist/golden-matcher.d.ts +0 -71
- package/dist/golden-matcher.d.ts.map +0 -1
- package/dist/governance/eu-ai-act.d.ts +0 -37
- package/dist/governance/eu-ai-act.d.ts.map +0 -1
- package/dist/governance/index.d.ts.map +0 -1
- package/dist/governance/nist-ai-rmf.d.ts +0 -15
- package/dist/governance/nist-ai-rmf.d.ts.map +0 -1
- package/dist/governance/soc2.d.ts +0 -12
- package/dist/governance/soc2.d.ts.map +0 -1
- package/dist/governance/types.d.ts +0 -66
- package/dist/governance/types.d.ts.map +0 -1
- package/dist/harness-optimizer.d.ts +0 -82
- package/dist/harness-optimizer.d.ts.map +0 -1
- package/dist/held-out-gate.d.ts +0 -135
- package/dist/held-out-gate.d.ts.map +0 -1
- package/dist/hosted/client.d.ts +0 -73
- package/dist/hosted/client.d.ts.map +0 -1
- package/dist/hosted/from-env.test.d.ts +0 -8
- package/dist/hosted/from-env.test.d.ts.map +0 -1
- package/dist/hosted/index.d.ts.map +0 -1
- package/dist/hosted/types.d.ts +0 -159
- package/dist/hosted/types.d.ts.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/integrity/backend-integrity.d.ts +0 -71
- package/dist/integrity/backend-integrity.d.ts.map +0 -1
- package/dist/integrity/preflight.d.ts +0 -72
- package/dist/integrity/preflight.d.ts.map +0 -1
- package/dist/integrity/preflight.test.d.ts +0 -2
- package/dist/integrity/preflight.test.d.ts.map +0 -1
- package/dist/integrity/single-backend.d.ts +0 -67
- package/dist/integrity/single-backend.d.ts.map +0 -1
- package/dist/intent-match-judge.d.ts +0 -69
- package/dist/intent-match-judge.d.ts.map +0 -1
- package/dist/intent-match-judge.test.d.ts +0 -2
- package/dist/intent-match-judge.test.d.ts.map +0 -1
- package/dist/judge-calibration.d.ts.map +0 -1
- package/dist/judge-ensemble.d.ts +0 -66
- package/dist/judge-ensemble.d.ts.map +0 -1
- package/dist/judge-ensemble.test.d.ts +0 -8
- package/dist/judge-ensemble.test.d.ts.map +0 -1
- package/dist/judge-families.d.ts +0 -38
- package/dist/judge-families.d.ts.map +0 -1
- package/dist/judge-panel.d.ts +0 -65
- package/dist/judge-panel.d.ts.map +0 -1
- package/dist/judge-retry.d.ts +0 -70
- package/dist/judge-retry.d.ts.map +0 -1
- package/dist/judge-runner.d.ts +0 -36
- package/dist/judge-runner.d.ts.map +0 -1
- package/dist/judge-runner.test.d.ts +0 -2
- package/dist/judge-runner.test.d.ts.map +0 -1
- package/dist/judges.d.ts +0 -74
- package/dist/judges.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.d.ts +0 -89
- package/dist/keyword-coverage-judge.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.test.d.ts +0 -2
- package/dist/keyword-coverage-judge.test.d.ts.map +0 -1
- package/dist/knowledge/index.d.ts.map +0 -1
- package/dist/knowledge/readiness.d.ts +0 -26
- package/dist/knowledge/readiness.d.ts.map +0 -1
- package/dist/knowledge/types.d.ts +0 -75
- package/dist/knowledge/types.d.ts.map +0 -1
- package/dist/live-proof.d.ts +0 -62
- package/dist/live-proof.d.ts.map +0 -1
- package/dist/llm-client.d.ts.map +0 -1
- package/dist/llm-client.test.d.ts +0 -2
- package/dist/llm-client.test.d.ts.map +0 -1
- package/dist/locked-jsonl-appender.d.ts.map +0 -1
- package/dist/matrix/aggregation.d.ts +0 -16
- package/dist/matrix/aggregation.d.ts.map +0 -1
- package/dist/matrix/index.d.ts.map +0 -1
- package/dist/matrix/runner.d.ts +0 -15
- package/dist/matrix/runner.d.ts.map +0 -1
- package/dist/matrix/types.d.ts.map +0 -1
- package/dist/meta-eval/calibration.d.ts +0 -47
- package/dist/meta-eval/calibration.d.ts.map +0 -1
- package/dist/meta-eval/correlation-study.d.ts +0 -53
- package/dist/meta-eval/correlation-study.d.ts.map +0 -1
- package/dist/meta-eval/index.d.ts.map +0 -1
- package/dist/meta-eval/outcome-store.d.ts.map +0 -1
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +0 -1
- package/dist/meta-eval/sentinel.d.ts +0 -169
- package/dist/meta-eval/sentinel.d.ts.map +0 -1
- package/dist/metrics.d.ts +0 -63
- package/dist/metrics.d.ts.map +0 -1
- package/dist/model-seats.d.ts +0 -71
- package/dist/model-seats.d.ts.map +0 -1
- package/dist/model-seats.test.d.ts +0 -2
- package/dist/model-seats.test.d.ts.map +0 -1
- package/dist/muffled-gate-scanner.d.ts +0 -102
- package/dist/muffled-gate-scanner.d.ts.map +0 -1
- package/dist/multi-layer-verifier.d.ts.map +0 -1
- package/dist/multi-layer-verifier.test.d.ts +0 -2
- package/dist/multi-layer-verifier.test.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.d.ts +0 -80
- package/dist/multi-toolchain-layer.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.test.d.ts +0 -2
- package/dist/multi-toolchain-layer.test.d.ts.map +0 -1
- package/dist/multishot/default-tools.d.ts +0 -34
- package/dist/multishot/default-tools.d.ts.map +0 -1
- package/dist/multishot/index.d.ts.map +0 -1
- package/dist/multishot/judges.d.ts +0 -32
- package/dist/multishot/judges.d.ts.map +0 -1
- package/dist/multishot/matrix.d.ts +0 -107
- package/dist/multishot/matrix.d.ts.map +0 -1
- package/dist/multishot/multishot.d.ts +0 -23
- package/dist/multishot/multishot.d.ts.map +0 -1
- package/dist/multishot/router.d.ts +0 -37
- package/dist/multishot/router.d.ts.map +0 -1
- package/dist/multishot/types.d.ts +0 -60
- package/dist/multishot/types.d.ts.map +0 -1
- package/dist/observability.d.ts +0 -71
- package/dist/observability.d.ts.map +0 -1
- package/dist/oracle.d.ts +0 -55
- package/dist/oracle.d.ts.map +0 -1
- package/dist/orthogonality.d.ts +0 -35
- package/dist/orthogonality.d.ts.map +0 -1
- package/dist/otel-pipeline.d.ts +0 -31
- package/dist/otel-pipeline.d.ts.map +0 -1
- package/dist/paraphrase.d.ts +0 -107
- package/dist/paraphrase.d.ts.map +0 -1
- package/dist/pareto.d.ts.map +0 -1
- package/dist/partition-held-out.d.ts +0 -70
- package/dist/partition-held-out.d.ts.map +0 -1
- package/dist/partition-held-out.test.d.ts +0 -2
- package/dist/partition-held-out.test.d.ts.map +0 -1
- package/dist/perf/index.d.ts.map +0 -1
- package/dist/perf/integrity.d.ts +0 -30
- package/dist/perf/integrity.d.ts.map +0 -1
- package/dist/perf/journey.d.ts +0 -45
- package/dist/perf/journey.d.ts.map +0 -1
- package/dist/perf/ratchet.d.ts +0 -47
- package/dist/perf/ratchet.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.d.ts +0 -31
- package/dist/pipelines/budget-breach.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.test.d.ts +0 -2
- package/dist/pipelines/budget-breach.test.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.d.ts +0 -38
- package/dist/pipelines/failure-cluster.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.test.d.ts +0 -2
- package/dist/pipelines/failure-cluster.test.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.d.ts +0 -26
- package/dist/pipelines/first-divergence.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.test.d.ts +0 -2
- package/dist/pipelines/first-divergence.test.d.ts.map +0 -1
- package/dist/pipelines/index.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.d.ts +0 -26
- package/dist/pipelines/judge-agreement.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.test.d.ts +0 -2
- package/dist/pipelines/judge-agreement.test.d.ts.map +0 -1
- package/dist/pipelines/regression.d.ts +0 -23
- package/dist/pipelines/regression.d.ts.map +0 -1
- package/dist/pipelines/regression.test.d.ts +0 -2
- package/dist/pipelines/regression.test.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.d.ts +0 -32
- package/dist/pipelines/stuck-loop.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.test.d.ts +0 -2
- package/dist/pipelines/stuck-loop.test.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.d.ts +0 -34
- package/dist/pipelines/tool-waste.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.test.d.ts +0 -2
- package/dist/pipelines/tool-waste.test.d.ts.map +0 -1
- package/dist/playbook.d.ts +0 -16
- package/dist/playbook.d.ts.map +0 -1
- package/dist/pr-review-benchmark.d.ts +0 -88
- package/dist/pr-review-benchmark.d.ts.map +0 -1
- package/dist/pr-review-benchmark.test.d.ts +0 -2
- package/dist/pr-review-benchmark.test.d.ts.map +0 -1
- package/dist/pre-registration.d.ts +0 -125
- package/dist/pre-registration.d.ts.map +0 -1
- package/dist/prm/builtin-rubrics.d.ts +0 -33
- package/dist/prm/builtin-rubrics.d.ts.map +0 -1
- package/dist/prm/index.d.ts.map +0 -1
- package/dist/prm/inference.d.ts +0 -29
- package/dist/prm/inference.d.ts.map +0 -1
- package/dist/prm/inference.test.d.ts +0 -2
- package/dist/prm/inference.test.d.ts.map +0 -1
- package/dist/prm/rubric.d.ts.map +0 -1
- package/dist/prm/training-export.d.ts +0 -38
- package/dist/prm/training-export.d.ts.map +0 -1
- package/dist/produced-state.d.ts +0 -63
- package/dist/produced-state.d.ts.map +0 -1
- package/dist/produced-state.test.d.ts +0 -8
- package/dist/produced-state.test.d.ts.map +0 -1
- package/dist/profile/baselines.d.ts +0 -37
- package/dist/profile/baselines.d.ts.map +0 -1
- package/dist/profile/index.d.ts +0 -105
- package/dist/profile/index.d.ts.map +0 -1
- package/dist/promotion-gate.d.ts +0 -93
- package/dist/promotion-gate.d.ts.map +0 -1
- package/dist/prompt-registry.d.ts +0 -41
- package/dist/prompt-registry.d.ts.map +0 -1
- package/dist/propose-review-control.d.ts +0 -49
- package/dist/propose-review-control.d.ts.map +0 -1
- package/dist/propose-review-control.test.d.ts +0 -2
- package/dist/propose-review-control.test.d.ts.map +0 -1
- package/dist/propose-review.d.ts +0 -155
- package/dist/propose-review.d.ts.map +0 -1
- package/dist/red-team.d.ts.map +0 -1
- package/dist/reference-replay-steering.d.ts +0 -11
- package/dist/reference-replay-steering.d.ts.map +0 -1
- package/dist/reference-replay.d.ts +0 -176
- package/dist/reference-replay.d.ts.map +0 -1
- package/dist/reflective-mutation.d.ts +0 -79
- package/dist/reflective-mutation.d.ts.map +0 -1
- package/dist/registry.d.ts +0 -31
- package/dist/registry.d.ts.map +0 -1
- package/dist/release-confidence.d.ts +0 -128
- package/dist/release-confidence.d.ts.map +0 -1
- package/dist/release-report.d.ts +0 -11
- package/dist/release-report.d.ts.map +0 -1
- package/dist/replay.d.ts +0 -120
- package/dist/replay.d.ts.map +0 -1
- package/dist/reporter.d.ts +0 -14
- package/dist/reporter.d.ts.map +0 -1
- package/dist/reporting.d.ts.map +0 -1
- package/dist/researcher.d.ts +0 -140
- package/dist/researcher.d.ts.map +0 -1
- package/dist/reviewer.d.ts +0 -118
- package/dist/reviewer.d.ts.map +0 -1
- package/dist/reviewer.test.d.ts +0 -2
- package/dist/reviewer.test.d.ts.map +0 -1
- package/dist/reward-model-export.d.ts +0 -60
- package/dist/reward-model-export.d.ts.map +0 -1
- package/dist/rl/active-curriculum.d.ts +0 -110
- package/dist/rl/active-curriculum.d.ts.map +0 -1
- package/dist/rl/adaptation-eval.d.ts +0 -109
- package/dist/rl/adaptation-eval.d.ts.map +0 -1
- package/dist/rl/adversarial.d.ts.map +0 -1
- package/dist/rl/compute-curves.d.ts +0 -127
- package/dist/rl/compute-curves.d.ts.map +0 -1
- package/dist/rl/contamination.d.ts +0 -117
- package/dist/rl/contamination.d.ts.map +0 -1
- package/dist/rl/corpus.d.ts +0 -55
- package/dist/rl/corpus.d.ts.map +0 -1
- package/dist/rl/corpus.test.d.ts +0 -2
- package/dist/rl/corpus.test.d.ts.map +0 -1
- package/dist/rl/dataset.d.ts +0 -102
- package/dist/rl/dataset.d.ts.map +0 -1
- package/dist/rl/dataset.test.d.ts +0 -2
- package/dist/rl/dataset.test.d.ts.map +0 -1
- package/dist/rl/exporters.d.ts +0 -141
- package/dist/rl/exporters.d.ts.map +0 -1
- package/dist/rl/index.d.ts +0 -49
- package/dist/rl/index.d.ts.map +0 -1
- package/dist/rl/off-policy.d.ts.map +0 -1
- package/dist/rl/predictive-validity-researcher.d.ts +0 -69
- package/dist/rl/predictive-validity-researcher.d.ts.map +0 -1
- package/dist/rl/preferences.d.ts +0 -141
- package/dist/rl/preferences.d.ts.map +0 -1
- package/dist/rl/process-reward.d.ts +0 -122
- package/dist/rl/process-reward.d.ts.map +0 -1
- package/dist/rl/reward-hacking.d.ts +0 -104
- package/dist/rl/reward-hacking.d.ts.map +0 -1
- package/dist/rl/rl-campaign.d.ts +0 -85
- package/dist/rl/rl-campaign.d.ts.map +0 -1
- package/dist/rl/run-record-adapters.d.ts +0 -56
- package/dist/rl/run-record-adapters.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.d.ts +0 -166
- package/dist/rl/sim-fidelity.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.test.d.ts +0 -2
- package/dist/rl/sim-fidelity.test.d.ts.map +0 -1
- package/dist/rl/tournament.d.ts +0 -115
- package/dist/rl/tournament.d.ts.map +0 -1
- package/dist/rl/verifiable-reward.d.ts +0 -124
- package/dist/rl/verifiable-reward.d.ts.map +0 -1
- package/dist/run-critic.d.ts +0 -23
- package/dist/run-critic.d.ts.map +0 -1
- package/dist/run-evidence.d.ts +0 -32
- package/dist/run-evidence.d.ts.map +0 -1
- package/dist/run-record.d.ts.map +0 -1
- package/dist/run-record.test.d.ts +0 -2
- package/dist/run-record.test.d.ts.map +0 -1
- package/dist/run-score.d.ts +0 -31
- package/dist/run-score.d.ts.map +0 -1
- package/dist/runtime-trajectory.d.ts +0 -47
- package/dist/runtime-trajectory.d.ts.map +0 -1
- package/dist/sandbox-harness.d.ts.map +0 -1
- package/dist/sandbox-harness.test.d.ts +0 -2
- package/dist/sandbox-harness.test.d.ts.map +0 -1
- package/dist/sandbox-pool.d.ts +0 -74
- package/dist/sandbox-pool.d.ts.map +0 -1
- package/dist/sandbox-pool.test.d.ts +0 -2
- package/dist/sandbox-pool.test.d.ts.map +0 -1
- package/dist/scorecard.d.ts +0 -133
- package/dist/scorecard.d.ts.map +0 -1
- package/dist/scorecard.test.d.ts +0 -2
- package/dist/scorecard.test.d.ts.map +0 -1
- package/dist/self-play.d.ts +0 -69
- package/dist/self-play.d.ts.map +0 -1
- package/dist/semantic-concept-judge.d.ts +0 -135
- package/dist/semantic-concept-judge.d.ts.map +0 -1
- package/dist/semantic-concept-judge.test.d.ts +0 -2
- package/dist/semantic-concept-judge.test.d.ts.map +0 -1
- package/dist/sequential.d.ts.map +0 -1
- package/dist/series-convergence.d.ts.map +0 -1
- package/dist/slo.d.ts +0 -48
- package/dist/slo.d.ts.map +0 -1
- package/dist/state-continuity.d.ts +0 -47
- package/dist/state-continuity.d.ts.map +0 -1
- package/dist/statistics.d.ts.map +0 -1
- package/dist/statistics.test.d.ts +0 -2
- package/dist/statistics.test.d.ts.map +0 -1
- package/dist/steering-optimizer.d.ts +0 -58
- package/dist/steering-optimizer.d.ts.map +0 -1
- package/dist/steering.d.ts +0 -24
- package/dist/steering.d.ts.map +0 -1
- package/dist/storyboard/code-edit.d.ts +0 -64
- package/dist/storyboard/code-edit.d.ts.map +0 -1
- package/dist/storyboard/code-edit.test.d.ts +0 -2
- package/dist/storyboard/code-edit.test.d.ts.map +0 -1
- package/dist/storyboard/index.d.ts.map +0 -1
- package/dist/storyboard/index.test.d.ts +0 -2
- package/dist/storyboard/index.test.d.ts.map +0 -1
- package/dist/summary-report.d.ts.map +0 -1
- package/dist/telemetry/client.d.ts +0 -35
- package/dist/telemetry/client.d.ts.map +0 -1
- package/dist/telemetry/index.d.ts.map +0 -1
- package/dist/telemetry/schema.d.ts +0 -61
- package/dist/telemetry/schema.d.ts.map +0 -1
- package/dist/telemetry/sink-fetch.d.ts +0 -39
- package/dist/telemetry/sink-fetch.d.ts.map +0 -1
- package/dist/telemetry/sink-file.d.ts.map +0 -1
- package/dist/test-graded-scenario.d.ts +0 -42
- package/dist/test-graded-scenario.d.ts.map +0 -1
- package/dist/testing.d.ts.map +0 -1
- package/dist/tool-use-metrics.d.ts +0 -35
- package/dist/tool-use-metrics.d.ts.map +0 -1
- package/dist/trace/capture-fetch.d.ts +0 -48
- package/dist/trace/capture-fetch.d.ts.map +0 -1
- package/dist/trace/capture-fetch.test.d.ts +0 -2
- package/dist/trace/capture-fetch.test.d.ts.map +0 -1
- package/dist/trace/emitter.d.ts.map +0 -1
- package/dist/trace/extract-usage.d.ts +0 -46
- package/dist/trace/extract-usage.d.ts.map +0 -1
- package/dist/trace/extract-usage.test.d.ts +0 -2
- package/dist/trace/extract-usage.test.d.ts.map +0 -1
- package/dist/trace/index.d.ts +0 -15
- package/dist/trace/index.d.ts.map +0 -1
- package/dist/trace/integrity.d.ts.map +0 -1
- package/dist/trace/otel-bridge.d.ts +0 -29
- package/dist/trace/otel-bridge.d.ts.map +0 -1
- package/dist/trace/otel-export.d.ts +0 -52
- package/dist/trace/otel-export.d.ts.map +0 -1
- package/dist/trace/otel.d.ts +0 -57
- package/dist/trace/otel.d.ts.map +0 -1
- package/dist/trace/otlp-attributes.d.ts +0 -17
- package/dist/trace/otlp-attributes.d.ts.map +0 -1
- package/dist/trace/query.d.ts +0 -29
- package/dist/trace/query.d.ts.map +0 -1
- package/dist/trace/query.test.d.ts +0 -2
- package/dist/trace/query.test.d.ts.map +0 -1
- package/dist/trace/raw-provider-sink.d.ts.map +0 -1
- package/dist/trace/redact.d.ts.map +0 -1
- package/dist/trace/schema.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.d.ts +0 -72
- package/dist/trace/store-to-otlp.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.test.d.ts +0 -2
- package/dist/trace/store-to-otlp.test.d.ts.map +0 -1
- package/dist/trace/store.d.ts.map +0 -1
- package/dist/trace/store.test.d.ts +0 -2
- package/dist/trace/store.test.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.test.d.ts +0 -2
- package/dist/trace-analyst/analyst.test.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.d.ts +0 -40
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +0 -2
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +0 -1
- package/dist/trace-analyst/hook.d.ts +0 -55
- package/dist/trace-analyst/hook.d.ts.map +0 -1
- package/dist/trace-analyst/index.d.ts +0 -18
- package/dist/trace-analyst/index.d.ts.map +0 -1
- package/dist/trace-analyst/insights.d.ts +0 -71
- package/dist/trace-analyst/insights.d.ts.map +0 -1
- package/dist/trace-analyst/insights.test.d.ts +0 -2
- package/dist/trace-analyst/insights.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-flatten.d.ts +0 -42
- package/dist/trace-analyst/otlp-flatten.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.d.ts +0 -85
- package/dist/trace-analyst/otlp-span.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-span.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.d.ts +0 -115
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/prompts.d.ts +0 -6
- package/dist/trace-analyst/prompts.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.d.ts +0 -126
- package/dist/trace-analyst/store-otlp.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.test.d.ts +0 -8
- package/dist/trace-analyst/store-otlp.test.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/store.d.ts +0 -63
- package/dist/trace-analyst/store.d.ts.map +0 -1
- package/dist/trace-analyst/tools.d.ts +0 -44
- package/dist/trace-analyst/tools.d.ts.map +0 -1
- package/dist/trace-analyst/tools.test.d.ts +0 -10
- package/dist/trace-analyst/tools.test.d.ts.map +0 -1
- package/dist/trace-analyst/types.d.ts.map +0 -1
- package/dist/trace-contracts.d.ts +0 -180
- package/dist/trace-contracts.d.ts.map +0 -1
- package/dist/traced-analyst.d.ts +0 -26
- package/dist/traced-analyst.d.ts.map +0 -1
- package/dist/traced-judges.d.ts +0 -27
- package/dist/traced-judges.d.ts.map +0 -1
- package/dist/traces.d.ts.map +0 -1
- package/dist/trajectory.d.ts.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/ui-finding.d.ts +0 -104
- package/dist/ui-finding.d.ts.map +0 -1
- package/dist/verdict-cache.d.ts +0 -78
- package/dist/verdict-cache.d.ts.map +0 -1
- package/dist/verdict-cache.test.d.ts +0 -2
- package/dist/verdict-cache.test.d.ts.map +0 -1
- package/dist/verdict.d.ts.map +0 -1
- package/dist/visual-diff.d.ts +0 -32
- package/dist/visual-diff.d.ts.map +0 -1
- package/dist/wire/handlers.d.ts +0 -54
- package/dist/wire/handlers.d.ts.map +0 -1
- package/dist/wire/index.d.ts.map +0 -1
- package/dist/wire/openapi.d.ts +0 -3
- package/dist/wire/openapi.d.ts.map +0 -1
- package/dist/wire/rpc.d.ts +0 -21
- package/dist/wire/rpc.d.ts.map +0 -1
- package/dist/wire/rubrics.d.ts +0 -34
- package/dist/wire/rubrics.d.ts.map +0 -1
- package/dist/wire/schemas.d.ts +0 -410
- package/dist/wire/schemas.d.ts.map +0 -1
- package/dist/wire/server.d.ts +0 -60
- package/dist/wire/server.d.ts.map +0 -1
- package/dist/workflow/event-schema.d.ts +0 -5
- package/dist/workflow/event-schema.d.ts.map +0 -1
- package/dist/workflow/feedback-pack.d.ts +0 -99
- package/dist/workflow/feedback-pack.d.ts.map +0 -1
- package/dist/workflow/index.d.ts.map +0 -1
- package/dist/workflow/intelligence-export.d.ts +0 -62
- package/dist/workflow/intelligence-export.d.ts.map +0 -1
- package/dist/workflow/partner-report.d.ts +0 -49
- package/dist/workflow/partner-report.d.ts.map +0 -1
- package/dist/workflow/phase-graph.d.ts +0 -43
- package/dist/workflow/phase-graph.d.ts.map +0 -1
- package/dist/workflow/promotion-gate.d.ts +0 -61
- package/dist/workflow/promotion-gate.d.ts.map +0 -1
- package/dist/workflow/run-record.d.ts +0 -12
- package/dist/workflow/run-record.d.ts.map +0 -1
- package/dist/workflow/runtime-adapter.d.ts +0 -20
- package/dist/workflow/runtime-adapter.d.ts.map +0 -1
- package/dist/workflow/sanitize.d.ts +0 -21
- package/dist/workflow/sanitize.d.ts.map +0 -1
- package/dist/workflow/schema.d.ts +0 -5
- package/dist/workflow/schema.d.ts.map +0 -1
- package/dist/workflow/summary.d.ts +0 -43
- package/dist/workflow/summary.d.ts.map +0 -1
- package/dist/workflow/trace-event-fields.d.ts +0 -6
- package/dist/workflow/trace-event-fields.d.ts.map +0 -1
- package/dist/workflow/trajectory.d.ts +0 -15
- package/dist/workflow/trajectory.d.ts.map +0 -1
- package/dist/workflow/types.d.ts +0 -68
- package/dist/workflow/types.d.ts.map +0 -1
- package/dist/workspace-inspector.d.ts +0 -67
- package/dist/workspace-inspector.d.ts.map +0 -1
- package/dist/wrangler-deploy-runner.test.d.ts +0 -2
- package/dist/wrangler-deploy-runner.test.d.ts.map +0 -1
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
* - drifting: recent trend is monotonic and beyond noise — regressing or improving
|
|
12
12
|
* - noisy: neither — keep iterating, but flag as untrustworthy for gating
|
|
13
13
|
*/
|
|
14
|
-
|
|
14
|
+
interface SeriesConvergenceOptions {
|
|
15
15
|
/** Window size for "recent" analysis (default 5). */
|
|
16
16
|
window?: number;
|
|
17
17
|
/** Coefficient-of-variation threshold below which the window is stabilized (default 0.05 = 5%). */
|
|
@@ -19,7 +19,7 @@ export interface SeriesConvergenceOptions {
|
|
|
19
19
|
/** Minimum monotone run length to call drift (default 3). */
|
|
20
20
|
driftRun?: number;
|
|
21
21
|
}
|
|
22
|
-
|
|
22
|
+
interface SeriesConvergenceResult {
|
|
23
23
|
state: 'stabilized' | 'drifting-up' | 'drifting-down' | 'noisy' | 'insufficient-data';
|
|
24
24
|
windowMean: number;
|
|
25
25
|
windowCv: number;
|
|
@@ -28,5 +28,6 @@ export interface SeriesConvergenceResult {
|
|
|
28
28
|
/** True when n ≥ window AND windowCv ≤ stableCv. */
|
|
29
29
|
stable: boolean;
|
|
30
30
|
}
|
|
31
|
-
|
|
32
|
-
|
|
31
|
+
declare function analyzeSeries(values: number[], options?: SeriesConvergenceOptions): SeriesConvergenceResult;
|
|
32
|
+
|
|
33
|
+
export { type SeriesConvergenceOptions as S, type SeriesConvergenceResult as a, analyzeSeries as b };
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Fleet telemetry envelope — agent-eval's portable observability shape.
|
|
3
|
+
*
|
|
4
|
+
* Designed so any consumer (Node CLI, Cloudflare Worker, Lambda, browser
|
|
5
|
+
* extension) can emit structured rows describing one unit of work — a page
|
|
6
|
+
* audit, a tool call, an evolve round, a full agent run — to a central sink.
|
|
7
|
+
*
|
|
8
|
+
* The schema is intentionally a strict superset of agent-eval's `Run` shape
|
|
9
|
+
* so a future TraceStore adapter can promote envelopes into traces without
|
|
10
|
+
* translation.
|
|
11
|
+
*/
|
|
12
|
+
declare const TELEMETRY_SCHEMA_VERSION = 1;
|
|
13
|
+
/** Discriminator for the unit of work this envelope describes. */
|
|
14
|
+
type TelemetryKind = 'agent-run' | 'design-audit-page' | 'design-audit-run' | 'design-evolve-round' | 'design-evolve-run' | 'gepa-trial' | 'gepa-generation' | 'tool-call' | 'judge-verdict' | 'custom';
|
|
15
|
+
interface TelemetryEnvelope {
|
|
16
|
+
schemaVersion: typeof TELEMETRY_SCHEMA_VERSION;
|
|
17
|
+
envelopeId: string;
|
|
18
|
+
runId: string;
|
|
19
|
+
timestamp: string;
|
|
20
|
+
parentRunId?: string;
|
|
21
|
+
source: TelemetrySource;
|
|
22
|
+
model?: TelemetryModel;
|
|
23
|
+
kind: TelemetryKind;
|
|
24
|
+
ok: boolean;
|
|
25
|
+
durationMs: number;
|
|
26
|
+
data: Record<string, unknown>;
|
|
27
|
+
metrics: Record<string, number>;
|
|
28
|
+
tags?: Record<string, string>;
|
|
29
|
+
error?: string;
|
|
30
|
+
}
|
|
31
|
+
interface TelemetrySource {
|
|
32
|
+
/** Repo identity — basename of cwd plus git remote if discoverable. */
|
|
33
|
+
repo: string;
|
|
34
|
+
cwd: string;
|
|
35
|
+
gitSha?: string;
|
|
36
|
+
gitBranch?: string;
|
|
37
|
+
cliVersion: string;
|
|
38
|
+
/** What was invoked, e.g. `design-audit`, `bad run`, `gepa --target`. */
|
|
39
|
+
invocation: string;
|
|
40
|
+
/** Sanitised argv minus secrets. */
|
|
41
|
+
argv?: string[];
|
|
42
|
+
/**
|
|
43
|
+
* Multi-tenant identity. Set when the consumer runs inside a hosted
|
|
44
|
+
* product so a fleet rollup can group by tenant without leaking customer
|
|
45
|
+
* URLs or PII.
|
|
46
|
+
*/
|
|
47
|
+
tenantId?: string;
|
|
48
|
+
/** Optional sub-tenant identity (project, suite, walkthrough, customer). */
|
|
49
|
+
customerId?: string;
|
|
50
|
+
/** SHA-256 (12 hex) of the API key used to authenticate this run, when applicable. */
|
|
51
|
+
apiKeyHash?: string;
|
|
52
|
+
}
|
|
53
|
+
interface TelemetryModel {
|
|
54
|
+
provider: string;
|
|
55
|
+
name: string;
|
|
56
|
+
/** SHA-256 (12 hex chars) of the prompt(s) used. */
|
|
57
|
+
promptHash?: string;
|
|
58
|
+
/** SHA-256 (12 hex chars) of the composed rubric body, if applicable. */
|
|
59
|
+
rubricHash?: string;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Workers-safe telemetry sinks — only `fetch` and pure JS. No `fs`, no
|
|
64
|
+
* `child_process`. Safe to import from a Cloudflare Worker, Lambda, edge
|
|
65
|
+
* function, or browser extension.
|
|
66
|
+
*
|
|
67
|
+
* For Node-only file persistence, import from '@tangle-network/agent-eval/telemetry/file'.
|
|
68
|
+
*/
|
|
69
|
+
|
|
70
|
+
interface TelemetrySink {
|
|
71
|
+
emit(envelope: TelemetryEnvelope): Promise<void> | void;
|
|
72
|
+
close?(): Promise<void> | void;
|
|
73
|
+
}
|
|
74
|
+
/** Best-effort POST to a remote collector. Fire-and-forget; never throws. */
|
|
75
|
+
declare class HttpTelemetrySink implements TelemetrySink {
|
|
76
|
+
private readonly endpoint;
|
|
77
|
+
private readonly bearer?;
|
|
78
|
+
private inflight;
|
|
79
|
+
constructor(endpoint: string, bearer?: string | undefined);
|
|
80
|
+
emit(envelope: TelemetryEnvelope): void;
|
|
81
|
+
close(): Promise<void>;
|
|
82
|
+
}
|
|
83
|
+
/** Fanout to multiple sinks — failures in one do not affect others. */
|
|
84
|
+
declare class FanoutTelemetrySink implements TelemetrySink {
|
|
85
|
+
private readonly sinks;
|
|
86
|
+
constructor(sinks: TelemetrySink[]);
|
|
87
|
+
emit(envelope: TelemetryEnvelope): void;
|
|
88
|
+
close(): Promise<void>;
|
|
89
|
+
}
|
|
90
|
+
/** No-op sink — used when telemetry is explicitly disabled. */
|
|
91
|
+
declare class NullTelemetrySink implements TelemetrySink {
|
|
92
|
+
emit(): void;
|
|
93
|
+
}
|
|
94
|
+
/** In-memory sink — useful for tests + downstream adapters. */
|
|
95
|
+
declare class InMemoryTelemetrySink implements TelemetrySink {
|
|
96
|
+
readonly envelopes: TelemetryEnvelope[];
|
|
97
|
+
emit(envelope: TelemetryEnvelope): void;
|
|
98
|
+
clear(): void;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
export { FanoutTelemetrySink as F, HttpTelemetrySink as H, InMemoryTelemetrySink as I, NullTelemetrySink as N, type TelemetryKind as T, type TelemetryModel as a, type TelemetrySource as b, type TelemetrySink as c, TELEMETRY_SCHEMA_VERSION as d, type TelemetryEnvelope as e };
|
|
@@ -1,15 +1,16 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import
|
|
1
|
+
import { d as ContinuousAgreementOptions, C as ContinuousAgreement } from './judge-calibration-0p2QcWNE.js';
|
|
2
|
+
import { J as JudgeScore } from './types-C7DGg5ex.js';
|
|
3
|
+
|
|
3
4
|
/** Identity: dimensions already follow "higher = better" by prompt convention
|
|
4
5
|
* (inverted dims like hallucination are scored 10 = best at the source). */
|
|
5
|
-
|
|
6
|
+
declare const normalizeScores: (scores: JudgeScore[]) => JudgeScore[];
|
|
6
7
|
/** Weighted mean — falls back to uniform weights when omitted */
|
|
7
|
-
|
|
8
|
+
declare function weightedMean(scores: {
|
|
8
9
|
score: number;
|
|
9
10
|
weight?: number;
|
|
10
11
|
}[]): number;
|
|
11
12
|
/** Bootstrap confidence interval */
|
|
12
|
-
|
|
13
|
+
declare function confidenceInterval(scores: number[], confidence?: number, opts?: {
|
|
13
14
|
seed?: number;
|
|
14
15
|
resamples?: number;
|
|
15
16
|
}): {
|
|
@@ -23,24 +24,24 @@ export declare function confidenceInterval(scores: number[], confidence?: number
|
|
|
23
24
|
* Each inner array is one judge's scores for all items.
|
|
24
25
|
* All arrays must have the same length (same items scored).
|
|
25
26
|
*/
|
|
26
|
-
|
|
27
|
+
declare function interRaterReliability(judgeScores: JudgeScore[][]): number;
|
|
27
28
|
/**
|
|
28
29
|
* Mann-Whitney U test for comparing two independent groups.
|
|
29
30
|
* Returns U statistic and approximate p-value (normal approximation).
|
|
30
31
|
*/
|
|
31
|
-
|
|
32
|
+
declare function mannWhitneyU(a: number[], b: number[]): {
|
|
32
33
|
u: number;
|
|
33
34
|
p: number;
|
|
34
35
|
};
|
|
35
36
|
/** Partial credit: returns 0-1 ratio of current toward target */
|
|
36
|
-
|
|
37
|
+
declare function partialCredit(current: number, target: number): number;
|
|
37
38
|
/**
|
|
38
39
|
* Paired t-test — before/after measurements on the SAME items.
|
|
39
40
|
* Pairing removes inter-item variance, giving tighter significance than
|
|
40
41
|
* an unpaired test when comparing prompt v1 vs prompt v2 on identical
|
|
41
42
|
* scenarios.
|
|
42
43
|
*/
|
|
43
|
-
|
|
44
|
+
declare function pairedTTest(before: number[], after: number[]): {
|
|
44
45
|
t: number;
|
|
45
46
|
df: number;
|
|
46
47
|
p: number;
|
|
@@ -49,7 +50,7 @@ export declare function pairedTTest(before: number[], after: number[]): {
|
|
|
49
50
|
* Wilcoxon signed-rank test — paired non-parametric alternative.
|
|
50
51
|
* Use when the differences aren't normally distributed.
|
|
51
52
|
*/
|
|
52
|
-
|
|
53
|
+
declare function wilcoxonSignedRank(before: number[], after: number[]): {
|
|
53
54
|
w: number;
|
|
54
55
|
p: number;
|
|
55
56
|
};
|
|
@@ -58,8 +59,8 @@ export declare function wilcoxonSignedRank(before: number[], after: number[]): {
|
|
|
58
59
|
* Positive d means group b has higher mean than group a.
|
|
59
60
|
* Rule of thumb: |d| < 0.2 negligible, 0.2–0.5 small, 0.5–0.8 medium, > 0.8 large.
|
|
60
61
|
*/
|
|
61
|
-
|
|
62
|
-
|
|
62
|
+
declare function cohensD(a: number[], b: number[]): number;
|
|
63
|
+
type CliffsMagnitude = 'negligible' | 'small' | 'medium' | 'large';
|
|
63
64
|
/**
|
|
64
65
|
* Cliff's delta — a non-parametric effect size for two independent samples.
|
|
65
66
|
* `δ = (#(after > before) − #(after < before)) / (n_before · n_after)`,
|
|
@@ -70,14 +71,14 @@ export type CliffsMagnitude = 'negligible' | 'small' | 'medium' | 'large';
|
|
|
70
71
|
* `pairedBootstrap` / `wilcoxonSignedRank` for the non-parametric reporting
|
|
71
72
|
* path. Returns 0 when either sample is empty.
|
|
72
73
|
*/
|
|
73
|
-
|
|
74
|
+
declare function cliffsDelta(before: number[], after: number[]): number;
|
|
74
75
|
/**
|
|
75
76
|
* Map a Cliff's delta to a qualitative magnitude using the standard
|
|
76
77
|
* Romano et al. thresholds (|δ|): <0.147 negligible, <0.33 small,
|
|
77
78
|
* <0.474 medium, else large.
|
|
78
79
|
*/
|
|
79
|
-
|
|
80
|
-
|
|
80
|
+
declare function interpretCliffs(delta: number): CliffsMagnitude;
|
|
81
|
+
interface WeightedCompositeInput {
|
|
81
82
|
/** Per-dimension scores (typically 0..1). */
|
|
82
83
|
dims: Record<string, number>;
|
|
83
84
|
/** Weight per dimension. Every weighted dimension MUST be present in
|
|
@@ -88,7 +89,7 @@ export interface WeightedCompositeInput {
|
|
|
88
89
|
/** Optional pass threshold; when set, the result reports `pass`. */
|
|
89
90
|
threshold?: number;
|
|
90
91
|
}
|
|
91
|
-
|
|
92
|
+
interface WeightedCompositeResult {
|
|
92
93
|
composite: number;
|
|
93
94
|
pass?: boolean;
|
|
94
95
|
}
|
|
@@ -101,8 +102,8 @@ export interface WeightedCompositeResult {
|
|
|
101
102
|
* weight is negative, or if the weights sum to 0 — none of which can produce
|
|
102
103
|
* a meaningful composite.
|
|
103
104
|
*/
|
|
104
|
-
|
|
105
|
-
|
|
105
|
+
declare function weightedComposite(input: WeightedCompositeInput): WeightedCompositeResult;
|
|
106
|
+
interface CorpusScoreRecord {
|
|
106
107
|
/** Stable identifier for the rated item (scenario, span, turn, …). */
|
|
107
108
|
itemId: string;
|
|
108
109
|
/** Identifier for the judge that produced this score. */
|
|
@@ -112,14 +113,14 @@ export interface CorpusScoreRecord {
|
|
|
112
113
|
/** Numeric score; must be finite. */
|
|
113
114
|
score: number;
|
|
114
115
|
}
|
|
115
|
-
|
|
116
|
+
interface CorpusAgreementPerDimension extends ContinuousAgreement {
|
|
116
117
|
dimension: string;
|
|
117
118
|
/** Item IDs that contributed to this dimension's matrix (every judge scored them). */
|
|
118
119
|
itemIds: string[];
|
|
119
120
|
/** Judge IDs that contributed to this dimension's matrix. */
|
|
120
121
|
judgeIds: string[];
|
|
121
122
|
}
|
|
122
|
-
|
|
123
|
+
interface CorpusAgreementReport {
|
|
123
124
|
/** Per-dimension ICC(2,1) + κ_w + Pearson + Spearman + bootstrap CIs. */
|
|
124
125
|
perDimension: CorpusAgreementPerDimension[];
|
|
125
126
|
/** Mean ICC across dimensions (NaN if no dimension yielded a finite ICC). */
|
|
@@ -131,7 +132,7 @@ export interface CorpusAgreementReport {
|
|
|
131
132
|
/** Judges seen across the corpus (sorted). */
|
|
132
133
|
judgeIds: string[];
|
|
133
134
|
}
|
|
134
|
-
|
|
135
|
+
interface CorpusAgreementOptions extends ContinuousAgreementOptions {
|
|
135
136
|
/**
|
|
136
137
|
* Restrict the audit to these dimensions. Default = every dimension
|
|
137
138
|
* that appears in the input. A dimension named here but absent from
|
|
@@ -161,7 +162,7 @@ export interface CorpusAgreementOptions extends ContinuousAgreementOptions {
|
|
|
161
162
|
* another dimension throws (would silently shrink the matrix).
|
|
162
163
|
* - Duplicate (itemId, judgeName, dimension) records throw.
|
|
163
164
|
*/
|
|
164
|
-
|
|
165
|
+
declare function corpusInterRaterAgreement(records: CorpusScoreRecord[], opts?: CorpusAgreementOptions): CorpusAgreementReport;
|
|
165
166
|
/**
|
|
166
167
|
* Convenience adapter for `JudgeScore[]` data keyed externally by item.
|
|
167
168
|
*
|
|
@@ -170,7 +171,7 @@ export declare function corpusInterRaterAgreement(records: CorpusScoreRecord[],
|
|
|
170
171
|
* agreement without manually flattening. `itemId` must be unique per
|
|
171
172
|
* row of `itemsScores`.
|
|
172
173
|
*/
|
|
173
|
-
|
|
174
|
+
declare function corpusInterRaterAgreementFromJudgeScores(itemsScores: Array<{
|
|
174
175
|
itemId: string;
|
|
175
176
|
scores: JudgeScore[];
|
|
176
177
|
}>, opts?: CorpusAgreementOptions): CorpusAgreementReport;
|
|
@@ -180,7 +181,7 @@ export declare function corpusInterRaterAgreementFromJudgeScores(itemsScores: Ar
|
|
|
180
181
|
* n = 2 * ( (z_{1-α/2} + z_{1-β}) / d )^2
|
|
181
182
|
* where d is Cohen's d. Returns Infinity for effect ≤ 0.
|
|
182
183
|
*/
|
|
183
|
-
|
|
184
|
+
declare function requiredSampleSize(opts: {
|
|
184
185
|
effect: number;
|
|
185
186
|
alpha?: number;
|
|
186
187
|
power?: number;
|
|
@@ -192,7 +193,7 @@ export declare function requiredSampleSize(opts: {
|
|
|
192
193
|
* sd(deltas) for score units; treat as a lower bound — Wilcoxon and bootstrap
|
|
193
194
|
* have asymptotic relative efficiency below 1 vs the t-test on heavy tails.
|
|
194
195
|
*/
|
|
195
|
-
|
|
196
|
+
declare function pairedMde(opts: {
|
|
196
197
|
nPaired: number;
|
|
197
198
|
alpha?: number;
|
|
198
199
|
power?: number;
|
|
@@ -212,7 +213,7 @@ export declare function pairedMde(opts: {
|
|
|
212
213
|
* approximation, treat the result as a lower bound and prefer the discordant-pair
|
|
213
214
|
* floor.
|
|
214
215
|
*/
|
|
215
|
-
|
|
216
|
+
declare function mcnemarRequiredN(opts: {
|
|
216
217
|
p10: number;
|
|
217
218
|
p01: number;
|
|
218
219
|
alpha?: number;
|
|
@@ -224,7 +225,7 @@ export declare function mcnemarRequiredN(opts: {
|
|
|
224
225
|
* of {@link mcnemarRequiredN} (same Lachin asymptotic model, same parameters).
|
|
225
226
|
* Returns a value in [0, 1]; equals `alpha` when there is no effect.
|
|
226
227
|
*/
|
|
227
|
-
|
|
228
|
+
declare function mcnemarPower(opts: {
|
|
228
229
|
p10: number;
|
|
229
230
|
p01: number;
|
|
230
231
|
nPairs: number;
|
|
@@ -232,7 +233,7 @@ export declare function mcnemarPower(opts: {
|
|
|
232
233
|
twoSided?: boolean;
|
|
233
234
|
}): number;
|
|
234
235
|
/** Bonferroni adjustment: multiply every p-value by the test count, clamp at 1. */
|
|
235
|
-
|
|
236
|
+
declare function bonferroni(pValues: number[], alpha?: number): {
|
|
236
237
|
adjusted: number[];
|
|
237
238
|
significant: boolean[];
|
|
238
239
|
};
|
|
@@ -240,11 +241,11 @@ export declare function bonferroni(pValues: number[], alpha?: number): {
|
|
|
240
241
|
* Benjamini–Hochberg false discovery rate. Returns adjusted q-values and
|
|
241
242
|
* significance at the target FDR; handles ties and preserves q monotonicity.
|
|
242
243
|
*/
|
|
243
|
-
|
|
244
|
+
declare function benjaminiHochberg(pValues: number[], fdr?: number): {
|
|
244
245
|
qValues: number[];
|
|
245
246
|
significant: boolean[];
|
|
246
247
|
};
|
|
247
|
-
|
|
248
|
+
interface PairedBootstrapResult {
|
|
248
249
|
/** Number of paired observations. */
|
|
249
250
|
n: number;
|
|
250
251
|
/** Median of paired deltas (after − before). */
|
|
@@ -260,7 +261,7 @@ export interface PairedBootstrapResult {
|
|
|
260
261
|
/** Number of bootstrap resamples used. */
|
|
261
262
|
resamples: number;
|
|
262
263
|
}
|
|
263
|
-
|
|
264
|
+
interface PairedBootstrapOptions {
|
|
264
265
|
/** Confidence level. Default 0.95. */
|
|
265
266
|
confidence?: number;
|
|
266
267
|
/** Bootstrap resample count. Default 2000. */
|
|
@@ -276,9 +277,9 @@ export interface PairedBootstrapOptions {
|
|
|
276
277
|
* lower bound is what the promotion gate checks — `low > threshold` means the
|
|
277
278
|
* gain is real at the confidence level. Throws on unequal sample sizes.
|
|
278
279
|
*/
|
|
279
|
-
|
|
280
|
+
declare function pairedBootstrap(before: number[], after: number[], opts?: PairedBootstrapOptions): PairedBootstrapResult;
|
|
280
281
|
/** A binomial proportion estimate with a confidence interval. */
|
|
281
|
-
|
|
282
|
+
interface ProportionInterval {
|
|
282
283
|
/** Point estimate successes / n (0 when n = 0). */
|
|
283
284
|
estimate: number;
|
|
284
285
|
/** Lower bound, clamped to [0, 1]. */
|
|
@@ -293,9 +294,9 @@ export interface ProportionInterval {
|
|
|
293
294
|
* CI — the continuous `confidenceInterval` assumes the wrong distribution for a
|
|
294
295
|
* proportion. `n = 0 ⇒ {0, 0, 0}`.
|
|
295
296
|
*/
|
|
296
|
-
|
|
297
|
+
declare function wilson(successes: number, n: number, confidence?: number): ProportionInterval;
|
|
297
298
|
/** Result of a McNemar paired-binary significance test. */
|
|
298
|
-
|
|
299
|
+
interface McNemarResult {
|
|
299
300
|
/** Total paired observations. */
|
|
300
301
|
n: number;
|
|
301
302
|
/** Discordant pairs (b + c) — the only ones that carry signal. */
|
|
@@ -321,9 +322,9 @@ export interface McNemarResult {
|
|
|
321
322
|
* reference). Inputs are paired 0/1 (or boolean) arrays, control first to match
|
|
322
323
|
* the module's (before, after) convention. Throws on unequal lengths.
|
|
323
324
|
*/
|
|
324
|
-
|
|
325
|
+
declare function mcnemar(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>): McNemarResult;
|
|
325
326
|
/** A paired binary effect size (treatment rate − control rate) with a CI. */
|
|
326
|
-
|
|
327
|
+
interface RiskDifferenceResult {
|
|
327
328
|
/** Total paired observations. */
|
|
328
329
|
n: number;
|
|
329
330
|
/** Discordant pairs: treatment-win count. */
|
|
@@ -347,7 +348,7 @@ export interface RiskDifferenceResult {
|
|
|
347
348
|
* the interval by ignoring the pairing). Inputs are paired 0/1 (or boolean)
|
|
348
349
|
* arrays, control first. Throws on unequal lengths.
|
|
349
350
|
*/
|
|
350
|
-
|
|
351
|
+
declare function pairedRiskDifference(control: ArrayLike<number | boolean>, treatment: ArrayLike<number | boolean>, confidence?: number): RiskDifferenceResult;
|
|
351
352
|
/**
|
|
352
353
|
* Unbiased pass@k for code generation (Chen et al. 2021, "Evaluating Large
|
|
353
354
|
* Language Models Trained on Code"). Given `n` independent samples for one
|
|
@@ -358,8 +359,8 @@ export declare function pairedRiskDifference(control: ArrayLike<number | boolean
|
|
|
358
359
|
* the suite for the corpus pass@k. Computed in the numerically stable product
|
|
359
360
|
* form. Requires 1 ≤ k ≤ n and 0 ≤ c ≤ n.
|
|
360
361
|
*/
|
|
361
|
-
|
|
362
|
-
|
|
362
|
+
declare function passAtK(n: number, c: number, k: number): number;
|
|
363
|
+
interface EProcessOptions {
|
|
363
364
|
/** Type-I error budget. The process decides when wealth ≥ 1/alpha
|
|
364
365
|
* (Ville's inequality). Default 0.05. */
|
|
365
366
|
alpha?: number;
|
|
@@ -372,7 +373,7 @@ export interface EProcessOptions {
|
|
|
372
373
|
* A pre-registered minEffect shifts this — see `sequentialPairedGate`. */
|
|
373
374
|
nullMean?: number;
|
|
374
375
|
}
|
|
375
|
-
|
|
376
|
+
interface EProcessStep {
|
|
376
377
|
/** Current wealth W_n — the e-value against H0 after n observations. */
|
|
377
378
|
wealth: number;
|
|
378
379
|
/** Observations consumed so far. */
|
|
@@ -380,7 +381,7 @@ export interface EProcessStep {
|
|
|
380
381
|
/** True from the first n where W_n ≥ 1/alpha onward (sticky). */
|
|
381
382
|
decided: boolean;
|
|
382
383
|
}
|
|
383
|
-
|
|
384
|
+
interface EProcessState extends EProcessStep {
|
|
384
385
|
alpha: number;
|
|
385
386
|
maxBet: number;
|
|
386
387
|
nullMean: number;
|
|
@@ -389,7 +390,7 @@ export interface EProcessState extends EProcessStep {
|
|
|
389
390
|
/** Observation count at the first threshold crossing; undefined until decided. */
|
|
390
391
|
decidedAtN?: number;
|
|
391
392
|
}
|
|
392
|
-
|
|
393
|
+
interface EProcess {
|
|
393
394
|
/** Consume one observation x ∈ [0,1]. Throws on non-finite / out-of-range
|
|
394
395
|
* input — a silent clamp would corrupt the type-I guarantee. */
|
|
395
396
|
update(x: number): EProcessStep;
|
|
@@ -422,10 +423,11 @@ export interface EProcess {
|
|
|
422
423
|
* wealth keeps updating after the crossing (the e-process remains valid), but
|
|
423
424
|
* the decision time is the first crossing.
|
|
424
425
|
*/
|
|
425
|
-
|
|
426
|
+
declare function eProcess(opts?: EProcessOptions): EProcess;
|
|
426
427
|
/** Tiny seedable PRNG (mulberry32) — deterministic resampling/shuffling, not
|
|
427
428
|
* cryptographic. Exported so e-process shuffles and bootstrap resampling
|
|
428
429
|
* share ONE PRNG implementation; a seed is REQUIRED (unseeded randomness in
|
|
429
430
|
* gate verdicts is non-reproducible by construction). */
|
|
430
|
-
|
|
431
|
-
|
|
431
|
+
declare function mulberry32(seed: number): () => number;
|
|
432
|
+
|
|
433
|
+
export { mulberry32 as A, normalizeScores as B, type CorpusAgreementReport as C, pairedMde as D, type EProcessState as E, pairedRiskDifference as F, pairedTTest as G, partialCredit as H, passAtK as I, requiredSampleSize as J, weightedComposite as K, weightedMean as L, type McNemarResult as M, wilson as N, type PairedBootstrapOptions as P, type RiskDifferenceResult as R, type WeightedCompositeInput as W, type PairedBootstrapResult as a, benjaminiHochberg as b, type CliffsMagnitude as c, type CorpusAgreementOptions as d, type CorpusAgreementPerDimension as e, type CorpusScoreRecord as f, type EProcess as g, type EProcessOptions as h, type EProcessStep as i, type ProportionInterval as j, type WeightedCompositeResult as k, bonferroni as l, cliffsDelta as m, cohensD as n, confidenceInterval as o, pairedBootstrap as p, corpusInterRaterAgreement as q, corpusInterRaterAgreementFromJudgeScores as r, eProcess as s, interRaterReliability as t, interpretCliffs as u, mannWhitneyU as v, wilcoxonSignedRank as w, mcnemar as x, mcnemarPower as y, mcnemarRequiredN as z };
|
|
@@ -1,16 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
* Two implementations ship in the core:
|
|
5
|
-
* - InMemoryTraceStore: dev + tests, fully in-process
|
|
6
|
-
* - FileSystemTraceStore: NDJSON append-only files per entity, suitable
|
|
7
|
-
* for long-running CI jobs; rolled over at 32MB
|
|
8
|
-
*
|
|
9
|
-
* Downstream adapters (DuckDB, Langfuse, R2 parquet) implement this same
|
|
10
|
-
* interface — the rest of the framework is storage-agnostic.
|
|
11
|
-
*/
|
|
12
|
-
import type { Artifact, BudgetLedgerEntry, EventKind, Run, RunStatus, Span, SpanKind, TraceEvent } from './schema';
|
|
13
|
-
export interface RunFilter {
|
|
1
|
+
import { R as Run, S as Span, a as TraceEvent, A as Artifact, B as BudgetLedgerEntry, f as RunStatus, g as RunLayer, c as SpanKind, E as EventKind } from './schema-m0gsnbt3.js';
|
|
2
|
+
|
|
3
|
+
interface RunFilter {
|
|
14
4
|
scenarioId?: string;
|
|
15
5
|
variantId?: string;
|
|
16
6
|
status?: RunStatus;
|
|
@@ -23,9 +13,9 @@ export interface RunFilter {
|
|
|
23
13
|
parentRunId?: string;
|
|
24
14
|
projectId?: string;
|
|
25
15
|
chatId?: string;
|
|
26
|
-
layer?:
|
|
16
|
+
layer?: RunLayer;
|
|
27
17
|
}
|
|
28
|
-
|
|
18
|
+
interface SpanFilter {
|
|
29
19
|
runId?: string;
|
|
30
20
|
parentSpanId?: string;
|
|
31
21
|
kind?: SpanKind;
|
|
@@ -35,14 +25,14 @@ export interface SpanFilter {
|
|
|
35
25
|
since?: number;
|
|
36
26
|
until?: number;
|
|
37
27
|
}
|
|
38
|
-
|
|
28
|
+
interface EventFilter {
|
|
39
29
|
runId?: string;
|
|
40
30
|
spanId?: string;
|
|
41
31
|
kind?: EventKind;
|
|
42
32
|
since?: number;
|
|
43
33
|
until?: number;
|
|
44
34
|
}
|
|
45
|
-
|
|
35
|
+
interface TraceStore {
|
|
46
36
|
appendRun(run: Run): Promise<void>;
|
|
47
37
|
updateRun(runId: string, patch: Partial<Run>): Promise<void>;
|
|
48
38
|
appendSpan(span: Span): Promise<void>;
|
|
@@ -57,7 +47,7 @@ export interface TraceStore {
|
|
|
57
47
|
budget(runId: string): Promise<BudgetLedgerEntry[]>;
|
|
58
48
|
artifacts(runId: string): Promise<Artifact[]>;
|
|
59
49
|
}
|
|
60
|
-
|
|
50
|
+
declare class InMemoryTraceStore implements TraceStore {
|
|
61
51
|
private runs;
|
|
62
52
|
private allSpans;
|
|
63
53
|
private allEvents;
|
|
@@ -77,12 +67,12 @@ export declare class InMemoryTraceStore implements TraceStore {
|
|
|
77
67
|
budget(runId: string): Promise<BudgetLedgerEntry[]>;
|
|
78
68
|
artifacts(runId: string): Promise<Artifact[]>;
|
|
79
69
|
}
|
|
80
|
-
|
|
70
|
+
interface FileSystemTraceStoreOptions {
|
|
81
71
|
dir: string;
|
|
82
72
|
/** Roll over NDJSON files when they exceed this size in bytes. Default 32 MB. */
|
|
83
73
|
maxBytes?: number;
|
|
84
74
|
}
|
|
85
|
-
|
|
75
|
+
declare class FileSystemTraceStore implements TraceStore {
|
|
86
76
|
private dir;
|
|
87
77
|
private maxBytes;
|
|
88
78
|
/** Lazy in-memory index for queries — populated on first read. */
|
|
@@ -122,4 +112,5 @@ export declare class FileSystemTraceStore implements TraceStore {
|
|
|
122
112
|
budget(runId: string): Promise<BudgetLedgerEntry[]>;
|
|
123
113
|
artifacts(runId: string): Promise<Artifact[]>;
|
|
124
114
|
}
|
|
125
|
-
|
|
115
|
+
|
|
116
|
+
export { type EventFilter as E, FileSystemTraceStore as F, InMemoryTraceStore as I, type RunFilter as R, type SpanFilter as S, type TraceStore as T, type FileSystemTraceStoreOptions as a };
|