@tangle-network/agent-eval 0.95.0 → 0.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/adapters/http.d.ts +17 -10
- package/dist/adapters/langchain.d.ts +14 -7
- package/dist/adapters/otel.d.ts +25 -13
- package/dist/{rl/adversarial.d.ts → adversarial-DIVcDoI_.d.ts} +7 -6
- package/dist/analyst/index.d.ts +236 -28
- package/dist/analyst/index.js +1 -1
- package/dist/{trace-analyst/analyst.d.ts → analyst-C8HHvfJp.d.ts} +14 -12
- package/dist/{contract/analyze-runs.d.ts → analyze-runs-DtT6F_6T.d.ts} +10 -9
- package/dist/authenticity/index.d.ts +16 -15
- package/dist/{baseline.d.ts → baseline-Bbid3WoO.d.ts} +44 -8
- package/dist/belief-state/index.d.ts +605 -14
- package/dist/benchmarks/index.d.ts +5 -23
- package/dist/builder-eval/index.d.ts +250 -5
- package/dist/calibration-BPmzuVPk.d.ts +101 -0
- package/dist/campaign/index.d.ts +1452 -38
- package/dist/{chunk-AQ5WQAIV.js → chunk-3NHEO6ZC.js} +2 -2
- package/dist/chunk-3NHEO6ZC.js.map +1 -0
- package/dist/cli.d.ts +0 -2
- package/dist/{contract/intake/code-agent-session.d.ts → code-agent-session-CPHRCb4-.d.ts} +17 -15
- package/dist/contract/index.d.ts +824 -107
- package/dist/contract/index.js +145 -1
- package/dist/contract/index.js.map +1 -1
- package/dist/control-Doncu-B_.d.ts +259 -0
- package/dist/{control-runtime.d.ts → control-runtime-Acf9CGhw.d.ts} +26 -23
- package/dist/control.d.ts +10 -11
- package/dist/corpus-D4YW9UoJ.d.ts +560 -0
- package/dist/{cost-ledger.d.ts → cost-ledger-DuSqlw5B.d.ts} +11 -10
- package/dist/{counterfactual.d.ts → counterfactual-DlOz8PBx.d.ts} +13 -12
- package/dist/{dataset.d.ts → dataset-BbGkaN2I.d.ts} +14 -11
- package/dist/{analyst/registry.d.ts → default-registry-GyE8X5SP.d.ts} +37 -8
- package/dist/diagnose.d.ts +252 -1
- package/dist/{trace/emitter.d.ts → emitter-C2rqGH_l.d.ts} +12 -9
- package/dist/{errors.d.ts → errors-CzMUYo7b.d.ts} +11 -10
- package/dist/failure-cluster-DH9Flgcf.d.ts +76 -0
- package/dist/{feedback-trajectory.d.ts → feedback-trajectory-BxY0cKfs.d.ts} +38 -36
- package/dist/fuzz.d.ts +547 -1
- package/dist/gepa-Dprxvz8r.d.ts +414 -0
- package/dist/governance/index.d.ts +135 -5
- package/dist/harness-optimizer-mOl9XX_O.d.ts +106 -0
- package/dist/hosted/index.d.ts +239 -10
- package/dist/index-_Y4oNOOb.d.ts +159 -0
- package/dist/index.d.ts +5660 -277
- package/dist/index.js +1 -1
- package/dist/{contract/insight-report.d.ts → insight-report-BnRjTibG.d.ts} +19 -16
- package/dist/{trace/integrity.d.ts → integrity-D2t12mMw.d.ts} +14 -11
- package/dist/{judge-calibration.d.ts → judge-calibration-0p2QcWNE.d.ts} +17 -16
- package/dist/{analyst/kind-factory.d.ts → kind-factory-X3eDYbKn.d.ts} +61 -10
- package/dist/knowledge/index.d.ts +103 -3
- package/dist/{llm-client.d.ts → llm-client-Bj7g0rqu.d.ts} +23 -21
- package/dist/matrix/index.d.ts +30 -12
- package/dist/meta-eval/index.d.ts +182 -6
- package/dist/{multi-layer-verifier.d.ts → multi-layer-verifier-DUZXrPDA.d.ts} +15 -12
- package/dist/multishot/index.d.ts +290 -7
- package/dist/{rl/off-policy.d.ts → off-policy-DiwuKKg7.d.ts} +9 -8
- package/dist/openapi.json +1 -1
- package/dist/{meta-eval/outcome-store.d.ts → outcome-store-rnXLEqSn.d.ts} +8 -7
- package/dist/{pareto.d.ts → pareto-E-pembql.d.ts} +10 -9
- package/dist/perf/index.d.ts +119 -13
- package/dist/pipelines/index.d.ts +173 -8
- package/dist/pre-registration-nfUdc9EQ.d.ts +483 -0
- package/dist/prm/index.d.ts +104 -5
- package/dist/provenance-CIxfBnkl.d.ts +426 -0
- package/dist/query-B7GGjRox.d.ts +32 -0
- package/dist/{trace/raw-provider-sink.d.ts → raw-provider-sink-C46HDghv.d.ts} +14 -13
- package/dist/{red-team.d.ts → red-team-BWdoyleI.d.ts} +16 -14
- package/dist/{trace/redact.d.ts → redact-B40YG2M_.d.ts} +8 -7
- package/dist/release-report-pidWUMZ2.d.ts +233 -0
- package/dist/reporting.d.ts +16 -15
- package/dist/{eval-campaign.d.ts → researcher-Jr8ME1dZ.d.ts} +156 -34
- package/dist/rl.d.ts +1193 -1
- package/dist/{prm/rubric.d.ts → rubric-Cc6UHvUb.d.ts} +13 -10
- package/dist/{meta-eval/rubric-predictive-validity.d.ts → rubric-predictive-validity-C2hDKM8Z.d.ts} +11 -8
- package/dist/run-critic-CmMf05uV.d.ts +56 -0
- package/dist/{run-record.d.ts → run-record-CP2ObebC.d.ts} +117 -15
- package/dist/runtime-trajectory-BOUUjI0y.d.ts +49 -0
- package/dist/{trace/schema.d.ts → schema-m0gsnbt3.d.ts} +30 -29
- package/dist/semantic-concept-judge-DSBB2Cfp.d.ts +624 -0
- package/dist/{sequential.d.ts → sequential-5iSVfzl2.d.ts} +10 -9
- package/dist/{series-convergence.d.ts → series-convergence-D5OWMBg6.d.ts} +5 -4
- package/dist/sink-fetch-B1Yg4Til.d.ts +101 -0
- package/dist/{statistics.d.ts → statistics-CCJpTGOS.d.ts} +48 -46
- package/dist/{trace/store.d.ts → store-BcFXE6LG.d.ts} +12 -21
- package/dist/{trace-analyst/types.d.ts → store-C1YxJDEK.d.ts} +74 -18
- package/dist/storyboard/index.d.ts +81 -16
- package/dist/{summary-report.d.ts → summary-report-CInXwsza.d.ts} +160 -23
- package/dist/telemetry/{sink-file.d.ts → file.d.ts} +7 -5
- package/dist/telemetry/index.d.ts +35 -17
- package/dist/{sandbox-harness.d.ts → test-graded-scenario-DeODGLra.d.ts} +60 -15
- package/dist/{locked-jsonl-appender.d.ts → testing-C21CHsq2.d.ts} +4 -3
- package/dist/testing.d.ts +1 -5
- package/dist/traces.d.ts +976 -4
- package/dist/{trajectory.d.ts → trajectory-2TkpSEVh.d.ts} +9 -8
- package/dist/{analyst/types.d.ts → types-B5x54y6n.d.ts} +112 -19
- package/dist/{campaign/types.d.ts → types-BMahhhio.d.ts} +47 -44
- package/dist/{matrix/types.d.ts → types-BUxNaJ8c.d.ts} +11 -9
- package/dist/{types.d.ts → types-C7DGg5ex.d.ts} +33 -31
- package/dist/{verdict.d.ts → verdict-C9MlYujm.d.ts} +3 -2
- package/dist/wire/index.d.ts +570 -13
- package/dist/workflow/index.d.ts +496 -22
- package/package.json +2 -2
- package/dist/action-policy.d.ts +0 -24
- package/dist/action-policy.d.ts.map +0 -1
- package/dist/action-policy.test.d.ts +0 -2
- package/dist/action-policy.test.d.ts.map +0 -1
- package/dist/active-learning.d.ts +0 -41
- package/dist/active-learning.d.ts.map +0 -1
- package/dist/adapters/http.d.ts.map +0 -1
- package/dist/adapters/langchain.d.ts.map +0 -1
- package/dist/adapters/otel.d.ts.map +0 -1
- package/dist/agent-profile-cell.d.ts +0 -101
- package/dist/agent-profile-cell.d.ts.map +0 -1
- package/dist/agent-profile.d.ts +0 -27
- package/dist/agent-profile.d.ts.map +0 -1
- package/dist/agent-profile.test.d.ts +0 -2
- package/dist/agent-profile.test.d.ts.map +0 -1
- package/dist/analyst/adapters.d.ts +0 -62
- package/dist/analyst/adapters.d.ts.map +0 -1
- package/dist/analyst/analyst.test.d.ts +0 -2
- package/dist/analyst/analyst.test.d.ts.map +0 -1
- package/dist/analyst/ax-service.d.ts +0 -27
- package/dist/analyst/ax-service.d.ts.map +0 -1
- package/dist/analyst/behavioral-analyst.d.ts +0 -28
- package/dist/analyst/behavioral-analyst.d.ts.map +0 -1
- package/dist/analyst/chat-client.d.ts +0 -91
- package/dist/analyst/chat-client.d.ts.map +0 -1
- package/dist/analyst/default-registry.d.ts +0 -27
- package/dist/analyst/default-registry.d.ts.map +0 -1
- package/dist/analyst/default-registry.test.d.ts +0 -2
- package/dist/analyst/default-registry.test.d.ts.map +0 -1
- package/dist/analyst/finding-signature.d.ts +0 -48
- package/dist/analyst/finding-signature.d.ts.map +0 -1
- package/dist/analyst/finding-subject.d.ts +0 -146
- package/dist/analyst/finding-subject.d.ts.map +0 -1
- package/dist/analyst/finding-subject.test.d.ts +0 -2
- package/dist/analyst/finding-subject.test.d.ts.map +0 -1
- package/dist/analyst/findings-store.d.ts +0 -75
- package/dist/analyst/findings-store.d.ts.map +0 -1
- package/dist/analyst/index.d.ts.map +0 -1
- package/dist/analyst/kind-factory.d.ts.map +0 -1
- package/dist/analyst/kinds/failure-mode.d.ts +0 -19
- package/dist/analyst/kinds/failure-mode.d.ts.map +0 -1
- package/dist/analyst/kinds/improvement.d.ts +0 -23
- package/dist/analyst/kinds/improvement.d.ts.map +0 -1
- package/dist/analyst/kinds/index.d.ts +0 -22
- package/dist/analyst/kinds/index.d.ts.map +0 -1
- package/dist/analyst/kinds/kinds.test.d.ts +0 -2
- package/dist/analyst/kinds/kinds.test.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-gap.d.ts +0 -28
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +0 -22
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.d.ts +0 -84
- package/dist/analyst/kinds/skill-usage.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.test.d.ts +0 -2
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.d.ts +0 -26
- package/dist/analyst/parse-tolerant.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.test.d.ts +0 -2
- package/dist/analyst/parse-tolerant.test.d.ts.map +0 -1
- package/dist/analyst/registry.budget.test.d.ts +0 -2
- package/dist/analyst/registry.budget.test.d.ts.map +0 -1
- package/dist/analyst/registry.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.d.ts +0 -35
- package/dist/analyst/steer-firewall.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.test.d.ts +0 -2
- package/dist/analyst/steer-firewall.test.d.ts.map +0 -1
- package/dist/analyst/structure-findings.d.ts +0 -37
- package/dist/analyst/structure-findings.d.ts.map +0 -1
- package/dist/analyst/structure-findings.test.d.ts +0 -2
- package/dist/analyst/structure-findings.test.d.ts.map +0 -1
- package/dist/analyst/tool-groups.d.ts +0 -34
- package/dist/analyst/tool-groups.d.ts.map +0 -1
- package/dist/analyst/types.d.ts.map +0 -1
- package/dist/anti-slop.d.ts +0 -59
- package/dist/anti-slop.d.ts.map +0 -1
- package/dist/artifact-validator.d.ts +0 -74
- package/dist/artifact-validator.d.ts.map +0 -1
- package/dist/attestation.d.ts +0 -63
- package/dist/attestation.d.ts.map +0 -1
- package/dist/attestation.test.d.ts +0 -2
- package/dist/attestation.test.d.ts.map +0 -1
- package/dist/authenticity/index.d.ts.map +0 -1
- package/dist/authenticity/index.test.d.ts +0 -2
- package/dist/authenticity/index.test.d.ts.map +0 -1
- package/dist/auto-pr.d.ts +0 -120
- package/dist/auto-pr.d.ts.map +0 -1
- package/dist/baseline.d.ts.map +0 -1
- package/dist/behavior-dsl.d.ts +0 -73
- package/dist/behavior-dsl.d.ts.map +0 -1
- package/dist/belief-state/calibration.d.ts +0 -10
- package/dist/belief-state/calibration.d.ts.map +0 -1
- package/dist/belief-state/calibration.test.d.ts +0 -2
- package/dist/belief-state/calibration.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.d.ts +0 -66
- package/dist/belief-state/code-agent-corpus.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.test.d.ts +0 -2
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.d.ts +0 -22
- package/dist/belief-state/code-agent-evidence.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.test.d.ts +0 -2
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/extract.d.ts +0 -7
- package/dist/belief-state/extract.d.ts.map +0 -1
- package/dist/belief-state/extract.test.d.ts +0 -2
- package/dist/belief-state/extract.test.d.ts.map +0 -1
- package/dist/belief-state/index.d.ts.map +0 -1
- package/dist/belief-state/ope.d.ts +0 -17
- package/dist/belief-state/ope.d.ts.map +0 -1
- package/dist/belief-state/ope.test.d.ts +0 -2
- package/dist/belief-state/ope.test.d.ts.map +0 -1
- package/dist/belief-state/phase0-measurement.d.ts +0 -55
- package/dist/belief-state/phase0-measurement.d.ts.map +0 -1
- package/dist/belief-state/report.d.ts +0 -17
- package/dist/belief-state/report.d.ts.map +0 -1
- package/dist/belief-state/report.test.d.ts +0 -2
- package/dist/belief-state/report.test.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.d.ts +0 -23
- package/dist/belief-state/research-evidence.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.test.d.ts +0 -2
- package/dist/belief-state/research-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +0 -32
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.d.ts +0 -87
- package/dist/belief-state/runtime-hooks.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.test.d.ts +0 -2
- package/dist/belief-state/runtime-hooks.test.d.ts.map +0 -1
- package/dist/belief-state/selective.d.ts +0 -15
- package/dist/belief-state/selective.d.ts.map +0 -1
- package/dist/belief-state/selective.test.d.ts +0 -2
- package/dist/belief-state/selective.test.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.d.ts +0 -80
- package/dist/belief-state/shadow-probe.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.test.d.ts +0 -2
- package/dist/belief-state/shadow-probe.test.d.ts.map +0 -1
- package/dist/belief-state/types.d.ts +0 -195
- package/dist/belief-state/types.d.ts.map +0 -1
- package/dist/belief-state/types.test.d.ts +0 -2
- package/dist/belief-state/types.test.d.ts.map +0 -1
- package/dist/benchmark.d.ts +0 -14
- package/dist/benchmark.d.ts.map +0 -1
- package/dist/benchmarks/index.d.ts.map +0 -1
- package/dist/benchmarks/routing/dataset.d.ts +0 -34
- package/dist/benchmarks/routing/dataset.d.ts.map +0 -1
- package/dist/benchmarks/routing/index.d.ts +0 -34
- package/dist/benchmarks/routing/index.d.ts.map +0 -1
- package/dist/benchmarks/types.d.ts +0 -49
- package/dist/benchmarks/types.d.ts.map +0 -1
- package/dist/bisector.d.ts +0 -81
- package/dist/bisector.d.ts.map +0 -1
- package/dist/budget-guard.d.ts +0 -31
- package/dist/budget-guard.d.ts.map +0 -1
- package/dist/builder-eval/builder-session.d.ts +0 -111
- package/dist/builder-eval/builder-session.d.ts.map +0 -1
- package/dist/builder-eval/correlation.d.ts +0 -32
- package/dist/builder-eval/correlation.d.ts.map +0 -1
- package/dist/builder-eval/index.d.ts.map +0 -1
- package/dist/builder-eval/project-registry.d.ts +0 -51
- package/dist/builder-eval/project-registry.d.ts.map +0 -1
- package/dist/builder-eval/three-layer-eval.d.ts +0 -55
- package/dist/builder-eval/three-layer-eval.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.d.ts +0 -108
- package/dist/campaign/analyst-surface.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.test.d.ts +0 -2
- package/dist/campaign/analyst-surface.test.d.ts.map +0 -1
- package/dist/campaign/auto-pr.d.ts +0 -46
- package/dist/campaign/auto-pr.d.ts.map +0 -1
- package/dist/campaign/distillation/agreement-judge.d.ts +0 -69
- package/dist/campaign/distillation/agreement-judge.d.ts.map +0 -1
- package/dist/campaign/distillation/cli.d.ts +0 -35
- package/dist/campaign/distillation/cli.d.ts.map +0 -1
- package/dist/campaign/distillation/distillation.test.d.ts +0 -2
- package/dist/campaign/distillation/distillation.test.d.ts.map +0 -1
- package/dist/campaign/distillation/gold-scenarios.d.ts +0 -54
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +0 -1
- package/dist/campaign/distillation/run-distillation.d.ts +0 -119
- package/dist/campaign/distillation/run-distillation.d.ts.map +0 -1
- package/dist/campaign/gates/compose.d.ts +0 -12
- package/dist/campaign/gates/compose.d.ts.map +0 -1
- package/dist/campaign/gates/default-production-gate.d.ts +0 -58
- package/dist/campaign/gates/default-production-gate.d.ts.map +0 -1
- package/dist/campaign/gates/heldout-gate.d.ts +0 -12
- package/dist/campaign/gates/heldout-gate.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.d.ts +0 -125
- package/dist/campaign/gates/promotion-policy.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.test.d.ts +0 -2
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.d.ts +0 -146
- package/dist/campaign/gates/sequential.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.test.d.ts +0 -2
- package/dist/campaign/gates/sequential.test.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.d.ts +0 -99
- package/dist/campaign/gates/statistical-heldout.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.test.d.ts +0 -2
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +0 -1
- package/dist/campaign/index.d.ts.map +0 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +0 -59
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +0 -1
- package/dist/campaign/presets/compare-proposers.d.ts +0 -146
- package/dist/campaign/presets/compare-proposers.d.ts.map +0 -1
- package/dist/campaign/presets/playback.d.ts +0 -120
- package/dist/campaign/presets/playback.d.ts.map +0 -1
- package/dist/campaign/presets/playback.test.d.ts +0 -2
- package/dist/campaign/presets/playback.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-eval.d.ts +0 -14
- package/dist/campaign/presets/run-eval.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.d.ts +0 -63
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +0 -2
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-optimization.d.ts +0 -92
- package/dist/campaign/presets/run-optimization.d.ts.map +0 -1
- package/dist/campaign/presets/run-profile-matrix.d.ts +0 -151
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +0 -1
- package/dist/campaign/presets/run-skill-opt.d.ts +0 -96
- package/dist/campaign/presets/run-skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/_findings-text.d.ts +0 -22
- package/dist/campaign/proposers/_findings-text.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.d.ts +0 -33
- package/dist/campaign/proposers/ace.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.test.d.ts +0 -2
- package/dist/campaign/proposers/ace.test.d.ts.map +0 -1
- package/dist/campaign/proposers/analysis-edit.d.ts +0 -32
- package/dist/campaign/proposers/analysis-edit.d.ts.map +0 -1
- package/dist/campaign/proposers/evolutionary.d.ts +0 -20
- package/dist/campaign/proposers/evolutionary.d.ts.map +0 -1
- package/dist/campaign/proposers/fapo.d.ts +0 -120
- package/dist/campaign/proposers/fapo.d.ts.map +0 -1
- package/dist/campaign/proposers/gepa.d.ts +0 -86
- package/dist/campaign/proposers/gepa.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.d.ts +0 -44
- package/dist/campaign/proposers/halo.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.test.d.ts +0 -2
- package/dist/campaign/proposers/halo.test.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.d.ts +0 -47
- package/dist/campaign/proposers/memory.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.test.d.ts +0 -2
- package/dist/campaign/proposers/memory.test.d.ts.map +0 -1
- package/dist/campaign/proposers/skill-opt.d.ts +0 -88
- package/dist/campaign/proposers/skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.d.ts +0 -48
- package/dist/campaign/proposers/trace-analyst.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.test.d.ts +0 -2
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +0 -1
- package/dist/campaign/provenance.d.ts +0 -185
- package/dist/campaign/provenance.d.ts.map +0 -1
- package/dist/campaign/run-campaign.d.ts +0 -90
- package/dist/campaign/run-campaign.d.ts.map +0 -1
- package/dist/campaign/score-utils.d.ts +0 -26
- package/dist/campaign/score-utils.d.ts.map +0 -1
- package/dist/campaign/skill-patch.d.ts +0 -62
- package/dist/campaign/skill-patch.d.ts.map +0 -1
- package/dist/campaign/storage.d.ts +0 -38
- package/dist/campaign/storage.d.ts.map +0 -1
- package/dist/campaign/types.d.ts.map +0 -1
- package/dist/campaign/worktree/index.d.ts +0 -53
- package/dist/campaign/worktree/index.d.ts.map +0 -1
- package/dist/canary.d.ts +0 -101
- package/dist/canary.d.ts.map +0 -1
- package/dist/causal-attribution.d.ts +0 -45
- package/dist/causal-attribution.d.ts.map +0 -1
- package/dist/chunk-AQ5WQAIV.js.map +0 -1
- package/dist/ci-gate.d.ts +0 -44
- package/dist/ci-gate.d.ts.map +0 -1
- package/dist/cli.d.ts.map +0 -1
- package/dist/client.d.ts +0 -77
- package/dist/client.d.ts.map +0 -1
- package/dist/client.test.d.ts +0 -2
- package/dist/client.test.d.ts.map +0 -1
- package/dist/command-runner.d.ts +0 -74
- package/dist/command-runner.d.ts.map +0 -1
- package/dist/command-runner.test.d.ts +0 -2
- package/dist/command-runner.test.d.ts.map +0 -1
- package/dist/completion-verifier.d.ts +0 -147
- package/dist/completion-verifier.d.ts.map +0 -1
- package/dist/completion-verifier.test.d.ts +0 -9
- package/dist/completion-verifier.test.d.ts.map +0 -1
- package/dist/concurrency.d.ts +0 -23
- package/dist/concurrency.d.ts.map +0 -1
- package/dist/contamination-guard.d.ts +0 -81
- package/dist/contamination-guard.d.ts.map +0 -1
- package/dist/contract/analyze-runs.d.ts.map +0 -1
- package/dist/contract/define-agent-eval.d.ts +0 -52
- package/dist/contract/define-agent-eval.d.ts.map +0 -1
- package/dist/contract/diff.d.ts +0 -114
- package/dist/contract/diff.d.ts.map +0 -1
- package/dist/contract/index.d.ts.map +0 -1
- package/dist/contract/insight-report.d.ts.map +0 -1
- package/dist/contract/insight-types-fwd.d.ts +0 -7
- package/dist/contract/insight-types-fwd.d.ts.map +0 -1
- package/dist/contract/intake/agent-trace.d.ts +0 -97
- package/dist/contract/intake/agent-trace.d.ts.map +0 -1
- package/dist/contract/intake/code-agent-session.d.ts.map +0 -1
- package/dist/contract/intake/feedback-table.d.ts +0 -87
- package/dist/contract/intake/feedback-table.d.ts.map +0 -1
- package/dist/contract/intake/index.d.ts +0 -22
- package/dist/contract/intake/index.d.ts.map +0 -1
- package/dist/contract/intake/otel-spans.d.ts +0 -33
- package/dist/contract/intake/otel-spans.d.ts.map +0 -1
- package/dist/contract/self-improve.d.ts +0 -284
- package/dist/contract/self-improve.d.ts.map +0 -1
- package/dist/control-runtime.d.ts.map +0 -1
- package/dist/control-runtime.test.d.ts +0 -2
- package/dist/control-runtime.test.d.ts.map +0 -1
- package/dist/control.d.ts.map +0 -1
- package/dist/convergence.d.ts +0 -29
- package/dist/convergence.d.ts.map +0 -1
- package/dist/cost-ledger.d.ts.map +0 -1
- package/dist/cost-ledger.test.d.ts +0 -2
- package/dist/cost-ledger.test.d.ts.map +0 -1
- package/dist/cost-report.d.ts +0 -43
- package/dist/cost-report.d.ts.map +0 -1
- package/dist/cost-report.test.d.ts +0 -2
- package/dist/cost-report.test.d.ts.map +0 -1
- package/dist/cost-tracker.d.ts +0 -76
- package/dist/cost-tracker.d.ts.map +0 -1
- package/dist/counterfactual.d.ts.map +0 -1
- package/dist/cross-trace-diff.d.ts +0 -56
- package/dist/cross-trace-diff.d.ts.map +0 -1
- package/dist/dataset.d.ts.map +0 -1
- package/dist/deploy-gate-layer.d.ts +0 -125
- package/dist/deploy-gate-layer.d.ts.map +0 -1
- package/dist/deploy-gate-layer.test.d.ts +0 -2
- package/dist/deploy-gate-layer.test.d.ts.map +0 -1
- package/dist/description-length-gate.d.ts +0 -119
- package/dist/description-length-gate.d.ts.map +0 -1
- package/dist/detectors/edge.test.d.ts +0 -2
- package/dist/detectors/edge.test.d.ts.map +0 -1
- package/dist/detectors/index.d.ts +0 -81
- package/dist/detectors/index.d.ts.map +0 -1
- package/dist/detectors/index.test.d.ts +0 -2
- package/dist/detectors/index.test.d.ts.map +0 -1
- package/dist/diagnose/causal-sweep.d.ts +0 -100
- package/dist/diagnose/causal-sweep.d.ts.map +0 -1
- package/dist/diagnose/index.d.ts +0 -36
- package/dist/diagnose/index.d.ts.map +0 -1
- package/dist/diagnose/remediation.d.ts +0 -68
- package/dist/diagnose/remediation.d.ts.map +0 -1
- package/dist/diagnose/repair.d.ts +0 -77
- package/dist/diagnose/repair.d.ts.map +0 -1
- package/dist/discover-personas.d.ts +0 -35
- package/dist/discover-personas.d.ts.map +0 -1
- package/dist/driver.d.ts +0 -95
- package/dist/driver.d.ts.map +0 -1
- package/dist/driver.test.d.ts +0 -8
- package/dist/driver.test.d.ts.map +0 -1
- package/dist/dual-agent-bench.d.ts +0 -81
- package/dist/dual-agent-bench.d.ts.map +0 -1
- package/dist/error-count-extractor.d.ts +0 -47
- package/dist/error-count-extractor.d.ts.map +0 -1
- package/dist/error-count-extractor.test.d.ts +0 -2
- package/dist/error-count-extractor.test.d.ts.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/eval-campaign.d.ts.map +0 -1
- package/dist/eval-campaign.test.d.ts +0 -2
- package/dist/eval-campaign.test.d.ts.map +0 -1
- package/dist/eval-tools.d.ts +0 -55
- package/dist/eval-tools.d.ts.map +0 -1
- package/dist/eval-trace-store.d.ts +0 -107
- package/dist/eval-trace-store.d.ts.map +0 -1
- package/dist/eval-trace-store.test.d.ts +0 -2
- package/dist/eval-trace-store.test.d.ts.map +0 -1
- package/dist/executor.d.ts +0 -38
- package/dist/executor.d.ts.map +0 -1
- package/dist/executor.test.d.ts +0 -10
- package/dist/executor.test.d.ts.map +0 -1
- package/dist/experiment-tracker.d.ts +0 -178
- package/dist/experiment-tracker.d.ts.map +0 -1
- package/dist/experiment-tracker.test.d.ts +0 -2
- package/dist/experiment-tracker.test.d.ts.map +0 -1
- package/dist/failure-taxonomy.d.ts +0 -38
- package/dist/failure-taxonomy.d.ts.map +0 -1
- package/dist/feedback-trajectory.d.ts.map +0 -1
- package/dist/feedback-trajectory.test.d.ts +0 -2
- package/dist/feedback-trajectory.test.d.ts.map +0 -1
- package/dist/flow-layer.d.ts +0 -90
- package/dist/flow-layer.d.ts.map +0 -1
- package/dist/flow-layer.test.d.ts +0 -2
- package/dist/flow-layer.test.d.ts.map +0 -1
- package/dist/fuzz/capsule.d.ts +0 -46
- package/dist/fuzz/capsule.d.ts.map +0 -1
- package/dist/fuzz/cube.d.ts +0 -36
- package/dist/fuzz/cube.d.ts.map +0 -1
- package/dist/fuzz/explorer-cost.test.d.ts +0 -2
- package/dist/fuzz/explorer-cost.test.d.ts.map +0 -1
- package/dist/fuzz/explorer.d.ts +0 -64
- package/dist/fuzz/explorer.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.d.ts +0 -16
- package/dist/fuzz/fuzz-agent.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.test.d.ts +0 -2
- package/dist/fuzz/fuzz-agent.test.d.ts.map +0 -1
- package/dist/fuzz/gates.d.ts +0 -33
- package/dist/fuzz/gates.d.ts.map +0 -1
- package/dist/fuzz/index.d.ts +0 -26
- package/dist/fuzz/index.d.ts.map +0 -1
- package/dist/fuzz/policies.d.ts +0 -28
- package/dist/fuzz/policies.d.ts.map +0 -1
- package/dist/fuzz/tools.d.ts +0 -20
- package/dist/fuzz/tools.d.ts.map +0 -1
- package/dist/fuzz/types.d.ts +0 -307
- package/dist/fuzz/types.d.ts.map +0 -1
- package/dist/golden-matcher.d.ts +0 -71
- package/dist/golden-matcher.d.ts.map +0 -1
- package/dist/governance/eu-ai-act.d.ts +0 -37
- package/dist/governance/eu-ai-act.d.ts.map +0 -1
- package/dist/governance/index.d.ts.map +0 -1
- package/dist/governance/nist-ai-rmf.d.ts +0 -15
- package/dist/governance/nist-ai-rmf.d.ts.map +0 -1
- package/dist/governance/soc2.d.ts +0 -12
- package/dist/governance/soc2.d.ts.map +0 -1
- package/dist/governance/types.d.ts +0 -66
- package/dist/governance/types.d.ts.map +0 -1
- package/dist/harness-optimizer.d.ts +0 -82
- package/dist/harness-optimizer.d.ts.map +0 -1
- package/dist/held-out-gate.d.ts +0 -135
- package/dist/held-out-gate.d.ts.map +0 -1
- package/dist/hosted/client.d.ts +0 -73
- package/dist/hosted/client.d.ts.map +0 -1
- package/dist/hosted/from-env.test.d.ts +0 -8
- package/dist/hosted/from-env.test.d.ts.map +0 -1
- package/dist/hosted/index.d.ts.map +0 -1
- package/dist/hosted/types.d.ts +0 -159
- package/dist/hosted/types.d.ts.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/integrity/backend-integrity.d.ts +0 -71
- package/dist/integrity/backend-integrity.d.ts.map +0 -1
- package/dist/integrity/preflight.d.ts +0 -72
- package/dist/integrity/preflight.d.ts.map +0 -1
- package/dist/integrity/preflight.test.d.ts +0 -2
- package/dist/integrity/preflight.test.d.ts.map +0 -1
- package/dist/integrity/single-backend.d.ts +0 -67
- package/dist/integrity/single-backend.d.ts.map +0 -1
- package/dist/intent-match-judge.d.ts +0 -69
- package/dist/intent-match-judge.d.ts.map +0 -1
- package/dist/intent-match-judge.test.d.ts +0 -2
- package/dist/intent-match-judge.test.d.ts.map +0 -1
- package/dist/judge-calibration.d.ts.map +0 -1
- package/dist/judge-ensemble.d.ts +0 -66
- package/dist/judge-ensemble.d.ts.map +0 -1
- package/dist/judge-ensemble.test.d.ts +0 -8
- package/dist/judge-ensemble.test.d.ts.map +0 -1
- package/dist/judge-families.d.ts +0 -38
- package/dist/judge-families.d.ts.map +0 -1
- package/dist/judge-panel.d.ts +0 -65
- package/dist/judge-panel.d.ts.map +0 -1
- package/dist/judge-retry.d.ts +0 -70
- package/dist/judge-retry.d.ts.map +0 -1
- package/dist/judge-runner.d.ts +0 -36
- package/dist/judge-runner.d.ts.map +0 -1
- package/dist/judge-runner.test.d.ts +0 -2
- package/dist/judge-runner.test.d.ts.map +0 -1
- package/dist/judges.d.ts +0 -74
- package/dist/judges.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.d.ts +0 -89
- package/dist/keyword-coverage-judge.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.test.d.ts +0 -2
- package/dist/keyword-coverage-judge.test.d.ts.map +0 -1
- package/dist/knowledge/index.d.ts.map +0 -1
- package/dist/knowledge/readiness.d.ts +0 -26
- package/dist/knowledge/readiness.d.ts.map +0 -1
- package/dist/knowledge/types.d.ts +0 -75
- package/dist/knowledge/types.d.ts.map +0 -1
- package/dist/live-proof.d.ts +0 -62
- package/dist/live-proof.d.ts.map +0 -1
- package/dist/llm-client.d.ts.map +0 -1
- package/dist/llm-client.test.d.ts +0 -2
- package/dist/llm-client.test.d.ts.map +0 -1
- package/dist/locked-jsonl-appender.d.ts.map +0 -1
- package/dist/matrix/aggregation.d.ts +0 -16
- package/dist/matrix/aggregation.d.ts.map +0 -1
- package/dist/matrix/index.d.ts.map +0 -1
- package/dist/matrix/runner.d.ts +0 -15
- package/dist/matrix/runner.d.ts.map +0 -1
- package/dist/matrix/types.d.ts.map +0 -1
- package/dist/meta-eval/calibration.d.ts +0 -47
- package/dist/meta-eval/calibration.d.ts.map +0 -1
- package/dist/meta-eval/correlation-study.d.ts +0 -53
- package/dist/meta-eval/correlation-study.d.ts.map +0 -1
- package/dist/meta-eval/index.d.ts.map +0 -1
- package/dist/meta-eval/outcome-store.d.ts.map +0 -1
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +0 -1
- package/dist/meta-eval/sentinel.d.ts +0 -169
- package/dist/meta-eval/sentinel.d.ts.map +0 -1
- package/dist/metrics.d.ts +0 -63
- package/dist/metrics.d.ts.map +0 -1
- package/dist/model-seats.d.ts +0 -71
- package/dist/model-seats.d.ts.map +0 -1
- package/dist/model-seats.test.d.ts +0 -2
- package/dist/model-seats.test.d.ts.map +0 -1
- package/dist/muffled-gate-scanner.d.ts +0 -102
- package/dist/muffled-gate-scanner.d.ts.map +0 -1
- package/dist/multi-layer-verifier.d.ts.map +0 -1
- package/dist/multi-layer-verifier.test.d.ts +0 -2
- package/dist/multi-layer-verifier.test.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.d.ts +0 -80
- package/dist/multi-toolchain-layer.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.test.d.ts +0 -2
- package/dist/multi-toolchain-layer.test.d.ts.map +0 -1
- package/dist/multishot/default-tools.d.ts +0 -34
- package/dist/multishot/default-tools.d.ts.map +0 -1
- package/dist/multishot/index.d.ts.map +0 -1
- package/dist/multishot/judges.d.ts +0 -32
- package/dist/multishot/judges.d.ts.map +0 -1
- package/dist/multishot/matrix.d.ts +0 -107
- package/dist/multishot/matrix.d.ts.map +0 -1
- package/dist/multishot/multishot.d.ts +0 -23
- package/dist/multishot/multishot.d.ts.map +0 -1
- package/dist/multishot/router.d.ts +0 -37
- package/dist/multishot/router.d.ts.map +0 -1
- package/dist/multishot/types.d.ts +0 -60
- package/dist/multishot/types.d.ts.map +0 -1
- package/dist/observability.d.ts +0 -71
- package/dist/observability.d.ts.map +0 -1
- package/dist/oracle.d.ts +0 -55
- package/dist/oracle.d.ts.map +0 -1
- package/dist/orthogonality.d.ts +0 -35
- package/dist/orthogonality.d.ts.map +0 -1
- package/dist/otel-pipeline.d.ts +0 -31
- package/dist/otel-pipeline.d.ts.map +0 -1
- package/dist/paraphrase.d.ts +0 -107
- package/dist/paraphrase.d.ts.map +0 -1
- package/dist/pareto.d.ts.map +0 -1
- package/dist/partition-held-out.d.ts +0 -70
- package/dist/partition-held-out.d.ts.map +0 -1
- package/dist/partition-held-out.test.d.ts +0 -2
- package/dist/partition-held-out.test.d.ts.map +0 -1
- package/dist/perf/index.d.ts.map +0 -1
- package/dist/perf/integrity.d.ts +0 -30
- package/dist/perf/integrity.d.ts.map +0 -1
- package/dist/perf/journey.d.ts +0 -45
- package/dist/perf/journey.d.ts.map +0 -1
- package/dist/perf/ratchet.d.ts +0 -47
- package/dist/perf/ratchet.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.d.ts +0 -31
- package/dist/pipelines/budget-breach.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.test.d.ts +0 -2
- package/dist/pipelines/budget-breach.test.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.d.ts +0 -38
- package/dist/pipelines/failure-cluster.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.test.d.ts +0 -2
- package/dist/pipelines/failure-cluster.test.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.d.ts +0 -26
- package/dist/pipelines/first-divergence.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.test.d.ts +0 -2
- package/dist/pipelines/first-divergence.test.d.ts.map +0 -1
- package/dist/pipelines/index.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.d.ts +0 -26
- package/dist/pipelines/judge-agreement.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.test.d.ts +0 -2
- package/dist/pipelines/judge-agreement.test.d.ts.map +0 -1
- package/dist/pipelines/regression.d.ts +0 -23
- package/dist/pipelines/regression.d.ts.map +0 -1
- package/dist/pipelines/regression.test.d.ts +0 -2
- package/dist/pipelines/regression.test.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.d.ts +0 -32
- package/dist/pipelines/stuck-loop.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.test.d.ts +0 -2
- package/dist/pipelines/stuck-loop.test.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.d.ts +0 -34
- package/dist/pipelines/tool-waste.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.test.d.ts +0 -2
- package/dist/pipelines/tool-waste.test.d.ts.map +0 -1
- package/dist/playbook.d.ts +0 -16
- package/dist/playbook.d.ts.map +0 -1
- package/dist/pr-review-benchmark.d.ts +0 -88
- package/dist/pr-review-benchmark.d.ts.map +0 -1
- package/dist/pr-review-benchmark.test.d.ts +0 -2
- package/dist/pr-review-benchmark.test.d.ts.map +0 -1
- package/dist/pre-registration.d.ts +0 -125
- package/dist/pre-registration.d.ts.map +0 -1
- package/dist/prm/builtin-rubrics.d.ts +0 -33
- package/dist/prm/builtin-rubrics.d.ts.map +0 -1
- package/dist/prm/index.d.ts.map +0 -1
- package/dist/prm/inference.d.ts +0 -29
- package/dist/prm/inference.d.ts.map +0 -1
- package/dist/prm/inference.test.d.ts +0 -2
- package/dist/prm/inference.test.d.ts.map +0 -1
- package/dist/prm/rubric.d.ts.map +0 -1
- package/dist/prm/training-export.d.ts +0 -38
- package/dist/prm/training-export.d.ts.map +0 -1
- package/dist/produced-state.d.ts +0 -63
- package/dist/produced-state.d.ts.map +0 -1
- package/dist/produced-state.test.d.ts +0 -8
- package/dist/produced-state.test.d.ts.map +0 -1
- package/dist/profile/baselines.d.ts +0 -37
- package/dist/profile/baselines.d.ts.map +0 -1
- package/dist/profile/index.d.ts +0 -105
- package/dist/profile/index.d.ts.map +0 -1
- package/dist/promotion-gate.d.ts +0 -93
- package/dist/promotion-gate.d.ts.map +0 -1
- package/dist/prompt-registry.d.ts +0 -41
- package/dist/prompt-registry.d.ts.map +0 -1
- package/dist/propose-review-control.d.ts +0 -49
- package/dist/propose-review-control.d.ts.map +0 -1
- package/dist/propose-review-control.test.d.ts +0 -2
- package/dist/propose-review-control.test.d.ts.map +0 -1
- package/dist/propose-review.d.ts +0 -155
- package/dist/propose-review.d.ts.map +0 -1
- package/dist/red-team.d.ts.map +0 -1
- package/dist/reference-replay-steering.d.ts +0 -11
- package/dist/reference-replay-steering.d.ts.map +0 -1
- package/dist/reference-replay.d.ts +0 -176
- package/dist/reference-replay.d.ts.map +0 -1
- package/dist/reflective-mutation.d.ts +0 -79
- package/dist/reflective-mutation.d.ts.map +0 -1
- package/dist/registry.d.ts +0 -31
- package/dist/registry.d.ts.map +0 -1
- package/dist/release-confidence.d.ts +0 -128
- package/dist/release-confidence.d.ts.map +0 -1
- package/dist/release-report.d.ts +0 -11
- package/dist/release-report.d.ts.map +0 -1
- package/dist/replay.d.ts +0 -120
- package/dist/replay.d.ts.map +0 -1
- package/dist/reporter.d.ts +0 -14
- package/dist/reporter.d.ts.map +0 -1
- package/dist/reporting.d.ts.map +0 -1
- package/dist/researcher.d.ts +0 -140
- package/dist/researcher.d.ts.map +0 -1
- package/dist/reviewer.d.ts +0 -118
- package/dist/reviewer.d.ts.map +0 -1
- package/dist/reviewer.test.d.ts +0 -2
- package/dist/reviewer.test.d.ts.map +0 -1
- package/dist/reward-model-export.d.ts +0 -60
- package/dist/reward-model-export.d.ts.map +0 -1
- package/dist/rl/active-curriculum.d.ts +0 -110
- package/dist/rl/active-curriculum.d.ts.map +0 -1
- package/dist/rl/adaptation-eval.d.ts +0 -109
- package/dist/rl/adaptation-eval.d.ts.map +0 -1
- package/dist/rl/adversarial.d.ts.map +0 -1
- package/dist/rl/compute-curves.d.ts +0 -127
- package/dist/rl/compute-curves.d.ts.map +0 -1
- package/dist/rl/contamination.d.ts +0 -117
- package/dist/rl/contamination.d.ts.map +0 -1
- package/dist/rl/corpus.d.ts +0 -55
- package/dist/rl/corpus.d.ts.map +0 -1
- package/dist/rl/corpus.test.d.ts +0 -2
- package/dist/rl/corpus.test.d.ts.map +0 -1
- package/dist/rl/dataset.d.ts +0 -102
- package/dist/rl/dataset.d.ts.map +0 -1
- package/dist/rl/dataset.test.d.ts +0 -2
- package/dist/rl/dataset.test.d.ts.map +0 -1
- package/dist/rl/exporters.d.ts +0 -141
- package/dist/rl/exporters.d.ts.map +0 -1
- package/dist/rl/index.d.ts +0 -49
- package/dist/rl/index.d.ts.map +0 -1
- package/dist/rl/off-policy.d.ts.map +0 -1
- package/dist/rl/predictive-validity-researcher.d.ts +0 -69
- package/dist/rl/predictive-validity-researcher.d.ts.map +0 -1
- package/dist/rl/preferences.d.ts +0 -141
- package/dist/rl/preferences.d.ts.map +0 -1
- package/dist/rl/process-reward.d.ts +0 -122
- package/dist/rl/process-reward.d.ts.map +0 -1
- package/dist/rl/reward-hacking.d.ts +0 -104
- package/dist/rl/reward-hacking.d.ts.map +0 -1
- package/dist/rl/rl-campaign.d.ts +0 -85
- package/dist/rl/rl-campaign.d.ts.map +0 -1
- package/dist/rl/run-record-adapters.d.ts +0 -56
- package/dist/rl/run-record-adapters.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.d.ts +0 -166
- package/dist/rl/sim-fidelity.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.test.d.ts +0 -2
- package/dist/rl/sim-fidelity.test.d.ts.map +0 -1
- package/dist/rl/tournament.d.ts +0 -115
- package/dist/rl/tournament.d.ts.map +0 -1
- package/dist/rl/verifiable-reward.d.ts +0 -124
- package/dist/rl/verifiable-reward.d.ts.map +0 -1
- package/dist/run-critic.d.ts +0 -23
- package/dist/run-critic.d.ts.map +0 -1
- package/dist/run-evidence.d.ts +0 -32
- package/dist/run-evidence.d.ts.map +0 -1
- package/dist/run-record.d.ts.map +0 -1
- package/dist/run-record.test.d.ts +0 -2
- package/dist/run-record.test.d.ts.map +0 -1
- package/dist/run-score.d.ts +0 -31
- package/dist/run-score.d.ts.map +0 -1
- package/dist/runtime-trajectory.d.ts +0 -47
- package/dist/runtime-trajectory.d.ts.map +0 -1
- package/dist/sandbox-harness.d.ts.map +0 -1
- package/dist/sandbox-harness.test.d.ts +0 -2
- package/dist/sandbox-harness.test.d.ts.map +0 -1
- package/dist/sandbox-pool.d.ts +0 -74
- package/dist/sandbox-pool.d.ts.map +0 -1
- package/dist/sandbox-pool.test.d.ts +0 -2
- package/dist/sandbox-pool.test.d.ts.map +0 -1
- package/dist/scorecard.d.ts +0 -133
- package/dist/scorecard.d.ts.map +0 -1
- package/dist/scorecard.test.d.ts +0 -2
- package/dist/scorecard.test.d.ts.map +0 -1
- package/dist/self-play.d.ts +0 -69
- package/dist/self-play.d.ts.map +0 -1
- package/dist/semantic-concept-judge.d.ts +0 -135
- package/dist/semantic-concept-judge.d.ts.map +0 -1
- package/dist/semantic-concept-judge.test.d.ts +0 -2
- package/dist/semantic-concept-judge.test.d.ts.map +0 -1
- package/dist/sequential.d.ts.map +0 -1
- package/dist/series-convergence.d.ts.map +0 -1
- package/dist/slo.d.ts +0 -48
- package/dist/slo.d.ts.map +0 -1
- package/dist/state-continuity.d.ts +0 -47
- package/dist/state-continuity.d.ts.map +0 -1
- package/dist/statistics.d.ts.map +0 -1
- package/dist/statistics.test.d.ts +0 -2
- package/dist/statistics.test.d.ts.map +0 -1
- package/dist/steering-optimizer.d.ts +0 -58
- package/dist/steering-optimizer.d.ts.map +0 -1
- package/dist/steering.d.ts +0 -24
- package/dist/steering.d.ts.map +0 -1
- package/dist/storyboard/code-edit.d.ts +0 -64
- package/dist/storyboard/code-edit.d.ts.map +0 -1
- package/dist/storyboard/code-edit.test.d.ts +0 -2
- package/dist/storyboard/code-edit.test.d.ts.map +0 -1
- package/dist/storyboard/index.d.ts.map +0 -1
- package/dist/storyboard/index.test.d.ts +0 -2
- package/dist/storyboard/index.test.d.ts.map +0 -1
- package/dist/summary-report.d.ts.map +0 -1
- package/dist/telemetry/client.d.ts +0 -35
- package/dist/telemetry/client.d.ts.map +0 -1
- package/dist/telemetry/index.d.ts.map +0 -1
- package/dist/telemetry/schema.d.ts +0 -61
- package/dist/telemetry/schema.d.ts.map +0 -1
- package/dist/telemetry/sink-fetch.d.ts +0 -39
- package/dist/telemetry/sink-fetch.d.ts.map +0 -1
- package/dist/telemetry/sink-file.d.ts.map +0 -1
- package/dist/test-graded-scenario.d.ts +0 -42
- package/dist/test-graded-scenario.d.ts.map +0 -1
- package/dist/testing.d.ts.map +0 -1
- package/dist/tool-use-metrics.d.ts +0 -35
- package/dist/tool-use-metrics.d.ts.map +0 -1
- package/dist/trace/capture-fetch.d.ts +0 -48
- package/dist/trace/capture-fetch.d.ts.map +0 -1
- package/dist/trace/capture-fetch.test.d.ts +0 -2
- package/dist/trace/capture-fetch.test.d.ts.map +0 -1
- package/dist/trace/emitter.d.ts.map +0 -1
- package/dist/trace/extract-usage.d.ts +0 -46
- package/dist/trace/extract-usage.d.ts.map +0 -1
- package/dist/trace/extract-usage.test.d.ts +0 -2
- package/dist/trace/extract-usage.test.d.ts.map +0 -1
- package/dist/trace/index.d.ts +0 -15
- package/dist/trace/index.d.ts.map +0 -1
- package/dist/trace/integrity.d.ts.map +0 -1
- package/dist/trace/otel-bridge.d.ts +0 -29
- package/dist/trace/otel-bridge.d.ts.map +0 -1
- package/dist/trace/otel-export.d.ts +0 -52
- package/dist/trace/otel-export.d.ts.map +0 -1
- package/dist/trace/otel.d.ts +0 -57
- package/dist/trace/otel.d.ts.map +0 -1
- package/dist/trace/otlp-attributes.d.ts +0 -17
- package/dist/trace/otlp-attributes.d.ts.map +0 -1
- package/dist/trace/query.d.ts +0 -29
- package/dist/trace/query.d.ts.map +0 -1
- package/dist/trace/query.test.d.ts +0 -2
- package/dist/trace/query.test.d.ts.map +0 -1
- package/dist/trace/raw-provider-sink.d.ts.map +0 -1
- package/dist/trace/redact.d.ts.map +0 -1
- package/dist/trace/schema.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.d.ts +0 -72
- package/dist/trace/store-to-otlp.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.test.d.ts +0 -2
- package/dist/trace/store-to-otlp.test.d.ts.map +0 -1
- package/dist/trace/store.d.ts.map +0 -1
- package/dist/trace/store.test.d.ts +0 -2
- package/dist/trace/store.test.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.test.d.ts +0 -2
- package/dist/trace-analyst/analyst.test.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.d.ts +0 -40
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +0 -2
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +0 -1
- package/dist/trace-analyst/hook.d.ts +0 -55
- package/dist/trace-analyst/hook.d.ts.map +0 -1
- package/dist/trace-analyst/index.d.ts +0 -18
- package/dist/trace-analyst/index.d.ts.map +0 -1
- package/dist/trace-analyst/insights.d.ts +0 -71
- package/dist/trace-analyst/insights.d.ts.map +0 -1
- package/dist/trace-analyst/insights.test.d.ts +0 -2
- package/dist/trace-analyst/insights.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-flatten.d.ts +0 -42
- package/dist/trace-analyst/otlp-flatten.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.d.ts +0 -85
- package/dist/trace-analyst/otlp-span.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-span.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.d.ts +0 -115
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/prompts.d.ts +0 -6
- package/dist/trace-analyst/prompts.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.d.ts +0 -126
- package/dist/trace-analyst/store-otlp.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.test.d.ts +0 -8
- package/dist/trace-analyst/store-otlp.test.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/store.d.ts +0 -63
- package/dist/trace-analyst/store.d.ts.map +0 -1
- package/dist/trace-analyst/tools.d.ts +0 -44
- package/dist/trace-analyst/tools.d.ts.map +0 -1
- package/dist/trace-analyst/tools.test.d.ts +0 -10
- package/dist/trace-analyst/tools.test.d.ts.map +0 -1
- package/dist/trace-analyst/types.d.ts.map +0 -1
- package/dist/trace-contracts.d.ts +0 -180
- package/dist/trace-contracts.d.ts.map +0 -1
- package/dist/traced-analyst.d.ts +0 -26
- package/dist/traced-analyst.d.ts.map +0 -1
- package/dist/traced-judges.d.ts +0 -27
- package/dist/traced-judges.d.ts.map +0 -1
- package/dist/traces.d.ts.map +0 -1
- package/dist/trajectory.d.ts.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/ui-finding.d.ts +0 -104
- package/dist/ui-finding.d.ts.map +0 -1
- package/dist/verdict-cache.d.ts +0 -78
- package/dist/verdict-cache.d.ts.map +0 -1
- package/dist/verdict-cache.test.d.ts +0 -2
- package/dist/verdict-cache.test.d.ts.map +0 -1
- package/dist/verdict.d.ts.map +0 -1
- package/dist/visual-diff.d.ts +0 -32
- package/dist/visual-diff.d.ts.map +0 -1
- package/dist/wire/handlers.d.ts +0 -54
- package/dist/wire/handlers.d.ts.map +0 -1
- package/dist/wire/index.d.ts.map +0 -1
- package/dist/wire/openapi.d.ts +0 -3
- package/dist/wire/openapi.d.ts.map +0 -1
- package/dist/wire/rpc.d.ts +0 -21
- package/dist/wire/rpc.d.ts.map +0 -1
- package/dist/wire/rubrics.d.ts +0 -34
- package/dist/wire/rubrics.d.ts.map +0 -1
- package/dist/wire/schemas.d.ts +0 -410
- package/dist/wire/schemas.d.ts.map +0 -1
- package/dist/wire/server.d.ts +0 -60
- package/dist/wire/server.d.ts.map +0 -1
- package/dist/workflow/event-schema.d.ts +0 -5
- package/dist/workflow/event-schema.d.ts.map +0 -1
- package/dist/workflow/feedback-pack.d.ts +0 -99
- package/dist/workflow/feedback-pack.d.ts.map +0 -1
- package/dist/workflow/index.d.ts.map +0 -1
- package/dist/workflow/intelligence-export.d.ts +0 -62
- package/dist/workflow/intelligence-export.d.ts.map +0 -1
- package/dist/workflow/partner-report.d.ts +0 -49
- package/dist/workflow/partner-report.d.ts.map +0 -1
- package/dist/workflow/phase-graph.d.ts +0 -43
- package/dist/workflow/phase-graph.d.ts.map +0 -1
- package/dist/workflow/promotion-gate.d.ts +0 -61
- package/dist/workflow/promotion-gate.d.ts.map +0 -1
- package/dist/workflow/run-record.d.ts +0 -12
- package/dist/workflow/run-record.d.ts.map +0 -1
- package/dist/workflow/runtime-adapter.d.ts +0 -20
- package/dist/workflow/runtime-adapter.d.ts.map +0 -1
- package/dist/workflow/sanitize.d.ts +0 -21
- package/dist/workflow/sanitize.d.ts.map +0 -1
- package/dist/workflow/schema.d.ts +0 -5
- package/dist/workflow/schema.d.ts.map +0 -1
- package/dist/workflow/summary.d.ts +0 -43
- package/dist/workflow/summary.d.ts.map +0 -1
- package/dist/workflow/trace-event-fields.d.ts +0 -6
- package/dist/workflow/trace-event-fields.d.ts.map +0 -1
- package/dist/workflow/trajectory.d.ts +0 -15
- package/dist/workflow/trajectory.d.ts.map +0 -1
- package/dist/workflow/types.d.ts +0 -68
- package/dist/workflow/types.d.ts.map +0 -1
- package/dist/workspace-inspector.d.ts +0 -67
- package/dist/workspace-inspector.d.ts.map +0 -1
- package/dist/wrangler-deploy-runner.test.d.ts +0 -2
- package/dist/wrangler-deploy-runner.test.d.ts.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { G as GainDistributionBin, P as ParetoFigureSpec } from './summary-report-CInXwsza.js';
|
|
2
|
+
import { C as ContinuousAgreement } from './judge-calibration-0p2QcWNE.js';
|
|
3
|
+
|
|
1
4
|
/**
|
|
2
5
|
* # InsightReport — the rigorous decision packet for any set of agent runs.
|
|
3
6
|
*
|
|
@@ -27,9 +30,8 @@
|
|
|
27
30
|
* Consumers read the `recommendations` array first — that's the
|
|
28
31
|
* actionable layer, ranked by priority. The numeric sections back it up.
|
|
29
32
|
*/
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
export interface InsightReport {
|
|
33
|
+
|
|
34
|
+
interface InsightReport {
|
|
33
35
|
/** Number of runs analyzed. */
|
|
34
36
|
n: number;
|
|
35
37
|
/** Composite-score distribution across all runs. Always present. */
|
|
@@ -94,7 +96,7 @@ export interface InsightReport {
|
|
|
94
96
|
recommendations: Recommendation[];
|
|
95
97
|
}
|
|
96
98
|
/** Distributional summary of a scalar-valued metric. */
|
|
97
|
-
|
|
99
|
+
interface ScalarDistribution {
|
|
98
100
|
/** Sample count after dropping non-finite values. */
|
|
99
101
|
n: number;
|
|
100
102
|
mean: number;
|
|
@@ -114,7 +116,7 @@ export interface ScalarDistribution {
|
|
|
114
116
|
score: number;
|
|
115
117
|
}>;
|
|
116
118
|
}
|
|
117
|
-
|
|
119
|
+
interface JudgeInsight {
|
|
118
120
|
/** Number of times this judge scored a run. */
|
|
119
121
|
n: number;
|
|
120
122
|
/** Mean composite over this judge's runs. */
|
|
@@ -132,7 +134,7 @@ export interface JudgeInsight {
|
|
|
132
134
|
* quality? */
|
|
133
135
|
verbosityBias?: number;
|
|
134
136
|
}
|
|
135
|
-
|
|
137
|
+
interface InterRaterInsight {
|
|
136
138
|
/** Number of raters whose scores were aggregated. */
|
|
137
139
|
raters: number;
|
|
138
140
|
/** Number of runs every rater scored. */
|
|
@@ -151,7 +153,7 @@ export interface InterRaterInsight {
|
|
|
151
153
|
range: number;
|
|
152
154
|
}>;
|
|
153
155
|
}
|
|
154
|
-
|
|
156
|
+
interface LiftInsight {
|
|
155
157
|
baselineMean: number;
|
|
156
158
|
candidateMean: number;
|
|
157
159
|
/** Candidate − baseline. */
|
|
@@ -169,7 +171,7 @@ export interface LiftInsight {
|
|
|
169
171
|
/** Sample size needed to detect the observed delta at 80% power. */
|
|
170
172
|
requiredN: number;
|
|
171
173
|
}
|
|
172
|
-
|
|
174
|
+
interface FailureClusterInsight {
|
|
173
175
|
/** All clusters identified by the registry, ranked by share descending. */
|
|
174
176
|
clusters: Array<{
|
|
175
177
|
id: string;
|
|
@@ -188,7 +190,7 @@ export interface FailureClusterInsight {
|
|
|
188
190
|
* is computed directly from the tags the harness already recorded — so a
|
|
189
191
|
* customer ingesting one batch with no judge/analyst still learns which
|
|
190
192
|
* named failure dominates. */
|
|
191
|
-
|
|
193
|
+
interface FailureModeTally {
|
|
192
194
|
/** The `failureMode` tag. */
|
|
193
195
|
mode: string;
|
|
194
196
|
/** Number of runs carrying this tag. */
|
|
@@ -196,7 +198,7 @@ export interface FailureModeTally {
|
|
|
196
198
|
/** Share of the whole corpus, 0..1. */
|
|
197
199
|
share: number;
|
|
198
200
|
}
|
|
199
|
-
|
|
201
|
+
interface ContaminationInsight {
|
|
200
202
|
/** Canary phrases that leaked into outputs. */
|
|
201
203
|
leaks: number;
|
|
202
204
|
/** Holdout audit verdict — did any holdout-tagged run end up in the
|
|
@@ -208,7 +210,7 @@ export interface ContaminationInsight {
|
|
|
208
210
|
matched: string;
|
|
209
211
|
}>;
|
|
210
212
|
}
|
|
211
|
-
|
|
213
|
+
interface OutcomeCorrelationInsight {
|
|
212
214
|
/** What outcome the consumer is correlating against (e.g.
|
|
213
215
|
* `'engagement_rate'`, `'approval_rate'`, `'downstream_pass'`). */
|
|
214
216
|
metric: string;
|
|
@@ -225,7 +227,7 @@ export interface OutcomeCorrelationInsight {
|
|
|
225
227
|
r2: number;
|
|
226
228
|
};
|
|
227
229
|
}
|
|
228
|
-
|
|
230
|
+
interface ReleaseSummary {
|
|
229
231
|
/** Overall verdict across axes — fail if any axis fails, else warn if any
|
|
230
232
|
* warns, else pass. */
|
|
231
233
|
status: 'pass' | 'warn' | 'fail';
|
|
@@ -238,7 +240,7 @@ export interface ReleaseSummary {
|
|
|
238
240
|
* consumers can post-process to populate. */
|
|
239
241
|
issues: string[];
|
|
240
242
|
}
|
|
241
|
-
|
|
243
|
+
interface MetricDelta {
|
|
242
244
|
/** Current-period mean. */
|
|
243
245
|
current: number;
|
|
244
246
|
/** Baseline-period mean. */
|
|
@@ -262,7 +264,7 @@ export interface MetricDelta {
|
|
|
262
264
|
* tiny from triggering recommendations. */
|
|
263
265
|
significant: boolean;
|
|
264
266
|
}
|
|
265
|
-
|
|
267
|
+
interface PriorPeriodComparison {
|
|
266
268
|
/** Sample counts. */
|
|
267
269
|
baselineN: number;
|
|
268
270
|
currentN: number;
|
|
@@ -278,7 +280,7 @@ export interface PriorPeriodComparison {
|
|
|
278
280
|
/** Metric names where current is significantly BETTER than baseline. */
|
|
279
281
|
improvedMetrics: string[];
|
|
280
282
|
}
|
|
281
|
-
|
|
283
|
+
interface Recommendation {
|
|
282
284
|
priority: 'critical' | 'high' | 'medium' | 'low';
|
|
283
285
|
kind: 'ship' | 'hold' | 'investigate' | 'fix' | 'recalibrate' | 'expand-corpus';
|
|
284
286
|
title: string;
|
|
@@ -286,4 +288,5 @@ export interface Recommendation {
|
|
|
286
288
|
/** Optional pointer back into the report for the evidence. */
|
|
287
289
|
evidencePath?: string;
|
|
288
290
|
}
|
|
289
|
-
|
|
291
|
+
|
|
292
|
+
export type { FailureClusterInsight as F, InsightReport as I, JudgeInsight as J, LiftInsight as L, OutcomeCorrelationInsight as O, Recommendation as R, ScalarDistribution as S, InterRaterInsight as a, ReleaseSummary as b };
|
|
@@ -1,3 +1,7 @@
|
|
|
1
|
+
import { C as CaptureIntegrityError } from './errors-CzMUYo7b.js';
|
|
2
|
+
import { R as RawProviderSink } from './raw-provider-sink-C46HDghv.js';
|
|
3
|
+
import { T as TraceStore } from './store-BcFXE6LG.js';
|
|
4
|
+
|
|
1
5
|
/**
|
|
2
6
|
* Run-completion integrity check — at end of run, verify the expected event
|
|
3
7
|
* types were actually captured. The point is the launch-review failure mode:
|
|
@@ -18,10 +22,8 @@
|
|
|
18
22
|
* the caller chooses the failure mode (throw, mark run failed, log warning).
|
|
19
23
|
* `throwIfRunIncomplete` is the convenient strict mode.
|
|
20
24
|
*/
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
import type { TraceStore } from './store';
|
|
24
|
-
export interface RunIntegrityExpectations {
|
|
25
|
+
|
|
26
|
+
interface RunIntegrityExpectations {
|
|
25
27
|
/** Minimum LLM span count. Default 0 (no requirement). */
|
|
26
28
|
llmSpansMin?: number;
|
|
27
29
|
/** Minimum judge span count. Default 0. */
|
|
@@ -44,13 +46,13 @@ export interface RunIntegrityExpectations {
|
|
|
44
46
|
/** Run outcome must be set (not null/undefined). Default false. */
|
|
45
47
|
requireOutcome?: boolean;
|
|
46
48
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
+
type RunIntegrityIssueCode = 'no_run' | 'missing_llm_spans' | 'missing_judge_spans' | 'missing_tool_spans' | 'missing_raw_events' | 'no_raw_sink' | 'orphan_llm_span' | 'missing_outcome';
|
|
50
|
+
interface RunIntegrityIssue {
|
|
49
51
|
code: RunIntegrityIssueCode;
|
|
50
52
|
message: string;
|
|
51
53
|
detail?: Record<string, unknown>;
|
|
52
54
|
}
|
|
53
|
-
|
|
55
|
+
interface RunIntegrityReport {
|
|
54
56
|
ok: boolean;
|
|
55
57
|
runId: string;
|
|
56
58
|
llmSpanCount: number;
|
|
@@ -68,11 +70,12 @@ export interface RunIntegrityReport {
|
|
|
68
70
|
};
|
|
69
71
|
issues: RunIntegrityIssue[];
|
|
70
72
|
}
|
|
71
|
-
|
|
73
|
+
declare class RunIntegrityError extends CaptureIntegrityError {
|
|
72
74
|
readonly report: RunIntegrityReport;
|
|
73
75
|
constructor(report: RunIntegrityReport);
|
|
74
76
|
}
|
|
75
|
-
|
|
77
|
+
declare function assertRunCaptured(store: TraceStore, runId: string, expectations?: RunIntegrityExpectations): Promise<RunIntegrityReport>;
|
|
76
78
|
/** Strict mode: throws `RunIntegrityError` when the report isn't ok. */
|
|
77
|
-
|
|
78
|
-
|
|
79
|
+
declare function throwIfRunIncomplete(report: RunIntegrityReport): void;
|
|
80
|
+
|
|
81
|
+
export { type RunIntegrityExpectations as R, type RunIntegrityReport as a, RunIntegrityError as b, type RunIntegrityIssue as c, type RunIntegrityIssueCode as d, assertRunCaptured as e, throwIfRunIncomplete as t };
|
|
@@ -18,19 +18,19 @@
|
|
|
18
18
|
* Returns actionable diagnostics, not a single number. Consumers then
|
|
19
19
|
* decide whether to trust the judge, retrain it, or add a tie-breaker.
|
|
20
20
|
*/
|
|
21
|
-
|
|
21
|
+
interface GoldenItem {
|
|
22
22
|
itemId: string;
|
|
23
23
|
humanScore: number;
|
|
24
24
|
/** Optional group used for per-group bias audits (e.g. model-of-output family). */
|
|
25
25
|
group?: string;
|
|
26
26
|
}
|
|
27
|
-
|
|
27
|
+
interface CandidateScore {
|
|
28
28
|
itemId: string;
|
|
29
29
|
score: number;
|
|
30
30
|
/** Optional — enables positional-bias analysis (did order matter?). */
|
|
31
31
|
positionOfAInput?: 'first' | 'second';
|
|
32
32
|
}
|
|
33
|
-
|
|
33
|
+
interface CalibrationResult {
|
|
34
34
|
n: number;
|
|
35
35
|
pearson: number;
|
|
36
36
|
/** Cohen's κ with quadratic weights over integer-rounded scores. */
|
|
@@ -45,8 +45,8 @@ export interface CalibrationResult {
|
|
|
45
45
|
delta: number;
|
|
46
46
|
}>;
|
|
47
47
|
}
|
|
48
|
-
|
|
49
|
-
|
|
48
|
+
declare function calibrateJudge(golden: GoldenItem[], candidate: CandidateScore[]): CalibrationResult;
|
|
49
|
+
interface PositionalBiasResult {
|
|
50
50
|
/**
|
|
51
51
|
* Score delta (first-position - second-position) averaged across items
|
|
52
52
|
* presented in both positions. Non-zero = positional bias.
|
|
@@ -58,17 +58,17 @@ export interface PositionalBiasResult {
|
|
|
58
58
|
* Feed the same items to the judge twice with A/B swapped and pass all
|
|
59
59
|
* results here. Items that don't appear in both positions are ignored.
|
|
60
60
|
*/
|
|
61
|
-
|
|
62
|
-
|
|
61
|
+
declare function positionalBias(scores: CandidateScore[]): PositionalBiasResult;
|
|
62
|
+
interface VerbosityBiasResult {
|
|
63
63
|
/** Pearson correlation between output length and score. Strong positive = verbosity bias. */
|
|
64
64
|
pearson: number;
|
|
65
65
|
n: number;
|
|
66
66
|
}
|
|
67
|
-
|
|
67
|
+
declare function verbosityBias(samples: Array<{
|
|
68
68
|
outputLen: number;
|
|
69
69
|
score: number;
|
|
70
70
|
}>): VerbosityBiasResult;
|
|
71
|
-
|
|
71
|
+
interface SelfPreferenceResult {
|
|
72
72
|
/** Mean judge score when judge's family matches output's family. */
|
|
73
73
|
inFamilyMean: number;
|
|
74
74
|
outOfFamilyMean: number;
|
|
@@ -80,11 +80,11 @@ export interface SelfPreferenceResult {
|
|
|
80
80
|
* model X (in-family) and model Y (out-of-family). Non-zero delta
|
|
81
81
|
* indicates self-preference.
|
|
82
82
|
*/
|
|
83
|
-
|
|
83
|
+
declare function selfPreference(samples: Array<{
|
|
84
84
|
score: number;
|
|
85
85
|
inFamily: boolean;
|
|
86
86
|
}>): SelfPreferenceResult;
|
|
87
|
-
|
|
87
|
+
interface ContinuousAgreement {
|
|
88
88
|
/** Cohen's κ_w with quadratic weights, computed on raw [0,1] scores. */
|
|
89
89
|
weightedKappa: number;
|
|
90
90
|
/** ICC(2,1): two-way random effects, absolute agreement, single rater. */
|
|
@@ -103,7 +103,7 @@ export interface ContinuousAgreement {
|
|
|
103
103
|
/** Number of raters. */
|
|
104
104
|
raters: number;
|
|
105
105
|
}
|
|
106
|
-
|
|
106
|
+
interface ContinuousAgreementOptions {
|
|
107
107
|
/** Bootstrap iterations. Default 1000. Set to 0 to skip CIs (CI = [NaN, NaN]). */
|
|
108
108
|
bootstrap?: number;
|
|
109
109
|
/** κ weighting scheme. Default 'quadratic'. */
|
|
@@ -120,8 +120,8 @@ export interface ContinuousAgreementOptions {
|
|
|
120
120
|
* are dropped. Returns NaN metrics if fewer than 2 raters or 2 complete
|
|
121
121
|
* items remain.
|
|
122
122
|
*/
|
|
123
|
-
|
|
124
|
-
|
|
123
|
+
declare function continuousAgreement(scores: number[][], opts?: ContinuousAgreementOptions): ContinuousAgreement;
|
|
124
|
+
interface ContinuousCalibrationResult extends CalibrationResult {
|
|
125
125
|
/** Cohen's κ_w computed on raw (un-rounded) scores. */
|
|
126
126
|
weightedKappaContinuous: number;
|
|
127
127
|
/** ICC(2,1) treating golden + candidate as two raters. */
|
|
@@ -137,5 +137,6 @@ export interface ContinuousCalibrationResult extends CalibrationResult {
|
|
|
137
137
|
* agreement metrics. The old fields (n, pearson, kappa, mae, worstItems)
|
|
138
138
|
* are preserved unchanged so existing callers continue to work.
|
|
139
139
|
*/
|
|
140
|
-
|
|
141
|
-
|
|
140
|
+
declare function calibrateJudgeContinuous(golden: GoldenItem[], candidate: CandidateScore[], opts?: ContinuousAgreementOptions): ContinuousCalibrationResult;
|
|
141
|
+
|
|
142
|
+
export { type ContinuousAgreement as C, type GoldenItem as G, type PositionalBiasResult as P, type SelfPreferenceResult as S, type VerbosityBiasResult as V, type CalibrationResult as a, type ContinuousCalibrationResult as b, type CandidateScore as c, type ContinuousAgreementOptions as d, calibrateJudge as e, calibrateJudgeContinuous as f, continuousAgreement as g, positionalBias as p, selfPreference as s, verbosityBias as v };
|
|
@@ -1,3 +1,56 @@
|
|
|
1
|
+
import { AxAIService, AxFunction } from '@ax-llm/ax';
|
|
2
|
+
import { T as TraceAnalysisStore } from './store-C1YxJDEK.js';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import { g as AnalystCost, a as AnalystContext, A as Analyst } from './types-B5x54y6n.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Typed Ax output for analyst findings.
|
|
8
|
+
*
|
|
9
|
+
* Replaces the legacy `findings:string[]` pattern (where every bullet
|
|
10
|
+
* became a flat-severity `AnalystFinding`) with a structured object
|
|
11
|
+
* array. Ax binds the field as `findings:json[]` so the provider emits
|
|
12
|
+
* native structured output; at the kind-factory boundary we Zod-validate
|
|
13
|
+
* each emitted finding so malformed rows fail loud instead of being
|
|
14
|
+
* silently lifted with default severity.
|
|
15
|
+
*
|
|
16
|
+
* Why not `f.object().array()` directly in the signature? The Ax
|
|
17
|
+
* signature string `question:string -> findings:json[]` already lets
|
|
18
|
+
* the provider emit JSON arrays. A Zod boundary is required either
|
|
19
|
+
* way (the provider can return any JSON), and Zod gives us a single
|
|
20
|
+
* validation surface independent of which Ax version is installed.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
declare const ANALYST_SEVERITIES: readonly ["critical", "high", "medium", "low", "info"];
|
|
24
|
+
declare const RawAnalystFindingSchema: z.ZodObject<{
|
|
25
|
+
severity: z.ZodEnum<{
|
|
26
|
+
info: "info";
|
|
27
|
+
critical: "critical";
|
|
28
|
+
medium: "medium";
|
|
29
|
+
low: "low";
|
|
30
|
+
high: "high";
|
|
31
|
+
}>;
|
|
32
|
+
claim: z.ZodString;
|
|
33
|
+
subject: z.ZodOptional<z.ZodString>;
|
|
34
|
+
evidence_uri: z.ZodString;
|
|
35
|
+
evidence_excerpt: z.ZodOptional<z.ZodString>;
|
|
36
|
+
confidence: z.ZodNumber;
|
|
37
|
+
rationale: z.ZodOptional<z.ZodString>;
|
|
38
|
+
recommended_action: z.ZodOptional<z.ZodString>;
|
|
39
|
+
}, z.core.$strict>;
|
|
40
|
+
type RawAnalystFinding = z.infer<typeof RawAnalystFindingSchema>;
|
|
41
|
+
/**
|
|
42
|
+
* Description embedded into the actor prompt so the LLM knows what
|
|
43
|
+
* shape to emit. Kept here so kinds share one source of truth rather
|
|
44
|
+
* than restating the schema in every prompt.
|
|
45
|
+
*/
|
|
46
|
+
declare const RAW_FINDING_SCHEMA_PROMPT = "Each finding MUST be a JSON object with these fields:\n - severity: one of \"critical\" | \"high\" | \"medium\" | \"low\" | \"info\"\n - claim: one-sentence statement (max 2000 chars)\n - subject?: the routing locus this finding is about. It MUST be one of the exact subject forms listed in this kind's instructions above (e.g. `system-prompt:<section>`, `agent-knowledge:wiki:<slug>`, `tool-doc:<tool>`). A free phrase, a bare noun, or any form not in that list is REJECTED at parse time and the finding is discarded \u2014 omit subject entirely rather than guess a form.\n - evidence_uri: REQUIRED, never blank. Exactly one of \"span://<trace_id>/<span_id>\" (trace evidence), \"artifact://<relative-path>\" (files), \"metric://<name>\" (named scalars) \u2014 ALWAYS cite a real id surfaced by the tools. If you have no citable id, do not emit the finding.\n - evidence_excerpt?: short quote (<=2000 chars) from the cited span/artifact\n - confidence: number 0..1 \u2014 0.9+ when backed by exact quotes, 0.6-0.8 for inferred patterns, <0.5 for speculative\n - rationale?: one or two sentences explaining the reasoning\n - recommended_action?: concrete change phrased as an imperative (\"Add ...\", \"Replace ...\", \"Stop ...\") \u2014 omit when the finding is purely descriptive\n\nEmit an empty array when the question has no findings to report. Do not fabricate evidence.";
|
|
47
|
+
/**
|
|
48
|
+
* Validate one row emitted by the LLM. Returns the typed finding on
|
|
49
|
+
* success; returns `null` and logs the reason on failure so the kind
|
|
50
|
+
* factory can skip-and-count rather than abort the whole analyst run.
|
|
51
|
+
*/
|
|
52
|
+
declare function parseRawFinding(row: unknown, log?: (msg: string, fields?: Record<string, unknown>) => void): RawAnalystFinding | null;
|
|
53
|
+
|
|
1
54
|
/**
|
|
2
55
|
* Analyst-kind factory — the typed way to define trace analysts.
|
|
3
56
|
*
|
|
@@ -23,15 +76,12 @@
|
|
|
23
76
|
* description programmatically. Stored on the kind, not the registry,
|
|
24
77
|
* because the right metric is kind-specific.
|
|
25
78
|
*/
|
|
26
|
-
|
|
27
|
-
import type { TraceAnalysisStore } from '../trace-analyst/store';
|
|
28
|
-
import { type RawAnalystFinding } from './finding-signature';
|
|
29
|
-
import type { Analyst, AnalystContext, AnalystCost } from './types';
|
|
79
|
+
|
|
30
80
|
/**
|
|
31
81
|
* Per-kind specification. The factory turns this into a regular
|
|
32
82
|
* `Analyst<TraceAnalysisStore>` ready for `AnalystRegistry.register()`.
|
|
33
83
|
*/
|
|
34
|
-
|
|
84
|
+
interface TraceAnalystKindSpec {
|
|
35
85
|
/** Stable id. Appears in finding_id, telemetry, and registry exclusions. */
|
|
36
86
|
id: string;
|
|
37
87
|
/** One-sentence description shown in `registry.list()`. */
|
|
@@ -68,11 +118,11 @@ export interface TraceAnalystKindSpec {
|
|
|
68
118
|
* is the ground-truth finding set a fitted prompt should produce on this
|
|
69
119
|
* input. Metric: kind-specific (default: F1 on `finding_id` overlap).
|
|
70
120
|
*/
|
|
71
|
-
|
|
121
|
+
interface TraceAnalystGolden {
|
|
72
122
|
question: string;
|
|
73
123
|
expected: ReadonlyArray<Omit<RawAnalystFinding, 'confidence'>>;
|
|
74
124
|
}
|
|
75
|
-
|
|
125
|
+
interface CreateTraceAnalystKindOpts {
|
|
76
126
|
/** AxAIService bound at registration time. */
|
|
77
127
|
ai: AxAIService;
|
|
78
128
|
/** Optional model override; falls back to the AI service's default. */
|
|
@@ -101,7 +151,7 @@ export interface CreateTraceAnalystKindOpts {
|
|
|
101
151
|
* `analyze()` call (the agent carries chat-log + usage state we don't
|
|
102
152
|
* want shared across analyst runs).
|
|
103
153
|
*/
|
|
104
|
-
|
|
154
|
+
declare function createTraceAnalystKind(spec: TraceAnalystKindSpec, opts: CreateTraceAnalystKindOpts): Analyst<TraceAnalysisStore>;
|
|
105
155
|
/**
|
|
106
156
|
* Render a compact prior-findings block the actor reads alongside its
|
|
107
157
|
* brief. Each row is one line so the actor can scan dozens cheaply.
|
|
@@ -116,5 +166,6 @@ export declare function createTraceAnalystKind(spec: TraceAnalystKindSpec, opts:
|
|
|
116
166
|
* Exported for tests + for consumers that build their own actor
|
|
117
167
|
* prompts (e.g. specialized analysts living outside the default kinds).
|
|
118
168
|
*/
|
|
119
|
-
|
|
120
|
-
|
|
169
|
+
declare function renderPriorFindings(prior: AnalystContext['priorFindings']): string;
|
|
170
|
+
|
|
171
|
+
export { ANALYST_SEVERITIES as A, type CreateTraceAnalystKindOpts as C, RAW_FINDING_SCHEMA_PROMPT as R, type TraceAnalystKindSpec as T, type RawAnalystFinding as a, RawAnalystFindingSchema as b, type TraceAnalystGolden as c, createTraceAnalystKind as d, parseRawFinding as p, renderPriorFindings as r };
|
|
@@ -1,3 +1,103 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
import { j as ControlSeverity, C as ControlEvalResult } from '../control-runtime-Acf9CGhw.js';
|
|
2
|
+
import { T as TraceEmitter } from '../emitter-C2rqGH_l.js';
|
|
3
|
+
import '../schema-m0gsnbt3.js';
|
|
4
|
+
import '../store-BcFXE6LG.js';
|
|
5
|
+
|
|
6
|
+
type KnowledgeRequirementCategory = 'user_specific' | 'company_specific' | 'domain_specific' | 'codebase_specific' | 'market_specific' | 'regulatory' | 'tool_api' | 'credential_or_secret' | 'runtime_environment' | 'preference' | 'historical_context';
|
|
7
|
+
type KnowledgeAcquisitionMode = 'ask_user' | 'search_web' | 'query_connector' | 'inspect_repo' | 'run_command' | 'infer_low_confidence' | 'not_available';
|
|
8
|
+
type KnowledgeImportance = 'blocking' | 'high' | 'medium' | 'low';
|
|
9
|
+
type KnowledgeFreshness = 'static' | 'monthly' | 'weekly' | 'daily' | 'realtime';
|
|
10
|
+
type KnowledgeSensitivity = 'public' | 'private' | 'secret';
|
|
11
|
+
type KnowledgeFallbackPolicy = 'block' | 'ask' | 'continue_with_caveat' | 'use_default';
|
|
12
|
+
interface KnowledgeRequirement {
|
|
13
|
+
id: string;
|
|
14
|
+
description: string;
|
|
15
|
+
requiredFor: string[];
|
|
16
|
+
category: KnowledgeRequirementCategory;
|
|
17
|
+
acquisitionMode: KnowledgeAcquisitionMode;
|
|
18
|
+
importance: KnowledgeImportance;
|
|
19
|
+
freshness: KnowledgeFreshness;
|
|
20
|
+
sensitivity: KnowledgeSensitivity;
|
|
21
|
+
confidenceNeeded: number;
|
|
22
|
+
currentConfidence: number;
|
|
23
|
+
evidenceIds: string[];
|
|
24
|
+
fallbackPolicy: KnowledgeFallbackPolicy;
|
|
25
|
+
/**
|
|
26
|
+
* ISO timestamp after which this requirement must be treated as stale.
|
|
27
|
+
* Stale requirements score as missing even when they still have evidence.
|
|
28
|
+
*/
|
|
29
|
+
validUntil?: string;
|
|
30
|
+
/** ISO timestamp for the last source-grounding or human verification pass. */
|
|
31
|
+
lastVerifiedAt?: string;
|
|
32
|
+
metadata?: Record<string, unknown>;
|
|
33
|
+
}
|
|
34
|
+
interface KnowledgeBundle {
|
|
35
|
+
taskId: string;
|
|
36
|
+
requirements: KnowledgeRequirement[];
|
|
37
|
+
evidenceIds: string[];
|
|
38
|
+
claimIds: string[];
|
|
39
|
+
wikiPageIds: string[];
|
|
40
|
+
userAnswers: Record<string, string>;
|
|
41
|
+
missing: KnowledgeRequirement[];
|
|
42
|
+
readinessScore: number;
|
|
43
|
+
metadata?: Record<string, unknown>;
|
|
44
|
+
}
|
|
45
|
+
type KnowledgeRecommendedAction = 'run_agent' | 'ask_user' | 'collect_web_data' | 'query_connectors' | 'inspect_repo' | 'build_domain_wiki' | 'continue_with_caveat' | 'abort_or_rescope';
|
|
46
|
+
interface KnowledgeReadinessReport {
|
|
47
|
+
taskId: string;
|
|
48
|
+
readinessScore: number;
|
|
49
|
+
blockingMissingRequirements: KnowledgeRequirement[];
|
|
50
|
+
nonBlockingGaps: KnowledgeRequirement[];
|
|
51
|
+
recommendedAction: KnowledgeRecommendedAction;
|
|
52
|
+
bundle: KnowledgeBundle;
|
|
53
|
+
severity: ControlSeverity;
|
|
54
|
+
reason: string;
|
|
55
|
+
}
|
|
56
|
+
interface UserQuestion {
|
|
57
|
+
id: string;
|
|
58
|
+
question: string;
|
|
59
|
+
reason: string;
|
|
60
|
+
requirementId: string;
|
|
61
|
+
importance: KnowledgeImportance;
|
|
62
|
+
answerType: 'free_text' | 'select_one' | 'multi_select' | 'file_upload' | 'credential' | 'url';
|
|
63
|
+
defaultIfSkipped?: string;
|
|
64
|
+
impactIfUnknown: string;
|
|
65
|
+
options?: string[];
|
|
66
|
+
metadata?: Record<string, unknown>;
|
|
67
|
+
}
|
|
68
|
+
interface DataAcquisitionPlan {
|
|
69
|
+
id: string;
|
|
70
|
+
requirementIds: string[];
|
|
71
|
+
mode: Exclude<KnowledgeAcquisitionMode, 'not_available' | 'infer_low_confidence'> | 'build_domain_wiki';
|
|
72
|
+
description: string;
|
|
73
|
+
priority: KnowledgeImportance;
|
|
74
|
+
expectedEvidenceIds?: string[];
|
|
75
|
+
questions?: UserQuestion[];
|
|
76
|
+
metadata?: Record<string, unknown>;
|
|
77
|
+
}
|
|
78
|
+
type KnowledgeResponsibleSurface = 'knowledge-requirements' | 'data-acquisition' | 'retrieval-policy' | 'user-question-policy';
|
|
79
|
+
|
|
80
|
+
interface ScoreKnowledgeReadinessOptions {
|
|
81
|
+
taskId: string;
|
|
82
|
+
requirements: KnowledgeRequirement[];
|
|
83
|
+
evidenceIds?: string[];
|
|
84
|
+
claimIds?: string[];
|
|
85
|
+
wikiPageIds?: string[];
|
|
86
|
+
userAnswers?: Record<string, string>;
|
|
87
|
+
metadata?: Record<string, unknown>;
|
|
88
|
+
now?: Date;
|
|
89
|
+
}
|
|
90
|
+
declare function scoreKnowledgeReadiness(options: ScoreKnowledgeReadinessOptions): KnowledgeReadinessReport;
|
|
91
|
+
declare function blockingKnowledgeEval(report: KnowledgeReadinessReport, options?: {
|
|
92
|
+
id?: string;
|
|
93
|
+
minimumScore?: number;
|
|
94
|
+
emitter?: TraceEmitter;
|
|
95
|
+
}): ControlEvalResult;
|
|
96
|
+
declare function knowledgeReadinessTracePayload(report: KnowledgeReadinessReport, options?: {
|
|
97
|
+
passed?: boolean;
|
|
98
|
+
minimumScore?: number;
|
|
99
|
+
}): Record<string, unknown>;
|
|
100
|
+
declare function userQuestionsForKnowledgeGaps(gaps: KnowledgeRequirement[]): UserQuestion[];
|
|
101
|
+
declare function acquisitionPlansForKnowledgeGaps(gaps: KnowledgeRequirement[]): DataAcquisitionPlan[];
|
|
102
|
+
|
|
103
|
+
export { type DataAcquisitionPlan, type KnowledgeAcquisitionMode, type KnowledgeBundle, type KnowledgeFallbackPolicy, type KnowledgeFreshness, type KnowledgeImportance, type KnowledgeReadinessReport, type KnowledgeRecommendedAction, type KnowledgeRequirement, type KnowledgeRequirementCategory, type KnowledgeResponsibleSurface, type KnowledgeSensitivity, type ScoreKnowledgeReadinessOptions, type UserQuestion, acquisitionPlansForKnowledgeGaps, blockingKnowledgeEval, knowledgeReadinessTracePayload, scoreKnowledgeReadiness, userQuestionsForKnowledgeGaps };
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { A as AgentEvalError, C as CaptureIntegrityError } from './errors-CzMUYo7b.js';
|
|
2
|
+
import { R as RawProviderSink, P as ProviderRedactor } from './raw-provider-sink-C46HDghv.js';
|
|
3
|
+
|
|
1
4
|
/**
|
|
2
5
|
* LLM client with graceful degrade.
|
|
3
6
|
*
|
|
@@ -19,9 +22,8 @@
|
|
|
19
22
|
* output (semantic concept judge, reviewer directives, critic scores). Primitives
|
|
20
23
|
* that need free-form text use `callLlm` and parse output themselves.
|
|
21
24
|
*/
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
export interface LlmMessage {
|
|
25
|
+
|
|
26
|
+
interface LlmMessage {
|
|
25
27
|
role: 'system' | 'user' | 'assistant';
|
|
26
28
|
/**
|
|
27
29
|
* Either a plain text content string OR a multimodal content array
|
|
@@ -38,7 +40,7 @@ export interface LlmMessage {
|
|
|
38
40
|
};
|
|
39
41
|
}>;
|
|
40
42
|
}
|
|
41
|
-
|
|
43
|
+
interface LlmCallRequest {
|
|
42
44
|
model: string;
|
|
43
45
|
messages: LlmMessage[];
|
|
44
46
|
/** Optional JSON-mode response format (response_format: json_object). */
|
|
@@ -53,14 +55,14 @@ export interface LlmCallRequest {
|
|
|
53
55
|
/** Per-call timeout, default 300s. */
|
|
54
56
|
timeoutMs?: number;
|
|
55
57
|
}
|
|
56
|
-
|
|
58
|
+
interface LlmUsage {
|
|
57
59
|
promptTokens: number;
|
|
58
60
|
completionTokens: number;
|
|
59
61
|
totalTokens: number;
|
|
60
62
|
/** Proxies populate this when prompt caching is on. */
|
|
61
63
|
cachedPromptTokens?: number;
|
|
62
64
|
}
|
|
63
|
-
|
|
65
|
+
interface LlmCallResult {
|
|
64
66
|
/** The text content of the first choice. Empty string if none. */
|
|
65
67
|
content: string;
|
|
66
68
|
usage: LlmUsage;
|
|
@@ -92,13 +94,13 @@ export interface LlmCallResult {
|
|
|
92
94
|
/** Raw response body. */
|
|
93
95
|
raw: Record<string, unknown>;
|
|
94
96
|
}
|
|
95
|
-
|
|
97
|
+
declare class LlmCallError extends AgentEvalError {
|
|
96
98
|
readonly status: number;
|
|
97
99
|
readonly body: string;
|
|
98
100
|
readonly model: string;
|
|
99
101
|
constructor(message: string, status: number, body: string, model: string);
|
|
100
102
|
}
|
|
101
|
-
|
|
103
|
+
interface LlmClientOptions {
|
|
102
104
|
/** Base URL (without trailing slash). Must end at the `/v1` prefix. */
|
|
103
105
|
baseUrl?: string;
|
|
104
106
|
/** Bearer token — either `apiKey` or `bearer` populates `Authorization: Bearer ...`. */
|
|
@@ -163,39 +165,38 @@ export interface LlmClientOptions {
|
|
|
163
165
|
* treated identically whether it surfaces in the HTTP client or a
|
|
164
166
|
* TCloud-backed judge.
|
|
165
167
|
*/
|
|
166
|
-
|
|
168
|
+
declare function isTransientLlmError(err: unknown): boolean;
|
|
167
169
|
/** Exponential backoff: 500ms, 1s, 2s, 4s, ... capped at 16s. Attempt is 0-indexed. */
|
|
168
|
-
|
|
170
|
+
declare function backoffMs(attempt: number): number;
|
|
169
171
|
/**
|
|
170
172
|
* Strip a ```json / ``` code fence if the model emitted one.
|
|
171
173
|
* Idempotent for naked JSON. Some models (claude-code via router, certain
|
|
172
174
|
* deepseek models) wrap output even under json_object.
|
|
173
175
|
*/
|
|
174
|
-
|
|
175
|
-
export declare function extractJsonPayload(raw: string): string;
|
|
176
|
+
declare function stripFencedJson(raw: string): string;
|
|
176
177
|
/**
|
|
177
178
|
* Low-level call. Returns raw content + usage + cost. Retries on transient
|
|
178
179
|
* failures; does NOT degrade schema here — callers that want graceful
|
|
179
180
|
* degrade use `callLlmJson`.
|
|
180
181
|
*/
|
|
181
|
-
|
|
182
|
+
declare function callLlm(req: LlmCallRequest, opts?: LlmClientOptions): Promise<LlmCallResult>;
|
|
182
183
|
/**
|
|
183
184
|
* Structured-output call. Returns parsed JSON plus the raw result envelope.
|
|
184
185
|
* Degrades `jsonSchema` → `jsonMode` on a 400 that names the schema param —
|
|
185
186
|
* critical for deepseek-v3/v4, kimi-k2.6, and other models that don't accept
|
|
186
187
|
* the `response_format.json_schema` shape but DO accept `json_object`.
|
|
187
188
|
*/
|
|
188
|
-
|
|
189
|
+
declare function callLlmJson<T = unknown>(req: LlmCallRequest, opts?: LlmClientOptions): Promise<{
|
|
189
190
|
value: T;
|
|
190
191
|
result: LlmCallResult;
|
|
191
192
|
}>;
|
|
192
|
-
|
|
193
|
-
|
|
193
|
+
type LlmRouteAssertionReason = 'no_explicit_base_url' | 'base_url_blocked' | 'base_url_not_allowed' | 'no_auth' | 'wrong_provider';
|
|
194
|
+
declare class LlmRouteAssertionError extends CaptureIntegrityError {
|
|
194
195
|
readonly reason: LlmRouteAssertionReason;
|
|
195
196
|
readonly baseUrl: string;
|
|
196
197
|
constructor(message: string, reason: LlmRouteAssertionReason, baseUrl: string);
|
|
197
198
|
}
|
|
198
|
-
|
|
199
|
+
interface LlmRouteRequirements {
|
|
199
200
|
/**
|
|
200
201
|
* Throw if `opts.baseUrl` is undefined, i.e. the call would fall back to
|
|
201
202
|
* `DEFAULT_BASE_URL`. Set this for evaluation runs where silently using
|
|
@@ -227,7 +228,7 @@ export interface LlmRouteRequirements {
|
|
|
227
228
|
* Throws `LlmRouteAssertionError`. Pure — no I/O — so it's safe to call
|
|
228
229
|
* from constructors and CI gates.
|
|
229
230
|
*/
|
|
230
|
-
|
|
231
|
+
declare function assertLlmRoute(opts: LlmClientOptions, req?: LlmRouteRequirements): void;
|
|
231
232
|
/**
|
|
232
233
|
* Probe whether a model is reachable. Returns latency + null error on
|
|
233
234
|
* success; `ok=false` + error message on any failure (HTTP, timeout,
|
|
@@ -239,7 +240,7 @@ export declare function assertLlmRoute(opts: LlmClientOptions, req?: LlmRouteReq
|
|
|
239
240
|
* for short prompts, so don't tighten this further. We don't validate
|
|
240
241
|
* content; HTTP 200 means reachable.
|
|
241
242
|
*/
|
|
242
|
-
|
|
243
|
+
declare function probeLlm(model: string, opts?: LlmClientOptions & {
|
|
243
244
|
timeoutMs?: number;
|
|
244
245
|
}): Promise<{
|
|
245
246
|
ok: boolean;
|
|
@@ -251,7 +252,7 @@ export declare function probeLlm(model: string, opts?: LlmClientOptions & {
|
|
|
251
252
|
* Thin wrapper around the free functions; exists for callers that want
|
|
252
253
|
* to inject a single configured instance into multiple primitives.
|
|
253
254
|
*/
|
|
254
|
-
|
|
255
|
+
declare class LlmClient {
|
|
255
256
|
private readonly opts;
|
|
256
257
|
constructor(opts?: LlmClientOptions);
|
|
257
258
|
call(req: LlmCallRequest, per?: LlmClientOptions): Promise<LlmCallResult>;
|
|
@@ -260,4 +261,5 @@ export declare class LlmClient {
|
|
|
260
261
|
result: LlmCallResult;
|
|
261
262
|
}>;
|
|
262
263
|
}
|
|
263
|
-
|
|
264
|
+
|
|
265
|
+
export { type LlmClientOptions as L, type LlmRouteRequirements as a, type LlmCallRequest as b, type LlmCallResult as c, LlmCallError as d, LlmClient as e, type LlmMessage as f, LlmRouteAssertionError as g, type LlmUsage as h, assertLlmRoute as i, backoffMs as j, callLlm as k, callLlmJson as l, isTransientLlmError as m, probeLlm as p, stripFencedJson as s };
|