@tangle-network/agent-eval 0.95.0 → 0.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/adapters/http.d.ts +17 -10
- package/dist/adapters/langchain.d.ts +14 -7
- package/dist/adapters/otel.d.ts +25 -13
- package/dist/{rl/adversarial.d.ts → adversarial-DIVcDoI_.d.ts} +7 -6
- package/dist/analyst/index.d.ts +236 -28
- package/dist/analyst/index.js +1 -1
- package/dist/{trace-analyst/analyst.d.ts → analyst-C8HHvfJp.d.ts} +14 -12
- package/dist/{contract/analyze-runs.d.ts → analyze-runs-DtT6F_6T.d.ts} +10 -9
- package/dist/authenticity/index.d.ts +16 -15
- package/dist/{baseline.d.ts → baseline-Bbid3WoO.d.ts} +44 -8
- package/dist/belief-state/index.d.ts +605 -14
- package/dist/benchmarks/index.d.ts +5 -23
- package/dist/builder-eval/index.d.ts +250 -5
- package/dist/calibration-BPmzuVPk.d.ts +101 -0
- package/dist/campaign/index.d.ts +1452 -38
- package/dist/{chunk-AQ5WQAIV.js → chunk-3NHEO6ZC.js} +2 -2
- package/dist/chunk-3NHEO6ZC.js.map +1 -0
- package/dist/cli.d.ts +0 -2
- package/dist/{contract/intake/code-agent-session.d.ts → code-agent-session-CPHRCb4-.d.ts} +17 -15
- package/dist/contract/index.d.ts +824 -107
- package/dist/contract/index.js +145 -1
- package/dist/contract/index.js.map +1 -1
- package/dist/control-Doncu-B_.d.ts +259 -0
- package/dist/{control-runtime.d.ts → control-runtime-Acf9CGhw.d.ts} +26 -23
- package/dist/control.d.ts +10 -11
- package/dist/corpus-D4YW9UoJ.d.ts +560 -0
- package/dist/{cost-ledger.d.ts → cost-ledger-DuSqlw5B.d.ts} +11 -10
- package/dist/{counterfactual.d.ts → counterfactual-DlOz8PBx.d.ts} +13 -12
- package/dist/{dataset.d.ts → dataset-BbGkaN2I.d.ts} +14 -11
- package/dist/{analyst/registry.d.ts → default-registry-GyE8X5SP.d.ts} +37 -8
- package/dist/diagnose.d.ts +252 -1
- package/dist/{trace/emitter.d.ts → emitter-C2rqGH_l.d.ts} +12 -9
- package/dist/{errors.d.ts → errors-CzMUYo7b.d.ts} +11 -10
- package/dist/failure-cluster-DH9Flgcf.d.ts +76 -0
- package/dist/{feedback-trajectory.d.ts → feedback-trajectory-BxY0cKfs.d.ts} +38 -36
- package/dist/fuzz.d.ts +547 -1
- package/dist/gepa-Dprxvz8r.d.ts +414 -0
- package/dist/governance/index.d.ts +135 -5
- package/dist/harness-optimizer-mOl9XX_O.d.ts +106 -0
- package/dist/hosted/index.d.ts +239 -10
- package/dist/index-_Y4oNOOb.d.ts +159 -0
- package/dist/index.d.ts +5660 -277
- package/dist/index.js +1 -1
- package/dist/{contract/insight-report.d.ts → insight-report-BnRjTibG.d.ts} +19 -16
- package/dist/{trace/integrity.d.ts → integrity-D2t12mMw.d.ts} +14 -11
- package/dist/{judge-calibration.d.ts → judge-calibration-0p2QcWNE.d.ts} +17 -16
- package/dist/{analyst/kind-factory.d.ts → kind-factory-X3eDYbKn.d.ts} +61 -10
- package/dist/knowledge/index.d.ts +103 -3
- package/dist/{llm-client.d.ts → llm-client-Bj7g0rqu.d.ts} +23 -21
- package/dist/matrix/index.d.ts +30 -12
- package/dist/meta-eval/index.d.ts +182 -6
- package/dist/{multi-layer-verifier.d.ts → multi-layer-verifier-DUZXrPDA.d.ts} +15 -12
- package/dist/multishot/index.d.ts +290 -7
- package/dist/{rl/off-policy.d.ts → off-policy-DiwuKKg7.d.ts} +9 -8
- package/dist/openapi.json +1 -1
- package/dist/{meta-eval/outcome-store.d.ts → outcome-store-rnXLEqSn.d.ts} +8 -7
- package/dist/{pareto.d.ts → pareto-E-pembql.d.ts} +10 -9
- package/dist/perf/index.d.ts +119 -13
- package/dist/pipelines/index.d.ts +173 -8
- package/dist/pre-registration-nfUdc9EQ.d.ts +483 -0
- package/dist/prm/index.d.ts +104 -5
- package/dist/provenance-CIxfBnkl.d.ts +426 -0
- package/dist/query-B7GGjRox.d.ts +32 -0
- package/dist/{trace/raw-provider-sink.d.ts → raw-provider-sink-C46HDghv.d.ts} +14 -13
- package/dist/{red-team.d.ts → red-team-BWdoyleI.d.ts} +16 -14
- package/dist/{trace/redact.d.ts → redact-B40YG2M_.d.ts} +8 -7
- package/dist/release-report-pidWUMZ2.d.ts +233 -0
- package/dist/reporting.d.ts +16 -15
- package/dist/{eval-campaign.d.ts → researcher-Jr8ME1dZ.d.ts} +156 -34
- package/dist/rl.d.ts +1193 -1
- package/dist/{prm/rubric.d.ts → rubric-Cc6UHvUb.d.ts} +13 -10
- package/dist/{meta-eval/rubric-predictive-validity.d.ts → rubric-predictive-validity-C2hDKM8Z.d.ts} +11 -8
- package/dist/run-critic-CmMf05uV.d.ts +56 -0
- package/dist/{run-record.d.ts → run-record-CP2ObebC.d.ts} +117 -15
- package/dist/runtime-trajectory-BOUUjI0y.d.ts +49 -0
- package/dist/{trace/schema.d.ts → schema-m0gsnbt3.d.ts} +30 -29
- package/dist/semantic-concept-judge-DSBB2Cfp.d.ts +624 -0
- package/dist/{sequential.d.ts → sequential-5iSVfzl2.d.ts} +10 -9
- package/dist/{series-convergence.d.ts → series-convergence-D5OWMBg6.d.ts} +5 -4
- package/dist/sink-fetch-B1Yg4Til.d.ts +101 -0
- package/dist/{statistics.d.ts → statistics-CCJpTGOS.d.ts} +48 -46
- package/dist/{trace/store.d.ts → store-BcFXE6LG.d.ts} +12 -21
- package/dist/{trace-analyst/types.d.ts → store-C1YxJDEK.d.ts} +74 -18
- package/dist/storyboard/index.d.ts +81 -16
- package/dist/{summary-report.d.ts → summary-report-CInXwsza.d.ts} +160 -23
- package/dist/telemetry/{sink-file.d.ts → file.d.ts} +7 -5
- package/dist/telemetry/index.d.ts +35 -17
- package/dist/{sandbox-harness.d.ts → test-graded-scenario-DeODGLra.d.ts} +60 -15
- package/dist/{locked-jsonl-appender.d.ts → testing-C21CHsq2.d.ts} +4 -3
- package/dist/testing.d.ts +1 -5
- package/dist/traces.d.ts +976 -4
- package/dist/{trajectory.d.ts → trajectory-2TkpSEVh.d.ts} +9 -8
- package/dist/{analyst/types.d.ts → types-B5x54y6n.d.ts} +112 -19
- package/dist/{campaign/types.d.ts → types-BMahhhio.d.ts} +47 -44
- package/dist/{matrix/types.d.ts → types-BUxNaJ8c.d.ts} +11 -9
- package/dist/{types.d.ts → types-C7DGg5ex.d.ts} +33 -31
- package/dist/{verdict.d.ts → verdict-C9MlYujm.d.ts} +3 -2
- package/dist/wire/index.d.ts +570 -13
- package/dist/workflow/index.d.ts +496 -22
- package/package.json +2 -2
- package/dist/action-policy.d.ts +0 -24
- package/dist/action-policy.d.ts.map +0 -1
- package/dist/action-policy.test.d.ts +0 -2
- package/dist/action-policy.test.d.ts.map +0 -1
- package/dist/active-learning.d.ts +0 -41
- package/dist/active-learning.d.ts.map +0 -1
- package/dist/adapters/http.d.ts.map +0 -1
- package/dist/adapters/langchain.d.ts.map +0 -1
- package/dist/adapters/otel.d.ts.map +0 -1
- package/dist/agent-profile-cell.d.ts +0 -101
- package/dist/agent-profile-cell.d.ts.map +0 -1
- package/dist/agent-profile.d.ts +0 -27
- package/dist/agent-profile.d.ts.map +0 -1
- package/dist/agent-profile.test.d.ts +0 -2
- package/dist/agent-profile.test.d.ts.map +0 -1
- package/dist/analyst/adapters.d.ts +0 -62
- package/dist/analyst/adapters.d.ts.map +0 -1
- package/dist/analyst/analyst.test.d.ts +0 -2
- package/dist/analyst/analyst.test.d.ts.map +0 -1
- package/dist/analyst/ax-service.d.ts +0 -27
- package/dist/analyst/ax-service.d.ts.map +0 -1
- package/dist/analyst/behavioral-analyst.d.ts +0 -28
- package/dist/analyst/behavioral-analyst.d.ts.map +0 -1
- package/dist/analyst/chat-client.d.ts +0 -91
- package/dist/analyst/chat-client.d.ts.map +0 -1
- package/dist/analyst/default-registry.d.ts +0 -27
- package/dist/analyst/default-registry.d.ts.map +0 -1
- package/dist/analyst/default-registry.test.d.ts +0 -2
- package/dist/analyst/default-registry.test.d.ts.map +0 -1
- package/dist/analyst/finding-signature.d.ts +0 -48
- package/dist/analyst/finding-signature.d.ts.map +0 -1
- package/dist/analyst/finding-subject.d.ts +0 -146
- package/dist/analyst/finding-subject.d.ts.map +0 -1
- package/dist/analyst/finding-subject.test.d.ts +0 -2
- package/dist/analyst/finding-subject.test.d.ts.map +0 -1
- package/dist/analyst/findings-store.d.ts +0 -75
- package/dist/analyst/findings-store.d.ts.map +0 -1
- package/dist/analyst/index.d.ts.map +0 -1
- package/dist/analyst/kind-factory.d.ts.map +0 -1
- package/dist/analyst/kinds/failure-mode.d.ts +0 -19
- package/dist/analyst/kinds/failure-mode.d.ts.map +0 -1
- package/dist/analyst/kinds/improvement.d.ts +0 -23
- package/dist/analyst/kinds/improvement.d.ts.map +0 -1
- package/dist/analyst/kinds/index.d.ts +0 -22
- package/dist/analyst/kinds/index.d.ts.map +0 -1
- package/dist/analyst/kinds/kinds.test.d.ts +0 -2
- package/dist/analyst/kinds/kinds.test.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-gap.d.ts +0 -28
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +0 -22
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.d.ts +0 -84
- package/dist/analyst/kinds/skill-usage.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.test.d.ts +0 -2
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.d.ts +0 -26
- package/dist/analyst/parse-tolerant.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.test.d.ts +0 -2
- package/dist/analyst/parse-tolerant.test.d.ts.map +0 -1
- package/dist/analyst/registry.budget.test.d.ts +0 -2
- package/dist/analyst/registry.budget.test.d.ts.map +0 -1
- package/dist/analyst/registry.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.d.ts +0 -35
- package/dist/analyst/steer-firewall.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.test.d.ts +0 -2
- package/dist/analyst/steer-firewall.test.d.ts.map +0 -1
- package/dist/analyst/structure-findings.d.ts +0 -37
- package/dist/analyst/structure-findings.d.ts.map +0 -1
- package/dist/analyst/structure-findings.test.d.ts +0 -2
- package/dist/analyst/structure-findings.test.d.ts.map +0 -1
- package/dist/analyst/tool-groups.d.ts +0 -34
- package/dist/analyst/tool-groups.d.ts.map +0 -1
- package/dist/analyst/types.d.ts.map +0 -1
- package/dist/anti-slop.d.ts +0 -59
- package/dist/anti-slop.d.ts.map +0 -1
- package/dist/artifact-validator.d.ts +0 -74
- package/dist/artifact-validator.d.ts.map +0 -1
- package/dist/attestation.d.ts +0 -63
- package/dist/attestation.d.ts.map +0 -1
- package/dist/attestation.test.d.ts +0 -2
- package/dist/attestation.test.d.ts.map +0 -1
- package/dist/authenticity/index.d.ts.map +0 -1
- package/dist/authenticity/index.test.d.ts +0 -2
- package/dist/authenticity/index.test.d.ts.map +0 -1
- package/dist/auto-pr.d.ts +0 -120
- package/dist/auto-pr.d.ts.map +0 -1
- package/dist/baseline.d.ts.map +0 -1
- package/dist/behavior-dsl.d.ts +0 -73
- package/dist/behavior-dsl.d.ts.map +0 -1
- package/dist/belief-state/calibration.d.ts +0 -10
- package/dist/belief-state/calibration.d.ts.map +0 -1
- package/dist/belief-state/calibration.test.d.ts +0 -2
- package/dist/belief-state/calibration.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.d.ts +0 -66
- package/dist/belief-state/code-agent-corpus.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.test.d.ts +0 -2
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.d.ts +0 -22
- package/dist/belief-state/code-agent-evidence.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.test.d.ts +0 -2
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/extract.d.ts +0 -7
- package/dist/belief-state/extract.d.ts.map +0 -1
- package/dist/belief-state/extract.test.d.ts +0 -2
- package/dist/belief-state/extract.test.d.ts.map +0 -1
- package/dist/belief-state/index.d.ts.map +0 -1
- package/dist/belief-state/ope.d.ts +0 -17
- package/dist/belief-state/ope.d.ts.map +0 -1
- package/dist/belief-state/ope.test.d.ts +0 -2
- package/dist/belief-state/ope.test.d.ts.map +0 -1
- package/dist/belief-state/phase0-measurement.d.ts +0 -55
- package/dist/belief-state/phase0-measurement.d.ts.map +0 -1
- package/dist/belief-state/report.d.ts +0 -17
- package/dist/belief-state/report.d.ts.map +0 -1
- package/dist/belief-state/report.test.d.ts +0 -2
- package/dist/belief-state/report.test.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.d.ts +0 -23
- package/dist/belief-state/research-evidence.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.test.d.ts +0 -2
- package/dist/belief-state/research-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +0 -32
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.d.ts +0 -87
- package/dist/belief-state/runtime-hooks.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.test.d.ts +0 -2
- package/dist/belief-state/runtime-hooks.test.d.ts.map +0 -1
- package/dist/belief-state/selective.d.ts +0 -15
- package/dist/belief-state/selective.d.ts.map +0 -1
- package/dist/belief-state/selective.test.d.ts +0 -2
- package/dist/belief-state/selective.test.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.d.ts +0 -80
- package/dist/belief-state/shadow-probe.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.test.d.ts +0 -2
- package/dist/belief-state/shadow-probe.test.d.ts.map +0 -1
- package/dist/belief-state/types.d.ts +0 -195
- package/dist/belief-state/types.d.ts.map +0 -1
- package/dist/belief-state/types.test.d.ts +0 -2
- package/dist/belief-state/types.test.d.ts.map +0 -1
- package/dist/benchmark.d.ts +0 -14
- package/dist/benchmark.d.ts.map +0 -1
- package/dist/benchmarks/index.d.ts.map +0 -1
- package/dist/benchmarks/routing/dataset.d.ts +0 -34
- package/dist/benchmarks/routing/dataset.d.ts.map +0 -1
- package/dist/benchmarks/routing/index.d.ts +0 -34
- package/dist/benchmarks/routing/index.d.ts.map +0 -1
- package/dist/benchmarks/types.d.ts +0 -49
- package/dist/benchmarks/types.d.ts.map +0 -1
- package/dist/bisector.d.ts +0 -81
- package/dist/bisector.d.ts.map +0 -1
- package/dist/budget-guard.d.ts +0 -31
- package/dist/budget-guard.d.ts.map +0 -1
- package/dist/builder-eval/builder-session.d.ts +0 -111
- package/dist/builder-eval/builder-session.d.ts.map +0 -1
- package/dist/builder-eval/correlation.d.ts +0 -32
- package/dist/builder-eval/correlation.d.ts.map +0 -1
- package/dist/builder-eval/index.d.ts.map +0 -1
- package/dist/builder-eval/project-registry.d.ts +0 -51
- package/dist/builder-eval/project-registry.d.ts.map +0 -1
- package/dist/builder-eval/three-layer-eval.d.ts +0 -55
- package/dist/builder-eval/three-layer-eval.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.d.ts +0 -108
- package/dist/campaign/analyst-surface.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.test.d.ts +0 -2
- package/dist/campaign/analyst-surface.test.d.ts.map +0 -1
- package/dist/campaign/auto-pr.d.ts +0 -46
- package/dist/campaign/auto-pr.d.ts.map +0 -1
- package/dist/campaign/distillation/agreement-judge.d.ts +0 -69
- package/dist/campaign/distillation/agreement-judge.d.ts.map +0 -1
- package/dist/campaign/distillation/cli.d.ts +0 -35
- package/dist/campaign/distillation/cli.d.ts.map +0 -1
- package/dist/campaign/distillation/distillation.test.d.ts +0 -2
- package/dist/campaign/distillation/distillation.test.d.ts.map +0 -1
- package/dist/campaign/distillation/gold-scenarios.d.ts +0 -54
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +0 -1
- package/dist/campaign/distillation/run-distillation.d.ts +0 -119
- package/dist/campaign/distillation/run-distillation.d.ts.map +0 -1
- package/dist/campaign/gates/compose.d.ts +0 -12
- package/dist/campaign/gates/compose.d.ts.map +0 -1
- package/dist/campaign/gates/default-production-gate.d.ts +0 -58
- package/dist/campaign/gates/default-production-gate.d.ts.map +0 -1
- package/dist/campaign/gates/heldout-gate.d.ts +0 -12
- package/dist/campaign/gates/heldout-gate.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.d.ts +0 -125
- package/dist/campaign/gates/promotion-policy.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.test.d.ts +0 -2
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.d.ts +0 -146
- package/dist/campaign/gates/sequential.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.test.d.ts +0 -2
- package/dist/campaign/gates/sequential.test.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.d.ts +0 -99
- package/dist/campaign/gates/statistical-heldout.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.test.d.ts +0 -2
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +0 -1
- package/dist/campaign/index.d.ts.map +0 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +0 -59
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +0 -1
- package/dist/campaign/presets/compare-proposers.d.ts +0 -146
- package/dist/campaign/presets/compare-proposers.d.ts.map +0 -1
- package/dist/campaign/presets/playback.d.ts +0 -120
- package/dist/campaign/presets/playback.d.ts.map +0 -1
- package/dist/campaign/presets/playback.test.d.ts +0 -2
- package/dist/campaign/presets/playback.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-eval.d.ts +0 -14
- package/dist/campaign/presets/run-eval.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.d.ts +0 -63
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +0 -2
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-optimization.d.ts +0 -92
- package/dist/campaign/presets/run-optimization.d.ts.map +0 -1
- package/dist/campaign/presets/run-profile-matrix.d.ts +0 -151
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +0 -1
- package/dist/campaign/presets/run-skill-opt.d.ts +0 -96
- package/dist/campaign/presets/run-skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/_findings-text.d.ts +0 -22
- package/dist/campaign/proposers/_findings-text.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.d.ts +0 -33
- package/dist/campaign/proposers/ace.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.test.d.ts +0 -2
- package/dist/campaign/proposers/ace.test.d.ts.map +0 -1
- package/dist/campaign/proposers/analysis-edit.d.ts +0 -32
- package/dist/campaign/proposers/analysis-edit.d.ts.map +0 -1
- package/dist/campaign/proposers/evolutionary.d.ts +0 -20
- package/dist/campaign/proposers/evolutionary.d.ts.map +0 -1
- package/dist/campaign/proposers/fapo.d.ts +0 -120
- package/dist/campaign/proposers/fapo.d.ts.map +0 -1
- package/dist/campaign/proposers/gepa.d.ts +0 -86
- package/dist/campaign/proposers/gepa.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.d.ts +0 -44
- package/dist/campaign/proposers/halo.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.test.d.ts +0 -2
- package/dist/campaign/proposers/halo.test.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.d.ts +0 -47
- package/dist/campaign/proposers/memory.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.test.d.ts +0 -2
- package/dist/campaign/proposers/memory.test.d.ts.map +0 -1
- package/dist/campaign/proposers/skill-opt.d.ts +0 -88
- package/dist/campaign/proposers/skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.d.ts +0 -48
- package/dist/campaign/proposers/trace-analyst.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.test.d.ts +0 -2
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +0 -1
- package/dist/campaign/provenance.d.ts +0 -185
- package/dist/campaign/provenance.d.ts.map +0 -1
- package/dist/campaign/run-campaign.d.ts +0 -90
- package/dist/campaign/run-campaign.d.ts.map +0 -1
- package/dist/campaign/score-utils.d.ts +0 -26
- package/dist/campaign/score-utils.d.ts.map +0 -1
- package/dist/campaign/skill-patch.d.ts +0 -62
- package/dist/campaign/skill-patch.d.ts.map +0 -1
- package/dist/campaign/storage.d.ts +0 -38
- package/dist/campaign/storage.d.ts.map +0 -1
- package/dist/campaign/types.d.ts.map +0 -1
- package/dist/campaign/worktree/index.d.ts +0 -53
- package/dist/campaign/worktree/index.d.ts.map +0 -1
- package/dist/canary.d.ts +0 -101
- package/dist/canary.d.ts.map +0 -1
- package/dist/causal-attribution.d.ts +0 -45
- package/dist/causal-attribution.d.ts.map +0 -1
- package/dist/chunk-AQ5WQAIV.js.map +0 -1
- package/dist/ci-gate.d.ts +0 -44
- package/dist/ci-gate.d.ts.map +0 -1
- package/dist/cli.d.ts.map +0 -1
- package/dist/client.d.ts +0 -77
- package/dist/client.d.ts.map +0 -1
- package/dist/client.test.d.ts +0 -2
- package/dist/client.test.d.ts.map +0 -1
- package/dist/command-runner.d.ts +0 -74
- package/dist/command-runner.d.ts.map +0 -1
- package/dist/command-runner.test.d.ts +0 -2
- package/dist/command-runner.test.d.ts.map +0 -1
- package/dist/completion-verifier.d.ts +0 -147
- package/dist/completion-verifier.d.ts.map +0 -1
- package/dist/completion-verifier.test.d.ts +0 -9
- package/dist/completion-verifier.test.d.ts.map +0 -1
- package/dist/concurrency.d.ts +0 -23
- package/dist/concurrency.d.ts.map +0 -1
- package/dist/contamination-guard.d.ts +0 -81
- package/dist/contamination-guard.d.ts.map +0 -1
- package/dist/contract/analyze-runs.d.ts.map +0 -1
- package/dist/contract/define-agent-eval.d.ts +0 -52
- package/dist/contract/define-agent-eval.d.ts.map +0 -1
- package/dist/contract/diff.d.ts +0 -114
- package/dist/contract/diff.d.ts.map +0 -1
- package/dist/contract/index.d.ts.map +0 -1
- package/dist/contract/insight-report.d.ts.map +0 -1
- package/dist/contract/insight-types-fwd.d.ts +0 -7
- package/dist/contract/insight-types-fwd.d.ts.map +0 -1
- package/dist/contract/intake/agent-trace.d.ts +0 -97
- package/dist/contract/intake/agent-trace.d.ts.map +0 -1
- package/dist/contract/intake/code-agent-session.d.ts.map +0 -1
- package/dist/contract/intake/feedback-table.d.ts +0 -87
- package/dist/contract/intake/feedback-table.d.ts.map +0 -1
- package/dist/contract/intake/index.d.ts +0 -22
- package/dist/contract/intake/index.d.ts.map +0 -1
- package/dist/contract/intake/otel-spans.d.ts +0 -33
- package/dist/contract/intake/otel-spans.d.ts.map +0 -1
- package/dist/contract/self-improve.d.ts +0 -284
- package/dist/contract/self-improve.d.ts.map +0 -1
- package/dist/control-runtime.d.ts.map +0 -1
- package/dist/control-runtime.test.d.ts +0 -2
- package/dist/control-runtime.test.d.ts.map +0 -1
- package/dist/control.d.ts.map +0 -1
- package/dist/convergence.d.ts +0 -29
- package/dist/convergence.d.ts.map +0 -1
- package/dist/cost-ledger.d.ts.map +0 -1
- package/dist/cost-ledger.test.d.ts +0 -2
- package/dist/cost-ledger.test.d.ts.map +0 -1
- package/dist/cost-report.d.ts +0 -43
- package/dist/cost-report.d.ts.map +0 -1
- package/dist/cost-report.test.d.ts +0 -2
- package/dist/cost-report.test.d.ts.map +0 -1
- package/dist/cost-tracker.d.ts +0 -76
- package/dist/cost-tracker.d.ts.map +0 -1
- package/dist/counterfactual.d.ts.map +0 -1
- package/dist/cross-trace-diff.d.ts +0 -56
- package/dist/cross-trace-diff.d.ts.map +0 -1
- package/dist/dataset.d.ts.map +0 -1
- package/dist/deploy-gate-layer.d.ts +0 -125
- package/dist/deploy-gate-layer.d.ts.map +0 -1
- package/dist/deploy-gate-layer.test.d.ts +0 -2
- package/dist/deploy-gate-layer.test.d.ts.map +0 -1
- package/dist/description-length-gate.d.ts +0 -119
- package/dist/description-length-gate.d.ts.map +0 -1
- package/dist/detectors/edge.test.d.ts +0 -2
- package/dist/detectors/edge.test.d.ts.map +0 -1
- package/dist/detectors/index.d.ts +0 -81
- package/dist/detectors/index.d.ts.map +0 -1
- package/dist/detectors/index.test.d.ts +0 -2
- package/dist/detectors/index.test.d.ts.map +0 -1
- package/dist/diagnose/causal-sweep.d.ts +0 -100
- package/dist/diagnose/causal-sweep.d.ts.map +0 -1
- package/dist/diagnose/index.d.ts +0 -36
- package/dist/diagnose/index.d.ts.map +0 -1
- package/dist/diagnose/remediation.d.ts +0 -68
- package/dist/diagnose/remediation.d.ts.map +0 -1
- package/dist/diagnose/repair.d.ts +0 -77
- package/dist/diagnose/repair.d.ts.map +0 -1
- package/dist/discover-personas.d.ts +0 -35
- package/dist/discover-personas.d.ts.map +0 -1
- package/dist/driver.d.ts +0 -95
- package/dist/driver.d.ts.map +0 -1
- package/dist/driver.test.d.ts +0 -8
- package/dist/driver.test.d.ts.map +0 -1
- package/dist/dual-agent-bench.d.ts +0 -81
- package/dist/dual-agent-bench.d.ts.map +0 -1
- package/dist/error-count-extractor.d.ts +0 -47
- package/dist/error-count-extractor.d.ts.map +0 -1
- package/dist/error-count-extractor.test.d.ts +0 -2
- package/dist/error-count-extractor.test.d.ts.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/eval-campaign.d.ts.map +0 -1
- package/dist/eval-campaign.test.d.ts +0 -2
- package/dist/eval-campaign.test.d.ts.map +0 -1
- package/dist/eval-tools.d.ts +0 -55
- package/dist/eval-tools.d.ts.map +0 -1
- package/dist/eval-trace-store.d.ts +0 -107
- package/dist/eval-trace-store.d.ts.map +0 -1
- package/dist/eval-trace-store.test.d.ts +0 -2
- package/dist/eval-trace-store.test.d.ts.map +0 -1
- package/dist/executor.d.ts +0 -38
- package/dist/executor.d.ts.map +0 -1
- package/dist/executor.test.d.ts +0 -10
- package/dist/executor.test.d.ts.map +0 -1
- package/dist/experiment-tracker.d.ts +0 -178
- package/dist/experiment-tracker.d.ts.map +0 -1
- package/dist/experiment-tracker.test.d.ts +0 -2
- package/dist/experiment-tracker.test.d.ts.map +0 -1
- package/dist/failure-taxonomy.d.ts +0 -38
- package/dist/failure-taxonomy.d.ts.map +0 -1
- package/dist/feedback-trajectory.d.ts.map +0 -1
- package/dist/feedback-trajectory.test.d.ts +0 -2
- package/dist/feedback-trajectory.test.d.ts.map +0 -1
- package/dist/flow-layer.d.ts +0 -90
- package/dist/flow-layer.d.ts.map +0 -1
- package/dist/flow-layer.test.d.ts +0 -2
- package/dist/flow-layer.test.d.ts.map +0 -1
- package/dist/fuzz/capsule.d.ts +0 -46
- package/dist/fuzz/capsule.d.ts.map +0 -1
- package/dist/fuzz/cube.d.ts +0 -36
- package/dist/fuzz/cube.d.ts.map +0 -1
- package/dist/fuzz/explorer-cost.test.d.ts +0 -2
- package/dist/fuzz/explorer-cost.test.d.ts.map +0 -1
- package/dist/fuzz/explorer.d.ts +0 -64
- package/dist/fuzz/explorer.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.d.ts +0 -16
- package/dist/fuzz/fuzz-agent.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.test.d.ts +0 -2
- package/dist/fuzz/fuzz-agent.test.d.ts.map +0 -1
- package/dist/fuzz/gates.d.ts +0 -33
- package/dist/fuzz/gates.d.ts.map +0 -1
- package/dist/fuzz/index.d.ts +0 -26
- package/dist/fuzz/index.d.ts.map +0 -1
- package/dist/fuzz/policies.d.ts +0 -28
- package/dist/fuzz/policies.d.ts.map +0 -1
- package/dist/fuzz/tools.d.ts +0 -20
- package/dist/fuzz/tools.d.ts.map +0 -1
- package/dist/fuzz/types.d.ts +0 -307
- package/dist/fuzz/types.d.ts.map +0 -1
- package/dist/golden-matcher.d.ts +0 -71
- package/dist/golden-matcher.d.ts.map +0 -1
- package/dist/governance/eu-ai-act.d.ts +0 -37
- package/dist/governance/eu-ai-act.d.ts.map +0 -1
- package/dist/governance/index.d.ts.map +0 -1
- package/dist/governance/nist-ai-rmf.d.ts +0 -15
- package/dist/governance/nist-ai-rmf.d.ts.map +0 -1
- package/dist/governance/soc2.d.ts +0 -12
- package/dist/governance/soc2.d.ts.map +0 -1
- package/dist/governance/types.d.ts +0 -66
- package/dist/governance/types.d.ts.map +0 -1
- package/dist/harness-optimizer.d.ts +0 -82
- package/dist/harness-optimizer.d.ts.map +0 -1
- package/dist/held-out-gate.d.ts +0 -135
- package/dist/held-out-gate.d.ts.map +0 -1
- package/dist/hosted/client.d.ts +0 -73
- package/dist/hosted/client.d.ts.map +0 -1
- package/dist/hosted/from-env.test.d.ts +0 -8
- package/dist/hosted/from-env.test.d.ts.map +0 -1
- package/dist/hosted/index.d.ts.map +0 -1
- package/dist/hosted/types.d.ts +0 -159
- package/dist/hosted/types.d.ts.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/integrity/backend-integrity.d.ts +0 -71
- package/dist/integrity/backend-integrity.d.ts.map +0 -1
- package/dist/integrity/preflight.d.ts +0 -72
- package/dist/integrity/preflight.d.ts.map +0 -1
- package/dist/integrity/preflight.test.d.ts +0 -2
- package/dist/integrity/preflight.test.d.ts.map +0 -1
- package/dist/integrity/single-backend.d.ts +0 -67
- package/dist/integrity/single-backend.d.ts.map +0 -1
- package/dist/intent-match-judge.d.ts +0 -69
- package/dist/intent-match-judge.d.ts.map +0 -1
- package/dist/intent-match-judge.test.d.ts +0 -2
- package/dist/intent-match-judge.test.d.ts.map +0 -1
- package/dist/judge-calibration.d.ts.map +0 -1
- package/dist/judge-ensemble.d.ts +0 -66
- package/dist/judge-ensemble.d.ts.map +0 -1
- package/dist/judge-ensemble.test.d.ts +0 -8
- package/dist/judge-ensemble.test.d.ts.map +0 -1
- package/dist/judge-families.d.ts +0 -38
- package/dist/judge-families.d.ts.map +0 -1
- package/dist/judge-panel.d.ts +0 -65
- package/dist/judge-panel.d.ts.map +0 -1
- package/dist/judge-retry.d.ts +0 -70
- package/dist/judge-retry.d.ts.map +0 -1
- package/dist/judge-runner.d.ts +0 -36
- package/dist/judge-runner.d.ts.map +0 -1
- package/dist/judge-runner.test.d.ts +0 -2
- package/dist/judge-runner.test.d.ts.map +0 -1
- package/dist/judges.d.ts +0 -74
- package/dist/judges.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.d.ts +0 -89
- package/dist/keyword-coverage-judge.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.test.d.ts +0 -2
- package/dist/keyword-coverage-judge.test.d.ts.map +0 -1
- package/dist/knowledge/index.d.ts.map +0 -1
- package/dist/knowledge/readiness.d.ts +0 -26
- package/dist/knowledge/readiness.d.ts.map +0 -1
- package/dist/knowledge/types.d.ts +0 -75
- package/dist/knowledge/types.d.ts.map +0 -1
- package/dist/live-proof.d.ts +0 -62
- package/dist/live-proof.d.ts.map +0 -1
- package/dist/llm-client.d.ts.map +0 -1
- package/dist/llm-client.test.d.ts +0 -2
- package/dist/llm-client.test.d.ts.map +0 -1
- package/dist/locked-jsonl-appender.d.ts.map +0 -1
- package/dist/matrix/aggregation.d.ts +0 -16
- package/dist/matrix/aggregation.d.ts.map +0 -1
- package/dist/matrix/index.d.ts.map +0 -1
- package/dist/matrix/runner.d.ts +0 -15
- package/dist/matrix/runner.d.ts.map +0 -1
- package/dist/matrix/types.d.ts.map +0 -1
- package/dist/meta-eval/calibration.d.ts +0 -47
- package/dist/meta-eval/calibration.d.ts.map +0 -1
- package/dist/meta-eval/correlation-study.d.ts +0 -53
- package/dist/meta-eval/correlation-study.d.ts.map +0 -1
- package/dist/meta-eval/index.d.ts.map +0 -1
- package/dist/meta-eval/outcome-store.d.ts.map +0 -1
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +0 -1
- package/dist/meta-eval/sentinel.d.ts +0 -169
- package/dist/meta-eval/sentinel.d.ts.map +0 -1
- package/dist/metrics.d.ts +0 -63
- package/dist/metrics.d.ts.map +0 -1
- package/dist/model-seats.d.ts +0 -71
- package/dist/model-seats.d.ts.map +0 -1
- package/dist/model-seats.test.d.ts +0 -2
- package/dist/model-seats.test.d.ts.map +0 -1
- package/dist/muffled-gate-scanner.d.ts +0 -102
- package/dist/muffled-gate-scanner.d.ts.map +0 -1
- package/dist/multi-layer-verifier.d.ts.map +0 -1
- package/dist/multi-layer-verifier.test.d.ts +0 -2
- package/dist/multi-layer-verifier.test.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.d.ts +0 -80
- package/dist/multi-toolchain-layer.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.test.d.ts +0 -2
- package/dist/multi-toolchain-layer.test.d.ts.map +0 -1
- package/dist/multishot/default-tools.d.ts +0 -34
- package/dist/multishot/default-tools.d.ts.map +0 -1
- package/dist/multishot/index.d.ts.map +0 -1
- package/dist/multishot/judges.d.ts +0 -32
- package/dist/multishot/judges.d.ts.map +0 -1
- package/dist/multishot/matrix.d.ts +0 -107
- package/dist/multishot/matrix.d.ts.map +0 -1
- package/dist/multishot/multishot.d.ts +0 -23
- package/dist/multishot/multishot.d.ts.map +0 -1
- package/dist/multishot/router.d.ts +0 -37
- package/dist/multishot/router.d.ts.map +0 -1
- package/dist/multishot/types.d.ts +0 -60
- package/dist/multishot/types.d.ts.map +0 -1
- package/dist/observability.d.ts +0 -71
- package/dist/observability.d.ts.map +0 -1
- package/dist/oracle.d.ts +0 -55
- package/dist/oracle.d.ts.map +0 -1
- package/dist/orthogonality.d.ts +0 -35
- package/dist/orthogonality.d.ts.map +0 -1
- package/dist/otel-pipeline.d.ts +0 -31
- package/dist/otel-pipeline.d.ts.map +0 -1
- package/dist/paraphrase.d.ts +0 -107
- package/dist/paraphrase.d.ts.map +0 -1
- package/dist/pareto.d.ts.map +0 -1
- package/dist/partition-held-out.d.ts +0 -70
- package/dist/partition-held-out.d.ts.map +0 -1
- package/dist/partition-held-out.test.d.ts +0 -2
- package/dist/partition-held-out.test.d.ts.map +0 -1
- package/dist/perf/index.d.ts.map +0 -1
- package/dist/perf/integrity.d.ts +0 -30
- package/dist/perf/integrity.d.ts.map +0 -1
- package/dist/perf/journey.d.ts +0 -45
- package/dist/perf/journey.d.ts.map +0 -1
- package/dist/perf/ratchet.d.ts +0 -47
- package/dist/perf/ratchet.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.d.ts +0 -31
- package/dist/pipelines/budget-breach.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.test.d.ts +0 -2
- package/dist/pipelines/budget-breach.test.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.d.ts +0 -38
- package/dist/pipelines/failure-cluster.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.test.d.ts +0 -2
- package/dist/pipelines/failure-cluster.test.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.d.ts +0 -26
- package/dist/pipelines/first-divergence.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.test.d.ts +0 -2
- package/dist/pipelines/first-divergence.test.d.ts.map +0 -1
- package/dist/pipelines/index.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.d.ts +0 -26
- package/dist/pipelines/judge-agreement.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.test.d.ts +0 -2
- package/dist/pipelines/judge-agreement.test.d.ts.map +0 -1
- package/dist/pipelines/regression.d.ts +0 -23
- package/dist/pipelines/regression.d.ts.map +0 -1
- package/dist/pipelines/regression.test.d.ts +0 -2
- package/dist/pipelines/regression.test.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.d.ts +0 -32
- package/dist/pipelines/stuck-loop.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.test.d.ts +0 -2
- package/dist/pipelines/stuck-loop.test.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.d.ts +0 -34
- package/dist/pipelines/tool-waste.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.test.d.ts +0 -2
- package/dist/pipelines/tool-waste.test.d.ts.map +0 -1
- package/dist/playbook.d.ts +0 -16
- package/dist/playbook.d.ts.map +0 -1
- package/dist/pr-review-benchmark.d.ts +0 -88
- package/dist/pr-review-benchmark.d.ts.map +0 -1
- package/dist/pr-review-benchmark.test.d.ts +0 -2
- package/dist/pr-review-benchmark.test.d.ts.map +0 -1
- package/dist/pre-registration.d.ts +0 -125
- package/dist/pre-registration.d.ts.map +0 -1
- package/dist/prm/builtin-rubrics.d.ts +0 -33
- package/dist/prm/builtin-rubrics.d.ts.map +0 -1
- package/dist/prm/index.d.ts.map +0 -1
- package/dist/prm/inference.d.ts +0 -29
- package/dist/prm/inference.d.ts.map +0 -1
- package/dist/prm/inference.test.d.ts +0 -2
- package/dist/prm/inference.test.d.ts.map +0 -1
- package/dist/prm/rubric.d.ts.map +0 -1
- package/dist/prm/training-export.d.ts +0 -38
- package/dist/prm/training-export.d.ts.map +0 -1
- package/dist/produced-state.d.ts +0 -63
- package/dist/produced-state.d.ts.map +0 -1
- package/dist/produced-state.test.d.ts +0 -8
- package/dist/produced-state.test.d.ts.map +0 -1
- package/dist/profile/baselines.d.ts +0 -37
- package/dist/profile/baselines.d.ts.map +0 -1
- package/dist/profile/index.d.ts +0 -105
- package/dist/profile/index.d.ts.map +0 -1
- package/dist/promotion-gate.d.ts +0 -93
- package/dist/promotion-gate.d.ts.map +0 -1
- package/dist/prompt-registry.d.ts +0 -41
- package/dist/prompt-registry.d.ts.map +0 -1
- package/dist/propose-review-control.d.ts +0 -49
- package/dist/propose-review-control.d.ts.map +0 -1
- package/dist/propose-review-control.test.d.ts +0 -2
- package/dist/propose-review-control.test.d.ts.map +0 -1
- package/dist/propose-review.d.ts +0 -155
- package/dist/propose-review.d.ts.map +0 -1
- package/dist/red-team.d.ts.map +0 -1
- package/dist/reference-replay-steering.d.ts +0 -11
- package/dist/reference-replay-steering.d.ts.map +0 -1
- package/dist/reference-replay.d.ts +0 -176
- package/dist/reference-replay.d.ts.map +0 -1
- package/dist/reflective-mutation.d.ts +0 -79
- package/dist/reflective-mutation.d.ts.map +0 -1
- package/dist/registry.d.ts +0 -31
- package/dist/registry.d.ts.map +0 -1
- package/dist/release-confidence.d.ts +0 -128
- package/dist/release-confidence.d.ts.map +0 -1
- package/dist/release-report.d.ts +0 -11
- package/dist/release-report.d.ts.map +0 -1
- package/dist/replay.d.ts +0 -120
- package/dist/replay.d.ts.map +0 -1
- package/dist/reporter.d.ts +0 -14
- package/dist/reporter.d.ts.map +0 -1
- package/dist/reporting.d.ts.map +0 -1
- package/dist/researcher.d.ts +0 -140
- package/dist/researcher.d.ts.map +0 -1
- package/dist/reviewer.d.ts +0 -118
- package/dist/reviewer.d.ts.map +0 -1
- package/dist/reviewer.test.d.ts +0 -2
- package/dist/reviewer.test.d.ts.map +0 -1
- package/dist/reward-model-export.d.ts +0 -60
- package/dist/reward-model-export.d.ts.map +0 -1
- package/dist/rl/active-curriculum.d.ts +0 -110
- package/dist/rl/active-curriculum.d.ts.map +0 -1
- package/dist/rl/adaptation-eval.d.ts +0 -109
- package/dist/rl/adaptation-eval.d.ts.map +0 -1
- package/dist/rl/adversarial.d.ts.map +0 -1
- package/dist/rl/compute-curves.d.ts +0 -127
- package/dist/rl/compute-curves.d.ts.map +0 -1
- package/dist/rl/contamination.d.ts +0 -117
- package/dist/rl/contamination.d.ts.map +0 -1
- package/dist/rl/corpus.d.ts +0 -55
- package/dist/rl/corpus.d.ts.map +0 -1
- package/dist/rl/corpus.test.d.ts +0 -2
- package/dist/rl/corpus.test.d.ts.map +0 -1
- package/dist/rl/dataset.d.ts +0 -102
- package/dist/rl/dataset.d.ts.map +0 -1
- package/dist/rl/dataset.test.d.ts +0 -2
- package/dist/rl/dataset.test.d.ts.map +0 -1
- package/dist/rl/exporters.d.ts +0 -141
- package/dist/rl/exporters.d.ts.map +0 -1
- package/dist/rl/index.d.ts +0 -49
- package/dist/rl/index.d.ts.map +0 -1
- package/dist/rl/off-policy.d.ts.map +0 -1
- package/dist/rl/predictive-validity-researcher.d.ts +0 -69
- package/dist/rl/predictive-validity-researcher.d.ts.map +0 -1
- package/dist/rl/preferences.d.ts +0 -141
- package/dist/rl/preferences.d.ts.map +0 -1
- package/dist/rl/process-reward.d.ts +0 -122
- package/dist/rl/process-reward.d.ts.map +0 -1
- package/dist/rl/reward-hacking.d.ts +0 -104
- package/dist/rl/reward-hacking.d.ts.map +0 -1
- package/dist/rl/rl-campaign.d.ts +0 -85
- package/dist/rl/rl-campaign.d.ts.map +0 -1
- package/dist/rl/run-record-adapters.d.ts +0 -56
- package/dist/rl/run-record-adapters.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.d.ts +0 -166
- package/dist/rl/sim-fidelity.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.test.d.ts +0 -2
- package/dist/rl/sim-fidelity.test.d.ts.map +0 -1
- package/dist/rl/tournament.d.ts +0 -115
- package/dist/rl/tournament.d.ts.map +0 -1
- package/dist/rl/verifiable-reward.d.ts +0 -124
- package/dist/rl/verifiable-reward.d.ts.map +0 -1
- package/dist/run-critic.d.ts +0 -23
- package/dist/run-critic.d.ts.map +0 -1
- package/dist/run-evidence.d.ts +0 -32
- package/dist/run-evidence.d.ts.map +0 -1
- package/dist/run-record.d.ts.map +0 -1
- package/dist/run-record.test.d.ts +0 -2
- package/dist/run-record.test.d.ts.map +0 -1
- package/dist/run-score.d.ts +0 -31
- package/dist/run-score.d.ts.map +0 -1
- package/dist/runtime-trajectory.d.ts +0 -47
- package/dist/runtime-trajectory.d.ts.map +0 -1
- package/dist/sandbox-harness.d.ts.map +0 -1
- package/dist/sandbox-harness.test.d.ts +0 -2
- package/dist/sandbox-harness.test.d.ts.map +0 -1
- package/dist/sandbox-pool.d.ts +0 -74
- package/dist/sandbox-pool.d.ts.map +0 -1
- package/dist/sandbox-pool.test.d.ts +0 -2
- package/dist/sandbox-pool.test.d.ts.map +0 -1
- package/dist/scorecard.d.ts +0 -133
- package/dist/scorecard.d.ts.map +0 -1
- package/dist/scorecard.test.d.ts +0 -2
- package/dist/scorecard.test.d.ts.map +0 -1
- package/dist/self-play.d.ts +0 -69
- package/dist/self-play.d.ts.map +0 -1
- package/dist/semantic-concept-judge.d.ts +0 -135
- package/dist/semantic-concept-judge.d.ts.map +0 -1
- package/dist/semantic-concept-judge.test.d.ts +0 -2
- package/dist/semantic-concept-judge.test.d.ts.map +0 -1
- package/dist/sequential.d.ts.map +0 -1
- package/dist/series-convergence.d.ts.map +0 -1
- package/dist/slo.d.ts +0 -48
- package/dist/slo.d.ts.map +0 -1
- package/dist/state-continuity.d.ts +0 -47
- package/dist/state-continuity.d.ts.map +0 -1
- package/dist/statistics.d.ts.map +0 -1
- package/dist/statistics.test.d.ts +0 -2
- package/dist/statistics.test.d.ts.map +0 -1
- package/dist/steering-optimizer.d.ts +0 -58
- package/dist/steering-optimizer.d.ts.map +0 -1
- package/dist/steering.d.ts +0 -24
- package/dist/steering.d.ts.map +0 -1
- package/dist/storyboard/code-edit.d.ts +0 -64
- package/dist/storyboard/code-edit.d.ts.map +0 -1
- package/dist/storyboard/code-edit.test.d.ts +0 -2
- package/dist/storyboard/code-edit.test.d.ts.map +0 -1
- package/dist/storyboard/index.d.ts.map +0 -1
- package/dist/storyboard/index.test.d.ts +0 -2
- package/dist/storyboard/index.test.d.ts.map +0 -1
- package/dist/summary-report.d.ts.map +0 -1
- package/dist/telemetry/client.d.ts +0 -35
- package/dist/telemetry/client.d.ts.map +0 -1
- package/dist/telemetry/index.d.ts.map +0 -1
- package/dist/telemetry/schema.d.ts +0 -61
- package/dist/telemetry/schema.d.ts.map +0 -1
- package/dist/telemetry/sink-fetch.d.ts +0 -39
- package/dist/telemetry/sink-fetch.d.ts.map +0 -1
- package/dist/telemetry/sink-file.d.ts.map +0 -1
- package/dist/test-graded-scenario.d.ts +0 -42
- package/dist/test-graded-scenario.d.ts.map +0 -1
- package/dist/testing.d.ts.map +0 -1
- package/dist/tool-use-metrics.d.ts +0 -35
- package/dist/tool-use-metrics.d.ts.map +0 -1
- package/dist/trace/capture-fetch.d.ts +0 -48
- package/dist/trace/capture-fetch.d.ts.map +0 -1
- package/dist/trace/capture-fetch.test.d.ts +0 -2
- package/dist/trace/capture-fetch.test.d.ts.map +0 -1
- package/dist/trace/emitter.d.ts.map +0 -1
- package/dist/trace/extract-usage.d.ts +0 -46
- package/dist/trace/extract-usage.d.ts.map +0 -1
- package/dist/trace/extract-usage.test.d.ts +0 -2
- package/dist/trace/extract-usage.test.d.ts.map +0 -1
- package/dist/trace/index.d.ts +0 -15
- package/dist/trace/index.d.ts.map +0 -1
- package/dist/trace/integrity.d.ts.map +0 -1
- package/dist/trace/otel-bridge.d.ts +0 -29
- package/dist/trace/otel-bridge.d.ts.map +0 -1
- package/dist/trace/otel-export.d.ts +0 -52
- package/dist/trace/otel-export.d.ts.map +0 -1
- package/dist/trace/otel.d.ts +0 -57
- package/dist/trace/otel.d.ts.map +0 -1
- package/dist/trace/otlp-attributes.d.ts +0 -17
- package/dist/trace/otlp-attributes.d.ts.map +0 -1
- package/dist/trace/query.d.ts +0 -29
- package/dist/trace/query.d.ts.map +0 -1
- package/dist/trace/query.test.d.ts +0 -2
- package/dist/trace/query.test.d.ts.map +0 -1
- package/dist/trace/raw-provider-sink.d.ts.map +0 -1
- package/dist/trace/redact.d.ts.map +0 -1
- package/dist/trace/schema.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.d.ts +0 -72
- package/dist/trace/store-to-otlp.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.test.d.ts +0 -2
- package/dist/trace/store-to-otlp.test.d.ts.map +0 -1
- package/dist/trace/store.d.ts.map +0 -1
- package/dist/trace/store.test.d.ts +0 -2
- package/dist/trace/store.test.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.test.d.ts +0 -2
- package/dist/trace-analyst/analyst.test.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.d.ts +0 -40
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +0 -2
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +0 -1
- package/dist/trace-analyst/hook.d.ts +0 -55
- package/dist/trace-analyst/hook.d.ts.map +0 -1
- package/dist/trace-analyst/index.d.ts +0 -18
- package/dist/trace-analyst/index.d.ts.map +0 -1
- package/dist/trace-analyst/insights.d.ts +0 -71
- package/dist/trace-analyst/insights.d.ts.map +0 -1
- package/dist/trace-analyst/insights.test.d.ts +0 -2
- package/dist/trace-analyst/insights.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-flatten.d.ts +0 -42
- package/dist/trace-analyst/otlp-flatten.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.d.ts +0 -85
- package/dist/trace-analyst/otlp-span.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-span.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.d.ts +0 -115
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/prompts.d.ts +0 -6
- package/dist/trace-analyst/prompts.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.d.ts +0 -126
- package/dist/trace-analyst/store-otlp.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.test.d.ts +0 -8
- package/dist/trace-analyst/store-otlp.test.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/store.d.ts +0 -63
- package/dist/trace-analyst/store.d.ts.map +0 -1
- package/dist/trace-analyst/tools.d.ts +0 -44
- package/dist/trace-analyst/tools.d.ts.map +0 -1
- package/dist/trace-analyst/tools.test.d.ts +0 -10
- package/dist/trace-analyst/tools.test.d.ts.map +0 -1
- package/dist/trace-analyst/types.d.ts.map +0 -1
- package/dist/trace-contracts.d.ts +0 -180
- package/dist/trace-contracts.d.ts.map +0 -1
- package/dist/traced-analyst.d.ts +0 -26
- package/dist/traced-analyst.d.ts.map +0 -1
- package/dist/traced-judges.d.ts +0 -27
- package/dist/traced-judges.d.ts.map +0 -1
- package/dist/traces.d.ts.map +0 -1
- package/dist/trajectory.d.ts.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/ui-finding.d.ts +0 -104
- package/dist/ui-finding.d.ts.map +0 -1
- package/dist/verdict-cache.d.ts +0 -78
- package/dist/verdict-cache.d.ts.map +0 -1
- package/dist/verdict-cache.test.d.ts +0 -2
- package/dist/verdict-cache.test.d.ts.map +0 -1
- package/dist/verdict.d.ts.map +0 -1
- package/dist/visual-diff.d.ts +0 -32
- package/dist/visual-diff.d.ts.map +0 -1
- package/dist/wire/handlers.d.ts +0 -54
- package/dist/wire/handlers.d.ts.map +0 -1
- package/dist/wire/index.d.ts.map +0 -1
- package/dist/wire/openapi.d.ts +0 -3
- package/dist/wire/openapi.d.ts.map +0 -1
- package/dist/wire/rpc.d.ts +0 -21
- package/dist/wire/rpc.d.ts.map +0 -1
- package/dist/wire/rubrics.d.ts +0 -34
- package/dist/wire/rubrics.d.ts.map +0 -1
- package/dist/wire/schemas.d.ts +0 -410
- package/dist/wire/schemas.d.ts.map +0 -1
- package/dist/wire/server.d.ts +0 -60
- package/dist/wire/server.d.ts.map +0 -1
- package/dist/workflow/event-schema.d.ts +0 -5
- package/dist/workflow/event-schema.d.ts.map +0 -1
- package/dist/workflow/feedback-pack.d.ts +0 -99
- package/dist/workflow/feedback-pack.d.ts.map +0 -1
- package/dist/workflow/index.d.ts.map +0 -1
- package/dist/workflow/intelligence-export.d.ts +0 -62
- package/dist/workflow/intelligence-export.d.ts.map +0 -1
- package/dist/workflow/partner-report.d.ts +0 -49
- package/dist/workflow/partner-report.d.ts.map +0 -1
- package/dist/workflow/phase-graph.d.ts +0 -43
- package/dist/workflow/phase-graph.d.ts.map +0 -1
- package/dist/workflow/promotion-gate.d.ts +0 -61
- package/dist/workflow/promotion-gate.d.ts.map +0 -1
- package/dist/workflow/run-record.d.ts +0 -12
- package/dist/workflow/run-record.d.ts.map +0 -1
- package/dist/workflow/runtime-adapter.d.ts +0 -20
- package/dist/workflow/runtime-adapter.d.ts.map +0 -1
- package/dist/workflow/sanitize.d.ts +0 -21
- package/dist/workflow/sanitize.d.ts.map +0 -1
- package/dist/workflow/schema.d.ts +0 -5
- package/dist/workflow/schema.d.ts.map +0 -1
- package/dist/workflow/summary.d.ts +0 -43
- package/dist/workflow/summary.d.ts.map +0 -1
- package/dist/workflow/trace-event-fields.d.ts +0 -6
- package/dist/workflow/trace-event-fields.d.ts.map +0 -1
- package/dist/workflow/trajectory.d.ts +0 -15
- package/dist/workflow/trajectory.d.ts.map +0 -1
- package/dist/workflow/types.d.ts +0 -68
- package/dist/workflow/types.d.ts.map +0 -1
- package/dist/workspace-inspector.d.ts +0 -67
- package/dist/workspace-inspector.d.ts.map +0 -1
- package/dist/wrangler-deploy-runner.test.d.ts +0 -2
- package/dist/wrangler-deploy-runner.test.d.ts.map +0 -1
|
@@ -13,12 +13,12 @@
|
|
|
13
13
|
* `ViewTraceOversized`.
|
|
14
14
|
*/
|
|
15
15
|
/** OTLP span kind (subset we actually use). */
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
type TraceAnalystSpanKind = 'AGENT' | 'LLM' | 'TOOL' | 'CHAIN' | 'GUARDRAIL' | 'SPAN' | 'UNKNOWN';
|
|
17
|
+
type TraceAnalystSpanStatus = 'OK' | 'ERROR' | 'UNSET';
|
|
18
18
|
/** Subset of OTLP span fields the analyst exposes to the agent. The
|
|
19
19
|
* store's job is to project upstream's full span shape down to this
|
|
20
20
|
* view — the analyst never sees vendor extensions directly. */
|
|
21
|
-
|
|
21
|
+
interface TraceAnalystSpan {
|
|
22
22
|
trace_id: string;
|
|
23
23
|
span_id: string;
|
|
24
24
|
parent_span_id: string | null;
|
|
@@ -37,7 +37,7 @@ export interface TraceAnalystSpan {
|
|
|
37
37
|
* callers must respect the per-attribute byte cap. */
|
|
38
38
|
attributes: Record<string, unknown>;
|
|
39
39
|
}
|
|
40
|
-
|
|
40
|
+
interface TraceAnalystTraceSummary {
|
|
41
41
|
trace_id: string;
|
|
42
42
|
service_name: string | null;
|
|
43
43
|
agent_name: string | null;
|
|
@@ -50,7 +50,7 @@ export interface TraceAnalystTraceSummary {
|
|
|
50
50
|
models: string[];
|
|
51
51
|
tools: string[];
|
|
52
52
|
}
|
|
53
|
-
|
|
53
|
+
interface TraceAnalystFilters {
|
|
54
54
|
/** Restrict to traces that contain at least one error span. */
|
|
55
55
|
has_errors?: boolean;
|
|
56
56
|
/** Match if any span's `service.name` is in this list. */
|
|
@@ -74,7 +74,7 @@ export interface TraceAnalystFilters {
|
|
|
74
74
|
* paths, durations) out of the span `status_message` so semantically identical
|
|
75
75
|
* failures collapse into one cluster. An analyst that accounts for every
|
|
76
76
|
* cluster has, by construction, covered every distinct failure mode. */
|
|
77
|
-
|
|
77
|
+
interface ErrorCluster {
|
|
78
78
|
/** Normalized status_message — the cluster key. */
|
|
79
79
|
signature: string;
|
|
80
80
|
/** A verbatim, un-normalized exemplar message (for exact-string citation). */
|
|
@@ -92,7 +92,7 @@ export interface ErrorCluster {
|
|
|
92
92
|
/** Real span ids carrying this signature (capped). */
|
|
93
93
|
exemplar_span_ids: string[];
|
|
94
94
|
}
|
|
95
|
-
|
|
95
|
+
interface DatasetOverview {
|
|
96
96
|
total_traces: number;
|
|
97
97
|
raw_jsonl_bytes: number;
|
|
98
98
|
services: string[];
|
|
@@ -115,7 +115,7 @@ export interface DatasetOverview {
|
|
|
115
115
|
latest: string;
|
|
116
116
|
} | null;
|
|
117
117
|
}
|
|
118
|
-
|
|
118
|
+
interface QueryTracesPage {
|
|
119
119
|
traces: TraceAnalystTraceSummary[];
|
|
120
120
|
total: number;
|
|
121
121
|
has_more: boolean;
|
|
@@ -123,12 +123,12 @@ export interface QueryTracesPage {
|
|
|
123
123
|
/** Full-trace view. When the response would exceed the per-call byte
|
|
124
124
|
* budget, `oversized` is populated INSTEAD of `spans` so the agent
|
|
125
125
|
* knows to switch to `searchTrace` / `viewSpans`. */
|
|
126
|
-
|
|
126
|
+
interface ViewTraceResult {
|
|
127
127
|
trace_id: string;
|
|
128
128
|
spans?: TraceAnalystSpan[];
|
|
129
129
|
oversized?: ViewTraceOversized;
|
|
130
130
|
}
|
|
131
|
-
|
|
131
|
+
interface ViewTraceOversized {
|
|
132
132
|
span_count: number;
|
|
133
133
|
/** Names with their counts, sorted desc. Capped at 20 entries. */
|
|
134
134
|
top_span_names: Array<[string, number]>;
|
|
@@ -136,7 +136,7 @@ export interface ViewTraceOversized {
|
|
|
136
136
|
span_response_bytes_max: number;
|
|
137
137
|
error_span_count: number;
|
|
138
138
|
}
|
|
139
|
-
|
|
139
|
+
interface ViewSpansResult {
|
|
140
140
|
trace_id: string;
|
|
141
141
|
spans: TraceAnalystSpan[];
|
|
142
142
|
/** Number of requested span ids that were not found in the trace. */
|
|
@@ -144,7 +144,7 @@ export interface ViewSpansResult {
|
|
|
144
144
|
/** Number of attribute fields truncated to fit the per-attribute cap. */
|
|
145
145
|
truncated_attribute_count: number;
|
|
146
146
|
}
|
|
147
|
-
|
|
147
|
+
interface SpanMatchRecord {
|
|
148
148
|
trace_id: string;
|
|
149
149
|
span_id: string;
|
|
150
150
|
span_name: string;
|
|
@@ -157,13 +157,13 @@ export interface SpanMatchRecord {
|
|
|
157
157
|
context_after: string;
|
|
158
158
|
match_offset: number;
|
|
159
159
|
}
|
|
160
|
-
|
|
160
|
+
interface SearchTraceResult {
|
|
161
161
|
trace_id: string;
|
|
162
162
|
hits: SpanMatchRecord[];
|
|
163
163
|
total_matches: number;
|
|
164
164
|
has_more: boolean;
|
|
165
165
|
}
|
|
166
|
-
|
|
166
|
+
interface SearchSpanResult {
|
|
167
167
|
trace_id: string;
|
|
168
168
|
span_id: string;
|
|
169
169
|
hits: SpanMatchRecord[];
|
|
@@ -171,7 +171,7 @@ export interface SearchSpanResult {
|
|
|
171
171
|
has_more: boolean;
|
|
172
172
|
}
|
|
173
173
|
/** Tunable byte budgets for bounded RLM tool output. */
|
|
174
|
-
|
|
174
|
+
interface TraceAnalystByteBudgets {
|
|
175
175
|
/** Max bytes any single tool response may emit. Hard ceiling enforced
|
|
176
176
|
* by the store; oversized → summary. Default 150_000. */
|
|
177
177
|
perCallByteCeiling: number;
|
|
@@ -185,8 +185,64 @@ export interface TraceAnalystByteBudgets {
|
|
|
185
185
|
* context window. Default 1024. */
|
|
186
186
|
perMatchTextBudget: number;
|
|
187
187
|
}
|
|
188
|
-
|
|
188
|
+
declare const DEFAULT_TRACE_ANALYST_BUDGETS: TraceAnalystByteBudgets;
|
|
189
189
|
/** Marker substituted in place of truncated string payloads. Callers
|
|
190
190
|
* parsing tool output can detect it deterministically. */
|
|
191
|
-
|
|
192
|
-
|
|
191
|
+
declare const TRACE_ANALYST_TRUNCATION_MARKER_PREFIX = "[trace-analyst truncated:";
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* `TraceAnalysisStore` — read-side interface the trace-analyst calls
|
|
195
|
+
* through. Six operations, all bounded:
|
|
196
|
+
*
|
|
197
|
+
* - `getOverview(filters?)` — dataset rollup + sample trace ids.
|
|
198
|
+
* - `queryTraces(filters?, limit, offset)` — paginated summaries.
|
|
199
|
+
* - `countTraces(filters?)` — cheap count without materialisation.
|
|
200
|
+
* - `viewTrace(trace_id, perAttrCap)` — full span list, oversized → summary.
|
|
201
|
+
* - `viewSpans(trace_id, span_ids, perAttrCap)` — surgical span fetch.
|
|
202
|
+
* - `searchTrace(trace_id, regex, max_matches)` — bounded regex hits.
|
|
203
|
+
* - `searchSpan(trace_id, span_id, regex, max_matches)` — single-span search.
|
|
204
|
+
*
|
|
205
|
+
* Multiple implementations ship in the core (`OtlpFileTraceStore`).
|
|
206
|
+
* Downstream callers can supply their own — e.g. a DuckDB-backed
|
|
207
|
+
* adapter or an in-memory adapter for tests — by implementing this
|
|
208
|
+
* interface.
|
|
209
|
+
*
|
|
210
|
+
* Filters compose with AND semantics. Empty/undefined fields impose
|
|
211
|
+
* no constraint. `regex_pattern` is the only opt-in raw-bytes scan —
|
|
212
|
+
* implementations may skip it via `count`/`overview` when not set.
|
|
213
|
+
*/
|
|
214
|
+
|
|
215
|
+
interface TraceAnalysisStore {
|
|
216
|
+
getOverview(filters?: TraceAnalystFilters): Promise<DatasetOverview>;
|
|
217
|
+
queryTraces(opts: {
|
|
218
|
+
filters?: TraceAnalystFilters;
|
|
219
|
+
limit: number;
|
|
220
|
+
offset?: number;
|
|
221
|
+
}): Promise<QueryTracesPage>;
|
|
222
|
+
countTraces(filters?: TraceAnalystFilters): Promise<number>;
|
|
223
|
+
viewTrace(opts: {
|
|
224
|
+
trace_id: string;
|
|
225
|
+
/** Override per-attribute byte cap. Defaults to discovery budget. */
|
|
226
|
+
per_attribute_byte_cap?: number;
|
|
227
|
+
}): Promise<ViewTraceResult>;
|
|
228
|
+
viewSpans(opts: {
|
|
229
|
+
trace_id: string;
|
|
230
|
+
span_ids: readonly string[];
|
|
231
|
+
/** Override per-attribute byte cap. Defaults to surgical budget. */
|
|
232
|
+
per_attribute_byte_cap?: number;
|
|
233
|
+
}): Promise<ViewSpansResult>;
|
|
234
|
+
searchTrace(opts: {
|
|
235
|
+
trace_id: string;
|
|
236
|
+
regex_pattern: string;
|
|
237
|
+
/** Hard cap on matches returned. Default 50. */
|
|
238
|
+
max_matches?: number;
|
|
239
|
+
}): Promise<SearchTraceResult>;
|
|
240
|
+
searchSpan(opts: {
|
|
241
|
+
trace_id: string;
|
|
242
|
+
span_id: string;
|
|
243
|
+
regex_pattern: string;
|
|
244
|
+
max_matches?: number;
|
|
245
|
+
}): Promise<SearchSpanResult>;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
export { DEFAULT_TRACE_ANALYST_BUDGETS as D, type ErrorCluster as E, type QueryTracesPage as Q, type SearchSpanResult as S, type TraceAnalysisStore as T, type ViewSpansResult as V, type TraceAnalystSpan as a, type DatasetOverview as b, type SearchTraceResult as c, type SpanMatchRecord as d, TRACE_ANALYST_TRUNCATION_MARKER_PREFIX as e, type TraceAnalystByteBudgets as f, type TraceAnalystFilters as g, type TraceAnalystSpanKind as h, type TraceAnalystSpanStatus as i, type TraceAnalystTraceSummary as j, type ViewTraceOversized as k, type ViewTraceResult as l };
|
|
@@ -1,3 +1,68 @@
|
|
|
1
|
+
import { S as Span } from '../schema-m0gsnbt3.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Code-edit extraction + the "watch the agent write code" animation.
|
|
5
|
+
*
|
|
6
|
+
* The base storyboard (./index) compresses a run into scenes but keeps scenes
|
|
7
|
+
* payload-free (title + narration + evidence span ids). To actually SHOW the
|
|
8
|
+
* code an agent wrote, this module pulls the concrete edit (path, diff,
|
|
9
|
+
* before/after, language) out of the raw edit-tool spans and renders a
|
|
10
|
+
* self-contained HTML replay that types each file out, character by character,
|
|
11
|
+
* like an editor recording.
|
|
12
|
+
*
|
|
13
|
+
* Additive on purpose: it reuses the canonical `Span` (trace/schema) and the
|
|
14
|
+
* `Storyboard` IR (./index) without modifying the reducer/compiler — so it
|
|
15
|
+
* composes with the existing pipeline and a Remotion consumer can read the same
|
|
16
|
+
* `CodeEdit[]` to render an MP4.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/** A concrete code change lifted from a tool span. */
|
|
20
|
+
interface CodeEdit {
|
|
21
|
+
/** Span the edit came from — back-reference into the trace. */
|
|
22
|
+
spanId: string;
|
|
23
|
+
path: string;
|
|
24
|
+
language?: string;
|
|
25
|
+
/** Unified diff, when the tool carried one. */
|
|
26
|
+
diff?: string;
|
|
27
|
+
/** Full file body before / after, when present — enables true keystroke
|
|
28
|
+
* animation rather than only flashing a diff. */
|
|
29
|
+
before?: string;
|
|
30
|
+
after?: string;
|
|
31
|
+
additions: number;
|
|
32
|
+
deletions: number;
|
|
33
|
+
startedAt: number;
|
|
34
|
+
}
|
|
35
|
+
/** Extract a CodeEdit from a single span, or undefined if it is not an edit. */
|
|
36
|
+
declare function codeEditFromSpan(span: Span): CodeEdit | undefined;
|
|
37
|
+
/** All code edits in a run, in chronological order. */
|
|
38
|
+
declare function extractCodeEdits(spans: readonly Span[]): CodeEdit[];
|
|
39
|
+
/** Pair each code-edit (`diff`) scene with its concrete edit, matched via the
|
|
40
|
+
* scene's evidence span ids. Scenes without a recoverable edit are dropped. */
|
|
41
|
+
declare function codeEditsForStoryboard(storyboard: Storyboard, spans: readonly Span[]): Array<{
|
|
42
|
+
scene: Scene;
|
|
43
|
+
edit: CodeEdit;
|
|
44
|
+
}>;
|
|
45
|
+
/** The text the animation types for an edit: prefer the full new file body,
|
|
46
|
+
* then the added lines of a diff, then a minimal placeholder so the scene is
|
|
47
|
+
* never blank. */
|
|
48
|
+
declare function editAnimationText(edit: CodeEdit): string;
|
|
49
|
+
interface CodeAnimationOptions {
|
|
50
|
+
/** Page title. Default "Agent writing code". */
|
|
51
|
+
title?: string;
|
|
52
|
+
/** Characters typed per animation tick. Higher = faster. Default 4. */
|
|
53
|
+
charsPerTick?: number;
|
|
54
|
+
/** Pause between files, ms. Default 900. */
|
|
55
|
+
pauseBetweenMs?: number;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Render a self-contained HTML page that animates the agent writing each file,
|
|
59
|
+
* character by character, with a filename tab, line numbers, and a blinking
|
|
60
|
+
* cursor — the shareable "watch it build" clip. No build step, no assets, no
|
|
61
|
+
* deps. The same `CodeEdit[]` feeds a Remotion MP4 renderer in a consumer
|
|
62
|
+
* package; this is the dependency-free baseline.
|
|
63
|
+
*/
|
|
64
|
+
declare function renderCodeAnimationHtml(source: readonly Span[] | readonly CodeEdit[], opts?: CodeAnimationOptions): string;
|
|
65
|
+
|
|
1
66
|
/**
|
|
2
67
|
* Storyboard — compile a raw agent trace into a small set of meaningful scenes,
|
|
3
68
|
* then render them as a shareable, watchable replay.
|
|
@@ -17,12 +82,12 @@
|
|
|
17
82
|
* the structure + visuals do not depend on one. Renderers that need a browser
|
|
18
83
|
* (Remotion, React) live in consumers; this module only emits strings.
|
|
19
84
|
*/
|
|
20
|
-
|
|
85
|
+
|
|
21
86
|
/** What the agent meaningfully did — the compressed vocabulary. */
|
|
22
|
-
|
|
87
|
+
type SemanticKind = 'understood_task' | 'user_message' | 'agent_reply' | 'reasoned' | 'ran_command' | 'read_file' | 'edited_code' | 'searched' | 'used_browser' | 'used_computer' | 'called_api' | 'called_tool' | 'evaluated' | 'observed_failure' | 'completed';
|
|
23
88
|
/** The modality-typed payload a scene shows. This is what makes any agent
|
|
24
89
|
* action — screen, browser, shell, code, API — actually viewable. */
|
|
25
|
-
|
|
90
|
+
type SceneVisual = {
|
|
26
91
|
type: 'screenshot';
|
|
27
92
|
src: string;
|
|
28
93
|
caption?: string;
|
|
@@ -56,7 +121,7 @@ export type SceneVisual = {
|
|
|
56
121
|
} | {
|
|
57
122
|
type: 'none';
|
|
58
123
|
};
|
|
59
|
-
|
|
124
|
+
interface SemanticEvent {
|
|
60
125
|
kind: SemanticKind;
|
|
61
126
|
/** One-line headline (the ticket/scene title). */
|
|
62
127
|
title: string;
|
|
@@ -71,7 +136,7 @@ export interface SemanticEvent {
|
|
|
71
136
|
startTs: number;
|
|
72
137
|
endTs: number;
|
|
73
138
|
}
|
|
74
|
-
|
|
139
|
+
interface ReduceOptions {
|
|
75
140
|
/** Collapse adjacent same-kind moments into one. Default true. */
|
|
76
141
|
collapseAdjacent?: boolean;
|
|
77
142
|
}
|
|
@@ -83,9 +148,9 @@ export interface ReduceOptions {
|
|
|
83
148
|
* compression step), carrying the latest visual; failures and conversation
|
|
84
149
|
* turns never collapse.
|
|
85
150
|
*/
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
151
|
+
declare function reduceToSemanticEvents(spans: readonly Span[], opts?: ReduceOptions): SemanticEvent[];
|
|
152
|
+
type SceneType = 'title_card' | 'prompt' | 'reply' | 'reasoning' | 'terminal' | 'file' | 'diff' | 'search' | 'browser' | 'computer' | 'api' | 'tool' | 'eval' | 'error' | 'summary';
|
|
153
|
+
interface Scene {
|
|
89
154
|
sceneType: SceneType;
|
|
90
155
|
title: string;
|
|
91
156
|
narration: string;
|
|
@@ -94,12 +159,12 @@ export interface Scene {
|
|
|
94
159
|
durationMs: number;
|
|
95
160
|
evidenceSpanIds: string[];
|
|
96
161
|
}
|
|
97
|
-
|
|
162
|
+
interface Storyboard {
|
|
98
163
|
title: string;
|
|
99
164
|
totalMs: number;
|
|
100
165
|
scenes: Scene[];
|
|
101
166
|
}
|
|
102
|
-
|
|
167
|
+
interface CompileOptions {
|
|
103
168
|
/** Headline for the title card. Default "Agent run". */
|
|
104
169
|
title?: string;
|
|
105
170
|
/** Max action scenes between the title + summary cards. Default 16. */
|
|
@@ -110,10 +175,10 @@ export interface CompileOptions {
|
|
|
110
175
|
* (importance 5) and edit (4) is always kept; the rest fill the budget by
|
|
111
176
|
* importance, then re-sort chronological. A title card opens, a summary closes.
|
|
112
177
|
*/
|
|
113
|
-
|
|
178
|
+
declare function compileStoryboard(events: readonly SemanticEvent[], opts?: CompileOptions): Storyboard;
|
|
114
179
|
/** Render the storyboard as a Markdown timeline — the human-readable shot list. */
|
|
115
|
-
|
|
116
|
-
|
|
180
|
+
declare function renderStoryboardMarkdown(storyboard: Storyboard): string;
|
|
181
|
+
interface HtmlRenderOptions {
|
|
117
182
|
/** Document title + on-page heading. Defaults to the storyboard title. */
|
|
118
183
|
title?: string;
|
|
119
184
|
}
|
|
@@ -124,6 +189,6 @@ export interface HtmlRenderOptions {
|
|
|
124
189
|
* / browser / prose); controls let a viewer scrub. This is the "free clip" a
|
|
125
190
|
* run produces; an MP4 is the same storyboard fed to Remotion in a consumer.
|
|
126
191
|
*/
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
192
|
+
declare function renderStoryboardHtml(storyboard: Storyboard, opts?: HtmlRenderOptions): string;
|
|
193
|
+
|
|
194
|
+
export { type CodeAnimationOptions, type CodeEdit, type CompileOptions, type HtmlRenderOptions, type ReduceOptions, type Scene, type SceneType, type SceneVisual, type SemanticEvent, type SemanticKind, type Storyboard, codeEditFromSpan, codeEditsForStoryboard, compileStoryboard, editAnimationText, extractCodeEdits, reduceToSemanticEvents, renderCodeAnimationHtml, renderStoryboardHtml, renderStoryboardMarkdown };
|
|
@@ -1,3 +1,141 @@
|
|
|
1
|
+
import { R as RunRecord } from './run-record-CP2ObebC.js';
|
|
2
|
+
import { F as FailureClusterReport } from './failure-cluster-DH9Flgcf.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* HeldOutGate — first-class held-out paired-delta promotion gate.
|
|
6
|
+
*
|
|
7
|
+
* Encodes the "honesty override" pattern that lived inline in
|
|
8
|
+
* `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.
|
|
9
|
+
* The optimizer's best-guess is one thing; what we should actually
|
|
10
|
+
* ship is another. The gate is the line between them.
|
|
11
|
+
*
|
|
12
|
+
* A candidate is promoted iff ALL three pass:
|
|
13
|
+
*
|
|
14
|
+
* 1. **Productive runs**: the candidate has at least
|
|
15
|
+
* `minProductiveRuns` paired observations on items where BOTH
|
|
16
|
+
* candidate and baseline produced a real (non-silent) score.
|
|
17
|
+
* 2. **Paired delta**: the lower bound of the bootstrap CI on the
|
|
18
|
+
* median per-item delta (candidate − baseline) on the HOLDOUT
|
|
19
|
+
* split is strictly greater than `pairedDeltaThreshold`.
|
|
20
|
+
* 3. **Overfit gap**: the candidate's gap between search-split
|
|
21
|
+
* score and holdout-split score is no worse (more positive)
|
|
22
|
+
* than the baseline's gap by more than `overfitGapThreshold`.
|
|
23
|
+
* "Better on search, worse on holdout" is the canonical
|
|
24
|
+
* overfit pattern; this catches it.
|
|
25
|
+
*
|
|
26
|
+
* The decision carries a machine-readable `rejectionCode` plus an
|
|
27
|
+
* `evidence` block with every number the gate looked at, so the
|
|
28
|
+
* downstream researcher / paper / dashboard can re-derive the
|
|
29
|
+
* verdict without re-running.
|
|
30
|
+
*
|
|
31
|
+
* See also:
|
|
32
|
+
* - `src/statistics.ts` for `pairedBootstrap` + `wilcoxonSignedRank`
|
|
33
|
+
* - `src/run-record.ts` for the input row schema
|
|
34
|
+
* - `src/reference-replay.ts` for the older, reference-replay-
|
|
35
|
+
* specific promotion path (still useful for replay-style evals).
|
|
36
|
+
*/
|
|
37
|
+
|
|
38
|
+
type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
|
|
39
|
+
interface HeldOutGateConfig {
|
|
40
|
+
/** Minimum number of paired (candidate, baseline) holdout observations
|
|
41
|
+
* required before the gate will even consider promoting. Default 3. */
|
|
42
|
+
minProductiveRuns?: number;
|
|
43
|
+
/** The bootstrap-CI lower bound on the median paired holdout delta
|
|
44
|
+
* must exceed this to promote. Default 0. */
|
|
45
|
+
pairedDeltaThreshold?: number;
|
|
46
|
+
/** Maximum allowed worsening of (search − holdout) gap relative to
|
|
47
|
+
* baseline. Default 0.15 (i.e. candidate may overfit by up to 15
|
|
48
|
+
* absolute score points more than baseline before rejection). */
|
|
49
|
+
overfitGapThreshold?: number;
|
|
50
|
+
/** Stable label of the baseline candidate. Required — paper-grade
|
|
51
|
+
* evaluation never compares two unlabelled candidates. */
|
|
52
|
+
baselineKey: string;
|
|
53
|
+
/** Confidence level for the bootstrap CI. Default 0.95. */
|
|
54
|
+
confidence?: number;
|
|
55
|
+
/** Bootstrap resamples. Default 2000. */
|
|
56
|
+
bootstrapResamples?: number;
|
|
57
|
+
/** Optional deterministic seed for the bootstrap. Default undefined
|
|
58
|
+
* (Math.random). */
|
|
59
|
+
seed?: number;
|
|
60
|
+
/**
|
|
61
|
+
* Hard ceiling on the candidate's median per-task USD cost. When the
|
|
62
|
+
* candidate clears the quality gates (paired-delta + overfit-gap) but
|
|
63
|
+
* its median cost exceeds this number, the gate rejects with
|
|
64
|
+
* `cost_ceiling`. Default `undefined` = no cost ceiling, behaving
|
|
65
|
+
* exactly like the pre-cost gate.
|
|
66
|
+
*
|
|
67
|
+
* This exists because "we ship the better prompt" is only an honest
|
|
68
|
+
* pitch when the better prompt also fits a customer-stated budget.
|
|
69
|
+
* Cost is read from `RunRecord.costUsd` (already mandatory on every
|
|
70
|
+
* run) so no new schema is required.
|
|
71
|
+
*/
|
|
72
|
+
costPerTaskCeiling?: number;
|
|
73
|
+
}
|
|
74
|
+
interface GateEvidence {
|
|
75
|
+
/** Number of paired (candidate, baseline) holdout observations used. */
|
|
76
|
+
productiveRuns: number;
|
|
77
|
+
/** Median of (candidate − baseline) paired holdout deltas. */
|
|
78
|
+
medianPairedDelta: number;
|
|
79
|
+
/** Bootstrap CI on the median paired holdout delta. */
|
|
80
|
+
pairedCI: {
|
|
81
|
+
low: number;
|
|
82
|
+
high: number;
|
|
83
|
+
};
|
|
84
|
+
/** Wilcoxon signed-rank p-value on the paired holdout deltas. */
|
|
85
|
+
pairedPValue: number;
|
|
86
|
+
/** Mean candidate score on the search split (NaN if none). */
|
|
87
|
+
searchScore: number;
|
|
88
|
+
/** Mean candidate score on the holdout split (NaN if none). */
|
|
89
|
+
holdoutScore: number;
|
|
90
|
+
/** Candidate (search − holdout) gap. */
|
|
91
|
+
overfitGap: number;
|
|
92
|
+
/** Baseline (search − holdout) gap. */
|
|
93
|
+
baselineOverfitGap: number;
|
|
94
|
+
/** Median per-task USD cost across the candidate's runs. Recorded
|
|
95
|
+
* even when no `costPerTaskCeiling` is configured so downstream
|
|
96
|
+
* dashboards (intelligence.tangle.tools) can render \$/task per
|
|
97
|
+
* generation regardless of gating policy. */
|
|
98
|
+
medianCandidateCost: number;
|
|
99
|
+
/** Median per-task USD cost across the baseline runs, for
|
|
100
|
+
* symmetric reporting. */
|
|
101
|
+
medianBaselineCost: number;
|
|
102
|
+
}
|
|
103
|
+
interface GateDecision {
|
|
104
|
+
/** Final promote/no-promote verdict. */
|
|
105
|
+
promote: boolean;
|
|
106
|
+
/** The candidate that was evaluated. */
|
|
107
|
+
candidateId: string;
|
|
108
|
+
/** The baseline it was compared against. */
|
|
109
|
+
baselineId: string;
|
|
110
|
+
/** Every number the gate looked at, for audit + paper export. */
|
|
111
|
+
evidence: GateEvidence;
|
|
112
|
+
/** Human-readable reason. */
|
|
113
|
+
reason: string;
|
|
114
|
+
/** Machine-readable rejection code, or null on promote. */
|
|
115
|
+
rejectionCode: HeldOutGateRejectionCode | null;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Held-out paired-delta promotion gate. Construct once with config,
|
|
119
|
+
* call `evaluate(candidateRuns, baselineRuns)` per (candidate,
|
|
120
|
+
* baseline) pair. Stateless across calls.
|
|
121
|
+
*/
|
|
122
|
+
declare class HeldOutGate {
|
|
123
|
+
private readonly minProductiveRuns;
|
|
124
|
+
private readonly pairedDeltaThreshold;
|
|
125
|
+
private readonly overfitGapThreshold;
|
|
126
|
+
private readonly baselineKey;
|
|
127
|
+
private readonly confidence;
|
|
128
|
+
private readonly resamples;
|
|
129
|
+
private readonly seed?;
|
|
130
|
+
private readonly costPerTaskCeiling?;
|
|
131
|
+
constructor(config: HeldOutGateConfig);
|
|
132
|
+
/** Decide whether `candidate` should replace `baseline`. Pairing
|
|
133
|
+
* is by (experimentId, seed) — identical experiment + seed pairs
|
|
134
|
+
* the candidate run with the matching baseline run. Pairs without
|
|
135
|
+
* a holdout score on both sides are dropped. */
|
|
136
|
+
evaluate(candidate: RunRecord[], baseline: RunRecord[]): GateDecision;
|
|
137
|
+
}
|
|
138
|
+
|
|
1
139
|
/**
|
|
2
140
|
* Reporting helpers — production summaries and paper-quality figures — sit alongside `reporter.ts` rather
|
|
3
141
|
* than replacing it.
|
|
@@ -22,10 +160,8 @@
|
|
|
22
160
|
* what you'd hand to vega-lite, plotly, matplotlib, or your own
|
|
23
161
|
* Canvas renderer to draw the actual figure.
|
|
24
162
|
*/
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
import type { RunRecord } from './run-record';
|
|
28
|
-
export interface SummaryTableOptions {
|
|
163
|
+
|
|
164
|
+
interface SummaryTableOptions {
|
|
29
165
|
/** Comparator candidate id. Wilcoxon + Cohen's d are computed
|
|
30
166
|
* versus this candidate. Required for paired stats columns. */
|
|
31
167
|
comparator?: string;
|
|
@@ -36,7 +172,7 @@ export interface SummaryTableOptions {
|
|
|
36
172
|
/** FDR for BH adjustment of the comparison p-values. Default 0.05. */
|
|
37
173
|
fdr?: number;
|
|
38
174
|
}
|
|
39
|
-
|
|
175
|
+
interface SummaryTableRow {
|
|
40
176
|
candidateId: string;
|
|
41
177
|
n: number;
|
|
42
178
|
mean: number;
|
|
@@ -47,7 +183,7 @@ export interface SummaryTableRow {
|
|
|
47
183
|
/** Cohen's d vs comparator. NaN if no comparator. */
|
|
48
184
|
cohensD: number;
|
|
49
185
|
}
|
|
50
|
-
|
|
186
|
+
interface SummaryTable {
|
|
51
187
|
rows: SummaryTableRow[];
|
|
52
188
|
comparator: string | null;
|
|
53
189
|
split: 'search' | 'holdout';
|
|
@@ -59,8 +195,8 @@ export interface SummaryTable {
|
|
|
59
195
|
* bootstrap CI on the chosen split, and (when a comparator is given)
|
|
60
196
|
* BH-adjusted Wilcoxon p + Cohen's d versus that comparator.
|
|
61
197
|
*/
|
|
62
|
-
|
|
63
|
-
|
|
198
|
+
declare function summaryTable(runs: RunRecord[], opts?: SummaryTableOptions): SummaryTable;
|
|
199
|
+
interface ParetoPoint {
|
|
64
200
|
candidateId: string;
|
|
65
201
|
/** Mean USD cost per run on the chosen split. */
|
|
66
202
|
cost: number;
|
|
@@ -75,7 +211,7 @@ export interface ParetoPoint {
|
|
|
75
211
|
* for it was passed in. */
|
|
76
212
|
gate?: 'promote' | 'reject_few_runs' | 'reject_negative_delta' | 'reject_overfit_gap' | null;
|
|
77
213
|
}
|
|
78
|
-
|
|
214
|
+
interface ParetoFigureSpec {
|
|
79
215
|
kind: 'pareto-cost-quality';
|
|
80
216
|
split: 'search' | 'holdout';
|
|
81
217
|
points: ParetoPoint[];
|
|
@@ -89,11 +225,11 @@ export interface ParetoFigureSpec {
|
|
|
89
225
|
* candidate id; if present, every point picks up the gate verdict
|
|
90
226
|
* for overlay.
|
|
91
227
|
*/
|
|
92
|
-
|
|
228
|
+
declare function paretoChart(runs: RunRecord[], opts?: {
|
|
93
229
|
split?: 'search' | 'holdout';
|
|
94
230
|
gateDecisions?: Record<string, GateDecision>;
|
|
95
231
|
}): ParetoFigureSpec;
|
|
96
|
-
|
|
232
|
+
interface GainDistributionBin {
|
|
97
233
|
/** Inclusive lower edge. */
|
|
98
234
|
lo: number;
|
|
99
235
|
/** Exclusive upper edge (or inclusive if it's the last bin). */
|
|
@@ -101,7 +237,7 @@ export interface GainDistributionBin {
|
|
|
101
237
|
/** Number of pairs whose delta lands in this bin. */
|
|
102
238
|
count: number;
|
|
103
239
|
}
|
|
104
|
-
|
|
240
|
+
interface GainDistributionFigureSpec {
|
|
105
241
|
kind: 'gain-distribution';
|
|
106
242
|
candidateId: string;
|
|
107
243
|
comparator: string;
|
|
@@ -115,7 +251,7 @@ export interface GainDistributionFigureSpec {
|
|
|
115
251
|
high: number;
|
|
116
252
|
};
|
|
117
253
|
}
|
|
118
|
-
|
|
254
|
+
interface GainDistributionOptions {
|
|
119
255
|
/** Number of histogram bins. Default 11 (so the centre is exact at 0). */
|
|
120
256
|
bins?: number;
|
|
121
257
|
/** Which split to use. Default 'holdout'. */
|
|
@@ -132,15 +268,15 @@ export interface GainDistributionOptions {
|
|
|
132
268
|
* comparator), histogrammed. Includes the bootstrap CI on the median
|
|
133
269
|
* delta — same primitive the promotion gate uses.
|
|
134
270
|
*/
|
|
135
|
-
|
|
136
|
-
|
|
271
|
+
declare function gainHistogram(runs: RunRecord[], candidateId: string, comparator: string, opts?: GainDistributionOptions): GainDistributionFigureSpec;
|
|
272
|
+
type ResearchReportDecision = 'promote' | 'hold' | 'reject' | 'equivalent' | 'needs_more_data';
|
|
137
273
|
/**
|
|
138
274
|
* Hard floor below which a paired comparison is treated as uninformative
|
|
139
275
|
* regardless of `minPairs`. Mirrors the lower limit on Wilcoxon signed-rank
|
|
140
276
|
* exact tables; below this the test has no power to separate effect sizes.
|
|
141
277
|
*/
|
|
142
|
-
|
|
143
|
-
|
|
278
|
+
declare const RESEARCH_REPORT_HARD_PAIR_FLOOR = 6;
|
|
279
|
+
interface ResearchReportOptions {
|
|
144
280
|
/** Human-readable report title. */
|
|
145
281
|
title?: string;
|
|
146
282
|
/** Comparator candidate id. Required for statistical decision guidance. */
|
|
@@ -197,14 +333,14 @@ export interface ResearchReportOptions {
|
|
|
197
333
|
*/
|
|
198
334
|
preregistrationHash?: string;
|
|
199
335
|
}
|
|
200
|
-
|
|
336
|
+
interface ResearchReportRecommendation {
|
|
201
337
|
decision: ResearchReportDecision;
|
|
202
338
|
candidateId: string | null;
|
|
203
339
|
rationale: string[];
|
|
204
340
|
risks: string[];
|
|
205
341
|
nextActions: string[];
|
|
206
342
|
}
|
|
207
|
-
|
|
343
|
+
interface ResearchReportCandidate {
|
|
208
344
|
candidateId: string;
|
|
209
345
|
n: number;
|
|
210
346
|
mean: number;
|
|
@@ -240,7 +376,7 @@ export interface ResearchReportCandidate {
|
|
|
240
376
|
decision: ResearchReportDecision;
|
|
241
377
|
decisionReason: string;
|
|
242
378
|
}
|
|
243
|
-
|
|
379
|
+
interface ResearchReportMethodology {
|
|
244
380
|
/**
|
|
245
381
|
* Plain-language assumptions the report depends on. Read these first when
|
|
246
382
|
* deciding whether the verdict is load-bearing for a launch decision.
|
|
@@ -255,7 +391,7 @@ export interface ResearchReportMethodology {
|
|
|
255
391
|
/** Citations for the methodological choices above. */
|
|
256
392
|
citations: string[];
|
|
257
393
|
}
|
|
258
|
-
|
|
394
|
+
interface ResearchReport {
|
|
259
395
|
kind: 'agent-eval-research-report';
|
|
260
396
|
title: string;
|
|
261
397
|
generatedAt: string;
|
|
@@ -304,5 +440,6 @@ export interface ResearchReport {
|
|
|
304
440
|
* Async because the fingerprint uses Web Crypto via `hashJson`; deterministic
|
|
305
441
|
* for any fixed `runs`, `seed`, and ROPE.
|
|
306
442
|
*/
|
|
307
|
-
|
|
308
|
-
|
|
443
|
+
declare function researchReport(runs: RunRecord[], opts?: ResearchReportOptions): Promise<ResearchReport>;
|
|
444
|
+
|
|
445
|
+
export { type GainDistributionBin as G, HeldOutGate as H, type ParetoFigureSpec as P, RESEARCH_REPORT_HARD_PAIR_FLOOR as R, type SummaryTable as S, type GainDistributionFigureSpec as a, type GainDistributionOptions as b, type ParetoPoint as c, type ResearchReport as d, type ResearchReportCandidate as e, type ResearchReportDecision as f, type ResearchReportMethodology as g, type ResearchReportOptions as h, type ResearchReportRecommendation as i, type SummaryTableOptions as j, type SummaryTableRow as k, gainHistogram as l, type GateDecision as m, type GateEvidence as n, type HeldOutGateConfig as o, paretoChart as p, type HeldOutGateRejectionCode as q, researchReport as r, summaryTable as s };
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
+
import { c as TelemetrySink, e as TelemetryEnvelope } from '../sink-fetch-B1Yg4Til.js';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* Node-only file sink. Imports `node:fs` — DO NOT import this from a Worker
|
|
3
5
|
* or edge runtime; use `./sink-fetch` instead.
|
|
4
6
|
*/
|
|
5
|
-
|
|
6
|
-
import type { TelemetrySink } from './sink-fetch';
|
|
7
|
+
|
|
7
8
|
/** Append envelopes to a JSONL file, partitioned by repo + date. */
|
|
8
|
-
|
|
9
|
+
declare class FileTelemetrySink implements TelemetrySink {
|
|
9
10
|
private readonly baseDir;
|
|
10
11
|
private streams;
|
|
11
12
|
constructor(baseDir: string);
|
|
@@ -13,5 +14,6 @@ export declare class FileTelemetrySink implements TelemetrySink {
|
|
|
13
14
|
close(): Promise<void>;
|
|
14
15
|
}
|
|
15
16
|
/** Default location for local telemetry, mirroring bad CLI's convention. */
|
|
16
|
-
|
|
17
|
-
|
|
17
|
+
declare function defaultTelemetryDir(homeDir: string, override?: string): string;
|
|
18
|
+
|
|
19
|
+
export { FileTelemetrySink, defaultTelemetryDir };
|