@tangle-network/agent-eval 0.95.0 → 0.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/adapters/http.d.ts +17 -10
- package/dist/adapters/langchain.d.ts +14 -7
- package/dist/adapters/otel.d.ts +25 -13
- package/dist/{rl/adversarial.d.ts → adversarial-DIVcDoI_.d.ts} +7 -6
- package/dist/analyst/index.d.ts +236 -28
- package/dist/analyst/index.js +1 -1
- package/dist/{trace-analyst/analyst.d.ts → analyst-C8HHvfJp.d.ts} +14 -12
- package/dist/{contract/analyze-runs.d.ts → analyze-runs-DtT6F_6T.d.ts} +10 -9
- package/dist/authenticity/index.d.ts +16 -15
- package/dist/{baseline.d.ts → baseline-Bbid3WoO.d.ts} +44 -8
- package/dist/belief-state/index.d.ts +605 -14
- package/dist/benchmarks/index.d.ts +5 -23
- package/dist/builder-eval/index.d.ts +250 -5
- package/dist/calibration-BPmzuVPk.d.ts +101 -0
- package/dist/campaign/index.d.ts +1452 -38
- package/dist/{chunk-AQ5WQAIV.js → chunk-3NHEO6ZC.js} +2 -2
- package/dist/chunk-3NHEO6ZC.js.map +1 -0
- package/dist/cli.d.ts +0 -2
- package/dist/{contract/intake/code-agent-session.d.ts → code-agent-session-CPHRCb4-.d.ts} +17 -15
- package/dist/contract/index.d.ts +824 -107
- package/dist/contract/index.js +145 -1
- package/dist/contract/index.js.map +1 -1
- package/dist/control-Doncu-B_.d.ts +259 -0
- package/dist/{control-runtime.d.ts → control-runtime-Acf9CGhw.d.ts} +26 -23
- package/dist/control.d.ts +10 -11
- package/dist/corpus-D4YW9UoJ.d.ts +560 -0
- package/dist/{cost-ledger.d.ts → cost-ledger-DuSqlw5B.d.ts} +11 -10
- package/dist/{counterfactual.d.ts → counterfactual-DlOz8PBx.d.ts} +13 -12
- package/dist/{dataset.d.ts → dataset-BbGkaN2I.d.ts} +14 -11
- package/dist/{analyst/registry.d.ts → default-registry-GyE8X5SP.d.ts} +37 -8
- package/dist/diagnose.d.ts +252 -1
- package/dist/{trace/emitter.d.ts → emitter-C2rqGH_l.d.ts} +12 -9
- package/dist/{errors.d.ts → errors-CzMUYo7b.d.ts} +11 -10
- package/dist/failure-cluster-DH9Flgcf.d.ts +76 -0
- package/dist/{feedback-trajectory.d.ts → feedback-trajectory-BxY0cKfs.d.ts} +38 -36
- package/dist/fuzz.d.ts +547 -1
- package/dist/gepa-Dprxvz8r.d.ts +414 -0
- package/dist/governance/index.d.ts +135 -5
- package/dist/harness-optimizer-mOl9XX_O.d.ts +106 -0
- package/dist/hosted/index.d.ts +239 -10
- package/dist/index-_Y4oNOOb.d.ts +159 -0
- package/dist/index.d.ts +5660 -277
- package/dist/index.js +1 -1
- package/dist/{contract/insight-report.d.ts → insight-report-BnRjTibG.d.ts} +19 -16
- package/dist/{trace/integrity.d.ts → integrity-D2t12mMw.d.ts} +14 -11
- package/dist/{judge-calibration.d.ts → judge-calibration-0p2QcWNE.d.ts} +17 -16
- package/dist/{analyst/kind-factory.d.ts → kind-factory-X3eDYbKn.d.ts} +61 -10
- package/dist/knowledge/index.d.ts +103 -3
- package/dist/{llm-client.d.ts → llm-client-Bj7g0rqu.d.ts} +23 -21
- package/dist/matrix/index.d.ts +30 -12
- package/dist/meta-eval/index.d.ts +182 -6
- package/dist/{multi-layer-verifier.d.ts → multi-layer-verifier-DUZXrPDA.d.ts} +15 -12
- package/dist/multishot/index.d.ts +290 -7
- package/dist/{rl/off-policy.d.ts → off-policy-DiwuKKg7.d.ts} +9 -8
- package/dist/openapi.json +1 -1
- package/dist/{meta-eval/outcome-store.d.ts → outcome-store-rnXLEqSn.d.ts} +8 -7
- package/dist/{pareto.d.ts → pareto-E-pembql.d.ts} +10 -9
- package/dist/perf/index.d.ts +119 -13
- package/dist/pipelines/index.d.ts +173 -8
- package/dist/pre-registration-nfUdc9EQ.d.ts +483 -0
- package/dist/prm/index.d.ts +104 -5
- package/dist/provenance-CIxfBnkl.d.ts +426 -0
- package/dist/query-B7GGjRox.d.ts +32 -0
- package/dist/{trace/raw-provider-sink.d.ts → raw-provider-sink-C46HDghv.d.ts} +14 -13
- package/dist/{red-team.d.ts → red-team-BWdoyleI.d.ts} +16 -14
- package/dist/{trace/redact.d.ts → redact-B40YG2M_.d.ts} +8 -7
- package/dist/release-report-pidWUMZ2.d.ts +233 -0
- package/dist/reporting.d.ts +16 -15
- package/dist/{eval-campaign.d.ts → researcher-Jr8ME1dZ.d.ts} +156 -34
- package/dist/rl.d.ts +1193 -1
- package/dist/{prm/rubric.d.ts → rubric-Cc6UHvUb.d.ts} +13 -10
- package/dist/{meta-eval/rubric-predictive-validity.d.ts → rubric-predictive-validity-C2hDKM8Z.d.ts} +11 -8
- package/dist/run-critic-CmMf05uV.d.ts +56 -0
- package/dist/{run-record.d.ts → run-record-CP2ObebC.d.ts} +117 -15
- package/dist/runtime-trajectory-BOUUjI0y.d.ts +49 -0
- package/dist/{trace/schema.d.ts → schema-m0gsnbt3.d.ts} +30 -29
- package/dist/semantic-concept-judge-DSBB2Cfp.d.ts +624 -0
- package/dist/{sequential.d.ts → sequential-5iSVfzl2.d.ts} +10 -9
- package/dist/{series-convergence.d.ts → series-convergence-D5OWMBg6.d.ts} +5 -4
- package/dist/sink-fetch-B1Yg4Til.d.ts +101 -0
- package/dist/{statistics.d.ts → statistics-CCJpTGOS.d.ts} +48 -46
- package/dist/{trace/store.d.ts → store-BcFXE6LG.d.ts} +12 -21
- package/dist/{trace-analyst/types.d.ts → store-C1YxJDEK.d.ts} +74 -18
- package/dist/storyboard/index.d.ts +81 -16
- package/dist/{summary-report.d.ts → summary-report-CInXwsza.d.ts} +160 -23
- package/dist/telemetry/{sink-file.d.ts → file.d.ts} +7 -5
- package/dist/telemetry/index.d.ts +35 -17
- package/dist/{sandbox-harness.d.ts → test-graded-scenario-DeODGLra.d.ts} +60 -15
- package/dist/{locked-jsonl-appender.d.ts → testing-C21CHsq2.d.ts} +4 -3
- package/dist/testing.d.ts +1 -5
- package/dist/traces.d.ts +976 -4
- package/dist/{trajectory.d.ts → trajectory-2TkpSEVh.d.ts} +9 -8
- package/dist/{analyst/types.d.ts → types-B5x54y6n.d.ts} +112 -19
- package/dist/{campaign/types.d.ts → types-BMahhhio.d.ts} +47 -44
- package/dist/{matrix/types.d.ts → types-BUxNaJ8c.d.ts} +11 -9
- package/dist/{types.d.ts → types-C7DGg5ex.d.ts} +33 -31
- package/dist/{verdict.d.ts → verdict-C9MlYujm.d.ts} +3 -2
- package/dist/wire/index.d.ts +570 -13
- package/dist/workflow/index.d.ts +496 -22
- package/package.json +2 -2
- package/dist/action-policy.d.ts +0 -24
- package/dist/action-policy.d.ts.map +0 -1
- package/dist/action-policy.test.d.ts +0 -2
- package/dist/action-policy.test.d.ts.map +0 -1
- package/dist/active-learning.d.ts +0 -41
- package/dist/active-learning.d.ts.map +0 -1
- package/dist/adapters/http.d.ts.map +0 -1
- package/dist/adapters/langchain.d.ts.map +0 -1
- package/dist/adapters/otel.d.ts.map +0 -1
- package/dist/agent-profile-cell.d.ts +0 -101
- package/dist/agent-profile-cell.d.ts.map +0 -1
- package/dist/agent-profile.d.ts +0 -27
- package/dist/agent-profile.d.ts.map +0 -1
- package/dist/agent-profile.test.d.ts +0 -2
- package/dist/agent-profile.test.d.ts.map +0 -1
- package/dist/analyst/adapters.d.ts +0 -62
- package/dist/analyst/adapters.d.ts.map +0 -1
- package/dist/analyst/analyst.test.d.ts +0 -2
- package/dist/analyst/analyst.test.d.ts.map +0 -1
- package/dist/analyst/ax-service.d.ts +0 -27
- package/dist/analyst/ax-service.d.ts.map +0 -1
- package/dist/analyst/behavioral-analyst.d.ts +0 -28
- package/dist/analyst/behavioral-analyst.d.ts.map +0 -1
- package/dist/analyst/chat-client.d.ts +0 -91
- package/dist/analyst/chat-client.d.ts.map +0 -1
- package/dist/analyst/default-registry.d.ts +0 -27
- package/dist/analyst/default-registry.d.ts.map +0 -1
- package/dist/analyst/default-registry.test.d.ts +0 -2
- package/dist/analyst/default-registry.test.d.ts.map +0 -1
- package/dist/analyst/finding-signature.d.ts +0 -48
- package/dist/analyst/finding-signature.d.ts.map +0 -1
- package/dist/analyst/finding-subject.d.ts +0 -146
- package/dist/analyst/finding-subject.d.ts.map +0 -1
- package/dist/analyst/finding-subject.test.d.ts +0 -2
- package/dist/analyst/finding-subject.test.d.ts.map +0 -1
- package/dist/analyst/findings-store.d.ts +0 -75
- package/dist/analyst/findings-store.d.ts.map +0 -1
- package/dist/analyst/index.d.ts.map +0 -1
- package/dist/analyst/kind-factory.d.ts.map +0 -1
- package/dist/analyst/kinds/failure-mode.d.ts +0 -19
- package/dist/analyst/kinds/failure-mode.d.ts.map +0 -1
- package/dist/analyst/kinds/improvement.d.ts +0 -23
- package/dist/analyst/kinds/improvement.d.ts.map +0 -1
- package/dist/analyst/kinds/index.d.ts +0 -22
- package/dist/analyst/kinds/index.d.ts.map +0 -1
- package/dist/analyst/kinds/kinds.test.d.ts +0 -2
- package/dist/analyst/kinds/kinds.test.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-gap.d.ts +0 -28
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +0 -22
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.d.ts +0 -84
- package/dist/analyst/kinds/skill-usage.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.test.d.ts +0 -2
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.d.ts +0 -26
- package/dist/analyst/parse-tolerant.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.test.d.ts +0 -2
- package/dist/analyst/parse-tolerant.test.d.ts.map +0 -1
- package/dist/analyst/registry.budget.test.d.ts +0 -2
- package/dist/analyst/registry.budget.test.d.ts.map +0 -1
- package/dist/analyst/registry.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.d.ts +0 -35
- package/dist/analyst/steer-firewall.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.test.d.ts +0 -2
- package/dist/analyst/steer-firewall.test.d.ts.map +0 -1
- package/dist/analyst/structure-findings.d.ts +0 -37
- package/dist/analyst/structure-findings.d.ts.map +0 -1
- package/dist/analyst/structure-findings.test.d.ts +0 -2
- package/dist/analyst/structure-findings.test.d.ts.map +0 -1
- package/dist/analyst/tool-groups.d.ts +0 -34
- package/dist/analyst/tool-groups.d.ts.map +0 -1
- package/dist/analyst/types.d.ts.map +0 -1
- package/dist/anti-slop.d.ts +0 -59
- package/dist/anti-slop.d.ts.map +0 -1
- package/dist/artifact-validator.d.ts +0 -74
- package/dist/artifact-validator.d.ts.map +0 -1
- package/dist/attestation.d.ts +0 -63
- package/dist/attestation.d.ts.map +0 -1
- package/dist/attestation.test.d.ts +0 -2
- package/dist/attestation.test.d.ts.map +0 -1
- package/dist/authenticity/index.d.ts.map +0 -1
- package/dist/authenticity/index.test.d.ts +0 -2
- package/dist/authenticity/index.test.d.ts.map +0 -1
- package/dist/auto-pr.d.ts +0 -120
- package/dist/auto-pr.d.ts.map +0 -1
- package/dist/baseline.d.ts.map +0 -1
- package/dist/behavior-dsl.d.ts +0 -73
- package/dist/behavior-dsl.d.ts.map +0 -1
- package/dist/belief-state/calibration.d.ts +0 -10
- package/dist/belief-state/calibration.d.ts.map +0 -1
- package/dist/belief-state/calibration.test.d.ts +0 -2
- package/dist/belief-state/calibration.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.d.ts +0 -66
- package/dist/belief-state/code-agent-corpus.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.test.d.ts +0 -2
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.d.ts +0 -22
- package/dist/belief-state/code-agent-evidence.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.test.d.ts +0 -2
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/extract.d.ts +0 -7
- package/dist/belief-state/extract.d.ts.map +0 -1
- package/dist/belief-state/extract.test.d.ts +0 -2
- package/dist/belief-state/extract.test.d.ts.map +0 -1
- package/dist/belief-state/index.d.ts.map +0 -1
- package/dist/belief-state/ope.d.ts +0 -17
- package/dist/belief-state/ope.d.ts.map +0 -1
- package/dist/belief-state/ope.test.d.ts +0 -2
- package/dist/belief-state/ope.test.d.ts.map +0 -1
- package/dist/belief-state/phase0-measurement.d.ts +0 -55
- package/dist/belief-state/phase0-measurement.d.ts.map +0 -1
- package/dist/belief-state/report.d.ts +0 -17
- package/dist/belief-state/report.d.ts.map +0 -1
- package/dist/belief-state/report.test.d.ts +0 -2
- package/dist/belief-state/report.test.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.d.ts +0 -23
- package/dist/belief-state/research-evidence.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.test.d.ts +0 -2
- package/dist/belief-state/research-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +0 -32
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.d.ts +0 -87
- package/dist/belief-state/runtime-hooks.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.test.d.ts +0 -2
- package/dist/belief-state/runtime-hooks.test.d.ts.map +0 -1
- package/dist/belief-state/selective.d.ts +0 -15
- package/dist/belief-state/selective.d.ts.map +0 -1
- package/dist/belief-state/selective.test.d.ts +0 -2
- package/dist/belief-state/selective.test.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.d.ts +0 -80
- package/dist/belief-state/shadow-probe.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.test.d.ts +0 -2
- package/dist/belief-state/shadow-probe.test.d.ts.map +0 -1
- package/dist/belief-state/types.d.ts +0 -195
- package/dist/belief-state/types.d.ts.map +0 -1
- package/dist/belief-state/types.test.d.ts +0 -2
- package/dist/belief-state/types.test.d.ts.map +0 -1
- package/dist/benchmark.d.ts +0 -14
- package/dist/benchmark.d.ts.map +0 -1
- package/dist/benchmarks/index.d.ts.map +0 -1
- package/dist/benchmarks/routing/dataset.d.ts +0 -34
- package/dist/benchmarks/routing/dataset.d.ts.map +0 -1
- package/dist/benchmarks/routing/index.d.ts +0 -34
- package/dist/benchmarks/routing/index.d.ts.map +0 -1
- package/dist/benchmarks/types.d.ts +0 -49
- package/dist/benchmarks/types.d.ts.map +0 -1
- package/dist/bisector.d.ts +0 -81
- package/dist/bisector.d.ts.map +0 -1
- package/dist/budget-guard.d.ts +0 -31
- package/dist/budget-guard.d.ts.map +0 -1
- package/dist/builder-eval/builder-session.d.ts +0 -111
- package/dist/builder-eval/builder-session.d.ts.map +0 -1
- package/dist/builder-eval/correlation.d.ts +0 -32
- package/dist/builder-eval/correlation.d.ts.map +0 -1
- package/dist/builder-eval/index.d.ts.map +0 -1
- package/dist/builder-eval/project-registry.d.ts +0 -51
- package/dist/builder-eval/project-registry.d.ts.map +0 -1
- package/dist/builder-eval/three-layer-eval.d.ts +0 -55
- package/dist/builder-eval/three-layer-eval.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.d.ts +0 -108
- package/dist/campaign/analyst-surface.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.test.d.ts +0 -2
- package/dist/campaign/analyst-surface.test.d.ts.map +0 -1
- package/dist/campaign/auto-pr.d.ts +0 -46
- package/dist/campaign/auto-pr.d.ts.map +0 -1
- package/dist/campaign/distillation/agreement-judge.d.ts +0 -69
- package/dist/campaign/distillation/agreement-judge.d.ts.map +0 -1
- package/dist/campaign/distillation/cli.d.ts +0 -35
- package/dist/campaign/distillation/cli.d.ts.map +0 -1
- package/dist/campaign/distillation/distillation.test.d.ts +0 -2
- package/dist/campaign/distillation/distillation.test.d.ts.map +0 -1
- package/dist/campaign/distillation/gold-scenarios.d.ts +0 -54
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +0 -1
- package/dist/campaign/distillation/run-distillation.d.ts +0 -119
- package/dist/campaign/distillation/run-distillation.d.ts.map +0 -1
- package/dist/campaign/gates/compose.d.ts +0 -12
- package/dist/campaign/gates/compose.d.ts.map +0 -1
- package/dist/campaign/gates/default-production-gate.d.ts +0 -58
- package/dist/campaign/gates/default-production-gate.d.ts.map +0 -1
- package/dist/campaign/gates/heldout-gate.d.ts +0 -12
- package/dist/campaign/gates/heldout-gate.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.d.ts +0 -125
- package/dist/campaign/gates/promotion-policy.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.test.d.ts +0 -2
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.d.ts +0 -146
- package/dist/campaign/gates/sequential.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.test.d.ts +0 -2
- package/dist/campaign/gates/sequential.test.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.d.ts +0 -99
- package/dist/campaign/gates/statistical-heldout.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.test.d.ts +0 -2
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +0 -1
- package/dist/campaign/index.d.ts.map +0 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +0 -59
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +0 -1
- package/dist/campaign/presets/compare-proposers.d.ts +0 -146
- package/dist/campaign/presets/compare-proposers.d.ts.map +0 -1
- package/dist/campaign/presets/playback.d.ts +0 -120
- package/dist/campaign/presets/playback.d.ts.map +0 -1
- package/dist/campaign/presets/playback.test.d.ts +0 -2
- package/dist/campaign/presets/playback.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-eval.d.ts +0 -14
- package/dist/campaign/presets/run-eval.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.d.ts +0 -63
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +0 -2
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-optimization.d.ts +0 -92
- package/dist/campaign/presets/run-optimization.d.ts.map +0 -1
- package/dist/campaign/presets/run-profile-matrix.d.ts +0 -151
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +0 -1
- package/dist/campaign/presets/run-skill-opt.d.ts +0 -96
- package/dist/campaign/presets/run-skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/_findings-text.d.ts +0 -22
- package/dist/campaign/proposers/_findings-text.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.d.ts +0 -33
- package/dist/campaign/proposers/ace.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.test.d.ts +0 -2
- package/dist/campaign/proposers/ace.test.d.ts.map +0 -1
- package/dist/campaign/proposers/analysis-edit.d.ts +0 -32
- package/dist/campaign/proposers/analysis-edit.d.ts.map +0 -1
- package/dist/campaign/proposers/evolutionary.d.ts +0 -20
- package/dist/campaign/proposers/evolutionary.d.ts.map +0 -1
- package/dist/campaign/proposers/fapo.d.ts +0 -120
- package/dist/campaign/proposers/fapo.d.ts.map +0 -1
- package/dist/campaign/proposers/gepa.d.ts +0 -86
- package/dist/campaign/proposers/gepa.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.d.ts +0 -44
- package/dist/campaign/proposers/halo.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.test.d.ts +0 -2
- package/dist/campaign/proposers/halo.test.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.d.ts +0 -47
- package/dist/campaign/proposers/memory.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.test.d.ts +0 -2
- package/dist/campaign/proposers/memory.test.d.ts.map +0 -1
- package/dist/campaign/proposers/skill-opt.d.ts +0 -88
- package/dist/campaign/proposers/skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.d.ts +0 -48
- package/dist/campaign/proposers/trace-analyst.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.test.d.ts +0 -2
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +0 -1
- package/dist/campaign/provenance.d.ts +0 -185
- package/dist/campaign/provenance.d.ts.map +0 -1
- package/dist/campaign/run-campaign.d.ts +0 -90
- package/dist/campaign/run-campaign.d.ts.map +0 -1
- package/dist/campaign/score-utils.d.ts +0 -26
- package/dist/campaign/score-utils.d.ts.map +0 -1
- package/dist/campaign/skill-patch.d.ts +0 -62
- package/dist/campaign/skill-patch.d.ts.map +0 -1
- package/dist/campaign/storage.d.ts +0 -38
- package/dist/campaign/storage.d.ts.map +0 -1
- package/dist/campaign/types.d.ts.map +0 -1
- package/dist/campaign/worktree/index.d.ts +0 -53
- package/dist/campaign/worktree/index.d.ts.map +0 -1
- package/dist/canary.d.ts +0 -101
- package/dist/canary.d.ts.map +0 -1
- package/dist/causal-attribution.d.ts +0 -45
- package/dist/causal-attribution.d.ts.map +0 -1
- package/dist/chunk-AQ5WQAIV.js.map +0 -1
- package/dist/ci-gate.d.ts +0 -44
- package/dist/ci-gate.d.ts.map +0 -1
- package/dist/cli.d.ts.map +0 -1
- package/dist/client.d.ts +0 -77
- package/dist/client.d.ts.map +0 -1
- package/dist/client.test.d.ts +0 -2
- package/dist/client.test.d.ts.map +0 -1
- package/dist/command-runner.d.ts +0 -74
- package/dist/command-runner.d.ts.map +0 -1
- package/dist/command-runner.test.d.ts +0 -2
- package/dist/command-runner.test.d.ts.map +0 -1
- package/dist/completion-verifier.d.ts +0 -147
- package/dist/completion-verifier.d.ts.map +0 -1
- package/dist/completion-verifier.test.d.ts +0 -9
- package/dist/completion-verifier.test.d.ts.map +0 -1
- package/dist/concurrency.d.ts +0 -23
- package/dist/concurrency.d.ts.map +0 -1
- package/dist/contamination-guard.d.ts +0 -81
- package/dist/contamination-guard.d.ts.map +0 -1
- package/dist/contract/analyze-runs.d.ts.map +0 -1
- package/dist/contract/define-agent-eval.d.ts +0 -52
- package/dist/contract/define-agent-eval.d.ts.map +0 -1
- package/dist/contract/diff.d.ts +0 -114
- package/dist/contract/diff.d.ts.map +0 -1
- package/dist/contract/index.d.ts.map +0 -1
- package/dist/contract/insight-report.d.ts.map +0 -1
- package/dist/contract/insight-types-fwd.d.ts +0 -7
- package/dist/contract/insight-types-fwd.d.ts.map +0 -1
- package/dist/contract/intake/agent-trace.d.ts +0 -97
- package/dist/contract/intake/agent-trace.d.ts.map +0 -1
- package/dist/contract/intake/code-agent-session.d.ts.map +0 -1
- package/dist/contract/intake/feedback-table.d.ts +0 -87
- package/dist/contract/intake/feedback-table.d.ts.map +0 -1
- package/dist/contract/intake/index.d.ts +0 -22
- package/dist/contract/intake/index.d.ts.map +0 -1
- package/dist/contract/intake/otel-spans.d.ts +0 -33
- package/dist/contract/intake/otel-spans.d.ts.map +0 -1
- package/dist/contract/self-improve.d.ts +0 -284
- package/dist/contract/self-improve.d.ts.map +0 -1
- package/dist/control-runtime.d.ts.map +0 -1
- package/dist/control-runtime.test.d.ts +0 -2
- package/dist/control-runtime.test.d.ts.map +0 -1
- package/dist/control.d.ts.map +0 -1
- package/dist/convergence.d.ts +0 -29
- package/dist/convergence.d.ts.map +0 -1
- package/dist/cost-ledger.d.ts.map +0 -1
- package/dist/cost-ledger.test.d.ts +0 -2
- package/dist/cost-ledger.test.d.ts.map +0 -1
- package/dist/cost-report.d.ts +0 -43
- package/dist/cost-report.d.ts.map +0 -1
- package/dist/cost-report.test.d.ts +0 -2
- package/dist/cost-report.test.d.ts.map +0 -1
- package/dist/cost-tracker.d.ts +0 -76
- package/dist/cost-tracker.d.ts.map +0 -1
- package/dist/counterfactual.d.ts.map +0 -1
- package/dist/cross-trace-diff.d.ts +0 -56
- package/dist/cross-trace-diff.d.ts.map +0 -1
- package/dist/dataset.d.ts.map +0 -1
- package/dist/deploy-gate-layer.d.ts +0 -125
- package/dist/deploy-gate-layer.d.ts.map +0 -1
- package/dist/deploy-gate-layer.test.d.ts +0 -2
- package/dist/deploy-gate-layer.test.d.ts.map +0 -1
- package/dist/description-length-gate.d.ts +0 -119
- package/dist/description-length-gate.d.ts.map +0 -1
- package/dist/detectors/edge.test.d.ts +0 -2
- package/dist/detectors/edge.test.d.ts.map +0 -1
- package/dist/detectors/index.d.ts +0 -81
- package/dist/detectors/index.d.ts.map +0 -1
- package/dist/detectors/index.test.d.ts +0 -2
- package/dist/detectors/index.test.d.ts.map +0 -1
- package/dist/diagnose/causal-sweep.d.ts +0 -100
- package/dist/diagnose/causal-sweep.d.ts.map +0 -1
- package/dist/diagnose/index.d.ts +0 -36
- package/dist/diagnose/index.d.ts.map +0 -1
- package/dist/diagnose/remediation.d.ts +0 -68
- package/dist/diagnose/remediation.d.ts.map +0 -1
- package/dist/diagnose/repair.d.ts +0 -77
- package/dist/diagnose/repair.d.ts.map +0 -1
- package/dist/discover-personas.d.ts +0 -35
- package/dist/discover-personas.d.ts.map +0 -1
- package/dist/driver.d.ts +0 -95
- package/dist/driver.d.ts.map +0 -1
- package/dist/driver.test.d.ts +0 -8
- package/dist/driver.test.d.ts.map +0 -1
- package/dist/dual-agent-bench.d.ts +0 -81
- package/dist/dual-agent-bench.d.ts.map +0 -1
- package/dist/error-count-extractor.d.ts +0 -47
- package/dist/error-count-extractor.d.ts.map +0 -1
- package/dist/error-count-extractor.test.d.ts +0 -2
- package/dist/error-count-extractor.test.d.ts.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/eval-campaign.d.ts.map +0 -1
- package/dist/eval-campaign.test.d.ts +0 -2
- package/dist/eval-campaign.test.d.ts.map +0 -1
- package/dist/eval-tools.d.ts +0 -55
- package/dist/eval-tools.d.ts.map +0 -1
- package/dist/eval-trace-store.d.ts +0 -107
- package/dist/eval-trace-store.d.ts.map +0 -1
- package/dist/eval-trace-store.test.d.ts +0 -2
- package/dist/eval-trace-store.test.d.ts.map +0 -1
- package/dist/executor.d.ts +0 -38
- package/dist/executor.d.ts.map +0 -1
- package/dist/executor.test.d.ts +0 -10
- package/dist/executor.test.d.ts.map +0 -1
- package/dist/experiment-tracker.d.ts +0 -178
- package/dist/experiment-tracker.d.ts.map +0 -1
- package/dist/experiment-tracker.test.d.ts +0 -2
- package/dist/experiment-tracker.test.d.ts.map +0 -1
- package/dist/failure-taxonomy.d.ts +0 -38
- package/dist/failure-taxonomy.d.ts.map +0 -1
- package/dist/feedback-trajectory.d.ts.map +0 -1
- package/dist/feedback-trajectory.test.d.ts +0 -2
- package/dist/feedback-trajectory.test.d.ts.map +0 -1
- package/dist/flow-layer.d.ts +0 -90
- package/dist/flow-layer.d.ts.map +0 -1
- package/dist/flow-layer.test.d.ts +0 -2
- package/dist/flow-layer.test.d.ts.map +0 -1
- package/dist/fuzz/capsule.d.ts +0 -46
- package/dist/fuzz/capsule.d.ts.map +0 -1
- package/dist/fuzz/cube.d.ts +0 -36
- package/dist/fuzz/cube.d.ts.map +0 -1
- package/dist/fuzz/explorer-cost.test.d.ts +0 -2
- package/dist/fuzz/explorer-cost.test.d.ts.map +0 -1
- package/dist/fuzz/explorer.d.ts +0 -64
- package/dist/fuzz/explorer.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.d.ts +0 -16
- package/dist/fuzz/fuzz-agent.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.test.d.ts +0 -2
- package/dist/fuzz/fuzz-agent.test.d.ts.map +0 -1
- package/dist/fuzz/gates.d.ts +0 -33
- package/dist/fuzz/gates.d.ts.map +0 -1
- package/dist/fuzz/index.d.ts +0 -26
- package/dist/fuzz/index.d.ts.map +0 -1
- package/dist/fuzz/policies.d.ts +0 -28
- package/dist/fuzz/policies.d.ts.map +0 -1
- package/dist/fuzz/tools.d.ts +0 -20
- package/dist/fuzz/tools.d.ts.map +0 -1
- package/dist/fuzz/types.d.ts +0 -307
- package/dist/fuzz/types.d.ts.map +0 -1
- package/dist/golden-matcher.d.ts +0 -71
- package/dist/golden-matcher.d.ts.map +0 -1
- package/dist/governance/eu-ai-act.d.ts +0 -37
- package/dist/governance/eu-ai-act.d.ts.map +0 -1
- package/dist/governance/index.d.ts.map +0 -1
- package/dist/governance/nist-ai-rmf.d.ts +0 -15
- package/dist/governance/nist-ai-rmf.d.ts.map +0 -1
- package/dist/governance/soc2.d.ts +0 -12
- package/dist/governance/soc2.d.ts.map +0 -1
- package/dist/governance/types.d.ts +0 -66
- package/dist/governance/types.d.ts.map +0 -1
- package/dist/harness-optimizer.d.ts +0 -82
- package/dist/harness-optimizer.d.ts.map +0 -1
- package/dist/held-out-gate.d.ts +0 -135
- package/dist/held-out-gate.d.ts.map +0 -1
- package/dist/hosted/client.d.ts +0 -73
- package/dist/hosted/client.d.ts.map +0 -1
- package/dist/hosted/from-env.test.d.ts +0 -8
- package/dist/hosted/from-env.test.d.ts.map +0 -1
- package/dist/hosted/index.d.ts.map +0 -1
- package/dist/hosted/types.d.ts +0 -159
- package/dist/hosted/types.d.ts.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/integrity/backend-integrity.d.ts +0 -71
- package/dist/integrity/backend-integrity.d.ts.map +0 -1
- package/dist/integrity/preflight.d.ts +0 -72
- package/dist/integrity/preflight.d.ts.map +0 -1
- package/dist/integrity/preflight.test.d.ts +0 -2
- package/dist/integrity/preflight.test.d.ts.map +0 -1
- package/dist/integrity/single-backend.d.ts +0 -67
- package/dist/integrity/single-backend.d.ts.map +0 -1
- package/dist/intent-match-judge.d.ts +0 -69
- package/dist/intent-match-judge.d.ts.map +0 -1
- package/dist/intent-match-judge.test.d.ts +0 -2
- package/dist/intent-match-judge.test.d.ts.map +0 -1
- package/dist/judge-calibration.d.ts.map +0 -1
- package/dist/judge-ensemble.d.ts +0 -66
- package/dist/judge-ensemble.d.ts.map +0 -1
- package/dist/judge-ensemble.test.d.ts +0 -8
- package/dist/judge-ensemble.test.d.ts.map +0 -1
- package/dist/judge-families.d.ts +0 -38
- package/dist/judge-families.d.ts.map +0 -1
- package/dist/judge-panel.d.ts +0 -65
- package/dist/judge-panel.d.ts.map +0 -1
- package/dist/judge-retry.d.ts +0 -70
- package/dist/judge-retry.d.ts.map +0 -1
- package/dist/judge-runner.d.ts +0 -36
- package/dist/judge-runner.d.ts.map +0 -1
- package/dist/judge-runner.test.d.ts +0 -2
- package/dist/judge-runner.test.d.ts.map +0 -1
- package/dist/judges.d.ts +0 -74
- package/dist/judges.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.d.ts +0 -89
- package/dist/keyword-coverage-judge.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.test.d.ts +0 -2
- package/dist/keyword-coverage-judge.test.d.ts.map +0 -1
- package/dist/knowledge/index.d.ts.map +0 -1
- package/dist/knowledge/readiness.d.ts +0 -26
- package/dist/knowledge/readiness.d.ts.map +0 -1
- package/dist/knowledge/types.d.ts +0 -75
- package/dist/knowledge/types.d.ts.map +0 -1
- package/dist/live-proof.d.ts +0 -62
- package/dist/live-proof.d.ts.map +0 -1
- package/dist/llm-client.d.ts.map +0 -1
- package/dist/llm-client.test.d.ts +0 -2
- package/dist/llm-client.test.d.ts.map +0 -1
- package/dist/locked-jsonl-appender.d.ts.map +0 -1
- package/dist/matrix/aggregation.d.ts +0 -16
- package/dist/matrix/aggregation.d.ts.map +0 -1
- package/dist/matrix/index.d.ts.map +0 -1
- package/dist/matrix/runner.d.ts +0 -15
- package/dist/matrix/runner.d.ts.map +0 -1
- package/dist/matrix/types.d.ts.map +0 -1
- package/dist/meta-eval/calibration.d.ts +0 -47
- package/dist/meta-eval/calibration.d.ts.map +0 -1
- package/dist/meta-eval/correlation-study.d.ts +0 -53
- package/dist/meta-eval/correlation-study.d.ts.map +0 -1
- package/dist/meta-eval/index.d.ts.map +0 -1
- package/dist/meta-eval/outcome-store.d.ts.map +0 -1
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +0 -1
- package/dist/meta-eval/sentinel.d.ts +0 -169
- package/dist/meta-eval/sentinel.d.ts.map +0 -1
- package/dist/metrics.d.ts +0 -63
- package/dist/metrics.d.ts.map +0 -1
- package/dist/model-seats.d.ts +0 -71
- package/dist/model-seats.d.ts.map +0 -1
- package/dist/model-seats.test.d.ts +0 -2
- package/dist/model-seats.test.d.ts.map +0 -1
- package/dist/muffled-gate-scanner.d.ts +0 -102
- package/dist/muffled-gate-scanner.d.ts.map +0 -1
- package/dist/multi-layer-verifier.d.ts.map +0 -1
- package/dist/multi-layer-verifier.test.d.ts +0 -2
- package/dist/multi-layer-verifier.test.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.d.ts +0 -80
- package/dist/multi-toolchain-layer.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.test.d.ts +0 -2
- package/dist/multi-toolchain-layer.test.d.ts.map +0 -1
- package/dist/multishot/default-tools.d.ts +0 -34
- package/dist/multishot/default-tools.d.ts.map +0 -1
- package/dist/multishot/index.d.ts.map +0 -1
- package/dist/multishot/judges.d.ts +0 -32
- package/dist/multishot/judges.d.ts.map +0 -1
- package/dist/multishot/matrix.d.ts +0 -107
- package/dist/multishot/matrix.d.ts.map +0 -1
- package/dist/multishot/multishot.d.ts +0 -23
- package/dist/multishot/multishot.d.ts.map +0 -1
- package/dist/multishot/router.d.ts +0 -37
- package/dist/multishot/router.d.ts.map +0 -1
- package/dist/multishot/types.d.ts +0 -60
- package/dist/multishot/types.d.ts.map +0 -1
- package/dist/observability.d.ts +0 -71
- package/dist/observability.d.ts.map +0 -1
- package/dist/oracle.d.ts +0 -55
- package/dist/oracle.d.ts.map +0 -1
- package/dist/orthogonality.d.ts +0 -35
- package/dist/orthogonality.d.ts.map +0 -1
- package/dist/otel-pipeline.d.ts +0 -31
- package/dist/otel-pipeline.d.ts.map +0 -1
- package/dist/paraphrase.d.ts +0 -107
- package/dist/paraphrase.d.ts.map +0 -1
- package/dist/pareto.d.ts.map +0 -1
- package/dist/partition-held-out.d.ts +0 -70
- package/dist/partition-held-out.d.ts.map +0 -1
- package/dist/partition-held-out.test.d.ts +0 -2
- package/dist/partition-held-out.test.d.ts.map +0 -1
- package/dist/perf/index.d.ts.map +0 -1
- package/dist/perf/integrity.d.ts +0 -30
- package/dist/perf/integrity.d.ts.map +0 -1
- package/dist/perf/journey.d.ts +0 -45
- package/dist/perf/journey.d.ts.map +0 -1
- package/dist/perf/ratchet.d.ts +0 -47
- package/dist/perf/ratchet.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.d.ts +0 -31
- package/dist/pipelines/budget-breach.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.test.d.ts +0 -2
- package/dist/pipelines/budget-breach.test.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.d.ts +0 -38
- package/dist/pipelines/failure-cluster.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.test.d.ts +0 -2
- package/dist/pipelines/failure-cluster.test.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.d.ts +0 -26
- package/dist/pipelines/first-divergence.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.test.d.ts +0 -2
- package/dist/pipelines/first-divergence.test.d.ts.map +0 -1
- package/dist/pipelines/index.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.d.ts +0 -26
- package/dist/pipelines/judge-agreement.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.test.d.ts +0 -2
- package/dist/pipelines/judge-agreement.test.d.ts.map +0 -1
- package/dist/pipelines/regression.d.ts +0 -23
- package/dist/pipelines/regression.d.ts.map +0 -1
- package/dist/pipelines/regression.test.d.ts +0 -2
- package/dist/pipelines/regression.test.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.d.ts +0 -32
- package/dist/pipelines/stuck-loop.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.test.d.ts +0 -2
- package/dist/pipelines/stuck-loop.test.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.d.ts +0 -34
- package/dist/pipelines/tool-waste.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.test.d.ts +0 -2
- package/dist/pipelines/tool-waste.test.d.ts.map +0 -1
- package/dist/playbook.d.ts +0 -16
- package/dist/playbook.d.ts.map +0 -1
- package/dist/pr-review-benchmark.d.ts +0 -88
- package/dist/pr-review-benchmark.d.ts.map +0 -1
- package/dist/pr-review-benchmark.test.d.ts +0 -2
- package/dist/pr-review-benchmark.test.d.ts.map +0 -1
- package/dist/pre-registration.d.ts +0 -125
- package/dist/pre-registration.d.ts.map +0 -1
- package/dist/prm/builtin-rubrics.d.ts +0 -33
- package/dist/prm/builtin-rubrics.d.ts.map +0 -1
- package/dist/prm/index.d.ts.map +0 -1
- package/dist/prm/inference.d.ts +0 -29
- package/dist/prm/inference.d.ts.map +0 -1
- package/dist/prm/inference.test.d.ts +0 -2
- package/dist/prm/inference.test.d.ts.map +0 -1
- package/dist/prm/rubric.d.ts.map +0 -1
- package/dist/prm/training-export.d.ts +0 -38
- package/dist/prm/training-export.d.ts.map +0 -1
- package/dist/produced-state.d.ts +0 -63
- package/dist/produced-state.d.ts.map +0 -1
- package/dist/produced-state.test.d.ts +0 -8
- package/dist/produced-state.test.d.ts.map +0 -1
- package/dist/profile/baselines.d.ts +0 -37
- package/dist/profile/baselines.d.ts.map +0 -1
- package/dist/profile/index.d.ts +0 -105
- package/dist/profile/index.d.ts.map +0 -1
- package/dist/promotion-gate.d.ts +0 -93
- package/dist/promotion-gate.d.ts.map +0 -1
- package/dist/prompt-registry.d.ts +0 -41
- package/dist/prompt-registry.d.ts.map +0 -1
- package/dist/propose-review-control.d.ts +0 -49
- package/dist/propose-review-control.d.ts.map +0 -1
- package/dist/propose-review-control.test.d.ts +0 -2
- package/dist/propose-review-control.test.d.ts.map +0 -1
- package/dist/propose-review.d.ts +0 -155
- package/dist/propose-review.d.ts.map +0 -1
- package/dist/red-team.d.ts.map +0 -1
- package/dist/reference-replay-steering.d.ts +0 -11
- package/dist/reference-replay-steering.d.ts.map +0 -1
- package/dist/reference-replay.d.ts +0 -176
- package/dist/reference-replay.d.ts.map +0 -1
- package/dist/reflective-mutation.d.ts +0 -79
- package/dist/reflective-mutation.d.ts.map +0 -1
- package/dist/registry.d.ts +0 -31
- package/dist/registry.d.ts.map +0 -1
- package/dist/release-confidence.d.ts +0 -128
- package/dist/release-confidence.d.ts.map +0 -1
- package/dist/release-report.d.ts +0 -11
- package/dist/release-report.d.ts.map +0 -1
- package/dist/replay.d.ts +0 -120
- package/dist/replay.d.ts.map +0 -1
- package/dist/reporter.d.ts +0 -14
- package/dist/reporter.d.ts.map +0 -1
- package/dist/reporting.d.ts.map +0 -1
- package/dist/researcher.d.ts +0 -140
- package/dist/researcher.d.ts.map +0 -1
- package/dist/reviewer.d.ts +0 -118
- package/dist/reviewer.d.ts.map +0 -1
- package/dist/reviewer.test.d.ts +0 -2
- package/dist/reviewer.test.d.ts.map +0 -1
- package/dist/reward-model-export.d.ts +0 -60
- package/dist/reward-model-export.d.ts.map +0 -1
- package/dist/rl/active-curriculum.d.ts +0 -110
- package/dist/rl/active-curriculum.d.ts.map +0 -1
- package/dist/rl/adaptation-eval.d.ts +0 -109
- package/dist/rl/adaptation-eval.d.ts.map +0 -1
- package/dist/rl/adversarial.d.ts.map +0 -1
- package/dist/rl/compute-curves.d.ts +0 -127
- package/dist/rl/compute-curves.d.ts.map +0 -1
- package/dist/rl/contamination.d.ts +0 -117
- package/dist/rl/contamination.d.ts.map +0 -1
- package/dist/rl/corpus.d.ts +0 -55
- package/dist/rl/corpus.d.ts.map +0 -1
- package/dist/rl/corpus.test.d.ts +0 -2
- package/dist/rl/corpus.test.d.ts.map +0 -1
- package/dist/rl/dataset.d.ts +0 -102
- package/dist/rl/dataset.d.ts.map +0 -1
- package/dist/rl/dataset.test.d.ts +0 -2
- package/dist/rl/dataset.test.d.ts.map +0 -1
- package/dist/rl/exporters.d.ts +0 -141
- package/dist/rl/exporters.d.ts.map +0 -1
- package/dist/rl/index.d.ts +0 -49
- package/dist/rl/index.d.ts.map +0 -1
- package/dist/rl/off-policy.d.ts.map +0 -1
- package/dist/rl/predictive-validity-researcher.d.ts +0 -69
- package/dist/rl/predictive-validity-researcher.d.ts.map +0 -1
- package/dist/rl/preferences.d.ts +0 -141
- package/dist/rl/preferences.d.ts.map +0 -1
- package/dist/rl/process-reward.d.ts +0 -122
- package/dist/rl/process-reward.d.ts.map +0 -1
- package/dist/rl/reward-hacking.d.ts +0 -104
- package/dist/rl/reward-hacking.d.ts.map +0 -1
- package/dist/rl/rl-campaign.d.ts +0 -85
- package/dist/rl/rl-campaign.d.ts.map +0 -1
- package/dist/rl/run-record-adapters.d.ts +0 -56
- package/dist/rl/run-record-adapters.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.d.ts +0 -166
- package/dist/rl/sim-fidelity.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.test.d.ts +0 -2
- package/dist/rl/sim-fidelity.test.d.ts.map +0 -1
- package/dist/rl/tournament.d.ts +0 -115
- package/dist/rl/tournament.d.ts.map +0 -1
- package/dist/rl/verifiable-reward.d.ts +0 -124
- package/dist/rl/verifiable-reward.d.ts.map +0 -1
- package/dist/run-critic.d.ts +0 -23
- package/dist/run-critic.d.ts.map +0 -1
- package/dist/run-evidence.d.ts +0 -32
- package/dist/run-evidence.d.ts.map +0 -1
- package/dist/run-record.d.ts.map +0 -1
- package/dist/run-record.test.d.ts +0 -2
- package/dist/run-record.test.d.ts.map +0 -1
- package/dist/run-score.d.ts +0 -31
- package/dist/run-score.d.ts.map +0 -1
- package/dist/runtime-trajectory.d.ts +0 -47
- package/dist/runtime-trajectory.d.ts.map +0 -1
- package/dist/sandbox-harness.d.ts.map +0 -1
- package/dist/sandbox-harness.test.d.ts +0 -2
- package/dist/sandbox-harness.test.d.ts.map +0 -1
- package/dist/sandbox-pool.d.ts +0 -74
- package/dist/sandbox-pool.d.ts.map +0 -1
- package/dist/sandbox-pool.test.d.ts +0 -2
- package/dist/sandbox-pool.test.d.ts.map +0 -1
- package/dist/scorecard.d.ts +0 -133
- package/dist/scorecard.d.ts.map +0 -1
- package/dist/scorecard.test.d.ts +0 -2
- package/dist/scorecard.test.d.ts.map +0 -1
- package/dist/self-play.d.ts +0 -69
- package/dist/self-play.d.ts.map +0 -1
- package/dist/semantic-concept-judge.d.ts +0 -135
- package/dist/semantic-concept-judge.d.ts.map +0 -1
- package/dist/semantic-concept-judge.test.d.ts +0 -2
- package/dist/semantic-concept-judge.test.d.ts.map +0 -1
- package/dist/sequential.d.ts.map +0 -1
- package/dist/series-convergence.d.ts.map +0 -1
- package/dist/slo.d.ts +0 -48
- package/dist/slo.d.ts.map +0 -1
- package/dist/state-continuity.d.ts +0 -47
- package/dist/state-continuity.d.ts.map +0 -1
- package/dist/statistics.d.ts.map +0 -1
- package/dist/statistics.test.d.ts +0 -2
- package/dist/statistics.test.d.ts.map +0 -1
- package/dist/steering-optimizer.d.ts +0 -58
- package/dist/steering-optimizer.d.ts.map +0 -1
- package/dist/steering.d.ts +0 -24
- package/dist/steering.d.ts.map +0 -1
- package/dist/storyboard/code-edit.d.ts +0 -64
- package/dist/storyboard/code-edit.d.ts.map +0 -1
- package/dist/storyboard/code-edit.test.d.ts +0 -2
- package/dist/storyboard/code-edit.test.d.ts.map +0 -1
- package/dist/storyboard/index.d.ts.map +0 -1
- package/dist/storyboard/index.test.d.ts +0 -2
- package/dist/storyboard/index.test.d.ts.map +0 -1
- package/dist/summary-report.d.ts.map +0 -1
- package/dist/telemetry/client.d.ts +0 -35
- package/dist/telemetry/client.d.ts.map +0 -1
- package/dist/telemetry/index.d.ts.map +0 -1
- package/dist/telemetry/schema.d.ts +0 -61
- package/dist/telemetry/schema.d.ts.map +0 -1
- package/dist/telemetry/sink-fetch.d.ts +0 -39
- package/dist/telemetry/sink-fetch.d.ts.map +0 -1
- package/dist/telemetry/sink-file.d.ts.map +0 -1
- package/dist/test-graded-scenario.d.ts +0 -42
- package/dist/test-graded-scenario.d.ts.map +0 -1
- package/dist/testing.d.ts.map +0 -1
- package/dist/tool-use-metrics.d.ts +0 -35
- package/dist/tool-use-metrics.d.ts.map +0 -1
- package/dist/trace/capture-fetch.d.ts +0 -48
- package/dist/trace/capture-fetch.d.ts.map +0 -1
- package/dist/trace/capture-fetch.test.d.ts +0 -2
- package/dist/trace/capture-fetch.test.d.ts.map +0 -1
- package/dist/trace/emitter.d.ts.map +0 -1
- package/dist/trace/extract-usage.d.ts +0 -46
- package/dist/trace/extract-usage.d.ts.map +0 -1
- package/dist/trace/extract-usage.test.d.ts +0 -2
- package/dist/trace/extract-usage.test.d.ts.map +0 -1
- package/dist/trace/index.d.ts +0 -15
- package/dist/trace/index.d.ts.map +0 -1
- package/dist/trace/integrity.d.ts.map +0 -1
- package/dist/trace/otel-bridge.d.ts +0 -29
- package/dist/trace/otel-bridge.d.ts.map +0 -1
- package/dist/trace/otel-export.d.ts +0 -52
- package/dist/trace/otel-export.d.ts.map +0 -1
- package/dist/trace/otel.d.ts +0 -57
- package/dist/trace/otel.d.ts.map +0 -1
- package/dist/trace/otlp-attributes.d.ts +0 -17
- package/dist/trace/otlp-attributes.d.ts.map +0 -1
- package/dist/trace/query.d.ts +0 -29
- package/dist/trace/query.d.ts.map +0 -1
- package/dist/trace/query.test.d.ts +0 -2
- package/dist/trace/query.test.d.ts.map +0 -1
- package/dist/trace/raw-provider-sink.d.ts.map +0 -1
- package/dist/trace/redact.d.ts.map +0 -1
- package/dist/trace/schema.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.d.ts +0 -72
- package/dist/trace/store-to-otlp.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.test.d.ts +0 -2
- package/dist/trace/store-to-otlp.test.d.ts.map +0 -1
- package/dist/trace/store.d.ts.map +0 -1
- package/dist/trace/store.test.d.ts +0 -2
- package/dist/trace/store.test.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.test.d.ts +0 -2
- package/dist/trace-analyst/analyst.test.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.d.ts +0 -40
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +0 -2
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +0 -1
- package/dist/trace-analyst/hook.d.ts +0 -55
- package/dist/trace-analyst/hook.d.ts.map +0 -1
- package/dist/trace-analyst/index.d.ts +0 -18
- package/dist/trace-analyst/index.d.ts.map +0 -1
- package/dist/trace-analyst/insights.d.ts +0 -71
- package/dist/trace-analyst/insights.d.ts.map +0 -1
- package/dist/trace-analyst/insights.test.d.ts +0 -2
- package/dist/trace-analyst/insights.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-flatten.d.ts +0 -42
- package/dist/trace-analyst/otlp-flatten.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.d.ts +0 -85
- package/dist/trace-analyst/otlp-span.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-span.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.d.ts +0 -115
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/prompts.d.ts +0 -6
- package/dist/trace-analyst/prompts.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.d.ts +0 -126
- package/dist/trace-analyst/store-otlp.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.test.d.ts +0 -8
- package/dist/trace-analyst/store-otlp.test.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/store.d.ts +0 -63
- package/dist/trace-analyst/store.d.ts.map +0 -1
- package/dist/trace-analyst/tools.d.ts +0 -44
- package/dist/trace-analyst/tools.d.ts.map +0 -1
- package/dist/trace-analyst/tools.test.d.ts +0 -10
- package/dist/trace-analyst/tools.test.d.ts.map +0 -1
- package/dist/trace-analyst/types.d.ts.map +0 -1
- package/dist/trace-contracts.d.ts +0 -180
- package/dist/trace-contracts.d.ts.map +0 -1
- package/dist/traced-analyst.d.ts +0 -26
- package/dist/traced-analyst.d.ts.map +0 -1
- package/dist/traced-judges.d.ts +0 -27
- package/dist/traced-judges.d.ts.map +0 -1
- package/dist/traces.d.ts.map +0 -1
- package/dist/trajectory.d.ts.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/ui-finding.d.ts +0 -104
- package/dist/ui-finding.d.ts.map +0 -1
- package/dist/verdict-cache.d.ts +0 -78
- package/dist/verdict-cache.d.ts.map +0 -1
- package/dist/verdict-cache.test.d.ts +0 -2
- package/dist/verdict-cache.test.d.ts.map +0 -1
- package/dist/verdict.d.ts.map +0 -1
- package/dist/visual-diff.d.ts +0 -32
- package/dist/visual-diff.d.ts.map +0 -1
- package/dist/wire/handlers.d.ts +0 -54
- package/dist/wire/handlers.d.ts.map +0 -1
- package/dist/wire/index.d.ts.map +0 -1
- package/dist/wire/openapi.d.ts +0 -3
- package/dist/wire/openapi.d.ts.map +0 -1
- package/dist/wire/rpc.d.ts +0 -21
- package/dist/wire/rpc.d.ts.map +0 -1
- package/dist/wire/rubrics.d.ts +0 -34
- package/dist/wire/rubrics.d.ts.map +0 -1
- package/dist/wire/schemas.d.ts +0 -410
- package/dist/wire/schemas.d.ts.map +0 -1
- package/dist/wire/server.d.ts +0 -60
- package/dist/wire/server.d.ts.map +0 -1
- package/dist/workflow/event-schema.d.ts +0 -5
- package/dist/workflow/event-schema.d.ts.map +0 -1
- package/dist/workflow/feedback-pack.d.ts +0 -99
- package/dist/workflow/feedback-pack.d.ts.map +0 -1
- package/dist/workflow/index.d.ts.map +0 -1
- package/dist/workflow/intelligence-export.d.ts +0 -62
- package/dist/workflow/intelligence-export.d.ts.map +0 -1
- package/dist/workflow/partner-report.d.ts +0 -49
- package/dist/workflow/partner-report.d.ts.map +0 -1
- package/dist/workflow/phase-graph.d.ts +0 -43
- package/dist/workflow/phase-graph.d.ts.map +0 -1
- package/dist/workflow/promotion-gate.d.ts +0 -61
- package/dist/workflow/promotion-gate.d.ts.map +0 -1
- package/dist/workflow/run-record.d.ts +0 -12
- package/dist/workflow/run-record.d.ts.map +0 -1
- package/dist/workflow/runtime-adapter.d.ts +0 -20
- package/dist/workflow/runtime-adapter.d.ts.map +0 -1
- package/dist/workflow/sanitize.d.ts +0 -21
- package/dist/workflow/sanitize.d.ts.map +0 -1
- package/dist/workflow/schema.d.ts +0 -5
- package/dist/workflow/schema.d.ts.map +0 -1
- package/dist/workflow/summary.d.ts +0 -43
- package/dist/workflow/summary.d.ts.map +0 -1
- package/dist/workflow/trace-event-fields.d.ts +0 -6
- package/dist/workflow/trace-event-fields.d.ts.map +0 -1
- package/dist/workflow/trajectory.d.ts +0 -15
- package/dist/workflow/trajectory.d.ts.map +0 -1
- package/dist/workflow/types.d.ts +0 -68
- package/dist/workflow/types.d.ts.map +0 -1
- package/dist/workspace-inspector.d.ts +0 -67
- package/dist/workspace-inspector.d.ts.map +0 -1
- package/dist/wrangler-deploy-runner.test.d.ts +0 -2
- package/dist/wrangler-deploy-runner.test.d.ts.map +0 -1
|
@@ -1,82 +0,0 @@
|
|
|
1
|
-
import { type Objective, type ParetoResult } from './pareto';
|
|
2
|
-
import { type RunTrace } from './run-critic';
|
|
3
|
-
import { type RunScore, type RunScoreWeights } from './run-score';
|
|
4
|
-
import type { SteeringBundle } from './steering';
|
|
5
|
-
export type HarnessIntervention = 'continue' | 'plan' | 'audit' | 'recover' | 'repair' | 'verify' | 'final_gate' | 'wait_for_measurement' | 'abort';
|
|
6
|
-
export interface WorkflowTopology {
|
|
7
|
-
id: string;
|
|
8
|
-
interventions: HarnessIntervention[];
|
|
9
|
-
maxParallelBranches?: number;
|
|
10
|
-
metadata?: Record<string, unknown>;
|
|
11
|
-
}
|
|
12
|
-
export interface MeasurementPolicy {
|
|
13
|
-
required: string[];
|
|
14
|
-
optional?: string[];
|
|
15
|
-
promoteOn?: Array<keyof RunScore | 'aggregate'>;
|
|
16
|
-
}
|
|
17
|
-
export interface HarnessVariant {
|
|
18
|
-
id: string;
|
|
19
|
-
steering?: SteeringBundle;
|
|
20
|
-
topology?: WorkflowTopology;
|
|
21
|
-
measurement?: MeasurementPolicy;
|
|
22
|
-
budgets?: Record<string, number>;
|
|
23
|
-
models?: Record<string, string>;
|
|
24
|
-
reviewers?: Record<string, string>;
|
|
25
|
-
metadata?: Record<string, unknown>;
|
|
26
|
-
}
|
|
27
|
-
export interface HarnessScenario {
|
|
28
|
-
id: string;
|
|
29
|
-
task: string;
|
|
30
|
-
split?: 'train' | 'validation' | 'test' | string;
|
|
31
|
-
metadata?: Record<string, unknown>;
|
|
32
|
-
}
|
|
33
|
-
export interface HarnessRunRequest {
|
|
34
|
-
variant: HarnessVariant;
|
|
35
|
-
scenario: HarnessScenario;
|
|
36
|
-
trialIndex: number;
|
|
37
|
-
}
|
|
38
|
-
export interface HarnessAdapter {
|
|
39
|
-
run(request: HarnessRunRequest): Promise<RunTrace>;
|
|
40
|
-
}
|
|
41
|
-
export interface HarnessRunResult {
|
|
42
|
-
variant: HarnessVariant;
|
|
43
|
-
scenario: HarnessScenario;
|
|
44
|
-
trialIndex: number;
|
|
45
|
-
trace: RunTrace;
|
|
46
|
-
score: RunScore;
|
|
47
|
-
aggregate: number;
|
|
48
|
-
}
|
|
49
|
-
export interface HarnessVariantReport {
|
|
50
|
-
variant: HarnessVariant;
|
|
51
|
-
runs: HarnessRunResult[];
|
|
52
|
-
aggregateMean: number;
|
|
53
|
-
passRate: number;
|
|
54
|
-
costUsdMean: number;
|
|
55
|
-
wallSecondsMean: number;
|
|
56
|
-
scoreMean: RunScore;
|
|
57
|
-
}
|
|
58
|
-
export interface HarnessSelection {
|
|
59
|
-
winner: HarnessVariantReport;
|
|
60
|
-
frontier: ParetoResult<HarnessVariantReport>;
|
|
61
|
-
reports: HarnessVariantReport[];
|
|
62
|
-
}
|
|
63
|
-
export interface HarnessExperimentResult {
|
|
64
|
-
results: HarnessRunResult[];
|
|
65
|
-
selection: HarnessSelection;
|
|
66
|
-
}
|
|
67
|
-
export interface HarnessExperimentConfig {
|
|
68
|
-
adapter: HarnessAdapter;
|
|
69
|
-
variants: HarnessVariant[];
|
|
70
|
-
scenarios: HarnessScenario[];
|
|
71
|
-
trialsPerScenario?: number;
|
|
72
|
-
parallelism?: number;
|
|
73
|
-
weights?: Partial<RunScoreWeights>;
|
|
74
|
-
objectives?: Array<Objective<HarnessVariantReport>>;
|
|
75
|
-
score?: (trace: RunTrace, request: HarnessRunRequest) => RunScore | Promise<RunScore>;
|
|
76
|
-
onResult?: (result: HarnessRunResult) => void | Promise<void>;
|
|
77
|
-
}
|
|
78
|
-
export declare const DEFAULT_HARNESS_OBJECTIVES: Array<Objective<HarnessVariantReport>>;
|
|
79
|
-
export declare function runHarnessExperiment(config: HarnessExperimentConfig): Promise<HarnessExperimentResult>;
|
|
80
|
-
export declare function selectHarnessVariant(results: HarnessRunResult[], objectives?: Array<Objective<HarnessVariantReport>>): HarnessSelection;
|
|
81
|
-
export declare function summarizeHarnessResults(results: HarnessRunResult[]): HarnessVariantReport[];
|
|
82
|
-
//# sourceMappingURL=harness-optimizer.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"harness-optimizer.d.ts","sourceRoot":"","sources":["../src/harness-optimizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,YAAY,EAAkB,MAAM,UAAU,CAAA;AAC5E,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,cAAc,CAAA;AACvD,OAAO,EAAqB,KAAK,QAAQ,EAAE,KAAK,eAAe,EAAE,MAAM,aAAa,CAAA;AACpF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAEhD,MAAM,MAAM,mBAAmB,GAC3B,UAAU,GACV,MAAM,GACN,OAAO,GACP,SAAS,GACT,QAAQ,GACR,QAAQ,GACR,YAAY,GACZ,sBAAsB,GACtB,OAAO,CAAA;AAEX,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAA;IACV,aAAa,EAAE,mBAAmB,EAAE,CAAA;IACpC,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;IACnB,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAA;CAChD;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAA;IACV,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IAC3B,WAAW,CAAC,EAAE,iBAAiB,CAAA;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,OAAO,GAAG,YAAY,GAAG,MAAM,GAAG,MAAM,CAAA;IAChD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,OAAO,EAAE,cAAc,CAAA;IACvB,QAAQ,EAAE,eAAe,CAAA;IACzB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;CACnD;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,cAAc,CAAA;IACvB,QAAQ,EAAE,eAAe,CAAA;IACzB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,QAAQ,CAAA;IACf,KAAK,EAAE,QAAQ,CAAA;IACf,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,cAAc,CAAA;IACvB,IAAI,EAAE,gBAAgB,EAAE,CAAA;IACxB,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,MAAM,CAAA;IAChB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,SAAS,EAAE,QAAQ,CAAA;CACpB;AAED,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,oBAAoB,CAAA;IAC5B,QAAQ,EAAE,YAAY,CAAC,oBAAoB,CAAC,CAAA;IAC5C,OAAO,EAAE,oBAAoB,EAAE,CAAA;CAChC;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,gBAAgB,EAAE,CAAA;IAC3B,SAAS,EAAE,gBAAgB,CAAA;CAC5B;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,cAAc,CAAA;IACvB,QAAQ,EAAE,cAAc,EAAE,CAAA;IAC1B,SAAS,EAAE,eAAe,EAAE,CAAA;IAC5B,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,CAAA;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAAC,CAAA;IACnD,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,iBAAiB,KAAK,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAA;IACrF,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC9D;AAED,eAAO,MAAM,0BAA0B,EAAE,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAK7E,CAAA;AAED,wBAAsB,oBAAoB,CACxC,MAAM,EAAE,uBAAuB,GAC9B,OAAO,CAAC,uBAAuB,CAAC,CAmBlC;AAED,wBAAgB,oBAAoB,CAClC,OAAO,EAAE,gBAAgB,EAAE,EAC3B,UAAU,GAAE,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAC,CAA8B,GAC9E,gBAAgB,CAQlB;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,oBAAoB,EAAE,CAoB3F"}
|
package/dist/held-out-gate.d.ts
DELETED
|
@@ -1,135 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* HeldOutGate — first-class held-out paired-delta promotion gate.
|
|
3
|
-
*
|
|
4
|
-
* Encodes the "honesty override" pattern that lived inline in
|
|
5
|
-
* `~/webb/redteam/scripts/agent-eval-autoresearch.ts:138–171`.
|
|
6
|
-
* The optimizer's best-guess is one thing; what we should actually
|
|
7
|
-
* ship is another. The gate is the line between them.
|
|
8
|
-
*
|
|
9
|
-
* A candidate is promoted iff ALL three pass:
|
|
10
|
-
*
|
|
11
|
-
* 1. **Productive runs**: the candidate has at least
|
|
12
|
-
* `minProductiveRuns` paired observations on items where BOTH
|
|
13
|
-
* candidate and baseline produced a real (non-silent) score.
|
|
14
|
-
* 2. **Paired delta**: the lower bound of the bootstrap CI on the
|
|
15
|
-
* median per-item delta (candidate − baseline) on the HOLDOUT
|
|
16
|
-
* split is strictly greater than `pairedDeltaThreshold`.
|
|
17
|
-
* 3. **Overfit gap**: the candidate's gap between search-split
|
|
18
|
-
* score and holdout-split score is no worse (more positive)
|
|
19
|
-
* than the baseline's gap by more than `overfitGapThreshold`.
|
|
20
|
-
* "Better on search, worse on holdout" is the canonical
|
|
21
|
-
* overfit pattern; this catches it.
|
|
22
|
-
*
|
|
23
|
-
* The decision carries a machine-readable `rejectionCode` plus an
|
|
24
|
-
* `evidence` block with every number the gate looked at, so the
|
|
25
|
-
* downstream researcher / paper / dashboard can re-derive the
|
|
26
|
-
* verdict without re-running.
|
|
27
|
-
*
|
|
28
|
-
* See also:
|
|
29
|
-
* - `src/statistics.ts` for `pairedBootstrap` + `wilcoxonSignedRank`
|
|
30
|
-
* - `src/run-record.ts` for the input row schema
|
|
31
|
-
* - `src/reference-replay.ts` for the older, reference-replay-
|
|
32
|
-
* specific promotion path (still useful for replay-style evals).
|
|
33
|
-
*/
|
|
34
|
-
import type { RunRecord } from './run-record';
|
|
35
|
-
export type HeldOutGateRejectionCode = 'few_runs' | 'negative_delta' | 'overfit_gap' | 'cost_ceiling';
|
|
36
|
-
export interface HeldOutGateConfig {
|
|
37
|
-
/** Minimum number of paired (candidate, baseline) holdout observations
|
|
38
|
-
* required before the gate will even consider promoting. Default 3. */
|
|
39
|
-
minProductiveRuns?: number;
|
|
40
|
-
/** The bootstrap-CI lower bound on the median paired holdout delta
|
|
41
|
-
* must exceed this to promote. Default 0. */
|
|
42
|
-
pairedDeltaThreshold?: number;
|
|
43
|
-
/** Maximum allowed worsening of (search − holdout) gap relative to
|
|
44
|
-
* baseline. Default 0.15 (i.e. candidate may overfit by up to 15
|
|
45
|
-
* absolute score points more than baseline before rejection). */
|
|
46
|
-
overfitGapThreshold?: number;
|
|
47
|
-
/** Stable label of the baseline candidate. Required — paper-grade
|
|
48
|
-
* evaluation never compares two unlabelled candidates. */
|
|
49
|
-
baselineKey: string;
|
|
50
|
-
/** Confidence level for the bootstrap CI. Default 0.95. */
|
|
51
|
-
confidence?: number;
|
|
52
|
-
/** Bootstrap resamples. Default 2000. */
|
|
53
|
-
bootstrapResamples?: number;
|
|
54
|
-
/** Optional deterministic seed for the bootstrap. Default undefined
|
|
55
|
-
* (Math.random). */
|
|
56
|
-
seed?: number;
|
|
57
|
-
/**
|
|
58
|
-
* Hard ceiling on the candidate's median per-task USD cost. When the
|
|
59
|
-
* candidate clears the quality gates (paired-delta + overfit-gap) but
|
|
60
|
-
* its median cost exceeds this number, the gate rejects with
|
|
61
|
-
* `cost_ceiling`. Default `undefined` = no cost ceiling, behaving
|
|
62
|
-
* exactly like the pre-cost gate.
|
|
63
|
-
*
|
|
64
|
-
* This exists because "we ship the better prompt" is only an honest
|
|
65
|
-
* pitch when the better prompt also fits a customer-stated budget.
|
|
66
|
-
* Cost is read from `RunRecord.costUsd` (already mandatory on every
|
|
67
|
-
* run) so no new schema is required.
|
|
68
|
-
*/
|
|
69
|
-
costPerTaskCeiling?: number;
|
|
70
|
-
}
|
|
71
|
-
export interface GateEvidence {
|
|
72
|
-
/** Number of paired (candidate, baseline) holdout observations used. */
|
|
73
|
-
productiveRuns: number;
|
|
74
|
-
/** Median of (candidate − baseline) paired holdout deltas. */
|
|
75
|
-
medianPairedDelta: number;
|
|
76
|
-
/** Bootstrap CI on the median paired holdout delta. */
|
|
77
|
-
pairedCI: {
|
|
78
|
-
low: number;
|
|
79
|
-
high: number;
|
|
80
|
-
};
|
|
81
|
-
/** Wilcoxon signed-rank p-value on the paired holdout deltas. */
|
|
82
|
-
pairedPValue: number;
|
|
83
|
-
/** Mean candidate score on the search split (NaN if none). */
|
|
84
|
-
searchScore: number;
|
|
85
|
-
/** Mean candidate score on the holdout split (NaN if none). */
|
|
86
|
-
holdoutScore: number;
|
|
87
|
-
/** Candidate (search − holdout) gap. */
|
|
88
|
-
overfitGap: number;
|
|
89
|
-
/** Baseline (search − holdout) gap. */
|
|
90
|
-
baselineOverfitGap: number;
|
|
91
|
-
/** Median per-task USD cost across the candidate's runs. Recorded
|
|
92
|
-
* even when no `costPerTaskCeiling` is configured so downstream
|
|
93
|
-
* dashboards (intelligence.tangle.tools) can render \$/task per
|
|
94
|
-
* generation regardless of gating policy. */
|
|
95
|
-
medianCandidateCost: number;
|
|
96
|
-
/** Median per-task USD cost across the baseline runs, for
|
|
97
|
-
* symmetric reporting. */
|
|
98
|
-
medianBaselineCost: number;
|
|
99
|
-
}
|
|
100
|
-
export interface GateDecision {
|
|
101
|
-
/** Final promote/no-promote verdict. */
|
|
102
|
-
promote: boolean;
|
|
103
|
-
/** The candidate that was evaluated. */
|
|
104
|
-
candidateId: string;
|
|
105
|
-
/** The baseline it was compared against. */
|
|
106
|
-
baselineId: string;
|
|
107
|
-
/** Every number the gate looked at, for audit + paper export. */
|
|
108
|
-
evidence: GateEvidence;
|
|
109
|
-
/** Human-readable reason. */
|
|
110
|
-
reason: string;
|
|
111
|
-
/** Machine-readable rejection code, or null on promote. */
|
|
112
|
-
rejectionCode: HeldOutGateRejectionCode | null;
|
|
113
|
-
}
|
|
114
|
-
/**
|
|
115
|
-
* Held-out paired-delta promotion gate. Construct once with config,
|
|
116
|
-
* call `evaluate(candidateRuns, baselineRuns)` per (candidate,
|
|
117
|
-
* baseline) pair. Stateless across calls.
|
|
118
|
-
*/
|
|
119
|
-
export declare class HeldOutGate {
|
|
120
|
-
private readonly minProductiveRuns;
|
|
121
|
-
private readonly pairedDeltaThreshold;
|
|
122
|
-
private readonly overfitGapThreshold;
|
|
123
|
-
private readonly baselineKey;
|
|
124
|
-
private readonly confidence;
|
|
125
|
-
private readonly resamples;
|
|
126
|
-
private readonly seed?;
|
|
127
|
-
private readonly costPerTaskCeiling?;
|
|
128
|
-
constructor(config: HeldOutGateConfig);
|
|
129
|
-
/** Decide whether `candidate` should replace `baseline`. Pairing
|
|
130
|
-
* is by (experimentId, seed) — identical experiment + seed pairs
|
|
131
|
-
* the candidate run with the matching baseline run. Pairs without
|
|
132
|
-
* a holdout score on both sides are dropped. */
|
|
133
|
-
evaluate(candidate: RunRecord[], baseline: RunRecord[]): GateDecision;
|
|
134
|
-
}
|
|
135
|
-
//# sourceMappingURL=held-out-gate.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"held-out-gate.d.ts","sourceRoot":"","sources":["../src/held-out-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAG7C,MAAM,MAAM,wBAAwB,GAChC,UAAU,GACV,gBAAgB,GAChB,aAAa,GACb,cAAc,CAAA;AAElB,MAAM,WAAW,iBAAiB;IAChC;4EACwE;IACxE,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B;kDAC8C;IAC9C,oBAAoB,CAAC,EAAE,MAAM,CAAA;IAC7B;;sEAEkE;IAClE,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B;+DAC2D;IAC3D,WAAW,EAAE,MAAM,CAAA;IACnB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,yCAAyC;IACzC,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B;yBACqB;IACrB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;;;;;;;;;OAWG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,YAAY;IAC3B,wEAAwE;IACxE,cAAc,EAAE,MAAM,CAAA;IACtB,8DAA8D;IAC9D,iBAAiB,EAAE,MAAM,CAAA;IACzB,uDAAuD;IACvD,QAAQ,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;IACvC,iEAAiE;IACjE,YAAY,EAAE,MAAM,CAAA;IACpB,8DAA8D;IAC9D,WAAW,EAAE,MAAM,CAAA;IACnB,+DAA+D;IAC/D,YAAY,EAAE,MAAM,CAAA;IACpB,wCAAwC;IACxC,UAAU,EAAE,MAAM,CAAA;IAClB,uCAAuC;IACvC,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;kDAG8C;IAC9C,mBAAmB,EAAE,MAAM,CAAA;IAC3B;+BAC2B;IAC3B,kBAAkB,EAAE,MAAM,CAAA;CAC3B;AAED,MAAM,WAAW,YAAY;IAC3B,wCAAwC;IACxC,OAAO,EAAE,OAAO,CAAA;IAChB,wCAAwC;IACxC,WAAW,EAAE,MAAM,CAAA;IACnB,4CAA4C;IAC5C,UAAU,EAAE,MAAM,CAAA;IAClB,iEAAiE;IACjE,QAAQ,EAAE,YAAY,CAAA;IACtB,6BAA6B;IAC7B,MAAM,EAAE,MAAM,CAAA;IACd,2DAA2D;IAC3D,aAAa,EAAE,wBAAwB,GAAG,IAAI,CAAA;CAC/C;AAED;;;;GAIG;AACH,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAQ;IAC1C,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAQ;IAC7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAQ;IAC5C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAQ;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAQ;IACnC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAQ;IAClC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAQ;IAC9B,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAQ;gBAEhC,MAAM,EAAE,iBAAiB;IAoBrC;;;qDAGiD;IACjD,QAAQ,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,YAAY;CAqJtE"}
|
package/dist/hosted/client.d.ts
DELETED
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* # Hosted-tier ingest client.
|
|
3
|
-
*
|
|
4
|
-
* Ships eval-run events + trace spans to any orchestrator (ours, a
|
|
5
|
-
* partner's self-hosted one, or a future open implementation) that
|
|
6
|
-
* speaks the wire format in `./types.ts`.
|
|
7
|
-
*
|
|
8
|
-
* Three modes:
|
|
9
|
-
* - **Ours:** point at `https://orchestrator.tangle.tools` (the host root —
|
|
10
|
-
* the client appends the versioned `/v1/ingest/...` path itself; a trailing
|
|
11
|
-
* `/v1` on the endpoint is tolerated and normalized away). We handle ingest
|
|
12
|
-
* + storage + dashboard.
|
|
13
|
-
* - **Self-hosted:** point at whatever URL runs the reference receiver
|
|
14
|
-
* from `examples/hosted-ingest-server/`.
|
|
15
|
-
* - **Off (default):** when `hostedTenant` is unset, nothing is sent.
|
|
16
|
-
* Everything stays local.
|
|
17
|
-
*/
|
|
18
|
-
import { type EvalRunEvent, type HostedWireVersion, type IngestResponse, type TraceSpanEvent } from './types';
|
|
19
|
-
export interface HostedTenant {
|
|
20
|
-
/** Orchestrator endpoint base URL (no trailing slash). Required. */
|
|
21
|
-
endpoint: string;
|
|
22
|
-
/** Bearer token issued by the orchestrator. Required. */
|
|
23
|
-
apiKey: string;
|
|
24
|
-
/** Tenant id — the orchestrator's primary key for this consumer. Required. */
|
|
25
|
-
tenantId: string;
|
|
26
|
-
/** Optional `fetch` override (auth wrappers, custom agent, test mocks). */
|
|
27
|
-
fetchImpl?: typeof fetch;
|
|
28
|
-
/** Per-call timeout in ms. Default 30s. */
|
|
29
|
-
timeoutMs?: number;
|
|
30
|
-
/** Retries on 5xx / network errors. Default 2. */
|
|
31
|
-
retries?: number;
|
|
32
|
-
}
|
|
33
|
-
export interface HostedClient {
|
|
34
|
-
ingestEvalRun(event: EvalRunEvent, idempotencyKey?: string): Promise<IngestResponse>;
|
|
35
|
-
ingestEvalRuns(events: EvalRunEvent[], idempotencyKey?: string): Promise<IngestResponse>;
|
|
36
|
-
ingestTraces(spans: TraceSpanEvent[], idempotencyKey?: string): Promise<IngestResponse>;
|
|
37
|
-
readonly tenant: HostedTenant;
|
|
38
|
-
readonly wireVersion: HostedWireVersion;
|
|
39
|
-
}
|
|
40
|
-
export declare function createHostedClient(tenant: HostedTenant): HostedClient;
|
|
41
|
-
/**
|
|
42
|
-
* Build a `HostedClient` from environment, or `undefined` when ingest is not
|
|
43
|
-
* configured — the canonical, fail-soft wiring every product uses so eval-run +
|
|
44
|
-
* trace provenance lands in the Intelligence dashboard with ONE call:
|
|
45
|
-
*
|
|
46
|
-
* const hosted = hostedClientFromEnv()
|
|
47
|
-
* // ...run the loop...
|
|
48
|
-
* await emitLoopProvenance({ ..., hostedClient: hosted }) // no-op if undefined
|
|
49
|
-
*
|
|
50
|
-
* Returns `undefined` (NOT an error) when any of endpoint / apiKey / tenantId is
|
|
51
|
-
* missing — so a product wires the ship call unconditionally and it stays a
|
|
52
|
-
* no-op until the env is set. Env precedence:
|
|
53
|
-
* - endpoint: `TANGLE_INGEST_URL` → `TANGLE_ORCHESTRATOR_URL`
|
|
54
|
-
* - apiKey: `TANGLE_INGEST_API_KEY` → `TANGLE_API_KEY`
|
|
55
|
-
* - tenantId: `TANGLE_TENANT_ID`
|
|
56
|
-
* A trailing slash on the endpoint is stripped. Pass `overrides` to supply any
|
|
57
|
-
* field directly (e.g. a fixed `tenantId` per product) — overrides win over env.
|
|
58
|
-
*/
|
|
59
|
-
/**
|
|
60
|
-
* Build a {@link HostedTenant} config from env — the input `selfImprove`'s
|
|
61
|
-
* `hostedTenant` and `emitLoopProvenance` take. Same env precedence + overrides
|
|
62
|
-
* as {@link hostedClientFromEnv}; returns `undefined` (not an error) when any of
|
|
63
|
-
* endpoint / apiKey / tenantId is missing, so a product wires
|
|
64
|
-
* `hostedTenant: hostedTenantFromEnv({ tenantId: 'my-agent' })` unconditionally
|
|
65
|
-
* and it stays off until the env is set.
|
|
66
|
-
*/
|
|
67
|
-
export declare function hostedTenantFromEnv(overrides?: Partial<HostedTenant> & {
|
|
68
|
-
env?: Record<string, string | undefined>;
|
|
69
|
-
}): HostedTenant | undefined;
|
|
70
|
-
export declare function hostedClientFromEnv(overrides?: Partial<HostedTenant> & {
|
|
71
|
-
env?: Record<string, string | undefined>;
|
|
72
|
-
}): HostedClient | undefined;
|
|
73
|
-
//# sourceMappingURL=client.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/hosted/client.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EACL,KAAK,YAAY,EAEjB,KAAK,iBAAiB,EAEtB,KAAK,cAAc,EAEnB,KAAK,cAAc,EACpB,MAAM,SAAS,CAAA;AAEhB,MAAM,WAAW,YAAY;IAC3B,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAA;IAChB,yDAAyD;IACzD,MAAM,EAAE,MAAM,CAAA;IACd,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,CAAA;IAChB,2EAA2E;IAC3E,SAAS,CAAC,EAAE,OAAO,KAAK,CAAA;IACxB,2CAA2C;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kDAAkD;IAClD,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,aAAa,CAAC,KAAK,EAAE,YAAY,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;IACpF,cAAc,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;IACxF,YAAY,CAAC,KAAK,EAAE,cAAc,EAAE,EAAE,cAAc,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAA;IACvF,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAA;IAC7B,QAAQ,CAAC,WAAW,EAAE,iBAAiB,CAAA;CACxC;AAuED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,YAAY,GAAG,YAAY,CAuBrE;AAED;;;;;;;;;;;;;;;;;GAiBG;AACH;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,GAAE,OAAO,CAAC,YAAY,CAAC,GAAG;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAA;CAAO,GACnF,YAAY,GAAG,SAAS,CAe1B;AAED,wBAAgB,mBAAmB,CACjC,SAAS,GAAE,OAAO,CAAC,YAAY,CAAC,GAAG;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAA;CAAO,GACnF,YAAY,GAAG,SAAS,CAG1B"}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* `hostedTenantFromEnv` is the env→HostedTenant map every product passes to
|
|
3
|
-
* `selfImprove({ hostedTenant })`. It must fail SOFT (undefined, not throw) when
|
|
4
|
-
* unconfigured, so a product wires it unconditionally and it stays off until the
|
|
5
|
-
* env is set.
|
|
6
|
-
*/
|
|
7
|
-
export {};
|
|
8
|
-
//# sourceMappingURL=from-env.test.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"from-env.test.d.ts","sourceRoot":"","sources":["../../src/hosted/from-env.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/hosted/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,kBAAkB,EAClB,KAAK,YAAY,EACjB,KAAK,YAAY,EACjB,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,UAAU,CAAA;AACjB,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,yBAAyB,EAC9B,KAAK,aAAa,EAClB,mBAAmB,EACnB,KAAK,mBAAmB,EACxB,KAAK,iBAAiB,EACtB,KAAK,qBAAqB,EAC1B,KAAK,cAAc,EACnB,KAAK,mBAAmB,EACxB,KAAK,cAAc,GACpB,MAAM,SAAS,CAAA"}
|
package/dist/hosted/types.d.ts
DELETED
|
@@ -1,159 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* # Hosted-tier wire format — the schema that EVERY orchestrator (ours,
|
|
3
|
-
* a partner's self-hosted one, a future open implementation) must accept.
|
|
4
|
-
*
|
|
5
|
-
* **Stability:** every type in this file is committed under semver. New
|
|
6
|
-
* minors only ADD optional fields. Breaking changes mean a major bump
|
|
7
|
-
* (`HostedWireVersion` literal increment).
|
|
8
|
-
*
|
|
9
|
-
* The wire format is two event streams in one transport:
|
|
10
|
-
*
|
|
11
|
-
* 1. **Eval-run events** (`POST /v1/ingest/eval-runs`). Posted when a
|
|
12
|
-
* campaign / improvement-loop completes (or per-generation if
|
|
13
|
-
* streaming). Carries the structured result + per-cell scores +
|
|
14
|
-
* surface diffs the orchestrator stores for the dashboard.
|
|
15
|
-
*
|
|
16
|
-
* 2. **Trace spans** (`POST /v1/ingest/traces`). Standard OTLP-shaped
|
|
17
|
-
* spans with a few additional attributes so the orchestrator can
|
|
18
|
-
* pivot from eval-run → underlying execution. Compatible with any
|
|
19
|
-
* OTel collector.
|
|
20
|
-
*
|
|
21
|
-
* Both endpoints are authenticated with a bearer token + a tenant id
|
|
22
|
-
* header. Tenants isolate everything downstream of ingest; no tenant
|
|
23
|
-
* ever sees another tenant's data.
|
|
24
|
-
*/
|
|
25
|
-
import type { GateDecision, MutableSurface } from '../campaign/types';
|
|
26
|
-
import type { InsightReport } from '../contract/insight-report';
|
|
27
|
-
export type { InsightReport } from '../contract/insight-report';
|
|
28
|
-
export declare const HOSTED_WIRE_VERSION: "2026-05-26.v1";
|
|
29
|
-
export type HostedWireVersion = typeof HOSTED_WIRE_VERSION;
|
|
30
|
-
/** Every ingest request carries these. */
|
|
31
|
-
export interface HostedIngestHeaders {
|
|
32
|
-
/** Bearer token. The orchestrator validates against the tenant key. */
|
|
33
|
-
authorization: `Bearer ${string}`;
|
|
34
|
-
/** Stable tenant id (the orchestrator-side primary key for the tenant). */
|
|
35
|
-
'x-tangle-tenant-id': string;
|
|
36
|
-
/** Wire-version pin so the server can reject incompatible payloads. */
|
|
37
|
-
'x-tangle-wire-version': HostedWireVersion;
|
|
38
|
-
/** Optional idempotency key for retry-safe ingest. */
|
|
39
|
-
'idempotency-key'?: string;
|
|
40
|
-
}
|
|
41
|
-
/** Lifecycle stages of an eval-run as the substrate reports them. */
|
|
42
|
-
export type EvalRunStatus = 'started' | 'baseline-complete' | 'generation-complete' | 'gate-decided' | 'finished' | 'errored';
|
|
43
|
-
export interface EvalRunCellScore {
|
|
44
|
-
/** Stable scenario id from the consumer's scenario set. */
|
|
45
|
-
scenarioId: string;
|
|
46
|
-
/** Repetition index when reps > 1; 0 for the default. */
|
|
47
|
-
rep: number;
|
|
48
|
-
/** Composite score across all judges + dimensions for this cell. */
|
|
49
|
-
compositeMean: number;
|
|
50
|
-
/** Per-judge → per-dimension scores; null where the judge did not run. */
|
|
51
|
-
dimensions: Record<string, Record<string, number>>;
|
|
52
|
-
/** Per-cell error message if the dispatch threw. Null on success. */
|
|
53
|
-
errorMessage?: string;
|
|
54
|
-
}
|
|
55
|
-
export interface EvalRunGenerationSnapshot {
|
|
56
|
-
/** Generation index. 0 is baseline. */
|
|
57
|
-
index: number;
|
|
58
|
-
/** Candidate surface fingerprint (stable hash) — pivot key into the
|
|
59
|
-
* trace stream to fetch the underlying execution. */
|
|
60
|
-
surfaceHash: string;
|
|
61
|
-
/** The candidate surface itself. May be omitted to avoid PII when the
|
|
62
|
-
* consumer prefers not to ship verbatim prompts. */
|
|
63
|
-
surface?: MutableSurface;
|
|
64
|
-
/** Per-cell scores for this generation. */
|
|
65
|
-
cells: EvalRunCellScore[];
|
|
66
|
-
/** Aggregate composite mean across all cells in this generation. */
|
|
67
|
-
compositeMean: number;
|
|
68
|
-
/** Total $ spent across this generation. */
|
|
69
|
-
costUsd: number;
|
|
70
|
-
/** Wall-clock duration of this generation. */
|
|
71
|
-
durationMs: number;
|
|
72
|
-
}
|
|
73
|
-
/**
|
|
74
|
-
* The top-level eval-run event. One ingest call per logical eval-run;
|
|
75
|
-
* generations stream in incrementally via repeated calls with the same
|
|
76
|
-
* `runId`. The orchestrator deduplicates by `(runId, generation.index)`.
|
|
77
|
-
*/
|
|
78
|
-
export interface EvalRunEvent {
|
|
79
|
-
/** Stable run id (the substrate's `runId`). UUID or substrate-generated. */
|
|
80
|
-
runId: string;
|
|
81
|
-
/** Where this run was happening — derived from `RunCampaignOptions.runDir`. */
|
|
82
|
-
runDir: string;
|
|
83
|
-
/** ISO-8601 timestamp the substrate recorded the event. */
|
|
84
|
-
timestamp: string;
|
|
85
|
-
/** Lifecycle stage this event represents. */
|
|
86
|
-
status: EvalRunStatus;
|
|
87
|
-
/** Free-form consumer tags (env, branch, model id, etc.). Searchable. */
|
|
88
|
-
labels: Record<string, string>;
|
|
89
|
-
/** Baseline campaign snapshot. Present when status >= baseline-complete. */
|
|
90
|
-
baseline?: EvalRunGenerationSnapshot;
|
|
91
|
-
/** Per-generation snapshots. Streams in; orchestrator appends. */
|
|
92
|
-
generations: EvalRunGenerationSnapshot[];
|
|
93
|
-
/** Final gate decision. Present when status >= gate-decided. */
|
|
94
|
-
gateDecision?: GateDecision;
|
|
95
|
-
/** Held-out lift = winner-on-holdout - baseline-on-holdout. */
|
|
96
|
-
holdoutLift?: number;
|
|
97
|
-
/** Total $ spent across baseline + every generation. */
|
|
98
|
-
totalCostUsd: number;
|
|
99
|
-
/** Total wall-clock duration. */
|
|
100
|
-
totalDurationMs: number;
|
|
101
|
-
/** Error message if status === 'errored'. */
|
|
102
|
-
errorMessage?: string;
|
|
103
|
-
/** Rigor packet emitted alongside the run — distributional summary,
|
|
104
|
-
* paired-bootstrap lift CI, judge stats, inter-rater agreement,
|
|
105
|
-
* contamination check, failure clusters (when an analyst is wired),
|
|
106
|
-
* outcome correlation (when downstream signal is supplied), and the
|
|
107
|
-
* recommendations the dashboard surfaces verbatim. Additive; older
|
|
108
|
-
* clients that don't know about this field continue to work. */
|
|
109
|
-
insightReport?: InsightReport;
|
|
110
|
-
}
|
|
111
|
-
/**
|
|
112
|
-
* OTel-shape span with a few additional attributes for eval-run pivoting.
|
|
113
|
-
* Compatible with any OTLP collector — `name`, `traceId`, `spanId`,
|
|
114
|
-
* `startTimeUnixNano`, `endTimeUnixNano`, `attributes` are stock OTel.
|
|
115
|
-
*/
|
|
116
|
-
export interface TraceSpanEvent {
|
|
117
|
-
traceId: string;
|
|
118
|
-
spanId: string;
|
|
119
|
-
parentSpanId?: string;
|
|
120
|
-
name: string;
|
|
121
|
-
startTimeUnixNano: number;
|
|
122
|
-
endTimeUnixNano: number;
|
|
123
|
-
attributes: Record<string, string | number | boolean>;
|
|
124
|
-
events?: Array<{
|
|
125
|
-
timeUnixNano: number;
|
|
126
|
-
name: string;
|
|
127
|
-
attributes?: Record<string, string | number | boolean>;
|
|
128
|
-
}>;
|
|
129
|
-
status?: {
|
|
130
|
-
code: 'OK' | 'ERROR' | 'UNSET';
|
|
131
|
-
message?: string;
|
|
132
|
-
};
|
|
133
|
-
/** Pivot back into the eval-run stream. */
|
|
134
|
-
'tangle.runId'?: string;
|
|
135
|
-
/** Pivot to the specific generation. */
|
|
136
|
-
'tangle.generation'?: number;
|
|
137
|
-
/** Pivot to the specific cell. */
|
|
138
|
-
'tangle.cellId'?: string;
|
|
139
|
-
/** Pivot to the specific scenario. */
|
|
140
|
-
'tangle.scenarioId'?: string;
|
|
141
|
-
}
|
|
142
|
-
export interface IngestEvalRunsRequest {
|
|
143
|
-
wireVersion: HostedWireVersion;
|
|
144
|
-
events: EvalRunEvent[];
|
|
145
|
-
}
|
|
146
|
-
export interface IngestTracesRequest {
|
|
147
|
-
wireVersion: HostedWireVersion;
|
|
148
|
-
spans: TraceSpanEvent[];
|
|
149
|
-
}
|
|
150
|
-
export interface IngestResponse {
|
|
151
|
-
/** Accepted events / spans count. */
|
|
152
|
-
accepted: number;
|
|
153
|
-
/** Rejected events with reasons (validation failures, dup idempotency key, etc.). */
|
|
154
|
-
rejected: Array<{
|
|
155
|
-
index: number;
|
|
156
|
-
reason: string;
|
|
157
|
-
}>;
|
|
158
|
-
}
|
|
159
|
-
//# sourceMappingURL=types.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/hosted/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAA;AACrE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAI/D,YAAY,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAE/D,eAAO,MAAM,mBAAmB,EAAG,eAAwB,CAAA;AAC3D,MAAM,MAAM,iBAAiB,GAAG,OAAO,mBAAmB,CAAA;AAI1D,0CAA0C;AAC1C,MAAM,WAAW,mBAAmB;IAClC,uEAAuE;IACvE,aAAa,EAAE,UAAU,MAAM,EAAE,CAAA;IACjC,2EAA2E;IAC3E,oBAAoB,EAAE,MAAM,CAAA;IAC5B,uEAAuE;IACvE,uBAAuB,EAAE,iBAAiB,CAAA;IAC1C,sDAAsD;IACtD,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAC3B;AAID,qEAAqE;AACrE,MAAM,MAAM,aAAa,GACrB,SAAS,GACT,mBAAmB,GACnB,qBAAqB,GACrB,cAAc,GACd,UAAU,GACV,SAAS,CAAA;AAEb,MAAM,WAAW,gBAAgB;IAC/B,2DAA2D;IAC3D,UAAU,EAAE,MAAM,CAAA;IAClB,yDAAyD;IACzD,GAAG,EAAE,MAAM,CAAA;IACX,oEAAoE;IACpE,aAAa,EAAE,MAAM,CAAA;IACrB,0EAA0E;IAC1E,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAClD,qEAAqE;IACrE,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,yBAAyB;IACxC,uCAAuC;IACvC,KAAK,EAAE,MAAM,CAAA;IACb;0DACsD;IACtD,WAAW,EAAE,MAAM,CAAA;IACnB;yDACqD;IACrD,OAAO,CAAC,EAAE,cAAc,CAAA;IACxB,2CAA2C;IAC3C,KAAK,EAAE,gBAAgB,EAAE,CAAA;IACzB,oEAAoE;IACpE,aAAa,EAAE,MAAM,CAAA;IACrB,4CAA4C;IAC5C,OAAO,EAAE,MAAM,CAAA;IACf,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAA;CACnB;AAED;;;;GAIG;AACH,MAAM,WAAW,YAAY;IAC3B,4EAA4E;IAC5E,KAAK,EAAE,MAAM,CAAA;IACb,+EAA+E;IAC/E,MAAM,EAAE,MAAM,CAAA;IACd,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAA;IACjB,6CAA6C;IAC7C,MAAM,EAAE,aAAa,CAAA;IACrB,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC9B,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,yBAAyB,CAAA;IACpC,kEAAkE;IAClE,WAAW,EAAE,yBAAyB,EAAE,CAAA;IACxC,gEAAgE;IAChE,YAAY,CAAC,EAAE,YAAY,CAAA;IAC3B,+DAA+D;IAC/D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,wDAAwD;IACxD,YAAY,EAAE,MAAM,CAAA;IACpB,iCAAiC;IACjC,eAAe,EAAE,MAAM,CAAA;IACvB,6CAA6C;IAC7C,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;;;;qEAKiE;IACjE,aAAa,CAAC,EAAE,aAAa,CAAA;CAC9B;AAID;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,IAAI,EAAE,MAAM,CAAA;IACZ,iBAAiB,EAAE,MAAM,CAAA;IACzB,eAAe,EAAE,MAAM,CAAA;IACvB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAA;IACrD,MAAM,CAAC,EAAE,KAAK,CAAC;QACb,YAAY,EAAE,MAAM,CAAA;QACpB,IAAI,EAAE,MAAM,CAAA;QACZ,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAA;KACvD,CAAC,CAAA;IACF,MAAM,CAAC,EAAE;QAAE,IAAI,EAAE,IAAI,GAAG,OAAO,GAAG,OAAO,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAC7D,2CAA2C;IAC3C,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,wCAAwC;IACxC,mBAAmB,CAAC,EAAE,MAAM,CAAA;IAC5B,kCAAkC;IAClC,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,sCAAsC;IACtC,mBAAmB,CAAC,EAAE,MAAM,CAAA;CAC7B;AAID,MAAM,WAAW,qBAAqB;IACpC,WAAW,EAAE,iBAAiB,CAAA;IAC9B,MAAM,EAAE,YAAY,EAAE,CAAA;CACvB;AAED,MAAM,WAAW,mBAAmB;IAClC,WAAW,EAAE,iBAAiB,CAAA;IAC9B,KAAK,EAAE,cAAc,EAAE,CAAA;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAA;IAChB,qFAAqF;IACrF,QAAQ,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACnD"}
|
package/dist/index.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,YAAY,EAAE,qBAAqB,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AAClF,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AACtD,YAAY,EACV,yBAAyB,EACzB,gBAAgB,EAChB,qBAAqB,EACrB,6BAA6B,EAC7B,0BAA0B,EAC1B,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EAClB,uBAAuB,GACxB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EACL,mBAAmB,EACnB,+BAA+B,EAC/B,4BAA4B,EAC5B,mBAAmB,EACnB,yBAAyB,EACzB,8BAA8B,EAC9B,qBAAqB,EACrB,2BAA2B,EAC3B,uBAAuB,EACvB,kBAAkB,EAClB,wBAAwB,EACxB,sBAAsB,GACvB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,KAAK,qBAAqB,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AAQlF,YAAY,EACV,YAAY,EACZ,UAAU,EACV,WAAW,EACX,YAAY,EACZ,aAAa,EACb,sBAAsB,EACtB,oBAAoB,EACpB,2BAA2B,EAC3B,iBAAiB,EACjB,mBAAmB,EACnB,uBAAuB,GACxB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAA;AACxD,OAAO,EACL,2BAA2B,EAC3B,KAAK,6BAA6B,GACnC,MAAM,4BAA4B,CAAA;AACnC,YAAY,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAA;AACpE,YAAY,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAA;AACnF,OAAO,EACL,KAAK,UAAU,EACf,iBAAiB,EACjB,YAAY,EACZ,KAAK,YAAY,EACjB,aAAa,EACb,KAAK,gBAAgB,GACtB,MAAM,0BAA0B,CAAA;AACjC,OAAO,EACL,KAAK,0BAA0B,EAC/B,sBAAsB,EACtB,mBAAmB,EACnB,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,GAC1B,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EACL,2BAA2B,EAC3B,sBAAsB,EACtB,qBAAqB,EACrB,uBAAuB,EACvB,6BAA6B,GAC9B,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAA;AACpF,OAAO,EACL,KAAK,YAAY,EACjB,eAAe,EACf,KAAK,sBAAsB,EAC3B,KAAK,YAAY,EACjB,KAAK,eAAe,GACrB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EACL,KAAK,OAAO,EACZ,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACtB,KAAK,eAAe,EACpB,gBAAgB,EAChB,KAAK,WAAW,EAChB,WAAW,GACZ,MAAM,iBAAiB,CAAA;AACxB,YAAY,EACV,YAAY,EACZ,UAAU,EACV,kBAAkB,EAClB,uBAAuB,EACvB,gCAAgC,EAChC,iCAAiC,EACjC,OAAO,GACR,MAAM,WAAW,CAAA;AAClB,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,2BAA2B,GAC5B,MAAM,WAAW,CAAA;AAClB,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACxD,YAAY,EACV,wBAAwB,EACxB,oBAAoB,EACpB,aAAa,EACb,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACnB,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,YAAY,GACb,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,GACf,MAAM,mBAAmB,CAAA;AAC1B,YAAY,EACV,aAAa,EACb,gBAAgB,EAChB,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,aAAa,CAAA;AACpB,OAAO,EACL,mBAAmB,EACnB,kBAAkB,EAClB,UAAU,EACV,sBAAsB,GACvB,MAAM,aAAa,CAAA;AACpB,YAAY,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAA;AAC9F,OAAO,EACL,WAAW,EACX,uBAAuB,EACvB,6BAA6B,EAC7B,kBAAkB,GACnB,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAA;AAIlD,OAAO,EACL,cAAc,EACd,qBAAqB,EACrB,WAAW,EACX,UAAU,EACV,aAAa,EACb,WAAW,EACX,eAAe,EACf,iBAAiB,GAClB,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,cAAc,EAAE,MAAM,YAAY,CAAA;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAC5C,YAAY,EACV,oBAAoB,EACpB,eAAe,EACf,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,oBAAoB,EACpB,eAAe,EACf,qBAAqB,EACrB,oBAAoB,EACpB,gBAAgB,EAChB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,wBAAwB,EACxB,uBAAuB,EACvB,qBAAqB,EACrB,kBAAkB,GACnB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EACL,mBAAmB,EACnB,8BAA8B,EAC9B,wBAAwB,EACxB,iCAAiC,EACjC,sCAAsC,EACtC,mCAAmC,EACnC,mCAAmC,EACnC,gCAAgC,EAChC,+BAA+B,EAC/B,8BAA8B,EAC9B,8BAA8B,EAC9B,0BAA0B,EAC1B,wBAAwB,EACxB,kCAAkC,EAClC,yBAAyB,EACzB,yBAAyB,GAC1B,MAAM,uBAAuB,CAAA;AAK9B,YAAY,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAA;AAC3E,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,yBAAyB,GAC1B,MAAM,+BAA+B,CAAA;AAItC,OAAO,EACL,kBAAkB,EAClB,KAAK,cAAc,EACnB,sBAAsB,EACtB,KAAK,sBAAsB,EAC3B,KAAK,gBAAgB,EACrB,eAAe,GAChB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EACL,KAAK,0BAA0B,EAC/B,mBAAmB,EACnB,KAAK,iBAAiB,EACtB,KAAK,uBAAuB,EAC5B,kBAAkB,EAClB,KAAK,kBAAkB,EACvB,KAAK,mBAAmB,GACzB,MAAM,4BAA4B,CAAA;AAEnC,OAAO,EACL,KAAK,wBAAwB,EAC7B,iBAAiB,EACjB,gBAAgB,EAChB,KAAK,WAAW,EAChB,WAAW,GACZ,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,gBAAgB,EAChB,kBAAkB,EAClB,cAAc,EACd,iBAAiB,EACjB,uBAAuB,EACvB,aAAa,EACb,eAAe,GAChB,MAAM,UAAU,CAAA;AACjB,cAAc,aAAa,CAAA;AAC3B,YAAY,EACV,iBAAiB,EACjB,eAAe,EACf,gBAAgB,EAChB,eAAe,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAC3C,OAAO,EACL,YAAY,EACZ,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,YAAY,GACb,MAAM,WAAW,CAAA;AAClB,YAAY,EACV,iBAAiB,EACjB,wBAAwB,EACxB,eAAe,EACf,sBAAsB,EACtB,eAAe,EACf,wBAAwB,EACxB,aAAa,EACb,oBAAoB,EACpB,gBAAgB,EAChB,cAAc,GACf,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EACL,sBAAsB,EACtB,iBAAiB,EACjB,+BAA+B,EAC/B,qBAAqB,EACrB,mBAAmB,EACnB,0BAA0B,GAC3B,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAC7C,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AAC1F,YAAY,EACV,4BAA4B,EAC5B,mBAAmB,GACpB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACL,qBAAqB,EACrB,cAAc,GACf,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACV,eAAe,EACf,sBAAsB,EACtB,2BAA2B,EAC3B,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,EACb,sBAAsB,EACtB,qBAAqB,EACrB,kBAAkB,EAClB,oBAAoB,EACpB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,cAAc,CAAA;AAErB,OAAO,EACL,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,OAAO,EACP,kBAAkB,EAClB,yBAAyB,EACzB,wCAAwC,EACxC,eAAe,EACf,qBAAqB,EACrB,YAAY,EACZ,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,eAAe,EACf,eAAe,EACf,SAAS,EACT,oBAAoB,EACpB,WAAW,EACX,aAAa,EACb,OAAO,EACP,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,EACZ,kBAAkB,EAClB,MAAM,GACP,MAAM,cAAc,CAAA;AAKrB,cAAc,iBAAiB,CAAA;AAC/B,YAAY,EACV,iBAAiB,EACjB,cAAc,EACd,gBAAgB,GACjB,MAAM,oCAAoC,CAAA;AAC3C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oCAAoC,CAAA;AACxE,YAAY,EACV,aAAa,EACb,cAAc,EACd,eAAe,EACf,qBAAqB,EACrB,WAAW,EACX,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,WAAW,EACX,UAAU,EACV,eAAe,EACf,WAAW,EACX,OAAO,EACP,UAAU,EACV,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,QAAQ,EACR,eAAe,EACf,QAAQ,EACR,YAAY,EACZ,cAAc,EACd,UAAU,EACV,IAAI,EACJ,WAAW,EACX,UAAU,GACX,MAAM,SAAS,CAAA;AAIhB,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAC9F,OAAO,EAAE,eAAe,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAClE,YAAY,EACV,QAAQ,IAAI,qBAAqB,EACjC,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,gBAAgB,GACjB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EACL,eAAe,EACf,iBAAiB,EACjB,WAAW,EACX,WAAW,EACX,UAAU,GACX,MAAM,sBAAsB,CAAA;AAC7B,YAAY,EACV,qBAAqB,EACrB,iBAAiB,EACjB,kBAAkB,EAClB,yBAAyB,EACzB,gBAAgB,EAChB,aAAa,EACb,gBAAgB,EAChB,WAAW,EACX,QAAQ,GACT,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EACL,iBAAiB,EACjB,2BAA2B,EAC3B,wBAAwB,EACxB,wBAAwB,EACxB,gBAAgB,GACjB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAA;AAClD,YAAY,EACV,oBAAoB,EACpB,eAAe,EACf,cAAc,EACd,iBAAiB,EACjB,uBAAuB,GACxB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnD,YAAY,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AACpE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAC1D,YAAY,EACV,cAAc,EACd,uBAAuB,EACvB,uBAAuB,EACvB,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,cAAc,EACd,oBAAoB,EACpB,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACL,0BAA0B,EAC1B,oBAAoB,EACpB,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,qBAAqB,CAAA;AAC5B,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACvE,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAA;AACzD,YAAY,EAAE,oBAAoB,EAAE,MAAM,eAAe,CAAA;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC7C,YAAY,EACV,iBAAiB,EACjB,gBAAgB,EAChB,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACL,aAAa,EACb,WAAW,EACX,WAAW,EACX,aAAa,EACb,aAAa,EACb,SAAS,GACV,MAAM,gBAAgB,CAAA;AACvB,YAAY,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,YAAY,CAAA;AACpE,YAAY,EACV,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AACvD,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AACrD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAA;AAC/D,YAAY,EACV,WAAW,EACX,iBAAiB,EACjB,SAAS,EACT,YAAY,EACZ,aAAa,EACb,mBAAmB,EACnB,mBAAmB,EACnB,iBAAiB,EACjB,MAAM,EACN,QAAQ,EACR,WAAW,EACX,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,EACZ,QAAQ,GACT,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAA;AACzB,YAAY,EACV,0BAA0B,EAC1B,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,GAC1B,MAAM,0BAA0B,CAAA;AACjC,OAAO,EACL,mCAAmC,EACnC,6BAA6B,GAC9B,MAAM,0BAA0B,CAAA;AACjC,YAAY,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,YAAY,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAA;AACnF,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACnF,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAA;AACpE,YAAY,EACV,yBAAyB,EACzB,0BAA0B,EAC1B,uBAAuB,EACvB,4BAA4B,EAC5B,wBAAwB,EACxB,uBAAuB,GACxB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAA;AACzF,YAAY,EACV,gBAAgB,EAChB,kBAAkB,EAClB,wBAAwB,EACxB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EACL,YAAY,EACZ,UAAU,EACV,0BAA0B,EAC1B,QAAQ,EACR,QAAQ,EACR,aAAa,GACd,MAAM,uBAAuB,CAAA;AAI9B,cAAc,SAAS,CAAA;AASvB,OAAO,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAC/D,YAAY,EACV,YAAY,EACZ,qBAAqB,EACrB,cAAc,EACd,WAAW,GACZ,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EACL,eAAe,EACf,aAAa,IAAI,qBAAqB,EACtC,eAAe,GAChB,MAAM,oBAAoB,CAAA;AAC3B,YAAY,EACV,uCAAuC,EACvC,mCAAmC,EACnC,gCAAgC,EAChC,0BAA0B,EAC1B,uBAAuB,EACvB,0BAA0B,GAC3B,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EACL,+BAA+B,EAC/B,gCAAgC,GACjC,MAAM,sBAAsB,CAAA;AAC7B,YAAY,EACV,aAAa,EACb,aAAa,EACb,oBAAoB,EACpB,aAAa,EACb,8BAA8B,EAC9B,gBAAgB,GACjB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACL,cAAc,EACd,mBAAmB,EACnB,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,uBAAuB,EACvB,gBAAgB,GACjB,MAAM,mBAAmB,CAAA;AAC1B,YAAY,EACV,oBAAoB,EACpB,mBAAmB,EACnB,kBAAkB,GACnB,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAA;AAC9D,YAAY,EAAE,SAAS,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAA;AAC1D,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAM9C,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AACvF,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAC/F,OAAO,EAAE,iBAAiB,EAAE,GAAG,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAChE,YAAY,EACV,aAAa,EACb,WAAW,EACX,eAAe,EACf,iBAAiB,EACjB,UAAU,EACV,SAAS,GACV,MAAM,eAAe,CAAA;AACtB,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AACvE,YAAY,EAAE,SAAS,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAA;AACrF,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAA;AAC5C,YAAY,EACV,mBAAmB,EACnB,gBAAgB,EAChB,eAAe,GAChB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EACL,cAAc,EACd,wBAAwB,EACxB,qBAAqB,EACrB,QAAQ,GACT,MAAM,oBAAoB,CAAA;AAC3B,YAAY,EACV,qBAAqB,EACrB,UAAU,EACV,oBAAoB,EACpB,aAAa,EACb,eAAe,EACf,eAAe,EACf,wBAAwB,EACxB,iBAAiB,EACjB,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,GACjB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EACL,sBAAsB,EACtB,iBAAiB,EACjB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,uBAAuB,GACxB,MAAM,sBAAsB,CAAA;AAC7B,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AACxF,OAAO,EACL,eAAe,EACf,0BAA0B,EAC1B,yBAAyB,EACzB,kBAAkB,EAClB,mBAAmB,EACnB,oBAAoB,EACpB,cAAc,EACd,mBAAmB,EACnB,iBAAiB,GAClB,MAAM,wBAAwB,CAAA;AAC/B,YAAY,EAAE,MAAM,EAAE,iBAAiB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AACrF,OAAO,EACL,eAAe,EACf,SAAS,EACT,UAAU,EACV,YAAY,EACZ,cAAc,EACd,WAAW,GACZ,MAAM,UAAU,CAAA;AACjB,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,UAAU,CAAA;AAClE,OAAO,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACpD,YAAY,EACV,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EACL,gBAAgB,EAChB,OAAO,EACP,UAAU,EACV,gBAAgB,GACjB,MAAM,sBAAsB,CAAA;AAE7B,YAAY,EACV,WAAW,EACX,oBAAoB,EACpB,iBAAiB,EACjB,SAAS,EACT,aAAa,EACb,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,gBAAgB,GACjB,MAAM,aAAa,CAAA;AACpB,OAAO,EACL,eAAe,EACf,aAAa,EACb,mBAAmB,EACnB,aAAa,EACb,UAAU,EACV,qBAAqB,GACtB,MAAM,aAAa,CAAA;AACpB,YAAY,EAAE,wBAAwB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAA;AAC7F,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,YAAY,EAAE,GAAG,EAAE,cAAc,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,OAAO,CAAA;AACvF,OAAO,EAAE,SAAS,EAAE,kBAAkB,EAAE,MAAM,OAAO,CAAA;AACrD,YAAY,EACV,eAAe,EACf,qBAAqB,EACrB,gBAAgB,EAChB,sBAAsB,GACvB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EACL,mBAAmB,EACnB,YAAY,EACZ,eAAe,EACf,cAAc,GACf,MAAM,oBAAoB,CAAA;AAU3B,YAAY,EACV,SAAS,EACT,mBAAmB,EACnB,iBAAiB,EACjB,MAAM,GACP,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,qBAAqB,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAC/D,YAAY,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAI/C,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AACpG,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAC7D,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAClF,OAAO,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,WAAW,CAAA;AAClE,YAAY,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AACvD,OAAO,EACL,cAAc,EACd,qBAAqB,EACrB,aAAa,EACb,cAAc,EACd,qBAAqB,GACtB,MAAM,uBAAuB,CAAA;AAC9B,YAAY,EACV,iBAAiB,EACjB,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,YAAY,EACZ,YAAY,GACb,MAAM,WAAW,CAAA;AAClB,OAAO,EAAE,OAAO,EAAE,kBAAkB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAA;AACtE,YAAY,EACV,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACnB,0BAA0B,EAC1B,2BAA2B,EAC3B,UAAU,EACV,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,qBAAqB,CAAA;AAE5B,OAAO,EACL,cAAc,EACd,wBAAwB,EACxB,mBAAmB,EACnB,cAAc,EACd,cAAc,EACd,aAAa,GACd,MAAM,qBAAqB,CAAA;AAC5B,YAAY,EACV,iBAAiB,EACjB,gBAAgB,EAChB,kBAAkB,EAClB,aAAa,GACd,MAAM,iBAAiB,CAAA;AACxB,OAAO,EACL,uBAAuB,EACvB,kBAAkB,EAClB,gBAAgB,GACjB,MAAM,iBAAiB,CAAA;AACxB,YAAY,EACV,OAAO,EACP,iCAAiC,EACjC,kCAAkC,EAClC,gBAAgB,GACjB,MAAM,cAAc,CAAA;AACrB,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,oBAAoB,EACpB,6BAA6B,EAC7B,uBAAuB,EACvB,sBAAsB,EACtB,WAAW,EACX,yBAAyB,GAC1B,MAAM,cAAc,CAAA;AACrB,YAAY,EACV,WAAW,EACX,eAAe,EACf,cAAc,EACd,cAAc,EACd,aAAa,GACd,MAAM,YAAY,CAAA;AACnB,OAAO,EACL,uBAAuB,EACvB,cAAc,EACd,aAAa,EACb,kBAAkB,EAClB,eAAe,GAChB,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,SAAS,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AACnF,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAM3D,YAAY,EAAE,aAAa,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AACzE,OAAO,EACL,MAAM,EACN,YAAY,EACZ,YAAY,GACb,MAAM,YAAY,CAAA;AAKnB,YAAY,EACV,qBAAqB,EACrB,sBAAsB,EACtB,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AAC9E,YAAY,EACV,WAAW,EACX,cAAc,EACd,qBAAqB,EACrB,eAAe,GAChB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnD,YAAY,EACV,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,kBAAkB,GACnB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,QAAQ,EACR,YAAY,EACZ,cAAc,GACf,MAAM,oBAAoB,CAAA;AAI3B,YAAY,EAAE,qBAAqB,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAChG,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAA;AAC3D,YAAY,EACV,uBAAuB,EACvB,kBAAkB,EAClB,aAAa,EACb,uBAAuB,GACxB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAA;AACxD,YAAY,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAA;AACjF,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,uBAAuB,CAAA;AAC9B,YAAY,EACV,iBAAiB,EACjB,cAAc,EACd,YAAY,EACZ,eAAe,EACf,gBAAgB,EAChB,cAAc,GACf,MAAM,aAAa,CAAA;AACpB,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AAIzC,cAAc,cAAc,CAAA;AAI5B,YAAY,EACV,aAAa,EACb,QAAQ,EACR,eAAe,EACf,gBAAgB,GACjB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AACrD,YAAY,EACV,YAAY,EACZ,oBAAoB,EACpB,YAAY,EACZ,eAAe,EACf,qBAAqB,EACrB,yBAAyB,GAC1B,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAA;AAC7F,YAAY,EACV,iBAAiB,EACjB,cAAc,EACd,aAAa,GACd,MAAM,yBAAyB,CAAA;AAChC,OAAO,EACL,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,yBAAyB,CAAA;AAChC,YAAY,EACV,UAAU,EACV,YAAY,EACZ,qBAAqB,EACrB,UAAU,EACV,oBAAoB,EACpB,QAAQ,EACR,QAAQ,GACT,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACxC,YAAY,EACV,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EACL,sBAAsB,EACtB,0BAA0B,EAC1B,mBAAmB,GACpB,MAAM,sBAAsB,CAAA;AAC7B,YAAY,EACV,kBAAkB,EAClB,sBAAsB,EACtB,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,0BAA0B,CAAA;AACjC,OAAO,EACL,gBAAgB,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,0BAA0B,GAC3B,MAAM,0BAA0B,CAAA;AACjC,YAAY,EACV,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,UAAU,EACV,oBAAoB,EACpB,QAAQ,GACT,MAAM,cAAc,CAAA;AACrB,OAAO,EACL,cAAc,EACd,SAAS,EACT,OAAO,EACP,WAAW,EACX,mBAAmB,EACnB,YAAY,EACZ,SAAS,EACT,sBAAsB,EACtB,QAAQ,EACR,eAAe,GAChB,MAAM,cAAc,CAAA;AACrB,YAAY,EACV,OAAO,EACP,KAAK,EACL,WAAW,EACX,WAAW,EACX,QAAQ,EACR,kBAAkB,EAClB,aAAa,EACb,aAAa,GACd,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EACL,mBAAmB,EACnB,kBAAkB,GACnB,MAAM,wBAAwB,CAAA;AAC/B,YAAY,EACV,UAAU,EACV,YAAY,EACZ,yBAAyB,GAC1B,MAAM,yBAAyB,CAAA;AAChC,OAAO,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAA;AAEhF,OAAO,EACL,sBAAsB,EACtB,8BAA8B,EAC9B,iCAAiC,EACjC,6BAA6B,EAC7B,4BAA4B,EAC5B,yBAAyB,EACzB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,oBAAoB,CAAA;AAC3B,YAAY,EACV,4BAA4B,EAC5B,mBAAmB,EACnB,cAAc,EACd,mBAAmB,EACnB,wBAAwB,EACxB,2BAA2B,GAC5B,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,mBAAmB,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAA;AACvE,YAAY,EACV,iBAAiB,EACjB,cAAc,EACd,WAAW,EACX,qBAAqB,EACrB,yBAAyB,EACzB,2BAA2B,EAC3B,0BAA0B,GAC3B,MAAM,0BAA0B,CAAA;AACjC,OAAO,EACL,0BAA0B,EAC1B,0BAA0B,EAC1B,uBAAuB,EACvB,8BAA8B,GAC/B,MAAM,0BAA0B,CAAA;AAIjC,OAAO,KAAK,UAAU,MAAM,oBAAoB,CAAA;AAChD,YAAY,EACV,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EACL,oBAAoB,EACpB,kBAAkB,IAAI,2BAA2B,GAClD,MAAM,oBAAoB,CAAA;AAC3B,YAAY,EACV,WAAW,EACX,UAAU,EACV,aAAa,EACb,YAAY,EACZ,cAAc,GACf,MAAM,UAAU,CAAA;AACjB,OAAO,EAAE,WAAW,EAAE,MAAM,UAAU,CAAA;AAEtC,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAA;AACrC,YAAY,EACV,0BAA0B,EAC1B,uBAAuB,EACvB,yBAAyB,EACzB,yBAAyB,EACzB,8BAA8B,GAC/B,MAAM,2BAA2B,CAAA;AAClC,OAAO,EACL,qBAAqB,EACrB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,2BAA2B,CAAA;AAClC,YAAY,EACV,iBAAiB,EACjB,uBAAuB,GACxB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAA;AACtD,YAAY,EACV,qBAAqB,EACrB,uBAAuB,EACvB,kBAAkB,EAClB,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,kBAAkB,EAClB,SAAS,GACV,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAA;AACjD,YAAY,EACV,cAAc,EACd,UAAU,EACV,WAAW,GACZ,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,wBAAwB,EACxB,YAAY,EACZ,SAAS,IAAI,eAAe,EAC5B,cAAc,GACf,MAAM,kBAAkB,CAAA;AACzB,YAAY,EACV,YAAY,EACZ,YAAY,EACZ,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAA;AAC7C,YAAY,EACV,iBAAiB,EACjB,gBAAgB,GACjB,MAAM,eAAe,CAAA;AACtB,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAA;AAC7D,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AAC9E,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEnD,OAAO,EAAE,gBAAgB,EAAE,0BAA0B,EAAE,WAAW,EAAE,MAAM,UAAU,CAAA;AACpF,YAAY,EACV,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,OAAO,GACR,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAE/D,YAAY,EACV,oBAAoB,EACpB,sBAAsB,EACtB,wBAAwB,EACxB,0BAA0B,EAC1B,wBAAwB,EACxB,wBAAwB,EACxB,mBAAmB,EACnB,sBAAsB,EACtB,gCAAgC,EAChC,mBAAmB,EACnB,oBAAoB,EACpB,sBAAsB,EACtB,4BAA4B,EAC5B,gCAAgC,EAChC,8BAA8B,EAC9B,kBAAkB,EAClB,yBAAyB,EACzB,yBAAyB,EACzB,uBAAuB,EACvB,uBAAuB,EACvB,4BAA4B,EAC5B,oBAAoB,EACpB,2BAA2B,EAC3B,oBAAoB,EACpB,8BAA8B,GAC/B,MAAM,oBAAoB,CAAA;AAC3B,YAAY,EAAE,kCAAkC,EAAE,MAAM,6BAA6B,CAAA;AACrF,OAAO,EACL,iCAAiC,EACjC,iCAAiC,GAClC,MAAM,6BAA6B,CAAA;AACpC,YAAY,EACV,iBAAiB,EACjB,kBAAkB,EAClB,UAAU,GACX,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EACL,qBAAqB,EACrB,2BAA2B,EAC3B,uBAAuB,GACxB,MAAM,uBAAuB,CAAA;AAC9B,YAAY,EACV,kBAAkB,EAClB,WAAW,EACX,qBAAqB,EACrB,yBAAyB,EACzB,sBAAsB,EACtB,sBAAsB,EACtB,wBAAwB,EACxB,0BAA0B,EAC1B,uBAAuB,EACvB,2BAA2B,EAC3B,oBAAoB,GACrB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,uBAAuB,EAAE,yBAAyB,EAAE,MAAM,sBAAsB,CAAA;AACzF,YAAY,EAAE,0BAA0B,EAAE,MAAM,kBAAkB,CAAA;AAClE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACtD,YAAY,EACV,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,UAAU,CAAA;AACjB,OAAO,EACL,iBAAiB,EACjB,eAAe,EACf,WAAW,EACX,oBAAoB,GACrB,MAAM,UAAU,CAAA;AACjB,YAAY,EACV,yBAAyB,EACzB,cAAc,EACd,gBAAgB,EAChB,WAAW,EACX,UAAU,EACV,cAAc,GACf,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAA;AAEjE,YAAY,EACV,iBAAiB,EACjB,gBAAgB,EAChB,UAAU,EACV,SAAS,EACT,WAAW,EACX,aAAa,GACd,MAAM,cAAc,CAAA;AACrB,OAAO,EACL,WAAW,EACX,kBAAkB,EAClB,wBAAwB,EACxB,kBAAkB,EAClB,iBAAiB,GAClB,MAAM,cAAc,CAAA;AACrB,YAAY,EACV,qBAAqB,EACrB,QAAQ,EACR,WAAW,EACX,WAAW,GACZ,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAClD,YAAY,EACV,wBAAwB,EACxB,6BAA6B,EAC7B,mBAAmB,EACnB,oBAAoB,EACpB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,cAAc,CAAA;AACrB,OAAO,EACL,gCAAgC,EAChC,oBAAoB,GACrB,MAAM,cAAc,CAAA;AACrB,YAAY,EACV,mBAAmB,EACnB,0BAA0B,EAC1B,uBAAuB,EACvB,gBAAgB,EAChB,WAAW,EACX,cAAc,EACd,uBAAuB,EACvB,sBAAsB,EACtB,yBAAyB,EACzB,qBAAqB,EACrB,4BAA4B,EAC5B,YAAY,EACZ,mBAAmB,EACnB,eAAe,GAChB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EACL,aAAa,EACb,WAAW,EACX,+BAA+B,EAC/B,cAAc,EACd,YAAY,GACb,MAAM,kBAAkB,CAAA;AAIzB,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AAC9E,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACpE,YAAY,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAA;AAC5D,OAAO,EAAE,mBAAmB,EAAE,MAAM,kBAAkB,CAAA;AACtD,YAAY,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACzD,OAAO,EAAE,UAAU,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AAMhE,YAAY,EACV,eAAe,EACf,0BAA0B,EAC1B,aAAa,EACb,kBAAkB,GACnB,MAAM,yCAAyC,CAAA;AAChD,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,MAAM,yCAAyC,CAAA;AAC7F,YAAY,EACV,YAAY,EACZ,SAAS,EACT,gBAAgB,GACjB,MAAM,wCAAwC,CAAA;AAC/C,OAAO,EACL,iBAAiB,EACjB,cAAc,EACd,SAAS,GACV,MAAM,wCAAwC,CAAA;AAC/C,YAAY,EACV,iBAAiB,EACjB,mBAAmB,EACnB,sBAAsB,EACtB,qBAAqB,GACtB,MAAM,0CAA0C,CAAA;AACjD,OAAO,EACL,wBAAwB,EACxB,0BAA0B,EAC1B,eAAe,GAChB,MAAM,0CAA0C,CAAA;AAKjD,OAAO,KAAK,OAAO,MAAM,iBAAiB,CAAA;AAI1C,YAAY,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAChE,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC9D,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AACzE,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAIxE,YAAY,EACV,qBAAqB,EACrB,uBAAuB,EACvB,cAAc,GACf,MAAM,eAAe,CAAA;AACtB,OAAO,EAAE,qBAAqB,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAA;AAMhF,YAAY,EACV,eAAe,EACf,kBAAkB,EAClB,WAAW,EACX,YAAY,EACZ,cAAc,EACd,cAAc,EACd,YAAY,EACZ,QAAQ,EACR,YAAY,GACb,MAAM,QAAQ,CAAA;AACf,OAAO,EACL,qBAAqB,EACrB,oBAAoB,EACpB,YAAY,EACZ,QAAQ,EACR,WAAW,EACX,gBAAgB,GACjB,MAAM,QAAQ,CAAA;AAKf,YAAY,EAAE,QAAQ,EAAE,eAAe,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAC1F,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA;AAKnD,YAAY,EACV,mBAAmB,EACnB,oBAAoB,EACpB,YAAY,EACZ,gBAAgB,EAChB,YAAY,EACZ,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,aAAa,EACb,WAAW,EACX,aAAa,GACd,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,qBAAqB,EACrB,SAAS,EACT,oBAAoB,EACpB,aAAa,GACd,MAAM,mBAAmB,CAAA;AAC1B,YAAY,EACV,WAAW,EACX,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,GAClB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EACL,WAAW,EACX,aAAa,EACb,WAAW,EACX,gBAAgB,EAChB,oBAAoB,GACrB,MAAM,iBAAiB,CAAA"}
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Backend-integrity guard: distinguish "agent failed" from "eval ran against
|
|
3
|
-
* a stub / unconfigured backend." Without this guard a canonical eval can
|
|
4
|
-
* silently report `0/N passed` and look like an agent-quality problem when
|
|
5
|
-
* the LLM was never actually called — the failure mode we just hit running
|
|
6
|
-
* the 4-vertical parallel eval (legal-sandbox-stub returned hard-coded 33-104
|
|
7
|
-
* char strings; gtm/creative defaulted to a cli-bridge that wasn't running).
|
|
8
|
-
*
|
|
9
|
-
* The shape:
|
|
10
|
-
*
|
|
11
|
-
* const report = summarizeBackendIntegrity(records)
|
|
12
|
-
* assertRealBackend(records) // throws BackendIntegrityError if 100% stub
|
|
13
|
-
*
|
|
14
|
-
* A record is "stub-mode" if its `tokenUsage.input === 0 && tokenUsage.output === 0`.
|
|
15
|
-
* (`costUsd` alone is unreliable — some backends successfully call LLMs but
|
|
16
|
-
* don't propagate pricing, producing real tokens with $0 cost.)
|
|
17
|
-
*
|
|
18
|
-
* Verdicts:
|
|
19
|
-
* - `real` — at least one record has nonzero token usage
|
|
20
|
-
* - `stub` — every record is stub-mode (eval ran blind)
|
|
21
|
-
* - `mixed` — some records real, some stub (partial backend failure;
|
|
22
|
-
* often the 429-cascade or auth-half-failed case)
|
|
23
|
-
*/
|
|
24
|
-
import { AgentEvalError } from '../errors';
|
|
25
|
-
import type { RunRecord } from '../run-record';
|
|
26
|
-
export interface BackendIntegrityReport {
|
|
27
|
-
/** Total records inspected. */
|
|
28
|
-
totalRecords: number;
|
|
29
|
-
/** Records with input=0 AND output=0 (a stub fingerprint). */
|
|
30
|
-
stubRecords: number;
|
|
31
|
-
/** Records with nonzero token usage (real LLM activity). */
|
|
32
|
-
realRecords: number;
|
|
33
|
-
/** Records where output>0 but costUsd=0 (real LLM, broken cost ledger). */
|
|
34
|
-
uncostedRecords: number;
|
|
35
|
-
/** Sum of input tokens across all records. */
|
|
36
|
-
totalInputTokens: number;
|
|
37
|
-
/** Sum of output tokens across all records. */
|
|
38
|
-
totalOutputTokens: number;
|
|
39
|
-
/** Sum of costUsd across all records. */
|
|
40
|
-
totalCostUsd: number;
|
|
41
|
-
/** Worst-case integrity verdict. */
|
|
42
|
-
verdict: 'real' | 'mixed' | 'stub';
|
|
43
|
-
/** Human-readable diagnosis suitable for terminal output. */
|
|
44
|
-
diagnosis: string;
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Error thrown when an integrity assertion fails. Caller can pattern-match
|
|
48
|
-
* by `code === 'AGENT_EVAL_BACKEND_STUB'` to differentiate from other
|
|
49
|
-
* errors.
|
|
50
|
-
*/
|
|
51
|
-
export declare class BackendIntegrityError extends AgentEvalError {
|
|
52
|
-
readonly report: BackendIntegrityReport;
|
|
53
|
-
constructor(message: string, report: BackendIntegrityReport);
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Inspect a batch of RunRecords and return an integrity report. Pure
|
|
57
|
-
* function — no I/O, no logging. The caller decides what to do with the
|
|
58
|
-
* verdict (print warning, throw, gate CI, etc.).
|
|
59
|
-
*/
|
|
60
|
-
export declare function summarizeBackendIntegrity(records: ReadonlyArray<RunRecord>): BackendIntegrityReport;
|
|
61
|
-
/**
|
|
62
|
-
* Throw BackendIntegrityError if the verdict is 'stub' — i.e. every record
|
|
63
|
-
* shows zero LLM activity. Non-strict callers can pass `{ allowMixed: false }`
|
|
64
|
-
* to also reject mixed verdicts (recommended for CI gates).
|
|
65
|
-
*
|
|
66
|
-
* Real backends pass through silently.
|
|
67
|
-
*/
|
|
68
|
-
export declare function assertRealBackend(records: ReadonlyArray<RunRecord>, opts?: {
|
|
69
|
-
allowMixed?: boolean;
|
|
70
|
-
}): BackendIntegrityReport;
|
|
71
|
-
//# sourceMappingURL=backend-integrity.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"backend-integrity.d.ts","sourceRoot":"","sources":["../../src/integrity/backend-integrity.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAA;AAC1C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAE9C,MAAM,WAAW,sBAAsB;IACrC,+BAA+B;IAC/B,YAAY,EAAE,MAAM,CAAA;IACpB,8DAA8D;IAC9D,WAAW,EAAE,MAAM,CAAA;IACnB,4DAA4D;IAC5D,WAAW,EAAE,MAAM,CAAA;IACnB,2EAA2E;IAC3E,eAAe,EAAE,MAAM,CAAA;IACvB,8CAA8C;IAC9C,gBAAgB,EAAE,MAAM,CAAA;IACxB,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAA;IACzB,yCAAyC;IACzC,YAAY,EAAE,MAAM,CAAA;IACpB,oCAAoC;IACpC,OAAO,EAAE,MAAM,GAAG,OAAO,GAAG,MAAM,CAAA;IAClC,6DAA6D;IAC7D,SAAS,EAAE,MAAM,CAAA;CAClB;AAED;;;;GAIG;AACH,qBAAa,qBAAsB,SAAQ,cAAc;aAGrC,MAAM,EAAE,sBAAsB;gBAD9C,OAAO,EAAE,MAAM,EACC,MAAM,EAAE,sBAAsB;CAIjD;AAUD;;;;GAIG;AACH,wBAAgB,yBAAyB,CACvC,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,GAChC,sBAAsB,CA6CxB;AAoCD;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,EACjC,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,OAAO,CAAA;CAAO,GAClC,sBAAsB,CAgBxB"}
|