@tangle-network/agent-eval 0.95.0 → 0.95.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/adapters/http.d.ts +17 -10
- package/dist/adapters/langchain.d.ts +14 -7
- package/dist/adapters/otel.d.ts +25 -13
- package/dist/{rl/adversarial.d.ts → adversarial-DIVcDoI_.d.ts} +7 -6
- package/dist/analyst/index.d.ts +236 -28
- package/dist/{trace-analyst/analyst.d.ts → analyst-C8HHvfJp.d.ts} +14 -12
- package/dist/{contract/analyze-runs.d.ts → analyze-runs-DtT6F_6T.d.ts} +10 -9
- package/dist/authenticity/index.d.ts +16 -15
- package/dist/{baseline.d.ts → baseline-Bbid3WoO.d.ts} +44 -8
- package/dist/belief-state/index.d.ts +605 -14
- package/dist/benchmarks/index.d.ts +5 -23
- package/dist/builder-eval/index.d.ts +250 -5
- package/dist/calibration-BPmzuVPk.d.ts +101 -0
- package/dist/campaign/index.d.ts +1452 -38
- package/dist/cli.d.ts +0 -2
- package/dist/{contract/intake/code-agent-session.d.ts → code-agent-session-CPHRCb4-.d.ts} +17 -15
- package/dist/contract/index.d.ts +688 -106
- package/dist/control-Doncu-B_.d.ts +259 -0
- package/dist/{control-runtime.d.ts → control-runtime-Acf9CGhw.d.ts} +26 -23
- package/dist/control.d.ts +10 -11
- package/dist/corpus-D4YW9UoJ.d.ts +560 -0
- package/dist/{cost-ledger.d.ts → cost-ledger-DuSqlw5B.d.ts} +11 -10
- package/dist/{counterfactual.d.ts → counterfactual-DlOz8PBx.d.ts} +13 -12
- package/dist/{dataset.d.ts → dataset-BbGkaN2I.d.ts} +14 -11
- package/dist/{analyst/registry.d.ts → default-registry-GyE8X5SP.d.ts} +37 -8
- package/dist/diagnose.d.ts +252 -1
- package/dist/{trace/emitter.d.ts → emitter-C2rqGH_l.d.ts} +12 -9
- package/dist/{errors.d.ts → errors-CzMUYo7b.d.ts} +11 -10
- package/dist/failure-cluster-DH9Flgcf.d.ts +76 -0
- package/dist/{feedback-trajectory.d.ts → feedback-trajectory-BxY0cKfs.d.ts} +38 -36
- package/dist/fuzz.d.ts +547 -1
- package/dist/gepa-C1NCIZ9o.d.ts +414 -0
- package/dist/governance/index.d.ts +135 -5
- package/dist/harness-optimizer-mOl9XX_O.d.ts +106 -0
- package/dist/hosted/index.d.ts +239 -10
- package/dist/index-_Y4oNOOb.d.ts +159 -0
- package/dist/index.d.ts +5660 -277
- package/dist/{contract/insight-report.d.ts → insight-report-BnRjTibG.d.ts} +19 -16
- package/dist/{trace/integrity.d.ts → integrity-D2t12mMw.d.ts} +14 -11
- package/dist/{judge-calibration.d.ts → judge-calibration-0p2QcWNE.d.ts} +17 -16
- package/dist/{analyst/kind-factory.d.ts → kind-factory-X3eDYbKn.d.ts} +61 -10
- package/dist/knowledge/index.d.ts +103 -3
- package/dist/{llm-client.d.ts → llm-client-Bj7g0rqu.d.ts} +23 -21
- package/dist/matrix/index.d.ts +30 -12
- package/dist/meta-eval/index.d.ts +182 -6
- package/dist/{multi-layer-verifier.d.ts → multi-layer-verifier-DUZXrPDA.d.ts} +15 -12
- package/dist/multishot/index.d.ts +290 -7
- package/dist/{rl/off-policy.d.ts → off-policy-DiwuKKg7.d.ts} +9 -8
- package/dist/openapi.json +1 -1
- package/dist/{meta-eval/outcome-store.d.ts → outcome-store-rnXLEqSn.d.ts} +8 -7
- package/dist/{pareto.d.ts → pareto-E-pembql.d.ts} +10 -9
- package/dist/perf/index.d.ts +119 -13
- package/dist/pipelines/index.d.ts +173 -8
- package/dist/pre-registration-nfUdc9EQ.d.ts +483 -0
- package/dist/prm/index.d.ts +104 -5
- package/dist/provenance-CncDq9qE.d.ts +426 -0
- package/dist/query-B7GGjRox.d.ts +32 -0
- package/dist/{trace/raw-provider-sink.d.ts → raw-provider-sink-C46HDghv.d.ts} +14 -13
- package/dist/{red-team.d.ts → red-team-BWdoyleI.d.ts} +16 -14
- package/dist/{trace/redact.d.ts → redact-B40YG2M_.d.ts} +8 -7
- package/dist/release-report-pidWUMZ2.d.ts +233 -0
- package/dist/reporting.d.ts +16 -15
- package/dist/{eval-campaign.d.ts → researcher-Jr8ME1dZ.d.ts} +156 -34
- package/dist/rl.d.ts +1193 -1
- package/dist/{prm/rubric.d.ts → rubric-Cc6UHvUb.d.ts} +13 -10
- package/dist/{meta-eval/rubric-predictive-validity.d.ts → rubric-predictive-validity-C2hDKM8Z.d.ts} +11 -8
- package/dist/run-critic-CmMf05uV.d.ts +56 -0
- package/dist/{run-record.d.ts → run-record-CP2ObebC.d.ts} +117 -15
- package/dist/runtime-trajectory-BOUUjI0y.d.ts +49 -0
- package/dist/{trace/schema.d.ts → schema-m0gsnbt3.d.ts} +30 -29
- package/dist/semantic-concept-judge-DSBB2Cfp.d.ts +624 -0
- package/dist/{sequential.d.ts → sequential-5iSVfzl2.d.ts} +10 -9
- package/dist/{series-convergence.d.ts → series-convergence-D5OWMBg6.d.ts} +5 -4
- package/dist/sink-fetch-B1Yg4Til.d.ts +101 -0
- package/dist/{statistics.d.ts → statistics-CCJpTGOS.d.ts} +48 -46
- package/dist/{trace/store.d.ts → store-BcFXE6LG.d.ts} +12 -21
- package/dist/{trace-analyst/types.d.ts → store-C1YxJDEK.d.ts} +74 -18
- package/dist/storyboard/index.d.ts +81 -16
- package/dist/{summary-report.d.ts → summary-report-CInXwsza.d.ts} +160 -23
- package/dist/telemetry/{sink-file.d.ts → file.d.ts} +7 -5
- package/dist/telemetry/index.d.ts +35 -17
- package/dist/{sandbox-harness.d.ts → test-graded-scenario-DeODGLra.d.ts} +60 -15
- package/dist/{locked-jsonl-appender.d.ts → testing-C21CHsq2.d.ts} +4 -3
- package/dist/testing.d.ts +1 -5
- package/dist/traces.d.ts +976 -4
- package/dist/{trajectory.d.ts → trajectory-2TkpSEVh.d.ts} +9 -8
- package/dist/{analyst/types.d.ts → types-B5x54y6n.d.ts} +112 -19
- package/dist/{matrix/types.d.ts → types-BUxNaJ8c.d.ts} +11 -9
- package/dist/{types.d.ts → types-C7DGg5ex.d.ts} +33 -31
- package/dist/{campaign/types.d.ts → types-DQRY8ZT-.d.ts} +47 -44
- package/dist/{verdict.d.ts → verdict-C9MlYujm.d.ts} +3 -2
- package/dist/wire/index.d.ts +570 -13
- package/dist/workflow/index.d.ts +496 -22
- package/package.json +2 -2
- package/dist/action-policy.d.ts +0 -24
- package/dist/action-policy.d.ts.map +0 -1
- package/dist/action-policy.test.d.ts +0 -2
- package/dist/action-policy.test.d.ts.map +0 -1
- package/dist/active-learning.d.ts +0 -41
- package/dist/active-learning.d.ts.map +0 -1
- package/dist/adapters/http.d.ts.map +0 -1
- package/dist/adapters/langchain.d.ts.map +0 -1
- package/dist/adapters/otel.d.ts.map +0 -1
- package/dist/agent-profile-cell.d.ts +0 -101
- package/dist/agent-profile-cell.d.ts.map +0 -1
- package/dist/agent-profile.d.ts +0 -27
- package/dist/agent-profile.d.ts.map +0 -1
- package/dist/agent-profile.test.d.ts +0 -2
- package/dist/agent-profile.test.d.ts.map +0 -1
- package/dist/analyst/adapters.d.ts +0 -62
- package/dist/analyst/adapters.d.ts.map +0 -1
- package/dist/analyst/analyst.test.d.ts +0 -2
- package/dist/analyst/analyst.test.d.ts.map +0 -1
- package/dist/analyst/ax-service.d.ts +0 -27
- package/dist/analyst/ax-service.d.ts.map +0 -1
- package/dist/analyst/behavioral-analyst.d.ts +0 -28
- package/dist/analyst/behavioral-analyst.d.ts.map +0 -1
- package/dist/analyst/chat-client.d.ts +0 -91
- package/dist/analyst/chat-client.d.ts.map +0 -1
- package/dist/analyst/default-registry.d.ts +0 -27
- package/dist/analyst/default-registry.d.ts.map +0 -1
- package/dist/analyst/default-registry.test.d.ts +0 -2
- package/dist/analyst/default-registry.test.d.ts.map +0 -1
- package/dist/analyst/finding-signature.d.ts +0 -48
- package/dist/analyst/finding-signature.d.ts.map +0 -1
- package/dist/analyst/finding-subject.d.ts +0 -146
- package/dist/analyst/finding-subject.d.ts.map +0 -1
- package/dist/analyst/finding-subject.test.d.ts +0 -2
- package/dist/analyst/finding-subject.test.d.ts.map +0 -1
- package/dist/analyst/findings-store.d.ts +0 -75
- package/dist/analyst/findings-store.d.ts.map +0 -1
- package/dist/analyst/index.d.ts.map +0 -1
- package/dist/analyst/kind-factory.d.ts.map +0 -1
- package/dist/analyst/kinds/failure-mode.d.ts +0 -19
- package/dist/analyst/kinds/failure-mode.d.ts.map +0 -1
- package/dist/analyst/kinds/improvement.d.ts +0 -23
- package/dist/analyst/kinds/improvement.d.ts.map +0 -1
- package/dist/analyst/kinds/index.d.ts +0 -22
- package/dist/analyst/kinds/index.d.ts.map +0 -1
- package/dist/analyst/kinds/kinds.test.d.ts +0 -2
- package/dist/analyst/kinds/kinds.test.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-gap.d.ts +0 -28
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +0 -22
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.d.ts +0 -84
- package/dist/analyst/kinds/skill-usage.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.test.d.ts +0 -2
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.d.ts +0 -26
- package/dist/analyst/parse-tolerant.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.test.d.ts +0 -2
- package/dist/analyst/parse-tolerant.test.d.ts.map +0 -1
- package/dist/analyst/registry.budget.test.d.ts +0 -2
- package/dist/analyst/registry.budget.test.d.ts.map +0 -1
- package/dist/analyst/registry.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.d.ts +0 -35
- package/dist/analyst/steer-firewall.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.test.d.ts +0 -2
- package/dist/analyst/steer-firewall.test.d.ts.map +0 -1
- package/dist/analyst/structure-findings.d.ts +0 -37
- package/dist/analyst/structure-findings.d.ts.map +0 -1
- package/dist/analyst/structure-findings.test.d.ts +0 -2
- package/dist/analyst/structure-findings.test.d.ts.map +0 -1
- package/dist/analyst/tool-groups.d.ts +0 -34
- package/dist/analyst/tool-groups.d.ts.map +0 -1
- package/dist/analyst/types.d.ts.map +0 -1
- package/dist/anti-slop.d.ts +0 -59
- package/dist/anti-slop.d.ts.map +0 -1
- package/dist/artifact-validator.d.ts +0 -74
- package/dist/artifact-validator.d.ts.map +0 -1
- package/dist/attestation.d.ts +0 -63
- package/dist/attestation.d.ts.map +0 -1
- package/dist/attestation.test.d.ts +0 -2
- package/dist/attestation.test.d.ts.map +0 -1
- package/dist/authenticity/index.d.ts.map +0 -1
- package/dist/authenticity/index.test.d.ts +0 -2
- package/dist/authenticity/index.test.d.ts.map +0 -1
- package/dist/auto-pr.d.ts +0 -120
- package/dist/auto-pr.d.ts.map +0 -1
- package/dist/baseline.d.ts.map +0 -1
- package/dist/behavior-dsl.d.ts +0 -73
- package/dist/behavior-dsl.d.ts.map +0 -1
- package/dist/belief-state/calibration.d.ts +0 -10
- package/dist/belief-state/calibration.d.ts.map +0 -1
- package/dist/belief-state/calibration.test.d.ts +0 -2
- package/dist/belief-state/calibration.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.d.ts +0 -66
- package/dist/belief-state/code-agent-corpus.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.test.d.ts +0 -2
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.d.ts +0 -22
- package/dist/belief-state/code-agent-evidence.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.test.d.ts +0 -2
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/extract.d.ts +0 -7
- package/dist/belief-state/extract.d.ts.map +0 -1
- package/dist/belief-state/extract.test.d.ts +0 -2
- package/dist/belief-state/extract.test.d.ts.map +0 -1
- package/dist/belief-state/index.d.ts.map +0 -1
- package/dist/belief-state/ope.d.ts +0 -17
- package/dist/belief-state/ope.d.ts.map +0 -1
- package/dist/belief-state/ope.test.d.ts +0 -2
- package/dist/belief-state/ope.test.d.ts.map +0 -1
- package/dist/belief-state/phase0-measurement.d.ts +0 -55
- package/dist/belief-state/phase0-measurement.d.ts.map +0 -1
- package/dist/belief-state/report.d.ts +0 -17
- package/dist/belief-state/report.d.ts.map +0 -1
- package/dist/belief-state/report.test.d.ts +0 -2
- package/dist/belief-state/report.test.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.d.ts +0 -23
- package/dist/belief-state/research-evidence.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.test.d.ts +0 -2
- package/dist/belief-state/research-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +0 -32
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.d.ts +0 -87
- package/dist/belief-state/runtime-hooks.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.test.d.ts +0 -2
- package/dist/belief-state/runtime-hooks.test.d.ts.map +0 -1
- package/dist/belief-state/selective.d.ts +0 -15
- package/dist/belief-state/selective.d.ts.map +0 -1
- package/dist/belief-state/selective.test.d.ts +0 -2
- package/dist/belief-state/selective.test.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.d.ts +0 -80
- package/dist/belief-state/shadow-probe.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.test.d.ts +0 -2
- package/dist/belief-state/shadow-probe.test.d.ts.map +0 -1
- package/dist/belief-state/types.d.ts +0 -195
- package/dist/belief-state/types.d.ts.map +0 -1
- package/dist/belief-state/types.test.d.ts +0 -2
- package/dist/belief-state/types.test.d.ts.map +0 -1
- package/dist/benchmark.d.ts +0 -14
- package/dist/benchmark.d.ts.map +0 -1
- package/dist/benchmarks/index.d.ts.map +0 -1
- package/dist/benchmarks/routing/dataset.d.ts +0 -34
- package/dist/benchmarks/routing/dataset.d.ts.map +0 -1
- package/dist/benchmarks/routing/index.d.ts +0 -34
- package/dist/benchmarks/routing/index.d.ts.map +0 -1
- package/dist/benchmarks/types.d.ts +0 -49
- package/dist/benchmarks/types.d.ts.map +0 -1
- package/dist/bisector.d.ts +0 -81
- package/dist/bisector.d.ts.map +0 -1
- package/dist/budget-guard.d.ts +0 -31
- package/dist/budget-guard.d.ts.map +0 -1
- package/dist/builder-eval/builder-session.d.ts +0 -111
- package/dist/builder-eval/builder-session.d.ts.map +0 -1
- package/dist/builder-eval/correlation.d.ts +0 -32
- package/dist/builder-eval/correlation.d.ts.map +0 -1
- package/dist/builder-eval/index.d.ts.map +0 -1
- package/dist/builder-eval/project-registry.d.ts +0 -51
- package/dist/builder-eval/project-registry.d.ts.map +0 -1
- package/dist/builder-eval/three-layer-eval.d.ts +0 -55
- package/dist/builder-eval/three-layer-eval.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.d.ts +0 -108
- package/dist/campaign/analyst-surface.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.test.d.ts +0 -2
- package/dist/campaign/analyst-surface.test.d.ts.map +0 -1
- package/dist/campaign/auto-pr.d.ts +0 -46
- package/dist/campaign/auto-pr.d.ts.map +0 -1
- package/dist/campaign/distillation/agreement-judge.d.ts +0 -69
- package/dist/campaign/distillation/agreement-judge.d.ts.map +0 -1
- package/dist/campaign/distillation/cli.d.ts +0 -35
- package/dist/campaign/distillation/cli.d.ts.map +0 -1
- package/dist/campaign/distillation/distillation.test.d.ts +0 -2
- package/dist/campaign/distillation/distillation.test.d.ts.map +0 -1
- package/dist/campaign/distillation/gold-scenarios.d.ts +0 -54
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +0 -1
- package/dist/campaign/distillation/run-distillation.d.ts +0 -119
- package/dist/campaign/distillation/run-distillation.d.ts.map +0 -1
- package/dist/campaign/gates/compose.d.ts +0 -12
- package/dist/campaign/gates/compose.d.ts.map +0 -1
- package/dist/campaign/gates/default-production-gate.d.ts +0 -58
- package/dist/campaign/gates/default-production-gate.d.ts.map +0 -1
- package/dist/campaign/gates/heldout-gate.d.ts +0 -12
- package/dist/campaign/gates/heldout-gate.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.d.ts +0 -125
- package/dist/campaign/gates/promotion-policy.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.test.d.ts +0 -2
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.d.ts +0 -146
- package/dist/campaign/gates/sequential.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.test.d.ts +0 -2
- package/dist/campaign/gates/sequential.test.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.d.ts +0 -99
- package/dist/campaign/gates/statistical-heldout.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.test.d.ts +0 -2
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +0 -1
- package/dist/campaign/index.d.ts.map +0 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +0 -59
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +0 -1
- package/dist/campaign/presets/compare-proposers.d.ts +0 -146
- package/dist/campaign/presets/compare-proposers.d.ts.map +0 -1
- package/dist/campaign/presets/playback.d.ts +0 -120
- package/dist/campaign/presets/playback.d.ts.map +0 -1
- package/dist/campaign/presets/playback.test.d.ts +0 -2
- package/dist/campaign/presets/playback.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-eval.d.ts +0 -14
- package/dist/campaign/presets/run-eval.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.d.ts +0 -63
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +0 -2
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-optimization.d.ts +0 -92
- package/dist/campaign/presets/run-optimization.d.ts.map +0 -1
- package/dist/campaign/presets/run-profile-matrix.d.ts +0 -151
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +0 -1
- package/dist/campaign/presets/run-skill-opt.d.ts +0 -96
- package/dist/campaign/presets/run-skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/_findings-text.d.ts +0 -22
- package/dist/campaign/proposers/_findings-text.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.d.ts +0 -33
- package/dist/campaign/proposers/ace.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.test.d.ts +0 -2
- package/dist/campaign/proposers/ace.test.d.ts.map +0 -1
- package/dist/campaign/proposers/analysis-edit.d.ts +0 -32
- package/dist/campaign/proposers/analysis-edit.d.ts.map +0 -1
- package/dist/campaign/proposers/evolutionary.d.ts +0 -20
- package/dist/campaign/proposers/evolutionary.d.ts.map +0 -1
- package/dist/campaign/proposers/fapo.d.ts +0 -120
- package/dist/campaign/proposers/fapo.d.ts.map +0 -1
- package/dist/campaign/proposers/gepa.d.ts +0 -86
- package/dist/campaign/proposers/gepa.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.d.ts +0 -44
- package/dist/campaign/proposers/halo.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.test.d.ts +0 -2
- package/dist/campaign/proposers/halo.test.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.d.ts +0 -47
- package/dist/campaign/proposers/memory.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.test.d.ts +0 -2
- package/dist/campaign/proposers/memory.test.d.ts.map +0 -1
- package/dist/campaign/proposers/skill-opt.d.ts +0 -88
- package/dist/campaign/proposers/skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.d.ts +0 -48
- package/dist/campaign/proposers/trace-analyst.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.test.d.ts +0 -2
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +0 -1
- package/dist/campaign/provenance.d.ts +0 -185
- package/dist/campaign/provenance.d.ts.map +0 -1
- package/dist/campaign/run-campaign.d.ts +0 -90
- package/dist/campaign/run-campaign.d.ts.map +0 -1
- package/dist/campaign/score-utils.d.ts +0 -26
- package/dist/campaign/score-utils.d.ts.map +0 -1
- package/dist/campaign/skill-patch.d.ts +0 -62
- package/dist/campaign/skill-patch.d.ts.map +0 -1
- package/dist/campaign/storage.d.ts +0 -38
- package/dist/campaign/storage.d.ts.map +0 -1
- package/dist/campaign/types.d.ts.map +0 -1
- package/dist/campaign/worktree/index.d.ts +0 -53
- package/dist/campaign/worktree/index.d.ts.map +0 -1
- package/dist/canary.d.ts +0 -101
- package/dist/canary.d.ts.map +0 -1
- package/dist/causal-attribution.d.ts +0 -45
- package/dist/causal-attribution.d.ts.map +0 -1
- package/dist/ci-gate.d.ts +0 -44
- package/dist/ci-gate.d.ts.map +0 -1
- package/dist/cli.d.ts.map +0 -1
- package/dist/client.d.ts +0 -77
- package/dist/client.d.ts.map +0 -1
- package/dist/client.test.d.ts +0 -2
- package/dist/client.test.d.ts.map +0 -1
- package/dist/command-runner.d.ts +0 -74
- package/dist/command-runner.d.ts.map +0 -1
- package/dist/command-runner.test.d.ts +0 -2
- package/dist/command-runner.test.d.ts.map +0 -1
- package/dist/completion-verifier.d.ts +0 -147
- package/dist/completion-verifier.d.ts.map +0 -1
- package/dist/completion-verifier.test.d.ts +0 -9
- package/dist/completion-verifier.test.d.ts.map +0 -1
- package/dist/concurrency.d.ts +0 -23
- package/dist/concurrency.d.ts.map +0 -1
- package/dist/contamination-guard.d.ts +0 -81
- package/dist/contamination-guard.d.ts.map +0 -1
- package/dist/contract/analyze-runs.d.ts.map +0 -1
- package/dist/contract/define-agent-eval.d.ts +0 -52
- package/dist/contract/define-agent-eval.d.ts.map +0 -1
- package/dist/contract/diff.d.ts +0 -114
- package/dist/contract/diff.d.ts.map +0 -1
- package/dist/contract/index.d.ts.map +0 -1
- package/dist/contract/insight-report.d.ts.map +0 -1
- package/dist/contract/insight-types-fwd.d.ts +0 -7
- package/dist/contract/insight-types-fwd.d.ts.map +0 -1
- package/dist/contract/intake/agent-trace.d.ts +0 -97
- package/dist/contract/intake/agent-trace.d.ts.map +0 -1
- package/dist/contract/intake/code-agent-session.d.ts.map +0 -1
- package/dist/contract/intake/feedback-table.d.ts +0 -87
- package/dist/contract/intake/feedback-table.d.ts.map +0 -1
- package/dist/contract/intake/index.d.ts +0 -22
- package/dist/contract/intake/index.d.ts.map +0 -1
- package/dist/contract/intake/otel-spans.d.ts +0 -33
- package/dist/contract/intake/otel-spans.d.ts.map +0 -1
- package/dist/contract/self-improve.d.ts +0 -284
- package/dist/contract/self-improve.d.ts.map +0 -1
- package/dist/control-runtime.d.ts.map +0 -1
- package/dist/control-runtime.test.d.ts +0 -2
- package/dist/control-runtime.test.d.ts.map +0 -1
- package/dist/control.d.ts.map +0 -1
- package/dist/convergence.d.ts +0 -29
- package/dist/convergence.d.ts.map +0 -1
- package/dist/cost-ledger.d.ts.map +0 -1
- package/dist/cost-ledger.test.d.ts +0 -2
- package/dist/cost-ledger.test.d.ts.map +0 -1
- package/dist/cost-report.d.ts +0 -43
- package/dist/cost-report.d.ts.map +0 -1
- package/dist/cost-report.test.d.ts +0 -2
- package/dist/cost-report.test.d.ts.map +0 -1
- package/dist/cost-tracker.d.ts +0 -76
- package/dist/cost-tracker.d.ts.map +0 -1
- package/dist/counterfactual.d.ts.map +0 -1
- package/dist/cross-trace-diff.d.ts +0 -56
- package/dist/cross-trace-diff.d.ts.map +0 -1
- package/dist/dataset.d.ts.map +0 -1
- package/dist/deploy-gate-layer.d.ts +0 -125
- package/dist/deploy-gate-layer.d.ts.map +0 -1
- package/dist/deploy-gate-layer.test.d.ts +0 -2
- package/dist/deploy-gate-layer.test.d.ts.map +0 -1
- package/dist/description-length-gate.d.ts +0 -119
- package/dist/description-length-gate.d.ts.map +0 -1
- package/dist/detectors/edge.test.d.ts +0 -2
- package/dist/detectors/edge.test.d.ts.map +0 -1
- package/dist/detectors/index.d.ts +0 -81
- package/dist/detectors/index.d.ts.map +0 -1
- package/dist/detectors/index.test.d.ts +0 -2
- package/dist/detectors/index.test.d.ts.map +0 -1
- package/dist/diagnose/causal-sweep.d.ts +0 -100
- package/dist/diagnose/causal-sweep.d.ts.map +0 -1
- package/dist/diagnose/index.d.ts +0 -36
- package/dist/diagnose/index.d.ts.map +0 -1
- package/dist/diagnose/remediation.d.ts +0 -68
- package/dist/diagnose/remediation.d.ts.map +0 -1
- package/dist/diagnose/repair.d.ts +0 -77
- package/dist/diagnose/repair.d.ts.map +0 -1
- package/dist/discover-personas.d.ts +0 -35
- package/dist/discover-personas.d.ts.map +0 -1
- package/dist/driver.d.ts +0 -95
- package/dist/driver.d.ts.map +0 -1
- package/dist/driver.test.d.ts +0 -8
- package/dist/driver.test.d.ts.map +0 -1
- package/dist/dual-agent-bench.d.ts +0 -81
- package/dist/dual-agent-bench.d.ts.map +0 -1
- package/dist/error-count-extractor.d.ts +0 -47
- package/dist/error-count-extractor.d.ts.map +0 -1
- package/dist/error-count-extractor.test.d.ts +0 -2
- package/dist/error-count-extractor.test.d.ts.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/eval-campaign.d.ts.map +0 -1
- package/dist/eval-campaign.test.d.ts +0 -2
- package/dist/eval-campaign.test.d.ts.map +0 -1
- package/dist/eval-tools.d.ts +0 -55
- package/dist/eval-tools.d.ts.map +0 -1
- package/dist/eval-trace-store.d.ts +0 -107
- package/dist/eval-trace-store.d.ts.map +0 -1
- package/dist/eval-trace-store.test.d.ts +0 -2
- package/dist/eval-trace-store.test.d.ts.map +0 -1
- package/dist/executor.d.ts +0 -38
- package/dist/executor.d.ts.map +0 -1
- package/dist/executor.test.d.ts +0 -10
- package/dist/executor.test.d.ts.map +0 -1
- package/dist/experiment-tracker.d.ts +0 -178
- package/dist/experiment-tracker.d.ts.map +0 -1
- package/dist/experiment-tracker.test.d.ts +0 -2
- package/dist/experiment-tracker.test.d.ts.map +0 -1
- package/dist/failure-taxonomy.d.ts +0 -38
- package/dist/failure-taxonomy.d.ts.map +0 -1
- package/dist/feedback-trajectory.d.ts.map +0 -1
- package/dist/feedback-trajectory.test.d.ts +0 -2
- package/dist/feedback-trajectory.test.d.ts.map +0 -1
- package/dist/flow-layer.d.ts +0 -90
- package/dist/flow-layer.d.ts.map +0 -1
- package/dist/flow-layer.test.d.ts +0 -2
- package/dist/flow-layer.test.d.ts.map +0 -1
- package/dist/fuzz/capsule.d.ts +0 -46
- package/dist/fuzz/capsule.d.ts.map +0 -1
- package/dist/fuzz/cube.d.ts +0 -36
- package/dist/fuzz/cube.d.ts.map +0 -1
- package/dist/fuzz/explorer-cost.test.d.ts +0 -2
- package/dist/fuzz/explorer-cost.test.d.ts.map +0 -1
- package/dist/fuzz/explorer.d.ts +0 -64
- package/dist/fuzz/explorer.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.d.ts +0 -16
- package/dist/fuzz/fuzz-agent.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.test.d.ts +0 -2
- package/dist/fuzz/fuzz-agent.test.d.ts.map +0 -1
- package/dist/fuzz/gates.d.ts +0 -33
- package/dist/fuzz/gates.d.ts.map +0 -1
- package/dist/fuzz/index.d.ts +0 -26
- package/dist/fuzz/index.d.ts.map +0 -1
- package/dist/fuzz/policies.d.ts +0 -28
- package/dist/fuzz/policies.d.ts.map +0 -1
- package/dist/fuzz/tools.d.ts +0 -20
- package/dist/fuzz/tools.d.ts.map +0 -1
- package/dist/fuzz/types.d.ts +0 -307
- package/dist/fuzz/types.d.ts.map +0 -1
- package/dist/golden-matcher.d.ts +0 -71
- package/dist/golden-matcher.d.ts.map +0 -1
- package/dist/governance/eu-ai-act.d.ts +0 -37
- package/dist/governance/eu-ai-act.d.ts.map +0 -1
- package/dist/governance/index.d.ts.map +0 -1
- package/dist/governance/nist-ai-rmf.d.ts +0 -15
- package/dist/governance/nist-ai-rmf.d.ts.map +0 -1
- package/dist/governance/soc2.d.ts +0 -12
- package/dist/governance/soc2.d.ts.map +0 -1
- package/dist/governance/types.d.ts +0 -66
- package/dist/governance/types.d.ts.map +0 -1
- package/dist/harness-optimizer.d.ts +0 -82
- package/dist/harness-optimizer.d.ts.map +0 -1
- package/dist/held-out-gate.d.ts +0 -135
- package/dist/held-out-gate.d.ts.map +0 -1
- package/dist/hosted/client.d.ts +0 -73
- package/dist/hosted/client.d.ts.map +0 -1
- package/dist/hosted/from-env.test.d.ts +0 -8
- package/dist/hosted/from-env.test.d.ts.map +0 -1
- package/dist/hosted/index.d.ts.map +0 -1
- package/dist/hosted/types.d.ts +0 -159
- package/dist/hosted/types.d.ts.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/integrity/backend-integrity.d.ts +0 -71
- package/dist/integrity/backend-integrity.d.ts.map +0 -1
- package/dist/integrity/preflight.d.ts +0 -72
- package/dist/integrity/preflight.d.ts.map +0 -1
- package/dist/integrity/preflight.test.d.ts +0 -2
- package/dist/integrity/preflight.test.d.ts.map +0 -1
- package/dist/integrity/single-backend.d.ts +0 -67
- package/dist/integrity/single-backend.d.ts.map +0 -1
- package/dist/intent-match-judge.d.ts +0 -69
- package/dist/intent-match-judge.d.ts.map +0 -1
- package/dist/intent-match-judge.test.d.ts +0 -2
- package/dist/intent-match-judge.test.d.ts.map +0 -1
- package/dist/judge-calibration.d.ts.map +0 -1
- package/dist/judge-ensemble.d.ts +0 -66
- package/dist/judge-ensemble.d.ts.map +0 -1
- package/dist/judge-ensemble.test.d.ts +0 -8
- package/dist/judge-ensemble.test.d.ts.map +0 -1
- package/dist/judge-families.d.ts +0 -38
- package/dist/judge-families.d.ts.map +0 -1
- package/dist/judge-panel.d.ts +0 -65
- package/dist/judge-panel.d.ts.map +0 -1
- package/dist/judge-retry.d.ts +0 -70
- package/dist/judge-retry.d.ts.map +0 -1
- package/dist/judge-runner.d.ts +0 -36
- package/dist/judge-runner.d.ts.map +0 -1
- package/dist/judge-runner.test.d.ts +0 -2
- package/dist/judge-runner.test.d.ts.map +0 -1
- package/dist/judges.d.ts +0 -74
- package/dist/judges.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.d.ts +0 -89
- package/dist/keyword-coverage-judge.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.test.d.ts +0 -2
- package/dist/keyword-coverage-judge.test.d.ts.map +0 -1
- package/dist/knowledge/index.d.ts.map +0 -1
- package/dist/knowledge/readiness.d.ts +0 -26
- package/dist/knowledge/readiness.d.ts.map +0 -1
- package/dist/knowledge/types.d.ts +0 -75
- package/dist/knowledge/types.d.ts.map +0 -1
- package/dist/live-proof.d.ts +0 -62
- package/dist/live-proof.d.ts.map +0 -1
- package/dist/llm-client.d.ts.map +0 -1
- package/dist/llm-client.test.d.ts +0 -2
- package/dist/llm-client.test.d.ts.map +0 -1
- package/dist/locked-jsonl-appender.d.ts.map +0 -1
- package/dist/matrix/aggregation.d.ts +0 -16
- package/dist/matrix/aggregation.d.ts.map +0 -1
- package/dist/matrix/index.d.ts.map +0 -1
- package/dist/matrix/runner.d.ts +0 -15
- package/dist/matrix/runner.d.ts.map +0 -1
- package/dist/matrix/types.d.ts.map +0 -1
- package/dist/meta-eval/calibration.d.ts +0 -47
- package/dist/meta-eval/calibration.d.ts.map +0 -1
- package/dist/meta-eval/correlation-study.d.ts +0 -53
- package/dist/meta-eval/correlation-study.d.ts.map +0 -1
- package/dist/meta-eval/index.d.ts.map +0 -1
- package/dist/meta-eval/outcome-store.d.ts.map +0 -1
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +0 -1
- package/dist/meta-eval/sentinel.d.ts +0 -169
- package/dist/meta-eval/sentinel.d.ts.map +0 -1
- package/dist/metrics.d.ts +0 -63
- package/dist/metrics.d.ts.map +0 -1
- package/dist/model-seats.d.ts +0 -71
- package/dist/model-seats.d.ts.map +0 -1
- package/dist/model-seats.test.d.ts +0 -2
- package/dist/model-seats.test.d.ts.map +0 -1
- package/dist/muffled-gate-scanner.d.ts +0 -102
- package/dist/muffled-gate-scanner.d.ts.map +0 -1
- package/dist/multi-layer-verifier.d.ts.map +0 -1
- package/dist/multi-layer-verifier.test.d.ts +0 -2
- package/dist/multi-layer-verifier.test.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.d.ts +0 -80
- package/dist/multi-toolchain-layer.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.test.d.ts +0 -2
- package/dist/multi-toolchain-layer.test.d.ts.map +0 -1
- package/dist/multishot/default-tools.d.ts +0 -34
- package/dist/multishot/default-tools.d.ts.map +0 -1
- package/dist/multishot/index.d.ts.map +0 -1
- package/dist/multishot/judges.d.ts +0 -32
- package/dist/multishot/judges.d.ts.map +0 -1
- package/dist/multishot/matrix.d.ts +0 -107
- package/dist/multishot/matrix.d.ts.map +0 -1
- package/dist/multishot/multishot.d.ts +0 -23
- package/dist/multishot/multishot.d.ts.map +0 -1
- package/dist/multishot/router.d.ts +0 -37
- package/dist/multishot/router.d.ts.map +0 -1
- package/dist/multishot/types.d.ts +0 -60
- package/dist/multishot/types.d.ts.map +0 -1
- package/dist/observability.d.ts +0 -71
- package/dist/observability.d.ts.map +0 -1
- package/dist/oracle.d.ts +0 -55
- package/dist/oracle.d.ts.map +0 -1
- package/dist/orthogonality.d.ts +0 -35
- package/dist/orthogonality.d.ts.map +0 -1
- package/dist/otel-pipeline.d.ts +0 -31
- package/dist/otel-pipeline.d.ts.map +0 -1
- package/dist/paraphrase.d.ts +0 -107
- package/dist/paraphrase.d.ts.map +0 -1
- package/dist/pareto.d.ts.map +0 -1
- package/dist/partition-held-out.d.ts +0 -70
- package/dist/partition-held-out.d.ts.map +0 -1
- package/dist/partition-held-out.test.d.ts +0 -2
- package/dist/partition-held-out.test.d.ts.map +0 -1
- package/dist/perf/index.d.ts.map +0 -1
- package/dist/perf/integrity.d.ts +0 -30
- package/dist/perf/integrity.d.ts.map +0 -1
- package/dist/perf/journey.d.ts +0 -45
- package/dist/perf/journey.d.ts.map +0 -1
- package/dist/perf/ratchet.d.ts +0 -47
- package/dist/perf/ratchet.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.d.ts +0 -31
- package/dist/pipelines/budget-breach.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.test.d.ts +0 -2
- package/dist/pipelines/budget-breach.test.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.d.ts +0 -38
- package/dist/pipelines/failure-cluster.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.test.d.ts +0 -2
- package/dist/pipelines/failure-cluster.test.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.d.ts +0 -26
- package/dist/pipelines/first-divergence.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.test.d.ts +0 -2
- package/dist/pipelines/first-divergence.test.d.ts.map +0 -1
- package/dist/pipelines/index.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.d.ts +0 -26
- package/dist/pipelines/judge-agreement.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.test.d.ts +0 -2
- package/dist/pipelines/judge-agreement.test.d.ts.map +0 -1
- package/dist/pipelines/regression.d.ts +0 -23
- package/dist/pipelines/regression.d.ts.map +0 -1
- package/dist/pipelines/regression.test.d.ts +0 -2
- package/dist/pipelines/regression.test.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.d.ts +0 -32
- package/dist/pipelines/stuck-loop.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.test.d.ts +0 -2
- package/dist/pipelines/stuck-loop.test.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.d.ts +0 -34
- package/dist/pipelines/tool-waste.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.test.d.ts +0 -2
- package/dist/pipelines/tool-waste.test.d.ts.map +0 -1
- package/dist/playbook.d.ts +0 -16
- package/dist/playbook.d.ts.map +0 -1
- package/dist/pr-review-benchmark.d.ts +0 -88
- package/dist/pr-review-benchmark.d.ts.map +0 -1
- package/dist/pr-review-benchmark.test.d.ts +0 -2
- package/dist/pr-review-benchmark.test.d.ts.map +0 -1
- package/dist/pre-registration.d.ts +0 -125
- package/dist/pre-registration.d.ts.map +0 -1
- package/dist/prm/builtin-rubrics.d.ts +0 -33
- package/dist/prm/builtin-rubrics.d.ts.map +0 -1
- package/dist/prm/index.d.ts.map +0 -1
- package/dist/prm/inference.d.ts +0 -29
- package/dist/prm/inference.d.ts.map +0 -1
- package/dist/prm/inference.test.d.ts +0 -2
- package/dist/prm/inference.test.d.ts.map +0 -1
- package/dist/prm/rubric.d.ts.map +0 -1
- package/dist/prm/training-export.d.ts +0 -38
- package/dist/prm/training-export.d.ts.map +0 -1
- package/dist/produced-state.d.ts +0 -63
- package/dist/produced-state.d.ts.map +0 -1
- package/dist/produced-state.test.d.ts +0 -8
- package/dist/produced-state.test.d.ts.map +0 -1
- package/dist/profile/baselines.d.ts +0 -37
- package/dist/profile/baselines.d.ts.map +0 -1
- package/dist/profile/index.d.ts +0 -105
- package/dist/profile/index.d.ts.map +0 -1
- package/dist/promotion-gate.d.ts +0 -93
- package/dist/promotion-gate.d.ts.map +0 -1
- package/dist/prompt-registry.d.ts +0 -41
- package/dist/prompt-registry.d.ts.map +0 -1
- package/dist/propose-review-control.d.ts +0 -49
- package/dist/propose-review-control.d.ts.map +0 -1
- package/dist/propose-review-control.test.d.ts +0 -2
- package/dist/propose-review-control.test.d.ts.map +0 -1
- package/dist/propose-review.d.ts +0 -155
- package/dist/propose-review.d.ts.map +0 -1
- package/dist/red-team.d.ts.map +0 -1
- package/dist/reference-replay-steering.d.ts +0 -11
- package/dist/reference-replay-steering.d.ts.map +0 -1
- package/dist/reference-replay.d.ts +0 -176
- package/dist/reference-replay.d.ts.map +0 -1
- package/dist/reflective-mutation.d.ts +0 -79
- package/dist/reflective-mutation.d.ts.map +0 -1
- package/dist/registry.d.ts +0 -31
- package/dist/registry.d.ts.map +0 -1
- package/dist/release-confidence.d.ts +0 -128
- package/dist/release-confidence.d.ts.map +0 -1
- package/dist/release-report.d.ts +0 -11
- package/dist/release-report.d.ts.map +0 -1
- package/dist/replay.d.ts +0 -120
- package/dist/replay.d.ts.map +0 -1
- package/dist/reporter.d.ts +0 -14
- package/dist/reporter.d.ts.map +0 -1
- package/dist/reporting.d.ts.map +0 -1
- package/dist/researcher.d.ts +0 -140
- package/dist/researcher.d.ts.map +0 -1
- package/dist/reviewer.d.ts +0 -118
- package/dist/reviewer.d.ts.map +0 -1
- package/dist/reviewer.test.d.ts +0 -2
- package/dist/reviewer.test.d.ts.map +0 -1
- package/dist/reward-model-export.d.ts +0 -60
- package/dist/reward-model-export.d.ts.map +0 -1
- package/dist/rl/active-curriculum.d.ts +0 -110
- package/dist/rl/active-curriculum.d.ts.map +0 -1
- package/dist/rl/adaptation-eval.d.ts +0 -109
- package/dist/rl/adaptation-eval.d.ts.map +0 -1
- package/dist/rl/adversarial.d.ts.map +0 -1
- package/dist/rl/compute-curves.d.ts +0 -127
- package/dist/rl/compute-curves.d.ts.map +0 -1
- package/dist/rl/contamination.d.ts +0 -117
- package/dist/rl/contamination.d.ts.map +0 -1
- package/dist/rl/corpus.d.ts +0 -55
- package/dist/rl/corpus.d.ts.map +0 -1
- package/dist/rl/corpus.test.d.ts +0 -2
- package/dist/rl/corpus.test.d.ts.map +0 -1
- package/dist/rl/dataset.d.ts +0 -102
- package/dist/rl/dataset.d.ts.map +0 -1
- package/dist/rl/dataset.test.d.ts +0 -2
- package/dist/rl/dataset.test.d.ts.map +0 -1
- package/dist/rl/exporters.d.ts +0 -141
- package/dist/rl/exporters.d.ts.map +0 -1
- package/dist/rl/index.d.ts +0 -49
- package/dist/rl/index.d.ts.map +0 -1
- package/dist/rl/off-policy.d.ts.map +0 -1
- package/dist/rl/predictive-validity-researcher.d.ts +0 -69
- package/dist/rl/predictive-validity-researcher.d.ts.map +0 -1
- package/dist/rl/preferences.d.ts +0 -141
- package/dist/rl/preferences.d.ts.map +0 -1
- package/dist/rl/process-reward.d.ts +0 -122
- package/dist/rl/process-reward.d.ts.map +0 -1
- package/dist/rl/reward-hacking.d.ts +0 -104
- package/dist/rl/reward-hacking.d.ts.map +0 -1
- package/dist/rl/rl-campaign.d.ts +0 -85
- package/dist/rl/rl-campaign.d.ts.map +0 -1
- package/dist/rl/run-record-adapters.d.ts +0 -56
- package/dist/rl/run-record-adapters.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.d.ts +0 -166
- package/dist/rl/sim-fidelity.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.test.d.ts +0 -2
- package/dist/rl/sim-fidelity.test.d.ts.map +0 -1
- package/dist/rl/tournament.d.ts +0 -115
- package/dist/rl/tournament.d.ts.map +0 -1
- package/dist/rl/verifiable-reward.d.ts +0 -124
- package/dist/rl/verifiable-reward.d.ts.map +0 -1
- package/dist/run-critic.d.ts +0 -23
- package/dist/run-critic.d.ts.map +0 -1
- package/dist/run-evidence.d.ts +0 -32
- package/dist/run-evidence.d.ts.map +0 -1
- package/dist/run-record.d.ts.map +0 -1
- package/dist/run-record.test.d.ts +0 -2
- package/dist/run-record.test.d.ts.map +0 -1
- package/dist/run-score.d.ts +0 -31
- package/dist/run-score.d.ts.map +0 -1
- package/dist/runtime-trajectory.d.ts +0 -47
- package/dist/runtime-trajectory.d.ts.map +0 -1
- package/dist/sandbox-harness.d.ts.map +0 -1
- package/dist/sandbox-harness.test.d.ts +0 -2
- package/dist/sandbox-harness.test.d.ts.map +0 -1
- package/dist/sandbox-pool.d.ts +0 -74
- package/dist/sandbox-pool.d.ts.map +0 -1
- package/dist/sandbox-pool.test.d.ts +0 -2
- package/dist/sandbox-pool.test.d.ts.map +0 -1
- package/dist/scorecard.d.ts +0 -133
- package/dist/scorecard.d.ts.map +0 -1
- package/dist/scorecard.test.d.ts +0 -2
- package/dist/scorecard.test.d.ts.map +0 -1
- package/dist/self-play.d.ts +0 -69
- package/dist/self-play.d.ts.map +0 -1
- package/dist/semantic-concept-judge.d.ts +0 -135
- package/dist/semantic-concept-judge.d.ts.map +0 -1
- package/dist/semantic-concept-judge.test.d.ts +0 -2
- package/dist/semantic-concept-judge.test.d.ts.map +0 -1
- package/dist/sequential.d.ts.map +0 -1
- package/dist/series-convergence.d.ts.map +0 -1
- package/dist/slo.d.ts +0 -48
- package/dist/slo.d.ts.map +0 -1
- package/dist/state-continuity.d.ts +0 -47
- package/dist/state-continuity.d.ts.map +0 -1
- package/dist/statistics.d.ts.map +0 -1
- package/dist/statistics.test.d.ts +0 -2
- package/dist/statistics.test.d.ts.map +0 -1
- package/dist/steering-optimizer.d.ts +0 -58
- package/dist/steering-optimizer.d.ts.map +0 -1
- package/dist/steering.d.ts +0 -24
- package/dist/steering.d.ts.map +0 -1
- package/dist/storyboard/code-edit.d.ts +0 -64
- package/dist/storyboard/code-edit.d.ts.map +0 -1
- package/dist/storyboard/code-edit.test.d.ts +0 -2
- package/dist/storyboard/code-edit.test.d.ts.map +0 -1
- package/dist/storyboard/index.d.ts.map +0 -1
- package/dist/storyboard/index.test.d.ts +0 -2
- package/dist/storyboard/index.test.d.ts.map +0 -1
- package/dist/summary-report.d.ts.map +0 -1
- package/dist/telemetry/client.d.ts +0 -35
- package/dist/telemetry/client.d.ts.map +0 -1
- package/dist/telemetry/index.d.ts.map +0 -1
- package/dist/telemetry/schema.d.ts +0 -61
- package/dist/telemetry/schema.d.ts.map +0 -1
- package/dist/telemetry/sink-fetch.d.ts +0 -39
- package/dist/telemetry/sink-fetch.d.ts.map +0 -1
- package/dist/telemetry/sink-file.d.ts.map +0 -1
- package/dist/test-graded-scenario.d.ts +0 -42
- package/dist/test-graded-scenario.d.ts.map +0 -1
- package/dist/testing.d.ts.map +0 -1
- package/dist/tool-use-metrics.d.ts +0 -35
- package/dist/tool-use-metrics.d.ts.map +0 -1
- package/dist/trace/capture-fetch.d.ts +0 -48
- package/dist/trace/capture-fetch.d.ts.map +0 -1
- package/dist/trace/capture-fetch.test.d.ts +0 -2
- package/dist/trace/capture-fetch.test.d.ts.map +0 -1
- package/dist/trace/emitter.d.ts.map +0 -1
- package/dist/trace/extract-usage.d.ts +0 -46
- package/dist/trace/extract-usage.d.ts.map +0 -1
- package/dist/trace/extract-usage.test.d.ts +0 -2
- package/dist/trace/extract-usage.test.d.ts.map +0 -1
- package/dist/trace/index.d.ts +0 -15
- package/dist/trace/index.d.ts.map +0 -1
- package/dist/trace/integrity.d.ts.map +0 -1
- package/dist/trace/otel-bridge.d.ts +0 -29
- package/dist/trace/otel-bridge.d.ts.map +0 -1
- package/dist/trace/otel-export.d.ts +0 -52
- package/dist/trace/otel-export.d.ts.map +0 -1
- package/dist/trace/otel.d.ts +0 -57
- package/dist/trace/otel.d.ts.map +0 -1
- package/dist/trace/otlp-attributes.d.ts +0 -17
- package/dist/trace/otlp-attributes.d.ts.map +0 -1
- package/dist/trace/query.d.ts +0 -29
- package/dist/trace/query.d.ts.map +0 -1
- package/dist/trace/query.test.d.ts +0 -2
- package/dist/trace/query.test.d.ts.map +0 -1
- package/dist/trace/raw-provider-sink.d.ts.map +0 -1
- package/dist/trace/redact.d.ts.map +0 -1
- package/dist/trace/schema.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.d.ts +0 -72
- package/dist/trace/store-to-otlp.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.test.d.ts +0 -2
- package/dist/trace/store-to-otlp.test.d.ts.map +0 -1
- package/dist/trace/store.d.ts.map +0 -1
- package/dist/trace/store.test.d.ts +0 -2
- package/dist/trace/store.test.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.test.d.ts +0 -2
- package/dist/trace-analyst/analyst.test.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.d.ts +0 -40
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +0 -2
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +0 -1
- package/dist/trace-analyst/hook.d.ts +0 -55
- package/dist/trace-analyst/hook.d.ts.map +0 -1
- package/dist/trace-analyst/index.d.ts +0 -18
- package/dist/trace-analyst/index.d.ts.map +0 -1
- package/dist/trace-analyst/insights.d.ts +0 -71
- package/dist/trace-analyst/insights.d.ts.map +0 -1
- package/dist/trace-analyst/insights.test.d.ts +0 -2
- package/dist/trace-analyst/insights.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-flatten.d.ts +0 -42
- package/dist/trace-analyst/otlp-flatten.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.d.ts +0 -85
- package/dist/trace-analyst/otlp-span.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-span.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.d.ts +0 -115
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/prompts.d.ts +0 -6
- package/dist/trace-analyst/prompts.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.d.ts +0 -126
- package/dist/trace-analyst/store-otlp.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.test.d.ts +0 -8
- package/dist/trace-analyst/store-otlp.test.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/store.d.ts +0 -63
- package/dist/trace-analyst/store.d.ts.map +0 -1
- package/dist/trace-analyst/tools.d.ts +0 -44
- package/dist/trace-analyst/tools.d.ts.map +0 -1
- package/dist/trace-analyst/tools.test.d.ts +0 -10
- package/dist/trace-analyst/tools.test.d.ts.map +0 -1
- package/dist/trace-analyst/types.d.ts.map +0 -1
- package/dist/trace-contracts.d.ts +0 -180
- package/dist/trace-contracts.d.ts.map +0 -1
- package/dist/traced-analyst.d.ts +0 -26
- package/dist/traced-analyst.d.ts.map +0 -1
- package/dist/traced-judges.d.ts +0 -27
- package/dist/traced-judges.d.ts.map +0 -1
- package/dist/traces.d.ts.map +0 -1
- package/dist/trajectory.d.ts.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/ui-finding.d.ts +0 -104
- package/dist/ui-finding.d.ts.map +0 -1
- package/dist/verdict-cache.d.ts +0 -78
- package/dist/verdict-cache.d.ts.map +0 -1
- package/dist/verdict-cache.test.d.ts +0 -2
- package/dist/verdict-cache.test.d.ts.map +0 -1
- package/dist/verdict.d.ts.map +0 -1
- package/dist/visual-diff.d.ts +0 -32
- package/dist/visual-diff.d.ts.map +0 -1
- package/dist/wire/handlers.d.ts +0 -54
- package/dist/wire/handlers.d.ts.map +0 -1
- package/dist/wire/index.d.ts.map +0 -1
- package/dist/wire/openapi.d.ts +0 -3
- package/dist/wire/openapi.d.ts.map +0 -1
- package/dist/wire/rpc.d.ts +0 -21
- package/dist/wire/rpc.d.ts.map +0 -1
- package/dist/wire/rubrics.d.ts +0 -34
- package/dist/wire/rubrics.d.ts.map +0 -1
- package/dist/wire/schemas.d.ts +0 -410
- package/dist/wire/schemas.d.ts.map +0 -1
- package/dist/wire/server.d.ts +0 -60
- package/dist/wire/server.d.ts.map +0 -1
- package/dist/workflow/event-schema.d.ts +0 -5
- package/dist/workflow/event-schema.d.ts.map +0 -1
- package/dist/workflow/feedback-pack.d.ts +0 -99
- package/dist/workflow/feedback-pack.d.ts.map +0 -1
- package/dist/workflow/index.d.ts.map +0 -1
- package/dist/workflow/intelligence-export.d.ts +0 -62
- package/dist/workflow/intelligence-export.d.ts.map +0 -1
- package/dist/workflow/partner-report.d.ts +0 -49
- package/dist/workflow/partner-report.d.ts.map +0 -1
- package/dist/workflow/phase-graph.d.ts +0 -43
- package/dist/workflow/phase-graph.d.ts.map +0 -1
- package/dist/workflow/promotion-gate.d.ts +0 -61
- package/dist/workflow/promotion-gate.d.ts.map +0 -1
- package/dist/workflow/run-record.d.ts +0 -12
- package/dist/workflow/run-record.d.ts.map +0 -1
- package/dist/workflow/runtime-adapter.d.ts +0 -20
- package/dist/workflow/runtime-adapter.d.ts.map +0 -1
- package/dist/workflow/sanitize.d.ts +0 -21
- package/dist/workflow/sanitize.d.ts.map +0 -1
- package/dist/workflow/schema.d.ts +0 -5
- package/dist/workflow/schema.d.ts.map +0 -1
- package/dist/workflow/summary.d.ts +0 -43
- package/dist/workflow/summary.d.ts.map +0 -1
- package/dist/workflow/trace-event-fields.d.ts +0 -6
- package/dist/workflow/trace-event-fields.d.ts.map +0 -1
- package/dist/workflow/trajectory.d.ts +0 -15
- package/dist/workflow/trajectory.d.ts.map +0 -1
- package/dist/workflow/types.d.ts +0 -68
- package/dist/workflow/types.d.ts.map +0 -1
- package/dist/workspace-inspector.d.ts +0 -67
- package/dist/workspace-inspector.d.ts.map +0 -1
- package/dist/wrangler-deploy-runner.test.d.ts +0 -2
- package/dist/wrangler-deploy-runner.test.d.ts.map +0 -1
|
@@ -1,284 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* # `selfImprove()` - the one-call improvement loop.
|
|
3
|
-
*
|
|
4
|
-
* The cheapest possible call site to run a real closed-loop self-
|
|
5
|
-
* improvement over your agent. Wraps `runImprovementLoop` with smart
|
|
6
|
-
* defaults and a budget-shaped options API; every escape hatch the
|
|
7
|
-
* substrate exposes is reachable from here without losing the
|
|
8
|
-
* one-function feel.
|
|
9
|
-
*
|
|
10
|
-
* Defaults:
|
|
11
|
-
* - In-memory storage (no filesystem touch).
|
|
12
|
-
* - `gepaProposer` reflective mutation with copywriting-flavored primitives
|
|
13
|
-
* (override `proposer` or `mutationPrimitives` for any domain).
|
|
14
|
-
* - `defaultProductionGate` with `deltaThreshold: 0.05`.
|
|
15
|
-
* - Held-out split = 25% of scenarios, deterministic by id hash.
|
|
16
|
-
* - 3 generations × population 2 (raise via `budget` for more search).
|
|
17
|
-
* - `autoOnPromote: 'none'` (we don't open PRs unless you ask).
|
|
18
|
-
*
|
|
19
|
-
* Want one-click? Provide `agent` + `scenarios` + `judge`. Done.
|
|
20
|
-
* Want distributed? Pass `cellPlacement` + an `httpDispatch`-backed
|
|
21
|
-
* agent. Want a code-tier surface? Pass a `MutableSurface` + your own
|
|
22
|
-
* `proposer`. Same function.
|
|
23
|
-
*/
|
|
24
|
-
import { type RunImprovementLoopResult } from '../campaign/presets/run-improvement-loop';
|
|
25
|
-
import type { RunOptimizationOptions } from '../campaign/presets/run-optimization';
|
|
26
|
-
import { type LoopProvenanceRecord } from '../campaign/provenance';
|
|
27
|
-
import { type CampaignStorage } from '../campaign/storage';
|
|
28
|
-
import type { DispatchContext, Gate, JudgeConfig, LabeledScenarioStore, MutableSurface, Scenario, SurfaceProposer } from '../campaign/types';
|
|
29
|
-
import { type HostedTenant } from '../hosted/client';
|
|
30
|
-
import type { RunRecord } from '../run-record';
|
|
31
|
-
import type { InsightReport } from './insight-report';
|
|
32
|
-
export interface SelfImproveBudget {
|
|
33
|
-
/** Hard $ ceiling across all cells in baseline + every generation. Cells
|
|
34
|
-
* beyond the ceiling are skipped (cost-aware, not aborted). */
|
|
35
|
-
dollars?: number;
|
|
36
|
-
/** How many improvement generations to explore. Default 3. Set 0 to
|
|
37
|
-
* skip improvement entirely (selfImprove becomes a baseline-only run). */
|
|
38
|
-
generations?: number;
|
|
39
|
-
/** Candidates the proposer emits per generation. Default 2. */
|
|
40
|
-
populationSize?: number;
|
|
41
|
-
/** Max concurrent cells across the loop. Default 2. */
|
|
42
|
-
maxConcurrency?: number;
|
|
43
|
-
/** Fraction of `scenarios` held out from training, used for the gate.
|
|
44
|
-
* Default 0.25. Ignored when `holdoutScenarios` is set explicitly. */
|
|
45
|
-
holdoutFraction?: number;
|
|
46
|
-
/** Explicit held-out scenarios; overrides `holdoutFraction`. */
|
|
47
|
-
holdoutScenarios?: Scenario[];
|
|
48
|
-
/** Per-scenario replicates per cell — raises bootstrap-CI tightness. Default 1. */
|
|
49
|
-
reps?: number;
|
|
50
|
-
/** Top-scoring candidates carried into the next generation. Default 2. */
|
|
51
|
-
promoteTopK?: number;
|
|
52
|
-
}
|
|
53
|
-
export interface SelfImproveLlm {
|
|
54
|
-
/** Endpoint base URL. Default Tangle Router. */
|
|
55
|
-
baseUrl?: string;
|
|
56
|
-
/** Bearer token. Default `process.env.OPENAI_API_KEY`. */
|
|
57
|
-
apiKey?: string;
|
|
58
|
-
/** Model id used by `gepaProposer` reflection. Default
|
|
59
|
-
* `anthropic/claude-sonnet-4.6`. */
|
|
60
|
-
model?: string;
|
|
61
|
-
}
|
|
62
|
-
export type SelfImproveProgressEvent = {
|
|
63
|
-
kind: 'baseline.started';
|
|
64
|
-
scenarios: number;
|
|
65
|
-
} | {
|
|
66
|
-
kind: 'baseline.completed';
|
|
67
|
-
compositeMean: number;
|
|
68
|
-
durationMs: number;
|
|
69
|
-
} | {
|
|
70
|
-
kind: 'generation.started';
|
|
71
|
-
index: number;
|
|
72
|
-
populationSize: number;
|
|
73
|
-
} | {
|
|
74
|
-
kind: 'generation.completed';
|
|
75
|
-
index: number;
|
|
76
|
-
bestComposite: number;
|
|
77
|
-
durationMs: number;
|
|
78
|
-
} | {
|
|
79
|
-
kind: 'gate.decided';
|
|
80
|
-
decision: string;
|
|
81
|
-
lift: number;
|
|
82
|
-
};
|
|
83
|
-
export interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
|
|
84
|
-
/**
|
|
85
|
-
* Your agent — a function that takes the current `MutableSurface`
|
|
86
|
-
* (typically a system prompt the loop is optimizing) plus the
|
|
87
|
-
* scenario + cell ctx, and returns the artifact your judge scores.
|
|
88
|
-
*
|
|
89
|
-
* Same shape as `RunOptimizationOptions.dispatchWithSurface`. Wrap a
|
|
90
|
-
* plain `Dispatch` if you don't have a surface seam:
|
|
91
|
-
*
|
|
92
|
-
* agent: (_surface, scenario, ctx) => yourPlainDispatch(scenario, ctx)
|
|
93
|
-
*
|
|
94
|
-
* That mode evaluates without mutating any surface — useful as a
|
|
95
|
-
* baseline-only run (set `budget.generations = 0`).
|
|
96
|
-
*/
|
|
97
|
-
agent: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
|
|
98
|
-
/** Scenarios to evaluate against. Train/holdout split is computed from
|
|
99
|
-
* these unless `budget.holdoutScenarios` is set explicitly. */
|
|
100
|
-
scenarios: TScenario[];
|
|
101
|
-
/** Judge that scores artifacts. Bring your own; use `langchainJudge`
|
|
102
|
-
* from `/adapters/langchain` for a Runnable-shaped one. */
|
|
103
|
-
judge: JudgeConfig<TArtifact, TScenario>;
|
|
104
|
-
/** Starting surface — system prompt, JSON config, anything `MutableSurface`
|
|
105
|
-
* accepts. The proposer mutates this each generation. */
|
|
106
|
-
baselineSurface: MutableSurface;
|
|
107
|
-
/** Budget + loop shape. All fields optional. */
|
|
108
|
-
budget?: SelfImproveBudget;
|
|
109
|
-
/** Custom surface proposer. Default is `gepaProposer` configured from `llm` +
|
|
110
|
-
* `mutationPrimitives`. */
|
|
111
|
-
proposer?: SurfaceProposer;
|
|
112
|
-
/** Default-proposer overrides — used when `proposer` is unset. */
|
|
113
|
-
mutationPrimitives?: string[];
|
|
114
|
-
proposerTarget?: string;
|
|
115
|
-
/** Custom gate. Default is `defaultProductionGate` with
|
|
116
|
-
* `deltaThreshold: 0.05` on the held-out split. */
|
|
117
|
-
gate?: Gate<TArtifact, TScenario>;
|
|
118
|
-
/** LLM config consumed by the default `gepaProposer`. Ignored if you pass
|
|
119
|
-
* your own `proposer`. */
|
|
120
|
-
llm?: SelfImproveLlm;
|
|
121
|
-
/** Storage backend. Default is DURABLE: when a real (non-`mem://`) `runDir`
|
|
122
|
-
* is available, the substrate defaults to `fsCampaignStorage()` so the
|
|
123
|
-
* provenance record + OTel spans survive the call. Pass
|
|
124
|
-
* `inMemoryCampaignStorage()` explicitly to opt OUT (tests, edge runtimes).
|
|
125
|
-
* Default when `runDir` is `mem://...` (or unset): in-memory. */
|
|
126
|
-
storage?: CampaignStorage;
|
|
127
|
-
/** Run directory (logical for in-memory storage, real path for fs).
|
|
128
|
-
* Default `mem://selfImprove-<timestamp>` (in-memory, non-durable). Pass a
|
|
129
|
-
* real path to persist the provenance record + spans. */
|
|
130
|
-
runDir?: string;
|
|
131
|
-
/**
|
|
132
|
-
* Worker call records for backend provenance. The agent is opaque to the
|
|
133
|
-
* substrate (it returns an artifact, not token usage), so to capture an
|
|
134
|
-
* `assertRealBackend`-grade verdict + worker call count + model in the
|
|
135
|
-
* provenance record, the agent reports its per-call `RunRecord`s here.
|
|
136
|
-
* Called once after the loop; return the records the agent accumulated.
|
|
137
|
-
* When unset, backend provenance is derived from campaign cells (cost only;
|
|
138
|
-
* verdict will read `stub` without token usage — the honest signal that no
|
|
139
|
-
* token channel was wired).
|
|
140
|
-
*/
|
|
141
|
-
collectWorkerRecords?: () => RunRecord[];
|
|
142
|
-
/** Fires once the durable provenance record + OTel spans are emitted.
|
|
143
|
-
* Receives the structured record for inline assertions / custom routing. */
|
|
144
|
-
onProvenance?: (record: LoopProvenanceRecord) => void;
|
|
145
|
-
/** Distributed execution seam — same as `RunCampaignOptions.cellPlacement`.
|
|
146
|
-
* Returns an opaque placement key the substrate forwards to your agent
|
|
147
|
-
* as `ctx.placement`. Combined with `httpDispatch` from
|
|
148
|
-
* `/adapters/http`, fans cells across regions. */
|
|
149
|
-
cellPlacement?: (input: {
|
|
150
|
-
scenario: TScenario;
|
|
151
|
-
rep: number;
|
|
152
|
-
generation?: number;
|
|
153
|
-
}) => string | undefined;
|
|
154
|
-
/** Streaming hook — fires on baseline + each generation + gate decision.
|
|
155
|
-
* Consumer routes events wherever (UI, dashboard, logs). */
|
|
156
|
-
onProgress?: (event: SelfImproveProgressEvent) => void;
|
|
157
|
-
/** Auto-promotion behavior on a ship decision. Default `'none'` — we
|
|
158
|
-
* return the winner; you ship it however you ship. `'pr'` opens a
|
|
159
|
-
* GitHub PR via `openAutoPr`; requires `ghOwner` + `ghRepo`. */
|
|
160
|
-
autoOnPromote?: 'pr' | 'none';
|
|
161
|
-
ghOwner?: string;
|
|
162
|
-
ghRepo?: string;
|
|
163
|
-
/**
|
|
164
|
-
* Opt-in: ship eval-run events to a hosted orchestrator (ours, your
|
|
165
|
-
* self-hosted one, or any compatible implementation of the
|
|
166
|
-
* `docs/hosted-ingest-spec.md` wire format). When set, the substrate
|
|
167
|
-
* POSTs the final `EvalRunEvent` to `${endpoint}/v1/ingest/eval-runs`
|
|
168
|
-
* after the loop completes. Failures are logged but do not fail the
|
|
169
|
-
* loop — local result is always returned.
|
|
170
|
-
*
|
|
171
|
-
* For our orchestrator: `{ endpoint: 'https://orchestrator.tangle.tools/v1', apiKey, tenantId }`.
|
|
172
|
-
*
|
|
173
|
-
* For your self-hosted: any URL serving the wire format. See
|
|
174
|
-
* `examples/hosted-ingest-server/` for the reference receiver.
|
|
175
|
-
*/
|
|
176
|
-
hostedTenant?: HostedTenant;
|
|
177
|
-
/** Free-form labels attached to the hosted event (env, branch, model id,
|
|
178
|
-
* etc.). Ignored when `hostedTenant` is unset. */
|
|
179
|
-
hostedLabels?: Record<string, string>;
|
|
180
|
-
/** Capture every artifact + judge score to this store (labeled-example
|
|
181
|
-
* corpus the proposer may read for few-shot, and the dataset you ship). Pass
|
|
182
|
-
* `'off'` to disable. Default: off. */
|
|
183
|
-
labeledStore?: LabeledScenarioStore | 'off';
|
|
184
|
-
/** Capture-source tag for `labeledStore`. Default `'eval-run'`. */
|
|
185
|
-
captureSource?: 'production-trace' | 'eval-run' | 'manual' | 'red-team' | 'synthetic';
|
|
186
|
-
/**
|
|
187
|
-
* Per-cell backend-integrity expectation — the fail-loud guard. A cell that
|
|
188
|
-
* produced an artifact but reported `costUsd === 0` AND zero tokens is a
|
|
189
|
-
* stub. Modes: `'assert'` throws on the first such cell, `'warn'` logs it,
|
|
190
|
-
* `'off'` skips the check (offline/replay). Default `'assert'` — `selfImprove`
|
|
191
|
-
* is the real-run path, so a stub fails loud rather than scoring a clean 0.
|
|
192
|
-
*/
|
|
193
|
-
expectUsage?: 'assert' | 'warn' | 'off';
|
|
194
|
-
/**
|
|
195
|
-
* Per-generation findings producer. After each
|
|
196
|
-
* generation is scored, this is called; whatever it returns REPLACES the
|
|
197
|
-
* proposer's `findings` for the next generation's `propose()`. Plug a
|
|
198
|
-
* trace-analyst registry / HALO here. When absent, findings stay
|
|
199
|
-
* `opts.findings`.
|
|
200
|
-
*/
|
|
201
|
-
analyzeGeneration?: RunOptimizationOptions<TScenario, TArtifact>['analyzeGeneration'];
|
|
202
|
-
/** Static findings forwarded to the proposer's `propose()` as `ctx.findings`
|
|
203
|
-
* (a findings-grounded proposer consumes them). Default: none. */
|
|
204
|
-
findings?: unknown[];
|
|
205
|
-
}
|
|
206
|
-
export interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
|
|
207
|
-
/** Composite mean across all scenarios, baseline run. */
|
|
208
|
-
baseline: {
|
|
209
|
-
compositeMean: number;
|
|
210
|
-
perScenario: Record<string, number>;
|
|
211
|
-
};
|
|
212
|
-
/** Composite mean on the held-out set, winner run. */
|
|
213
|
-
winner: {
|
|
214
|
-
compositeMean: number;
|
|
215
|
-
perScenario: Record<string, number>;
|
|
216
|
-
surface: MutableSurface;
|
|
217
|
-
/** Proposer label for the promoted change. Absent ⇒ winner == baseline or
|
|
218
|
-
* a bare-surface mutator. */
|
|
219
|
-
label?: string;
|
|
220
|
-
/** Proposer rationale — the "because Z" that motivated the promoted change.
|
|
221
|
-
* Threaded from the proposer's `ProposedCandidate` through the loop.
|
|
222
|
-
* Absent ⇒ winner == baseline. */
|
|
223
|
-
rationale?: string;
|
|
224
|
-
};
|
|
225
|
-
/** `winner.compositeMean - baselineOnHoldout.compositeMean`. Positive
|
|
226
|
-
* means the gate observed improvement. */
|
|
227
|
-
lift: number;
|
|
228
|
-
/** The explicit baseline→winner unified diff. Always present (empty string
|
|
229
|
-
* when winner == baseline). */
|
|
230
|
-
diff: string;
|
|
231
|
-
/** Durable, queryable provenance record: candidate→cell→gate→promote chain +
|
|
232
|
-
* rationale + diff + backend provenance. The artifact the hosted ingest
|
|
233
|
-
* path stores; the +lift RECOMPUTES from `record.heldOutLift`. */
|
|
234
|
-
provenance: LoopProvenanceRecord;
|
|
235
|
-
/** `defaultProductionGate.decide()` result. */
|
|
236
|
-
gateDecision: 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
|
|
237
|
-
/** Number of generations actually explored (may be less than the
|
|
238
|
-
* budget if the proposer gave up early). */
|
|
239
|
-
generationsExplored: number;
|
|
240
|
-
/** Wall-clock total. */
|
|
241
|
-
durationMs: number;
|
|
242
|
-
/** Total cost across baseline + every generation. */
|
|
243
|
-
totalCostUsd: number;
|
|
244
|
-
/**
|
|
245
|
-
* Rigor packet: distributional summary, paired-bootstrap lift CI,
|
|
246
|
-
* judge stats, contamination check, recommendations. Wired through
|
|
247
|
-
* `analyzeRuns()` on the baseline + winner cells of the campaign.
|
|
248
|
-
* Hosted-tier dashboards render this as the v3-vs-v4 decision view.
|
|
249
|
-
*/
|
|
250
|
-
insight: InsightReport;
|
|
251
|
-
/**
|
|
252
|
-
* Raw substrate result for advanced inspection — full per-generation
|
|
253
|
-
* candidates, full campaign artifacts, all judge scores. Useful for
|
|
254
|
-
* debugging or reporting beyond the summary.
|
|
255
|
-
*/
|
|
256
|
-
raw: RunImprovementLoopResult<TArtifact, TScenario>;
|
|
257
|
-
}
|
|
258
|
-
/**
|
|
259
|
-
* One-shot self-improvement loop. See module docstring for defaults +
|
|
260
|
-
* extension points.
|
|
261
|
-
*
|
|
262
|
-
* @example Minimum:
|
|
263
|
-
*
|
|
264
|
-
* const result = await selfImprove({
|
|
265
|
-
* agent: (surface, scenario, ctx) => myAgent(surface, scenario, ctx.signal),
|
|
266
|
-
* scenarios,
|
|
267
|
-
* judge,
|
|
268
|
-
* baselineSurface: DEFAULT_PROMPT,
|
|
269
|
-
* })
|
|
270
|
-
* console.log(`lift: ${result.lift.toFixed(3)} (${result.gateDecision})`)
|
|
271
|
-
*
|
|
272
|
-
* @example Distributed (workers in three regions):
|
|
273
|
-
*
|
|
274
|
-
* await selfImprove({
|
|
275
|
-
* agent: httpDispatch({ resolveUrl: ({ placement }) => REGION_URLS[placement!] }),
|
|
276
|
-
* scenarios,
|
|
277
|
-
* judge,
|
|
278
|
-
* baselineSurface: DEFAULT_PROMPT,
|
|
279
|
-
* cellPlacement: ({ scenario }) => scenario.region,
|
|
280
|
-
* budget: { maxConcurrency: 12 },
|
|
281
|
-
* })
|
|
282
|
-
*/
|
|
283
|
-
export declare function selfImprove<TScenario extends Scenario, TArtifact>(opts: SelfImproveOptions<TScenario, TArtifact>): Promise<SelfImproveResult<TScenario, TArtifact>>;
|
|
284
|
-
//# sourceMappingURL=self-improve.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"self-improve.d.ts","sourceRoot":"","sources":["../../src/contract/self-improve.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAIH,OAAO,EACL,KAAK,wBAAwB,EAE9B,MAAM,0CAA0C,CAAA;AACjD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAA;AAElF,OAAO,EAEL,KAAK,oBAAoB,EAE1B,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EACL,KAAK,eAAe,EAGrB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,KAAK,EAEV,eAAe,EACf,IAAI,EACJ,WAAW,EACX,oBAAoB,EACpB,cAAc,EACd,QAAQ,EACR,eAAe,EAChB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAsB,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAExE,OAAO,KAAK,EAAqB,SAAS,EAAE,MAAM,eAAe,CAAA;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErD,MAAM,WAAW,iBAAiB;IAChC;oEACgE;IAChE,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;+EAC2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB;2EACuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,gEAAgE;IAChE,gBAAgB,CAAC,EAAE,QAAQ,EAAE,CAAA;IAC7B,mFAAmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,0DAA0D;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;yCACqC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,wBAAwB,GAChC;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACzE;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC1F;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AAE5D,MAAM,WAAW,kBAAkB,CAAC,SAAS,SAAS,QAAQ,EAAE,SAAS;IACvE;;;;;;;;;;;;OAYG;IACH,KAAK,EAAE,CAAC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,SAAS,CAAC,CAAA;IAEjG;oEACgE;IAChE,SAAS,EAAE,SAAS,EAAE,CAAA;IAEtB;gEAC4D;IAC5D,KAAK,EAAE,WAAW,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IAExC;8DAC0D;IAC1D,eAAe,EAAE,cAAc,CAAA;IAE/B,gDAAgD;IAChD,MAAM,CAAC,EAAE,iBAAiB,CAAA;IAE1B;gCAC4B;IAC5B,QAAQ,CAAC,EAAE,eAAe,CAAA;IAE1B,kEAAkE;IAClE,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAA;IAEvB;wDACoD;IACpD,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IAEjC;+BAC2B;IAC3B,GAAG,CAAC,EAAE,cAAc,CAAA;IAEpB;;;;sEAIkE;IAClE,OAAO,CAAC,EAAE,eAAe,CAAA;IAEzB;;8DAE0D;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;;;;;;;OASG;IACH,oBAAoB,CAAC,EAAE,MAAM,SAAS,EAAE,CAAA;IAExC;iFAC6E;IAC7E,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,IAAI,CAAA;IAErD;;;uDAGmD;IACnD,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE;QACtB,QAAQ,EAAE,SAAS,CAAA;QACnB,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,KAAK,MAAM,GAAG,SAAS,CAAA;IAExB;iEAC6D;IAC7D,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,wBAAwB,KAAK,IAAI,CAAA;IAEtD;;qEAEiE;IACjE,aAAa,CAAC,EAAE,IAAI,GAAG,MAAM,CAAA;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;;;;;;;;;;OAYG;IACH,YAAY,CAAC,EAAE,YAAY,CAAA;IAE3B;uDACmD;IACnD,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAErC;;4CAEwC;IACxC,YAAY,CAAC,EAAE,oBAAoB,GAAG,KAAK,CAAA;IAE3C,mEAAmE;IACnE,aAAa,CAAC,EAAE,kBAAkB,GAAG,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,WAAW,CAAA;IAErF;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,QAAQ,GAAG,MAAM,GAAG,KAAK,CAAA;IAEvC;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,sBAAsB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,mBAAmB,CAAC,CAAA;IAErF;uEACmE;IACnE,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAA;CACrB;AAED,MAAM,WAAW,iBAAiB,CAAC,SAAS,SAAS,QAAQ,EAAE,SAAS;IACtE,yDAAyD;IACzD,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAA;QACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KACpC,CAAA;IACD,sDAAsD;IACtD,MAAM,EAAE;QACN,aAAa,EAAE,MAAM,CAAA;QACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACnC,OAAO,EAAE,cAAc,CAAA;QACvB;sCAC8B;QAC9B,KAAK,CAAC,EAAE,MAAM,CAAA;QACd;;2CAEmC;QACnC,SAAS,CAAC,EAAE,MAAM,CAAA;KACnB,CAAA;IACD;+CAC2C;IAC3C,IAAI,EAAE,MAAM,CAAA;IACZ;oCACgC;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ;;uEAEmE;IACnE,UAAU,EAAE,oBAAoB,CAAA;IAChC,+CAA+C;IAC/C,YAAY,EAAE,MAAM,GAAG,MAAM,GAAG,gBAAgB,GAAG,eAAe,GAAG,cAAc,CAAA;IACnF;iDAC6C;IAC7C,mBAAmB,EAAE,MAAM,CAAA;IAC3B,wBAAwB;IACxB,UAAU,EAAE,MAAM,CAAA;IAClB,qDAAqD;IACrD,YAAY,EAAE,MAAM,CAAA;IACpB;;;;;OAKG;IACH,OAAO,EAAE,aAAa,CAAA;IACtB;;;;OAIG;IACH,GAAG,EAAE,wBAAwB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;CACpD;AAkDD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,WAAW,CAAC,SAAS,SAAS,QAAQ,EAAE,SAAS,EACrE,IAAI,EAAE,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,GAC7C,OAAO,CAAC,iBAAiB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CA6LlD"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"control-runtime.d.ts","sourceRoot":"","sources":["../src/control-runtime.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAmB,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAClD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAE/C,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,UAAU,CAAA;AACvE,MAAM,MAAM,wBAAwB,GAAG,UAAU,GAAG,MAAM,CAAA;AAE1D,MAAM,WAAW,iBAAiB;IAChC,oCAAoC;IACpC,EAAE,EAAE,MAAM,CAAA;IACV,iCAAiC;IACjC,MAAM,EAAE,OAAO,CAAA;IACf,sDAAsD;IACtD,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,gFAAgF;IAChF,QAAQ,CAAC,EAAE,eAAe,CAAA;IAC1B,6BAA6B;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,4EAA4E;IAC5E,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,mBAAmB,CAAC,MAAM,EAAE,OAAO;IAClD;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B;;;OAGG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,0EAA0E;IAC1E,gBAAgB,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,MAAM,CAAA;IAC5C,+EAA+E;IAC/E,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,MAAM,CAAA;CAChD;AAED,MAAM,WAAW,cAAc,CAC7B,MAAM,EACN,OAAO,EACP,aAAa,EACb,KAAK,SAAS,iBAAiB,GAAG,iBAAiB;IAEnD,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,KAAK,EAAE,CAAA;IACd,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,EAAE,CAAA;IAC7D,MAAM,EAAE,aAAa,CAAA;IACrB,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,WAAW,EAAE,WAAW,CAAA;IACxB,OAAO,CAAC,EAAE,YAAY,CAAA;CACvB;AAED,MAAM,MAAM,eAAe,CAAC,OAAO,IAC/B;IACE,IAAI,EAAE,UAAU,CAAA;IAChB,MAAM,EAAE,OAAO,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB,GACD;IACE,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,OAAO,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;CACf,CAAA;AAEL,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,OAAO,CAAA;IACb,IAAI,EAAE,OAAO,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,YAAY,CAAC,EAAE,YAAY,CAAA;CAC5B;AAED,MAAM,WAAW,oBAAoB,CAAC,aAAa;IACjD,EAAE,EAAE,OAAO,CAAA;IACX,MAAM,CAAC,EAAE,aAAa,CAAA;IACtB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,GAAG,KAAK,GAAG,aAAa,GAAG,SAAS,GAAG,OAAO,CAAA;IACtF,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,WAAW,CAC1B,MAAM,EACN,OAAO,EACP,aAAa,EACb,KAAK,SAAS,iBAAiB,GAAG,iBAAiB;IAEnD,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,eAAe,CAAC,OAAO,CAAC,CAAA;IAClC,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,KAAK,EAAE,CAAA;IACpB,UAAU,EAAE,KAAK,EAAE,CAAA;IACnB,aAAa,CAAC,EAAE,oBAAoB,CAAC,aAAa,CAAC,CAAA;IACnD,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,gBAAgB,CAC/B,MAAM,EACN,OAAO,EACP,aAAa,EACb,KAAK,SAAS,iBAAiB,GAAG,iBAAiB;IAEnD,MAAM,EAAE,MAAM,CAAA;IACd,IAAI,EAAE,OAAO,CAAA;IACb,SAAS,EAAE,OAAO,CAAA;IAClB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,EAAE,CAAA;IAC3D,UAAU,EAAE,MAAM,GAAG,SAAS,CAAA;IAC9B,UAAU,EAAE,KAAK,EAAE,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,6FAA6F;IAC7F,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,YAAY,CAAC,EAAE,YAAY,CAAA;IAC3B,aAAa,EAAE,mBAAmB,EAAE,CAAA;IACpC,SAAS,EAAE,QAAQ,GAAG,aAAa,GAAG,QAAQ,GAAG,OAAO,GAAG,eAAe,CAAA;CAC3E;AAED,MAAM,WAAW,oBAAoB,CACnC,MAAM,EACN,OAAO,EACP,aAAa,EACb,KAAK,SAAS,iBAAiB,GAAG,iBAAiB;IAEnD,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;IAC/B,MAAM,CAAC,EAAE,WAAW,CAAA;IACpB,iGAAiG;IACjG,aAAa,CAAC,EAAE,wBAAwB,CAAA;IACxC;;;OAGG;IACH,gBAAgB,CAAC,EAAE,CAAC,GAAG,EAAE;QACvB,MAAM,EAAE,OAAO,CAAA;QACf,MAAM,EAAE,aAAa,CAAA;QACrB,KAAK,EAAE,MAAM,CAAA;QACb,KAAK,EAAE,KAAK,EAAE,CAAA;QACd,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,EAAE,CAAA;KAC9D,KAAK,MAAM,GAAG,SAAS,CAAA;IAExB,2FAA2F;IAC3F,OAAO,EAAE,CAAC,GAAG,EAAE;QACb,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,EAAE,CAAA;QAC7D,WAAW,EAAE,WAAW,CAAA;KACzB,KAAK,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,CAAA;IAE9B,gGAAgG;IAChG,QAAQ,EAAE,CAAC,GAAG,EAAE;QACd,MAAM,EAAE,MAAM,CAAA;QACd,KAAK,EAAE,MAAM,CAAA;QACb,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,EAAE,CAAA;QAC7D,WAAW,EAAE,WAAW,CAAA;KACzB,KAAK,OAAO,CAAC,KAAK,EAAE,CAAC,GAAG,KAAK,EAAE,CAAA;IAEhC,uGAAuG;IACvG,MAAM,EAAE,CACN,GAAG,EAAE,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,KACvD,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;IAEjE,iDAAiD;IACjD,GAAG,EAAE,CACH,MAAM,EAAE,OAAO,EACf,GAAG,EAAE,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,KACvD,OAAO,CAAC,aAAa,CAAC,GAAG,aAAa,CAAA;IAE3C,yEAAyE;IACzE,UAAU,CAAC,EAAE,CACX,GAAG,EAAE,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,KACvD,OAAO,CAAC,YAAY,CAAC,GAAG,YAAY,CAAA;IAEzC,0DAA0D;IAC1D,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IAE3F,kFAAkF;IAClF,YAAY,CAAC,EAAE,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAEnD,yEAAyE;IACzE,KAAK,CAAC,EAAE,UAAU,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAOD,wBAAsB,mBAAmB,CACvC,MAAM,EACN,OAAO,EACP,aAAa,EACb,KAAK,SAAS,iBAAiB,GAAG,iBAAiB,EAEnD,MAAM,EAAE,oBAAoB,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,GAClE,OAAO,CAAC,gBAAgB,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC,CAgmBlE;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,OAAO,EAC9C,kBAAkB,EAAE,MAAM,EAC1B,OAAO,GAAE,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,CAAM,GAC7E,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAEtC;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,OAAO,EAClD,kBAAkB,EAAE,MAAM,EAC1B,OAAO,GAAE,IAAI,CAAC,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,oBAAoB,CAAM,GAC7E,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAEtC;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,iBAAiB,EAAE,WAAW,CAAC,GAAG,iBAAiB,CAE5F;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,IAAI,CAAC,iBAAiB,EAAE,WAAW,CAAC,GAAG,iBAAiB,CAE7F;AAoDD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAIrE"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"control-runtime.test.d.ts","sourceRoot":"","sources":["../src/control-runtime.test.ts"],"names":[],"mappings":""}
|
package/dist/control.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"control.d.ts","sourceRoot":"","sources":["../src/control.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AACtD,YAAY,EACV,wBAAwB,EACxB,oBAAoB,EACpB,aAAa,EACb,cAAc,EACd,eAAe,EACf,iBAAiB,EACjB,gBAAgB,EAChB,oBAAoB,EACpB,mBAAmB,EACnB,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,YAAY,GACb,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,mBAAmB,EACnB,gBAAgB,EAChB,oBAAoB,EACpB,cAAc,GACf,MAAM,mBAAmB,CAAA;AAC1B,YAAY,EACV,mBAAmB,EACnB,mBAAmB,GACpB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACnD,YAAY,EACV,0BAA0B,EAC1B,0BAA0B,EAC1B,0BAA0B,EAC1B,yBAAyB,GAC1B,MAAM,0BAA0B,CAAA;AACjC,OAAO,EAAE,6BAA6B,EAAE,MAAM,0BAA0B,CAAA;AACxE,YAAY,EACV,4BAA4B,EAC5B,mBAAmB,GACpB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACL,qBAAqB,EACrB,cAAc,GACf,MAAM,gBAAgB,CAAA"}
|
package/dist/convergence.d.ts
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
import type { CompletionCriterion, DriverState } from './types';
|
|
2
|
-
/**
|
|
3
|
-
* ConvergenceTracker — tracks completion percentage over turns.
|
|
4
|
-
*
|
|
5
|
-
* Produces convergence curves showing how quickly the agent reaches
|
|
6
|
-
* completion criteria.
|
|
7
|
-
*/
|
|
8
|
-
export declare class ConvergenceTracker {
|
|
9
|
-
private criteria;
|
|
10
|
-
private history;
|
|
11
|
-
constructor(criteria: CompletionCriterion[]);
|
|
12
|
-
/** Evaluate criteria against current state, record result */
|
|
13
|
-
record(turn: number, state: DriverState): {
|
|
14
|
-
completionPercent: number;
|
|
15
|
-
complete: boolean;
|
|
16
|
-
criteriaStatus: Record<string, boolean | number>;
|
|
17
|
-
};
|
|
18
|
-
/** Get convergence curve */
|
|
19
|
-
getCurve(): number[];
|
|
20
|
-
/** Get full history with per-criterion status */
|
|
21
|
-
getHistory(): {
|
|
22
|
-
turn: number;
|
|
23
|
-
completionPercent: number;
|
|
24
|
-
criteriaStatus: Record<string, boolean | number>;
|
|
25
|
-
}[];
|
|
26
|
-
/** Find the turn where completion first reached 100% (or null) */
|
|
27
|
-
getTurnToCompletion(): number | null;
|
|
28
|
-
}
|
|
29
|
-
//# sourceMappingURL=convergence.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"convergence.d.ts","sourceRoot":"","sources":["../src/convergence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAE/D;;;;;GAKG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAuB;IACvC,OAAO,CAAC,OAAO,CAIP;gBAEI,QAAQ,EAAE,mBAAmB,EAAE;IAI3C,6DAA6D;IAC7D,MAAM,CACJ,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,WAAW,GACjB;QACD,iBAAiB,EAAE,MAAM,CAAA;QACzB,QAAQ,EAAE,OAAO,CAAA;QACjB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC,CAAA;KACjD;IA4BD,4BAA4B;IAC5B,QAAQ,IAAI,MAAM,EAAE;IAIpB,iDAAiD;IACjD,UAAU;cAnDF,MAAM;2BACO,MAAM;wBACT,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC;;IAqDlD,kEAAkE;IAClE,mBAAmB,IAAI,MAAM,GAAG,IAAI;CAIrC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cost-ledger.d.ts","sourceRoot":"","sources":["../src/cost-ledger.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH;gEACgE;AAChE,MAAM,MAAM,WAAW,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,QAAQ,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAA;AAE/F,MAAM,WAAW,SAAS;IACxB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAE1D;AAED,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAA;IACf,4EAA4E;IAC5E,WAAW,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,SAAS,GAAG,UAAU,CAQxE;AAED,MAAM,WAAW,eAAgB,SAAQ,SAAS;IAChD,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,WAAW,CAAA;IACpB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,OAAO,CAAA;IACpB,4DAA4D;IAC5D,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,0DAA0D;IAC1D,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC7B,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,WAAW,CAAA;IACpB,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,OAAO,EAAE,MAAM,CAAA;IACf,uEAAuE;IACvE,aAAa,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,qDAAqD;IACrD,SAAS,EAAE,aAAa,EAAE,CAAA;IAC1B;2DACuD;IACvD,cAAc,EAAE,MAAM,EAAE,CAAA;IACxB,4EAA4E;IAC5E,WAAW,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;GAMG;AACH,qBAAa,UAAU;IACrB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwB;IAChD,OAAO,CAAC,cAAc,CAAI;IAE1B;;;;OAIG;IACH,MAAM,CAAC,KAAK,EAAE;QACZ,KAAK,EAAE,MAAM,CAAA;QACb,OAAO,EAAE,WAAW,CAAA;QACpB,KAAK,EAAE,SAAS,CAAA;QAChB,aAAa,CAAC,EAAE,MAAM,CAAA;QACtB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAC7B,SAAS,CAAC,EAAE,MAAM,CAAA;KACnB,GAAG,eAAe;IAqBnB,+EAA+E;IAC/E,aAAa,CAAC,KAAK,SAAI,GAAG,IAAI;IAS9B,IAAI,IAAI,eAAe,EAAE;IAIzB,OAAO,IAAI,iBAAiB;IA0C5B,2EAA2E;IAC3E,oBAAoB,IAAI,MAAM,GAAG,IAAI;CAItC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cost-ledger.test.d.ts","sourceRoot":"","sources":["../src/cost-ledger.test.ts"],"names":[],"mappings":""}
|
package/dist/cost-report.d.ts
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Program cost report — a thin projection over `CostLedger.summary()` that
|
|
3
|
-
* adds the per-model rollup the summary lacks, plus `attachCostToReport`, the
|
|
4
|
-
* one way every artifact (capsules, campaign results, diagnose reports) gets
|
|
5
|
-
* its cost stamp.
|
|
6
|
-
*
|
|
7
|
-
* Honesty contract carried through from the ledger: `total.unknownEntries`
|
|
8
|
-
* and `perModel[].unpriced` surface the costUnknown axis — a $0 from an
|
|
9
|
-
* unpriced model is a lower bound, never a measured zero.
|
|
10
|
-
*/
|
|
11
|
-
import type { ChannelRollup, CostLedger } from './cost-ledger';
|
|
12
|
-
export interface ModelCostRollup {
|
|
13
|
-
model: string;
|
|
14
|
-
usd: number;
|
|
15
|
-
entries: number;
|
|
16
|
-
/** ≥1 entry for this model was costUnknown — `usd` is a lower bound. An
|
|
17
|
-
* `actualCostUsd` override clears the flag for that entry (the dollars are
|
|
18
|
-
* observed, even when the model has no pricing). */
|
|
19
|
-
unpriced: boolean;
|
|
20
|
-
}
|
|
21
|
-
export interface CostReport {
|
|
22
|
-
/** Per-channel breakdown — `CostLedgerSummary.byChannel` verbatim. */
|
|
23
|
-
perChannel: ChannelRollup[];
|
|
24
|
-
total: {
|
|
25
|
-
usd: number;
|
|
26
|
-
/** Entries whose cost was unknown — non-zero means `usd` is a lower bound. */
|
|
27
|
-
unknownEntries: number;
|
|
28
|
-
};
|
|
29
|
-
/** Per-model spend, sorted by model id. */
|
|
30
|
-
perModel: ModelCostRollup[];
|
|
31
|
-
}
|
|
32
|
-
/** Project a ledger into the program cost report. Pure — no I/O, no clock. */
|
|
33
|
-
export declare function costReport(ledger: CostLedger): CostReport;
|
|
34
|
-
/**
|
|
35
|
-
* Stamp a report-shaped object with its cost projection under the `cost` key.
|
|
36
|
-
* Generic so capsules, campaign results, and diagnose reports all stamp the
|
|
37
|
-
* same way. Throws when the report already carries a `cost` key — silently
|
|
38
|
-
* overwriting an existing stamp would corrupt the artifact's provenance.
|
|
39
|
-
*/
|
|
40
|
-
export declare function attachCostToReport<R extends object>(report: R, ledger: CostLedger): R & {
|
|
41
|
-
cost: CostReport;
|
|
42
|
-
};
|
|
43
|
-
//# sourceMappingURL=cost-report.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cost-report.d.ts","sourceRoot":"","sources":["../src/cost-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAG9D,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAA;IACb,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,MAAM,CAAA;IACf;;yDAEqD;IACrD,QAAQ,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,WAAW,UAAU;IACzB,sEAAsE;IACtE,UAAU,EAAE,aAAa,EAAE,CAAA;IAC3B,KAAK,EAAE;QACL,GAAG,EAAE,MAAM,CAAA;QACX,8EAA8E;QAC9E,cAAc,EAAE,MAAM,CAAA;KACvB,CAAA;IACD,2CAA2C;IAC3C,QAAQ,EAAE,eAAe,EAAE,CAAA;CAC5B;AAED,8EAA8E;AAC9E,wBAAgB,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,UAAU,CAuBzD;AAED;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,SAAS,MAAM,EACjD,MAAM,EAAE,CAAC,EACT,MAAM,EAAE,UAAU,GACjB,CAAC,GAAG;IAAE,IAAI,EAAE,UAAU,CAAA;CAAE,CAO1B"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cost-report.test.d.ts","sourceRoot":"","sources":["../src/cost-report.test.ts"],"names":[],"mappings":""}
|
package/dist/cost-tracker.d.ts
DELETED
|
@@ -1,76 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Cost tracker — token + USD accounting per scenario and per run.
|
|
3
|
-
*
|
|
4
|
-
* Adapted from generic usage-event accounting. Every
|
|
5
|
-
* optimizer needs to know "is the quality gain worth the cost delta?",
|
|
6
|
-
* and every dashboard needs dollars-per-completed-task. MODEL_PRICING
|
|
7
|
-
* from metrics.ts stays authoritative for estimate math; this module
|
|
8
|
-
* adds the aggregation + per-scenario roll-up that was duplicated
|
|
9
|
-
* across 4 verticals.
|
|
10
|
-
*/
|
|
11
|
-
export interface TokenSpec {
|
|
12
|
-
inputTokens: number;
|
|
13
|
-
outputTokens: number;
|
|
14
|
-
cachedTokens?: number;
|
|
15
|
-
reasoningTokens?: number;
|
|
16
|
-
}
|
|
17
|
-
export interface CostEntry extends TokenSpec {
|
|
18
|
-
scenarioId: string;
|
|
19
|
-
model: string;
|
|
20
|
-
/** Override estimate with an observed cost (e.g. from provider response). */
|
|
21
|
-
actualCostUsd?: number;
|
|
22
|
-
timestamp: number;
|
|
23
|
-
/** Free-form tags (variant id, round #, etc.). */
|
|
24
|
-
tags?: Record<string, string>;
|
|
25
|
-
}
|
|
26
|
-
export interface ScenarioCost {
|
|
27
|
-
scenarioId: string;
|
|
28
|
-
entries: CostEntry[];
|
|
29
|
-
totalInputTokens: number;
|
|
30
|
-
totalOutputTokens: number;
|
|
31
|
-
totalCachedTokens: number;
|
|
32
|
-
totalCostUsd: number;
|
|
33
|
-
/** Pass flag — set by consumer via markOutcome; used for cost-per-completed-task. */
|
|
34
|
-
completed?: boolean;
|
|
35
|
-
}
|
|
36
|
-
export declare class CostTracker {
|
|
37
|
-
private byScenario;
|
|
38
|
-
record(entry: Omit<CostEntry, 'timestamp'> & {
|
|
39
|
-
timestamp?: number;
|
|
40
|
-
}): CostEntry;
|
|
41
|
-
markOutcome(scenarioId: string, completed: boolean): void;
|
|
42
|
-
/**
|
|
43
|
-
* Convenience: record + markOutcome in one call from a
|
|
44
|
-
* `{ usage, verdict }`-shaped response (starter-foundry's
|
|
45
|
-
* `invokeMetaJudge` returns this shape; consumers that wrap any
|
|
46
|
-
* judge/critic can follow the same convention).
|
|
47
|
-
*
|
|
48
|
-
* `usage.model` must be present in `MODEL_PRICING` for cost math to
|
|
49
|
-
* populate; otherwise totalCostUsd stays at 0 for the entry but
|
|
50
|
-
* tokens still aggregate.
|
|
51
|
-
*/
|
|
52
|
-
recordVerdict(verdict: {
|
|
53
|
-
usage?: {
|
|
54
|
-
inputTokens: number;
|
|
55
|
-
outputTokens: number;
|
|
56
|
-
model: string;
|
|
57
|
-
cachedTokens?: number;
|
|
58
|
-
reasoningTokens?: number;
|
|
59
|
-
};
|
|
60
|
-
verdict?: 'pass' | 'fail' | 'borderline' | string;
|
|
61
|
-
}, scenarioId: string, tags?: Record<string, string>): CostEntry | null;
|
|
62
|
-
get(scenarioId: string): ScenarioCost | undefined;
|
|
63
|
-
list(): ScenarioCost[];
|
|
64
|
-
summary(): CostSummary;
|
|
65
|
-
}
|
|
66
|
-
export interface CostSummary {
|
|
67
|
-
scenarioCount: number;
|
|
68
|
-
completedCount: number;
|
|
69
|
-
totalInputTokens: number;
|
|
70
|
-
totalOutputTokens: number;
|
|
71
|
-
totalCostUsd: number;
|
|
72
|
-
avgCostPerScenarioUsd: number;
|
|
73
|
-
/** Total USD / completed scenarios — null when nothing completed. */
|
|
74
|
-
costPerCompletedTaskUsd: number | null;
|
|
75
|
-
}
|
|
76
|
-
//# sourceMappingURL=cost-tracker.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cost-tracker.d.ts","sourceRoot":"","sources":["../src/cost-tracker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,MAAM,WAAW,SAAS;IACxB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,SAAU,SAAQ,SAAS;IAC1C,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,6EAA6E;IAC7E,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC9B;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,OAAO,EAAE,SAAS,EAAE,CAAA;IACpB,gBAAgB,EAAE,MAAM,CAAA;IACxB,iBAAiB,EAAE,MAAM,CAAA;IACzB,iBAAiB,EAAE,MAAM,CAAA;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,qFAAqF;IACrF,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,UAAU,CAAkC;IAEpD,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,GAAG;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS;IAwB/E,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,GAAG,IAAI;IAMzD;;;;;;;;;OASG;IACH,aAAa,CACX,OAAO,EAAE;QACP,KAAK,CAAC,EAAE;YACN,WAAW,EAAE,MAAM,CAAA;YACnB,YAAY,EAAE,MAAM,CAAA;YACpB,KAAK,EAAE,MAAM,CAAA;YACb,YAAY,CAAC,EAAE,MAAM,CAAA;YACrB,eAAe,CAAC,EAAE,MAAM,CAAA;SACzB,CAAA;QACD,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,YAAY,GAAG,MAAM,CAAA;KAClD,EACD,UAAU,EAAE,MAAM,EAClB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC5B,SAAS,GAAG,IAAI;IAenB,GAAG,CAAC,UAAU,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIjD,IAAI,IAAI,YAAY,EAAE;IAItB,OAAO,IAAI,WAAW;CAiBvB;AAED,MAAM,WAAW,WAAW;IAC1B,aAAa,EAAE,MAAM,CAAA;IACrB,cAAc,EAAE,MAAM,CAAA;IACtB,gBAAgB,EAAE,MAAM,CAAA;IACxB,iBAAiB,EAAE,MAAM,CAAA;IACzB,YAAY,EAAE,MAAM,CAAA;IACpB,qBAAqB,EAAE,MAAM,CAAA;IAC7B,qEAAqE;IACrE,uBAAuB,EAAE,MAAM,GAAG,IAAI,CAAA;CACvC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"counterfactual.d.ts","sourceRoot":"","sources":["../src/counterfactual.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,KAAK,EAAW,IAAI,EAAY,MAAM,gBAAgB,CAAA;AAC7D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAmB,KAAK,UAAU,EAAE,KAAK,cAAc,EAAE,MAAM,cAAc,CAAA;AAEpF,MAAM,MAAM,sBAAsB,GAC9B;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAE,GAC5D;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GACtC;IAAE,IAAI,EAAE,uBAAuB,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAC9D;IACE,IAAI,EAAE,QAAQ,CAAA;IACd,EAAE,EAAE,MAAM,CAAA;IACV,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,cAAc,CAAA;CAChD,CAAA;AAEL,MAAM,WAAW,qBAAqB;IACpC,aAAa,EAAE,MAAM,CAAA;IACrB,kBAAkB,EAAE,UAAU,CAAA;IAC9B;0EACsE;IACtE,MAAM,EAAE,cAAc,EAAE,CAAA;IACxB,QAAQ,EAAE,sBAAsB,CAAA;IAChC;;8CAE0C;IAC1C,WAAW,EAAE,cAAc,CAAA;CAC5B;AAED,MAAM,WAAW,oBAAoB;IACnC,mBAAmB,EAAE,MAAM,CAAA;IAC3B,aAAa,EAAE,MAAM,CAAA;IACrB,QAAQ,EAAE,sBAAsB,CAAA;IAChC,gEAAgE;IAChE,KAAK,EAAE;QACL,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAA;QACnC,0BAA0B,EAAE,MAAM,GAAG,IAAI,CAAA;QACzC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;KAC1B,CAAA;CACF;AAED,MAAM,WAAW,oBAAoB;IACnC;;;;OAIG;IACH,WAAW,EAAE,CAAC,GAAG,EAAE,qBAAqB,EAAE,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;CAClF;AAED,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,UAAU,EACjB,aAAa,EAAE,MAAM,EACrB,QAAQ,EAAE,sBAAsB,EAChC,MAAM,EAAE,oBAAoB,GAC3B,OAAO,CAAC,oBAAoB,CAAC,CA6C/B;AA0BD;;;;;GAKG;AACH,wBAAgB,wBAAwB,CAAC,OAAO,EAAE,oBAAoB,EAAE,GAAG,KAAK,CAAC;IAC/E,YAAY,EAAE,sBAAsB,CAAC,MAAM,CAAC,CAAA;IAC5C,CAAC,EAAE,MAAM,CAAA;IACT,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;CACxB,CAAC,CA4BD;AAGD,YAAY,EAAE,IAAI,EAAE,CAAA"}
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Full cross-trace diff — align two trajectories step-by-step, report
|
|
3
|
-
* per-step score deltas, attribute a variant's total outcome lead to
|
|
4
|
-
* specific turns.
|
|
5
|
-
*
|
|
6
|
-
* 0.5 shipped `firstDivergenceView` (finds the first differing step).
|
|
7
|
-
* This does the heavier work: full alignment via LCS, per-step
|
|
8
|
-
* contribution to score delta using PRM verdicts when available,
|
|
9
|
-
* fallback to structural heuristics (latency, token count, tool
|
|
10
|
-
* outcome) otherwise.
|
|
11
|
-
*/
|
|
12
|
-
import type { TraceStore } from './trace/store';
|
|
13
|
-
import { type TrajectoryStep } from './trajectory';
|
|
14
|
-
export type AlignmentOp = {
|
|
15
|
-
op: 'match';
|
|
16
|
-
a: TrajectoryStep;
|
|
17
|
-
b: TrajectoryStep;
|
|
18
|
-
} | {
|
|
19
|
-
op: 'insert';
|
|
20
|
-
b: TrajectoryStep;
|
|
21
|
-
} | {
|
|
22
|
-
op: 'delete';
|
|
23
|
-
a: TrajectoryStep;
|
|
24
|
-
} | {
|
|
25
|
-
op: 'replace';
|
|
26
|
-
a: TrajectoryStep;
|
|
27
|
-
b: TrajectoryStep;
|
|
28
|
-
};
|
|
29
|
-
export interface StepAttribution {
|
|
30
|
-
op: AlignmentOp;
|
|
31
|
-
/** Difference in PRM score (or null when not scored by a matching judge). */
|
|
32
|
-
prmDelta: number | null;
|
|
33
|
-
/** Difference in latency (endedAt - startedAt). */
|
|
34
|
-
latencyDeltaMs: number | null;
|
|
35
|
-
/** Difference in token count (LLM spans). */
|
|
36
|
-
tokenDelta: number | null;
|
|
37
|
-
/** Reason this step is / isn't considered a contributor to the outcome delta. */
|
|
38
|
-
note: string;
|
|
39
|
-
}
|
|
40
|
-
export interface CrossTraceDiff {
|
|
41
|
-
runA: string;
|
|
42
|
-
runB: string;
|
|
43
|
-
alignment: AlignmentOp[];
|
|
44
|
-
attributions: StepAttribution[];
|
|
45
|
-
/** Total score delta (B - A). */
|
|
46
|
-
totalScoreDelta: number | null;
|
|
47
|
-
/** Sum of PRM deltas across matched/replaced steps. Close to
|
|
48
|
-
* `totalScoreDelta` when PRM covers the trajectory; gap indicates
|
|
49
|
-
* unmodeled variance. */
|
|
50
|
-
prmDeltaSum: number;
|
|
51
|
-
}
|
|
52
|
-
export interface CrossTraceDiffOptions {
|
|
53
|
-
stepEquals?: (a: TrajectoryStep, b: TrajectoryStep) => boolean;
|
|
54
|
-
}
|
|
55
|
-
export declare function crossTraceDiff(store: TraceStore, runA: string, runB: string, options?: CrossTraceDiffOptions): Promise<CrossTraceDiff>;
|
|
56
|
-
//# sourceMappingURL=cross-trace-diff.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cross-trace-diff.d.ts","sourceRoot":"","sources":["../src/cross-trace-diff.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC/C,OAAO,EAAmB,KAAK,cAAc,EAAE,MAAM,cAAc,CAAA;AAEnE,MAAM,MAAM,WAAW,GACnB;IAAE,EAAE,EAAE,OAAO,CAAC;IAAC,CAAC,EAAE,cAAc,CAAC;IAAC,CAAC,EAAE,cAAc,CAAA;CAAE,GACrD;IAAE,EAAE,EAAE,QAAQ,CAAC;IAAC,CAAC,EAAE,cAAc,CAAA;CAAE,GACnC;IAAE,EAAE,EAAE,QAAQ,CAAC;IAAC,CAAC,EAAE,cAAc,CAAA;CAAE,GACnC;IAAE,EAAE,EAAE,SAAS,CAAC;IAAC,CAAC,EAAE,cAAc,CAAC;IAAC,CAAC,EAAE,cAAc,CAAA;CAAE,CAAA;AAE3D,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,WAAW,CAAA;IACf,6EAA6E;IAC7E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,mDAAmD;IACnD,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,6CAA6C;IAC7C,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAA;CACb;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,WAAW,EAAE,CAAA;IACxB,YAAY,EAAE,eAAe,EAAE,CAAA;IAC/B,iCAAiC;IACjC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B;;8BAE0B;IAC1B,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC,EAAE,cAAc,KAAK,OAAO,CAAA;CAC/D;AAED,wBAAsB,cAAc,CAClC,KAAK,EAAE,UAAU,EACjB,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACZ,OAAO,GAAE,qBAA0B,GAClC,OAAO,CAAC,cAAc,CAAC,CAwBzB"}
|
package/dist/dataset.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../src/dataset.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,MAAM,YAAY,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,CAAA;AAC/D,MAAM,MAAM,iBAAiB,GAAG,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,SAAS,CAAA;AAEtE,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAA;IACV,6DAA6D;IAC7D,OAAO,EAAE,OAAO,CAAA;IAChB,KAAK,CAAC,EAAE,YAAY,CAAA;IACpB,UAAU,CAAC,EAAE,iBAAiB,CAAA;IAC9B,4EAA4E;IAC5E,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;;;;;OAQG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC9B;AAED,MAAM,WAAW,iBAAiB;IAChC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,4DAA4D;IAC5D,OAAO,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,iBAAiB,CAAA;IAC7B,+CAA+C;IAC/C,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa,EAAE,MAAM,CAAA;IACrB,WAAW,EAAE,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAA;CAC1C;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,CAAC,EAAE,YAAY,CAAA;IACpB,UAAU,CAAC,EAAE,iBAAiB,CAAA;IAC9B,gFAAgF;IAChF,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,mEAAmE;IACnE,MAAM,CAAC,EAAE,CAAC,QAAQ,EAAE,eAAe,KAAK,OAAO,CAAA;IAC/C,mEAAmE;IACnE,cAAc,CAAC,EAAE,OAAO,CAAA;CACzB;AAED,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAA;AAE1C,uFAAuF;AACvF,qBAAa,kBAAmB,SAAQ,eAAe;gBACzC,WAAW,EAAE,MAAM;CAKhC;AAED,qBAAa,OAAO;IAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,UAAU,EAAE,iBAAiB,CAAA;IACtC,OAAO,CAAC,SAAS,CAAmB;IACpC,OAAO,CAAC,MAAM,CAAS;gBAEX,IAAI,EAAE;QAChB,IAAI,EAAE,MAAM,CAAA;QACZ,UAAU,EAAE,iBAAiB,CAAA;QAC7B,SAAS,EAAE,eAAe,EAAE,CAAA;QAC5B,MAAM,CAAC,EAAE,OAAO,CAAA;KACjB;IAOD,4EAA4E;IAC5E,GAAG,IAAI,SAAS,eAAe,EAAE;IAIjC,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED;;;OAGG;IACH,KAAK,CAAC,OAAO,GAAE,YAAiB,GAAG,eAAe,EAAE;IAiBpD;;;OAGG;IACG,QAAQ,IAAI,OAAO,CAAC,eAAe,CAAC;IAe1C,4EAA4E;IAC5E,KAAK,CAAC,SAAS,GAAE,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAM,GAAG,OAAO;IAW1E,IAAI,IAAI,IAAI;IAIZ,GAAG,CAAC,QAAQ,EAAE,eAAe,GAAG,IAAI;IAQpC,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAOhC;;;OAGG;IACH,OAAO,IAAI,MAAM;IAQjB,MAAM,CAAC,SAAS,CACd,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,aAAa,GAAG,eAAe,GAAG,aAAa,CAAC,GAC/E,OAAO;CASX;AAID,wBAAsB,aAAa,CAAC,SAAS,EAAE,eAAe,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAWjF"}
|