@tangle-network/agent-eval 0.95.0 → 0.95.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/adapters/http.d.ts +17 -10
- package/dist/adapters/langchain.d.ts +14 -7
- package/dist/adapters/otel.d.ts +25 -13
- package/dist/{rl/adversarial.d.ts → adversarial-DIVcDoI_.d.ts} +7 -6
- package/dist/analyst/index.d.ts +236 -28
- package/dist/{trace-analyst/analyst.d.ts → analyst-C8HHvfJp.d.ts} +14 -12
- package/dist/{contract/analyze-runs.d.ts → analyze-runs-DtT6F_6T.d.ts} +10 -9
- package/dist/authenticity/index.d.ts +16 -15
- package/dist/{baseline.d.ts → baseline-Bbid3WoO.d.ts} +44 -8
- package/dist/belief-state/index.d.ts +605 -14
- package/dist/benchmarks/index.d.ts +5 -23
- package/dist/builder-eval/index.d.ts +250 -5
- package/dist/calibration-BPmzuVPk.d.ts +101 -0
- package/dist/campaign/index.d.ts +1452 -38
- package/dist/cli.d.ts +0 -2
- package/dist/{contract/intake/code-agent-session.d.ts → code-agent-session-CPHRCb4-.d.ts} +17 -15
- package/dist/contract/index.d.ts +688 -106
- package/dist/control-Doncu-B_.d.ts +259 -0
- package/dist/{control-runtime.d.ts → control-runtime-Acf9CGhw.d.ts} +26 -23
- package/dist/control.d.ts +10 -11
- package/dist/corpus-D4YW9UoJ.d.ts +560 -0
- package/dist/{cost-ledger.d.ts → cost-ledger-DuSqlw5B.d.ts} +11 -10
- package/dist/{counterfactual.d.ts → counterfactual-DlOz8PBx.d.ts} +13 -12
- package/dist/{dataset.d.ts → dataset-BbGkaN2I.d.ts} +14 -11
- package/dist/{analyst/registry.d.ts → default-registry-GyE8X5SP.d.ts} +37 -8
- package/dist/diagnose.d.ts +252 -1
- package/dist/{trace/emitter.d.ts → emitter-C2rqGH_l.d.ts} +12 -9
- package/dist/{errors.d.ts → errors-CzMUYo7b.d.ts} +11 -10
- package/dist/failure-cluster-DH9Flgcf.d.ts +76 -0
- package/dist/{feedback-trajectory.d.ts → feedback-trajectory-BxY0cKfs.d.ts} +38 -36
- package/dist/fuzz.d.ts +547 -1
- package/dist/gepa-C1NCIZ9o.d.ts +414 -0
- package/dist/governance/index.d.ts +135 -5
- package/dist/harness-optimizer-mOl9XX_O.d.ts +106 -0
- package/dist/hosted/index.d.ts +239 -10
- package/dist/index-_Y4oNOOb.d.ts +159 -0
- package/dist/index.d.ts +5660 -277
- package/dist/{contract/insight-report.d.ts → insight-report-BnRjTibG.d.ts} +19 -16
- package/dist/{trace/integrity.d.ts → integrity-D2t12mMw.d.ts} +14 -11
- package/dist/{judge-calibration.d.ts → judge-calibration-0p2QcWNE.d.ts} +17 -16
- package/dist/{analyst/kind-factory.d.ts → kind-factory-X3eDYbKn.d.ts} +61 -10
- package/dist/knowledge/index.d.ts +103 -3
- package/dist/{llm-client.d.ts → llm-client-Bj7g0rqu.d.ts} +23 -21
- package/dist/matrix/index.d.ts +30 -12
- package/dist/meta-eval/index.d.ts +182 -6
- package/dist/{multi-layer-verifier.d.ts → multi-layer-verifier-DUZXrPDA.d.ts} +15 -12
- package/dist/multishot/index.d.ts +290 -7
- package/dist/{rl/off-policy.d.ts → off-policy-DiwuKKg7.d.ts} +9 -8
- package/dist/openapi.json +1 -1
- package/dist/{meta-eval/outcome-store.d.ts → outcome-store-rnXLEqSn.d.ts} +8 -7
- package/dist/{pareto.d.ts → pareto-E-pembql.d.ts} +10 -9
- package/dist/perf/index.d.ts +119 -13
- package/dist/pipelines/index.d.ts +173 -8
- package/dist/pre-registration-nfUdc9EQ.d.ts +483 -0
- package/dist/prm/index.d.ts +104 -5
- package/dist/provenance-CncDq9qE.d.ts +426 -0
- package/dist/query-B7GGjRox.d.ts +32 -0
- package/dist/{trace/raw-provider-sink.d.ts → raw-provider-sink-C46HDghv.d.ts} +14 -13
- package/dist/{red-team.d.ts → red-team-BWdoyleI.d.ts} +16 -14
- package/dist/{trace/redact.d.ts → redact-B40YG2M_.d.ts} +8 -7
- package/dist/release-report-pidWUMZ2.d.ts +233 -0
- package/dist/reporting.d.ts +16 -15
- package/dist/{eval-campaign.d.ts → researcher-Jr8ME1dZ.d.ts} +156 -34
- package/dist/rl.d.ts +1193 -1
- package/dist/{prm/rubric.d.ts → rubric-Cc6UHvUb.d.ts} +13 -10
- package/dist/{meta-eval/rubric-predictive-validity.d.ts → rubric-predictive-validity-C2hDKM8Z.d.ts} +11 -8
- package/dist/run-critic-CmMf05uV.d.ts +56 -0
- package/dist/{run-record.d.ts → run-record-CP2ObebC.d.ts} +117 -15
- package/dist/runtime-trajectory-BOUUjI0y.d.ts +49 -0
- package/dist/{trace/schema.d.ts → schema-m0gsnbt3.d.ts} +30 -29
- package/dist/semantic-concept-judge-DSBB2Cfp.d.ts +624 -0
- package/dist/{sequential.d.ts → sequential-5iSVfzl2.d.ts} +10 -9
- package/dist/{series-convergence.d.ts → series-convergence-D5OWMBg6.d.ts} +5 -4
- package/dist/sink-fetch-B1Yg4Til.d.ts +101 -0
- package/dist/{statistics.d.ts → statistics-CCJpTGOS.d.ts} +48 -46
- package/dist/{trace/store.d.ts → store-BcFXE6LG.d.ts} +12 -21
- package/dist/{trace-analyst/types.d.ts → store-C1YxJDEK.d.ts} +74 -18
- package/dist/storyboard/index.d.ts +81 -16
- package/dist/{summary-report.d.ts → summary-report-CInXwsza.d.ts} +160 -23
- package/dist/telemetry/{sink-file.d.ts → file.d.ts} +7 -5
- package/dist/telemetry/index.d.ts +35 -17
- package/dist/{sandbox-harness.d.ts → test-graded-scenario-DeODGLra.d.ts} +60 -15
- package/dist/{locked-jsonl-appender.d.ts → testing-C21CHsq2.d.ts} +4 -3
- package/dist/testing.d.ts +1 -5
- package/dist/traces.d.ts +976 -4
- package/dist/{trajectory.d.ts → trajectory-2TkpSEVh.d.ts} +9 -8
- package/dist/{analyst/types.d.ts → types-B5x54y6n.d.ts} +112 -19
- package/dist/{matrix/types.d.ts → types-BUxNaJ8c.d.ts} +11 -9
- package/dist/{types.d.ts → types-C7DGg5ex.d.ts} +33 -31
- package/dist/{campaign/types.d.ts → types-DQRY8ZT-.d.ts} +47 -44
- package/dist/{verdict.d.ts → verdict-C9MlYujm.d.ts} +3 -2
- package/dist/wire/index.d.ts +570 -13
- package/dist/workflow/index.d.ts +496 -22
- package/package.json +2 -2
- package/dist/action-policy.d.ts +0 -24
- package/dist/action-policy.d.ts.map +0 -1
- package/dist/action-policy.test.d.ts +0 -2
- package/dist/action-policy.test.d.ts.map +0 -1
- package/dist/active-learning.d.ts +0 -41
- package/dist/active-learning.d.ts.map +0 -1
- package/dist/adapters/http.d.ts.map +0 -1
- package/dist/adapters/langchain.d.ts.map +0 -1
- package/dist/adapters/otel.d.ts.map +0 -1
- package/dist/agent-profile-cell.d.ts +0 -101
- package/dist/agent-profile-cell.d.ts.map +0 -1
- package/dist/agent-profile.d.ts +0 -27
- package/dist/agent-profile.d.ts.map +0 -1
- package/dist/agent-profile.test.d.ts +0 -2
- package/dist/agent-profile.test.d.ts.map +0 -1
- package/dist/analyst/adapters.d.ts +0 -62
- package/dist/analyst/adapters.d.ts.map +0 -1
- package/dist/analyst/analyst.test.d.ts +0 -2
- package/dist/analyst/analyst.test.d.ts.map +0 -1
- package/dist/analyst/ax-service.d.ts +0 -27
- package/dist/analyst/ax-service.d.ts.map +0 -1
- package/dist/analyst/behavioral-analyst.d.ts +0 -28
- package/dist/analyst/behavioral-analyst.d.ts.map +0 -1
- package/dist/analyst/chat-client.d.ts +0 -91
- package/dist/analyst/chat-client.d.ts.map +0 -1
- package/dist/analyst/default-registry.d.ts +0 -27
- package/dist/analyst/default-registry.d.ts.map +0 -1
- package/dist/analyst/default-registry.test.d.ts +0 -2
- package/dist/analyst/default-registry.test.d.ts.map +0 -1
- package/dist/analyst/finding-signature.d.ts +0 -48
- package/dist/analyst/finding-signature.d.ts.map +0 -1
- package/dist/analyst/finding-subject.d.ts +0 -146
- package/dist/analyst/finding-subject.d.ts.map +0 -1
- package/dist/analyst/finding-subject.test.d.ts +0 -2
- package/dist/analyst/finding-subject.test.d.ts.map +0 -1
- package/dist/analyst/findings-store.d.ts +0 -75
- package/dist/analyst/findings-store.d.ts.map +0 -1
- package/dist/analyst/index.d.ts.map +0 -1
- package/dist/analyst/kind-factory.d.ts.map +0 -1
- package/dist/analyst/kinds/failure-mode.d.ts +0 -19
- package/dist/analyst/kinds/failure-mode.d.ts.map +0 -1
- package/dist/analyst/kinds/improvement.d.ts +0 -23
- package/dist/analyst/kinds/improvement.d.ts.map +0 -1
- package/dist/analyst/kinds/index.d.ts +0 -22
- package/dist/analyst/kinds/index.d.ts.map +0 -1
- package/dist/analyst/kinds/kinds.test.d.ts +0 -2
- package/dist/analyst/kinds/kinds.test.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-gap.d.ts +0 -28
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +0 -1
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +0 -22
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.d.ts +0 -84
- package/dist/analyst/kinds/skill-usage.d.ts.map +0 -1
- package/dist/analyst/kinds/skill-usage.test.d.ts +0 -2
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.d.ts +0 -26
- package/dist/analyst/parse-tolerant.d.ts.map +0 -1
- package/dist/analyst/parse-tolerant.test.d.ts +0 -2
- package/dist/analyst/parse-tolerant.test.d.ts.map +0 -1
- package/dist/analyst/registry.budget.test.d.ts +0 -2
- package/dist/analyst/registry.budget.test.d.ts.map +0 -1
- package/dist/analyst/registry.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.d.ts +0 -35
- package/dist/analyst/steer-firewall.d.ts.map +0 -1
- package/dist/analyst/steer-firewall.test.d.ts +0 -2
- package/dist/analyst/steer-firewall.test.d.ts.map +0 -1
- package/dist/analyst/structure-findings.d.ts +0 -37
- package/dist/analyst/structure-findings.d.ts.map +0 -1
- package/dist/analyst/structure-findings.test.d.ts +0 -2
- package/dist/analyst/structure-findings.test.d.ts.map +0 -1
- package/dist/analyst/tool-groups.d.ts +0 -34
- package/dist/analyst/tool-groups.d.ts.map +0 -1
- package/dist/analyst/types.d.ts.map +0 -1
- package/dist/anti-slop.d.ts +0 -59
- package/dist/anti-slop.d.ts.map +0 -1
- package/dist/artifact-validator.d.ts +0 -74
- package/dist/artifact-validator.d.ts.map +0 -1
- package/dist/attestation.d.ts +0 -63
- package/dist/attestation.d.ts.map +0 -1
- package/dist/attestation.test.d.ts +0 -2
- package/dist/attestation.test.d.ts.map +0 -1
- package/dist/authenticity/index.d.ts.map +0 -1
- package/dist/authenticity/index.test.d.ts +0 -2
- package/dist/authenticity/index.test.d.ts.map +0 -1
- package/dist/auto-pr.d.ts +0 -120
- package/dist/auto-pr.d.ts.map +0 -1
- package/dist/baseline.d.ts.map +0 -1
- package/dist/behavior-dsl.d.ts +0 -73
- package/dist/behavior-dsl.d.ts.map +0 -1
- package/dist/belief-state/calibration.d.ts +0 -10
- package/dist/belief-state/calibration.d.ts.map +0 -1
- package/dist/belief-state/calibration.test.d.ts +0 -2
- package/dist/belief-state/calibration.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.d.ts +0 -66
- package/dist/belief-state/code-agent-corpus.d.ts.map +0 -1
- package/dist/belief-state/code-agent-corpus.test.d.ts +0 -2
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.d.ts +0 -22
- package/dist/belief-state/code-agent-evidence.d.ts.map +0 -1
- package/dist/belief-state/code-agent-evidence.test.d.ts +0 -2
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/extract.d.ts +0 -7
- package/dist/belief-state/extract.d.ts.map +0 -1
- package/dist/belief-state/extract.test.d.ts +0 -2
- package/dist/belief-state/extract.test.d.ts.map +0 -1
- package/dist/belief-state/index.d.ts.map +0 -1
- package/dist/belief-state/ope.d.ts +0 -17
- package/dist/belief-state/ope.d.ts.map +0 -1
- package/dist/belief-state/ope.test.d.ts +0 -2
- package/dist/belief-state/ope.test.d.ts.map +0 -1
- package/dist/belief-state/phase0-measurement.d.ts +0 -55
- package/dist/belief-state/phase0-measurement.d.ts.map +0 -1
- package/dist/belief-state/report.d.ts +0 -17
- package/dist/belief-state/report.d.ts.map +0 -1
- package/dist/belief-state/report.test.d.ts +0 -2
- package/dist/belief-state/report.test.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.d.ts +0 -23
- package/dist/belief-state/research-evidence.d.ts.map +0 -1
- package/dist/belief-state/research-evidence.test.d.ts +0 -2
- package/dist/belief-state/research-evidence.test.d.ts.map +0 -1
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +0 -32
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.d.ts +0 -87
- package/dist/belief-state/runtime-hooks.d.ts.map +0 -1
- package/dist/belief-state/runtime-hooks.test.d.ts +0 -2
- package/dist/belief-state/runtime-hooks.test.d.ts.map +0 -1
- package/dist/belief-state/selective.d.ts +0 -15
- package/dist/belief-state/selective.d.ts.map +0 -1
- package/dist/belief-state/selective.test.d.ts +0 -2
- package/dist/belief-state/selective.test.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.d.ts +0 -80
- package/dist/belief-state/shadow-probe.d.ts.map +0 -1
- package/dist/belief-state/shadow-probe.test.d.ts +0 -2
- package/dist/belief-state/shadow-probe.test.d.ts.map +0 -1
- package/dist/belief-state/types.d.ts +0 -195
- package/dist/belief-state/types.d.ts.map +0 -1
- package/dist/belief-state/types.test.d.ts +0 -2
- package/dist/belief-state/types.test.d.ts.map +0 -1
- package/dist/benchmark.d.ts +0 -14
- package/dist/benchmark.d.ts.map +0 -1
- package/dist/benchmarks/index.d.ts.map +0 -1
- package/dist/benchmarks/routing/dataset.d.ts +0 -34
- package/dist/benchmarks/routing/dataset.d.ts.map +0 -1
- package/dist/benchmarks/routing/index.d.ts +0 -34
- package/dist/benchmarks/routing/index.d.ts.map +0 -1
- package/dist/benchmarks/types.d.ts +0 -49
- package/dist/benchmarks/types.d.ts.map +0 -1
- package/dist/bisector.d.ts +0 -81
- package/dist/bisector.d.ts.map +0 -1
- package/dist/budget-guard.d.ts +0 -31
- package/dist/budget-guard.d.ts.map +0 -1
- package/dist/builder-eval/builder-session.d.ts +0 -111
- package/dist/builder-eval/builder-session.d.ts.map +0 -1
- package/dist/builder-eval/correlation.d.ts +0 -32
- package/dist/builder-eval/correlation.d.ts.map +0 -1
- package/dist/builder-eval/index.d.ts.map +0 -1
- package/dist/builder-eval/project-registry.d.ts +0 -51
- package/dist/builder-eval/project-registry.d.ts.map +0 -1
- package/dist/builder-eval/three-layer-eval.d.ts +0 -55
- package/dist/builder-eval/three-layer-eval.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.d.ts +0 -108
- package/dist/campaign/analyst-surface.d.ts.map +0 -1
- package/dist/campaign/analyst-surface.test.d.ts +0 -2
- package/dist/campaign/analyst-surface.test.d.ts.map +0 -1
- package/dist/campaign/auto-pr.d.ts +0 -46
- package/dist/campaign/auto-pr.d.ts.map +0 -1
- package/dist/campaign/distillation/agreement-judge.d.ts +0 -69
- package/dist/campaign/distillation/agreement-judge.d.ts.map +0 -1
- package/dist/campaign/distillation/cli.d.ts +0 -35
- package/dist/campaign/distillation/cli.d.ts.map +0 -1
- package/dist/campaign/distillation/distillation.test.d.ts +0 -2
- package/dist/campaign/distillation/distillation.test.d.ts.map +0 -1
- package/dist/campaign/distillation/gold-scenarios.d.ts +0 -54
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +0 -1
- package/dist/campaign/distillation/run-distillation.d.ts +0 -119
- package/dist/campaign/distillation/run-distillation.d.ts.map +0 -1
- package/dist/campaign/gates/compose.d.ts +0 -12
- package/dist/campaign/gates/compose.d.ts.map +0 -1
- package/dist/campaign/gates/default-production-gate.d.ts +0 -58
- package/dist/campaign/gates/default-production-gate.d.ts.map +0 -1
- package/dist/campaign/gates/heldout-gate.d.ts +0 -12
- package/dist/campaign/gates/heldout-gate.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.d.ts +0 -125
- package/dist/campaign/gates/promotion-policy.d.ts.map +0 -1
- package/dist/campaign/gates/promotion-policy.test.d.ts +0 -2
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.d.ts +0 -146
- package/dist/campaign/gates/sequential.d.ts.map +0 -1
- package/dist/campaign/gates/sequential.test.d.ts +0 -2
- package/dist/campaign/gates/sequential.test.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.d.ts +0 -99
- package/dist/campaign/gates/statistical-heldout.d.ts.map +0 -1
- package/dist/campaign/gates/statistical-heldout.test.d.ts +0 -2
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +0 -1
- package/dist/campaign/index.d.ts.map +0 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +0 -59
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +0 -1
- package/dist/campaign/presets/compare-proposers.d.ts +0 -146
- package/dist/campaign/presets/compare-proposers.d.ts.map +0 -1
- package/dist/campaign/presets/playback.d.ts +0 -120
- package/dist/campaign/presets/playback.d.ts.map +0 -1
- package/dist/campaign/presets/playback.test.d.ts +0 -2
- package/dist/campaign/presets/playback.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-eval.d.ts +0 -14
- package/dist/campaign/presets/run-eval.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.d.ts +0 -63
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +0 -1
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +0 -2
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +0 -1
- package/dist/campaign/presets/run-optimization.d.ts +0 -92
- package/dist/campaign/presets/run-optimization.d.ts.map +0 -1
- package/dist/campaign/presets/run-profile-matrix.d.ts +0 -151
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +0 -1
- package/dist/campaign/presets/run-skill-opt.d.ts +0 -96
- package/dist/campaign/presets/run-skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/_findings-text.d.ts +0 -22
- package/dist/campaign/proposers/_findings-text.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.d.ts +0 -33
- package/dist/campaign/proposers/ace.d.ts.map +0 -1
- package/dist/campaign/proposers/ace.test.d.ts +0 -2
- package/dist/campaign/proposers/ace.test.d.ts.map +0 -1
- package/dist/campaign/proposers/analysis-edit.d.ts +0 -32
- package/dist/campaign/proposers/analysis-edit.d.ts.map +0 -1
- package/dist/campaign/proposers/evolutionary.d.ts +0 -20
- package/dist/campaign/proposers/evolutionary.d.ts.map +0 -1
- package/dist/campaign/proposers/fapo.d.ts +0 -120
- package/dist/campaign/proposers/fapo.d.ts.map +0 -1
- package/dist/campaign/proposers/gepa.d.ts +0 -86
- package/dist/campaign/proposers/gepa.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.d.ts +0 -44
- package/dist/campaign/proposers/halo.d.ts.map +0 -1
- package/dist/campaign/proposers/halo.test.d.ts +0 -2
- package/dist/campaign/proposers/halo.test.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.d.ts +0 -47
- package/dist/campaign/proposers/memory.d.ts.map +0 -1
- package/dist/campaign/proposers/memory.test.d.ts +0 -2
- package/dist/campaign/proposers/memory.test.d.ts.map +0 -1
- package/dist/campaign/proposers/skill-opt.d.ts +0 -88
- package/dist/campaign/proposers/skill-opt.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.d.ts +0 -48
- package/dist/campaign/proposers/trace-analyst.d.ts.map +0 -1
- package/dist/campaign/proposers/trace-analyst.test.d.ts +0 -2
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +0 -1
- package/dist/campaign/provenance.d.ts +0 -185
- package/dist/campaign/provenance.d.ts.map +0 -1
- package/dist/campaign/run-campaign.d.ts +0 -90
- package/dist/campaign/run-campaign.d.ts.map +0 -1
- package/dist/campaign/score-utils.d.ts +0 -26
- package/dist/campaign/score-utils.d.ts.map +0 -1
- package/dist/campaign/skill-patch.d.ts +0 -62
- package/dist/campaign/skill-patch.d.ts.map +0 -1
- package/dist/campaign/storage.d.ts +0 -38
- package/dist/campaign/storage.d.ts.map +0 -1
- package/dist/campaign/types.d.ts.map +0 -1
- package/dist/campaign/worktree/index.d.ts +0 -53
- package/dist/campaign/worktree/index.d.ts.map +0 -1
- package/dist/canary.d.ts +0 -101
- package/dist/canary.d.ts.map +0 -1
- package/dist/causal-attribution.d.ts +0 -45
- package/dist/causal-attribution.d.ts.map +0 -1
- package/dist/ci-gate.d.ts +0 -44
- package/dist/ci-gate.d.ts.map +0 -1
- package/dist/cli.d.ts.map +0 -1
- package/dist/client.d.ts +0 -77
- package/dist/client.d.ts.map +0 -1
- package/dist/client.test.d.ts +0 -2
- package/dist/client.test.d.ts.map +0 -1
- package/dist/command-runner.d.ts +0 -74
- package/dist/command-runner.d.ts.map +0 -1
- package/dist/command-runner.test.d.ts +0 -2
- package/dist/command-runner.test.d.ts.map +0 -1
- package/dist/completion-verifier.d.ts +0 -147
- package/dist/completion-verifier.d.ts.map +0 -1
- package/dist/completion-verifier.test.d.ts +0 -9
- package/dist/completion-verifier.test.d.ts.map +0 -1
- package/dist/concurrency.d.ts +0 -23
- package/dist/concurrency.d.ts.map +0 -1
- package/dist/contamination-guard.d.ts +0 -81
- package/dist/contamination-guard.d.ts.map +0 -1
- package/dist/contract/analyze-runs.d.ts.map +0 -1
- package/dist/contract/define-agent-eval.d.ts +0 -52
- package/dist/contract/define-agent-eval.d.ts.map +0 -1
- package/dist/contract/diff.d.ts +0 -114
- package/dist/contract/diff.d.ts.map +0 -1
- package/dist/contract/index.d.ts.map +0 -1
- package/dist/contract/insight-report.d.ts.map +0 -1
- package/dist/contract/insight-types-fwd.d.ts +0 -7
- package/dist/contract/insight-types-fwd.d.ts.map +0 -1
- package/dist/contract/intake/agent-trace.d.ts +0 -97
- package/dist/contract/intake/agent-trace.d.ts.map +0 -1
- package/dist/contract/intake/code-agent-session.d.ts.map +0 -1
- package/dist/contract/intake/feedback-table.d.ts +0 -87
- package/dist/contract/intake/feedback-table.d.ts.map +0 -1
- package/dist/contract/intake/index.d.ts +0 -22
- package/dist/contract/intake/index.d.ts.map +0 -1
- package/dist/contract/intake/otel-spans.d.ts +0 -33
- package/dist/contract/intake/otel-spans.d.ts.map +0 -1
- package/dist/contract/self-improve.d.ts +0 -284
- package/dist/contract/self-improve.d.ts.map +0 -1
- package/dist/control-runtime.d.ts.map +0 -1
- package/dist/control-runtime.test.d.ts +0 -2
- package/dist/control-runtime.test.d.ts.map +0 -1
- package/dist/control.d.ts.map +0 -1
- package/dist/convergence.d.ts +0 -29
- package/dist/convergence.d.ts.map +0 -1
- package/dist/cost-ledger.d.ts.map +0 -1
- package/dist/cost-ledger.test.d.ts +0 -2
- package/dist/cost-ledger.test.d.ts.map +0 -1
- package/dist/cost-report.d.ts +0 -43
- package/dist/cost-report.d.ts.map +0 -1
- package/dist/cost-report.test.d.ts +0 -2
- package/dist/cost-report.test.d.ts.map +0 -1
- package/dist/cost-tracker.d.ts +0 -76
- package/dist/cost-tracker.d.ts.map +0 -1
- package/dist/counterfactual.d.ts.map +0 -1
- package/dist/cross-trace-diff.d.ts +0 -56
- package/dist/cross-trace-diff.d.ts.map +0 -1
- package/dist/dataset.d.ts.map +0 -1
- package/dist/deploy-gate-layer.d.ts +0 -125
- package/dist/deploy-gate-layer.d.ts.map +0 -1
- package/dist/deploy-gate-layer.test.d.ts +0 -2
- package/dist/deploy-gate-layer.test.d.ts.map +0 -1
- package/dist/description-length-gate.d.ts +0 -119
- package/dist/description-length-gate.d.ts.map +0 -1
- package/dist/detectors/edge.test.d.ts +0 -2
- package/dist/detectors/edge.test.d.ts.map +0 -1
- package/dist/detectors/index.d.ts +0 -81
- package/dist/detectors/index.d.ts.map +0 -1
- package/dist/detectors/index.test.d.ts +0 -2
- package/dist/detectors/index.test.d.ts.map +0 -1
- package/dist/diagnose/causal-sweep.d.ts +0 -100
- package/dist/diagnose/causal-sweep.d.ts.map +0 -1
- package/dist/diagnose/index.d.ts +0 -36
- package/dist/diagnose/index.d.ts.map +0 -1
- package/dist/diagnose/remediation.d.ts +0 -68
- package/dist/diagnose/remediation.d.ts.map +0 -1
- package/dist/diagnose/repair.d.ts +0 -77
- package/dist/diagnose/repair.d.ts.map +0 -1
- package/dist/discover-personas.d.ts +0 -35
- package/dist/discover-personas.d.ts.map +0 -1
- package/dist/driver.d.ts +0 -95
- package/dist/driver.d.ts.map +0 -1
- package/dist/driver.test.d.ts +0 -8
- package/dist/driver.test.d.ts.map +0 -1
- package/dist/dual-agent-bench.d.ts +0 -81
- package/dist/dual-agent-bench.d.ts.map +0 -1
- package/dist/error-count-extractor.d.ts +0 -47
- package/dist/error-count-extractor.d.ts.map +0 -1
- package/dist/error-count-extractor.test.d.ts +0 -2
- package/dist/error-count-extractor.test.d.ts.map +0 -1
- package/dist/errors.d.ts.map +0 -1
- package/dist/eval-campaign.d.ts.map +0 -1
- package/dist/eval-campaign.test.d.ts +0 -2
- package/dist/eval-campaign.test.d.ts.map +0 -1
- package/dist/eval-tools.d.ts +0 -55
- package/dist/eval-tools.d.ts.map +0 -1
- package/dist/eval-trace-store.d.ts +0 -107
- package/dist/eval-trace-store.d.ts.map +0 -1
- package/dist/eval-trace-store.test.d.ts +0 -2
- package/dist/eval-trace-store.test.d.ts.map +0 -1
- package/dist/executor.d.ts +0 -38
- package/dist/executor.d.ts.map +0 -1
- package/dist/executor.test.d.ts +0 -10
- package/dist/executor.test.d.ts.map +0 -1
- package/dist/experiment-tracker.d.ts +0 -178
- package/dist/experiment-tracker.d.ts.map +0 -1
- package/dist/experiment-tracker.test.d.ts +0 -2
- package/dist/experiment-tracker.test.d.ts.map +0 -1
- package/dist/failure-taxonomy.d.ts +0 -38
- package/dist/failure-taxonomy.d.ts.map +0 -1
- package/dist/feedback-trajectory.d.ts.map +0 -1
- package/dist/feedback-trajectory.test.d.ts +0 -2
- package/dist/feedback-trajectory.test.d.ts.map +0 -1
- package/dist/flow-layer.d.ts +0 -90
- package/dist/flow-layer.d.ts.map +0 -1
- package/dist/flow-layer.test.d.ts +0 -2
- package/dist/flow-layer.test.d.ts.map +0 -1
- package/dist/fuzz/capsule.d.ts +0 -46
- package/dist/fuzz/capsule.d.ts.map +0 -1
- package/dist/fuzz/cube.d.ts +0 -36
- package/dist/fuzz/cube.d.ts.map +0 -1
- package/dist/fuzz/explorer-cost.test.d.ts +0 -2
- package/dist/fuzz/explorer-cost.test.d.ts.map +0 -1
- package/dist/fuzz/explorer.d.ts +0 -64
- package/dist/fuzz/explorer.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.d.ts +0 -16
- package/dist/fuzz/fuzz-agent.d.ts.map +0 -1
- package/dist/fuzz/fuzz-agent.test.d.ts +0 -2
- package/dist/fuzz/fuzz-agent.test.d.ts.map +0 -1
- package/dist/fuzz/gates.d.ts +0 -33
- package/dist/fuzz/gates.d.ts.map +0 -1
- package/dist/fuzz/index.d.ts +0 -26
- package/dist/fuzz/index.d.ts.map +0 -1
- package/dist/fuzz/policies.d.ts +0 -28
- package/dist/fuzz/policies.d.ts.map +0 -1
- package/dist/fuzz/tools.d.ts +0 -20
- package/dist/fuzz/tools.d.ts.map +0 -1
- package/dist/fuzz/types.d.ts +0 -307
- package/dist/fuzz/types.d.ts.map +0 -1
- package/dist/golden-matcher.d.ts +0 -71
- package/dist/golden-matcher.d.ts.map +0 -1
- package/dist/governance/eu-ai-act.d.ts +0 -37
- package/dist/governance/eu-ai-act.d.ts.map +0 -1
- package/dist/governance/index.d.ts.map +0 -1
- package/dist/governance/nist-ai-rmf.d.ts +0 -15
- package/dist/governance/nist-ai-rmf.d.ts.map +0 -1
- package/dist/governance/soc2.d.ts +0 -12
- package/dist/governance/soc2.d.ts.map +0 -1
- package/dist/governance/types.d.ts +0 -66
- package/dist/governance/types.d.ts.map +0 -1
- package/dist/harness-optimizer.d.ts +0 -82
- package/dist/harness-optimizer.d.ts.map +0 -1
- package/dist/held-out-gate.d.ts +0 -135
- package/dist/held-out-gate.d.ts.map +0 -1
- package/dist/hosted/client.d.ts +0 -73
- package/dist/hosted/client.d.ts.map +0 -1
- package/dist/hosted/from-env.test.d.ts +0 -8
- package/dist/hosted/from-env.test.d.ts.map +0 -1
- package/dist/hosted/index.d.ts.map +0 -1
- package/dist/hosted/types.d.ts +0 -159
- package/dist/hosted/types.d.ts.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/integrity/backend-integrity.d.ts +0 -71
- package/dist/integrity/backend-integrity.d.ts.map +0 -1
- package/dist/integrity/preflight.d.ts +0 -72
- package/dist/integrity/preflight.d.ts.map +0 -1
- package/dist/integrity/preflight.test.d.ts +0 -2
- package/dist/integrity/preflight.test.d.ts.map +0 -1
- package/dist/integrity/single-backend.d.ts +0 -67
- package/dist/integrity/single-backend.d.ts.map +0 -1
- package/dist/intent-match-judge.d.ts +0 -69
- package/dist/intent-match-judge.d.ts.map +0 -1
- package/dist/intent-match-judge.test.d.ts +0 -2
- package/dist/intent-match-judge.test.d.ts.map +0 -1
- package/dist/judge-calibration.d.ts.map +0 -1
- package/dist/judge-ensemble.d.ts +0 -66
- package/dist/judge-ensemble.d.ts.map +0 -1
- package/dist/judge-ensemble.test.d.ts +0 -8
- package/dist/judge-ensemble.test.d.ts.map +0 -1
- package/dist/judge-families.d.ts +0 -38
- package/dist/judge-families.d.ts.map +0 -1
- package/dist/judge-panel.d.ts +0 -65
- package/dist/judge-panel.d.ts.map +0 -1
- package/dist/judge-retry.d.ts +0 -70
- package/dist/judge-retry.d.ts.map +0 -1
- package/dist/judge-runner.d.ts +0 -36
- package/dist/judge-runner.d.ts.map +0 -1
- package/dist/judge-runner.test.d.ts +0 -2
- package/dist/judge-runner.test.d.ts.map +0 -1
- package/dist/judges.d.ts +0 -74
- package/dist/judges.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.d.ts +0 -89
- package/dist/keyword-coverage-judge.d.ts.map +0 -1
- package/dist/keyword-coverage-judge.test.d.ts +0 -2
- package/dist/keyword-coverage-judge.test.d.ts.map +0 -1
- package/dist/knowledge/index.d.ts.map +0 -1
- package/dist/knowledge/readiness.d.ts +0 -26
- package/dist/knowledge/readiness.d.ts.map +0 -1
- package/dist/knowledge/types.d.ts +0 -75
- package/dist/knowledge/types.d.ts.map +0 -1
- package/dist/live-proof.d.ts +0 -62
- package/dist/live-proof.d.ts.map +0 -1
- package/dist/llm-client.d.ts.map +0 -1
- package/dist/llm-client.test.d.ts +0 -2
- package/dist/llm-client.test.d.ts.map +0 -1
- package/dist/locked-jsonl-appender.d.ts.map +0 -1
- package/dist/matrix/aggregation.d.ts +0 -16
- package/dist/matrix/aggregation.d.ts.map +0 -1
- package/dist/matrix/index.d.ts.map +0 -1
- package/dist/matrix/runner.d.ts +0 -15
- package/dist/matrix/runner.d.ts.map +0 -1
- package/dist/matrix/types.d.ts.map +0 -1
- package/dist/meta-eval/calibration.d.ts +0 -47
- package/dist/meta-eval/calibration.d.ts.map +0 -1
- package/dist/meta-eval/correlation-study.d.ts +0 -53
- package/dist/meta-eval/correlation-study.d.ts.map +0 -1
- package/dist/meta-eval/index.d.ts.map +0 -1
- package/dist/meta-eval/outcome-store.d.ts.map +0 -1
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +0 -1
- package/dist/meta-eval/sentinel.d.ts +0 -169
- package/dist/meta-eval/sentinel.d.ts.map +0 -1
- package/dist/metrics.d.ts +0 -63
- package/dist/metrics.d.ts.map +0 -1
- package/dist/model-seats.d.ts +0 -71
- package/dist/model-seats.d.ts.map +0 -1
- package/dist/model-seats.test.d.ts +0 -2
- package/dist/model-seats.test.d.ts.map +0 -1
- package/dist/muffled-gate-scanner.d.ts +0 -102
- package/dist/muffled-gate-scanner.d.ts.map +0 -1
- package/dist/multi-layer-verifier.d.ts.map +0 -1
- package/dist/multi-layer-verifier.test.d.ts +0 -2
- package/dist/multi-layer-verifier.test.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.d.ts +0 -80
- package/dist/multi-toolchain-layer.d.ts.map +0 -1
- package/dist/multi-toolchain-layer.test.d.ts +0 -2
- package/dist/multi-toolchain-layer.test.d.ts.map +0 -1
- package/dist/multishot/default-tools.d.ts +0 -34
- package/dist/multishot/default-tools.d.ts.map +0 -1
- package/dist/multishot/index.d.ts.map +0 -1
- package/dist/multishot/judges.d.ts +0 -32
- package/dist/multishot/judges.d.ts.map +0 -1
- package/dist/multishot/matrix.d.ts +0 -107
- package/dist/multishot/matrix.d.ts.map +0 -1
- package/dist/multishot/multishot.d.ts +0 -23
- package/dist/multishot/multishot.d.ts.map +0 -1
- package/dist/multishot/router.d.ts +0 -37
- package/dist/multishot/router.d.ts.map +0 -1
- package/dist/multishot/types.d.ts +0 -60
- package/dist/multishot/types.d.ts.map +0 -1
- package/dist/observability.d.ts +0 -71
- package/dist/observability.d.ts.map +0 -1
- package/dist/oracle.d.ts +0 -55
- package/dist/oracle.d.ts.map +0 -1
- package/dist/orthogonality.d.ts +0 -35
- package/dist/orthogonality.d.ts.map +0 -1
- package/dist/otel-pipeline.d.ts +0 -31
- package/dist/otel-pipeline.d.ts.map +0 -1
- package/dist/paraphrase.d.ts +0 -107
- package/dist/paraphrase.d.ts.map +0 -1
- package/dist/pareto.d.ts.map +0 -1
- package/dist/partition-held-out.d.ts +0 -70
- package/dist/partition-held-out.d.ts.map +0 -1
- package/dist/partition-held-out.test.d.ts +0 -2
- package/dist/partition-held-out.test.d.ts.map +0 -1
- package/dist/perf/index.d.ts.map +0 -1
- package/dist/perf/integrity.d.ts +0 -30
- package/dist/perf/integrity.d.ts.map +0 -1
- package/dist/perf/journey.d.ts +0 -45
- package/dist/perf/journey.d.ts.map +0 -1
- package/dist/perf/ratchet.d.ts +0 -47
- package/dist/perf/ratchet.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.d.ts +0 -31
- package/dist/pipelines/budget-breach.d.ts.map +0 -1
- package/dist/pipelines/budget-breach.test.d.ts +0 -2
- package/dist/pipelines/budget-breach.test.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.d.ts +0 -38
- package/dist/pipelines/failure-cluster.d.ts.map +0 -1
- package/dist/pipelines/failure-cluster.test.d.ts +0 -2
- package/dist/pipelines/failure-cluster.test.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.d.ts +0 -26
- package/dist/pipelines/first-divergence.d.ts.map +0 -1
- package/dist/pipelines/first-divergence.test.d.ts +0 -2
- package/dist/pipelines/first-divergence.test.d.ts.map +0 -1
- package/dist/pipelines/index.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.d.ts +0 -26
- package/dist/pipelines/judge-agreement.d.ts.map +0 -1
- package/dist/pipelines/judge-agreement.test.d.ts +0 -2
- package/dist/pipelines/judge-agreement.test.d.ts.map +0 -1
- package/dist/pipelines/regression.d.ts +0 -23
- package/dist/pipelines/regression.d.ts.map +0 -1
- package/dist/pipelines/regression.test.d.ts +0 -2
- package/dist/pipelines/regression.test.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.d.ts +0 -32
- package/dist/pipelines/stuck-loop.d.ts.map +0 -1
- package/dist/pipelines/stuck-loop.test.d.ts +0 -2
- package/dist/pipelines/stuck-loop.test.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.d.ts +0 -34
- package/dist/pipelines/tool-waste.d.ts.map +0 -1
- package/dist/pipelines/tool-waste.test.d.ts +0 -2
- package/dist/pipelines/tool-waste.test.d.ts.map +0 -1
- package/dist/playbook.d.ts +0 -16
- package/dist/playbook.d.ts.map +0 -1
- package/dist/pr-review-benchmark.d.ts +0 -88
- package/dist/pr-review-benchmark.d.ts.map +0 -1
- package/dist/pr-review-benchmark.test.d.ts +0 -2
- package/dist/pr-review-benchmark.test.d.ts.map +0 -1
- package/dist/pre-registration.d.ts +0 -125
- package/dist/pre-registration.d.ts.map +0 -1
- package/dist/prm/builtin-rubrics.d.ts +0 -33
- package/dist/prm/builtin-rubrics.d.ts.map +0 -1
- package/dist/prm/index.d.ts.map +0 -1
- package/dist/prm/inference.d.ts +0 -29
- package/dist/prm/inference.d.ts.map +0 -1
- package/dist/prm/inference.test.d.ts +0 -2
- package/dist/prm/inference.test.d.ts.map +0 -1
- package/dist/prm/rubric.d.ts.map +0 -1
- package/dist/prm/training-export.d.ts +0 -38
- package/dist/prm/training-export.d.ts.map +0 -1
- package/dist/produced-state.d.ts +0 -63
- package/dist/produced-state.d.ts.map +0 -1
- package/dist/produced-state.test.d.ts +0 -8
- package/dist/produced-state.test.d.ts.map +0 -1
- package/dist/profile/baselines.d.ts +0 -37
- package/dist/profile/baselines.d.ts.map +0 -1
- package/dist/profile/index.d.ts +0 -105
- package/dist/profile/index.d.ts.map +0 -1
- package/dist/promotion-gate.d.ts +0 -93
- package/dist/promotion-gate.d.ts.map +0 -1
- package/dist/prompt-registry.d.ts +0 -41
- package/dist/prompt-registry.d.ts.map +0 -1
- package/dist/propose-review-control.d.ts +0 -49
- package/dist/propose-review-control.d.ts.map +0 -1
- package/dist/propose-review-control.test.d.ts +0 -2
- package/dist/propose-review-control.test.d.ts.map +0 -1
- package/dist/propose-review.d.ts +0 -155
- package/dist/propose-review.d.ts.map +0 -1
- package/dist/red-team.d.ts.map +0 -1
- package/dist/reference-replay-steering.d.ts +0 -11
- package/dist/reference-replay-steering.d.ts.map +0 -1
- package/dist/reference-replay.d.ts +0 -176
- package/dist/reference-replay.d.ts.map +0 -1
- package/dist/reflective-mutation.d.ts +0 -79
- package/dist/reflective-mutation.d.ts.map +0 -1
- package/dist/registry.d.ts +0 -31
- package/dist/registry.d.ts.map +0 -1
- package/dist/release-confidence.d.ts +0 -128
- package/dist/release-confidence.d.ts.map +0 -1
- package/dist/release-report.d.ts +0 -11
- package/dist/release-report.d.ts.map +0 -1
- package/dist/replay.d.ts +0 -120
- package/dist/replay.d.ts.map +0 -1
- package/dist/reporter.d.ts +0 -14
- package/dist/reporter.d.ts.map +0 -1
- package/dist/reporting.d.ts.map +0 -1
- package/dist/researcher.d.ts +0 -140
- package/dist/researcher.d.ts.map +0 -1
- package/dist/reviewer.d.ts +0 -118
- package/dist/reviewer.d.ts.map +0 -1
- package/dist/reviewer.test.d.ts +0 -2
- package/dist/reviewer.test.d.ts.map +0 -1
- package/dist/reward-model-export.d.ts +0 -60
- package/dist/reward-model-export.d.ts.map +0 -1
- package/dist/rl/active-curriculum.d.ts +0 -110
- package/dist/rl/active-curriculum.d.ts.map +0 -1
- package/dist/rl/adaptation-eval.d.ts +0 -109
- package/dist/rl/adaptation-eval.d.ts.map +0 -1
- package/dist/rl/adversarial.d.ts.map +0 -1
- package/dist/rl/compute-curves.d.ts +0 -127
- package/dist/rl/compute-curves.d.ts.map +0 -1
- package/dist/rl/contamination.d.ts +0 -117
- package/dist/rl/contamination.d.ts.map +0 -1
- package/dist/rl/corpus.d.ts +0 -55
- package/dist/rl/corpus.d.ts.map +0 -1
- package/dist/rl/corpus.test.d.ts +0 -2
- package/dist/rl/corpus.test.d.ts.map +0 -1
- package/dist/rl/dataset.d.ts +0 -102
- package/dist/rl/dataset.d.ts.map +0 -1
- package/dist/rl/dataset.test.d.ts +0 -2
- package/dist/rl/dataset.test.d.ts.map +0 -1
- package/dist/rl/exporters.d.ts +0 -141
- package/dist/rl/exporters.d.ts.map +0 -1
- package/dist/rl/index.d.ts +0 -49
- package/dist/rl/index.d.ts.map +0 -1
- package/dist/rl/off-policy.d.ts.map +0 -1
- package/dist/rl/predictive-validity-researcher.d.ts +0 -69
- package/dist/rl/predictive-validity-researcher.d.ts.map +0 -1
- package/dist/rl/preferences.d.ts +0 -141
- package/dist/rl/preferences.d.ts.map +0 -1
- package/dist/rl/process-reward.d.ts +0 -122
- package/dist/rl/process-reward.d.ts.map +0 -1
- package/dist/rl/reward-hacking.d.ts +0 -104
- package/dist/rl/reward-hacking.d.ts.map +0 -1
- package/dist/rl/rl-campaign.d.ts +0 -85
- package/dist/rl/rl-campaign.d.ts.map +0 -1
- package/dist/rl/run-record-adapters.d.ts +0 -56
- package/dist/rl/run-record-adapters.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.d.ts +0 -166
- package/dist/rl/sim-fidelity.d.ts.map +0 -1
- package/dist/rl/sim-fidelity.test.d.ts +0 -2
- package/dist/rl/sim-fidelity.test.d.ts.map +0 -1
- package/dist/rl/tournament.d.ts +0 -115
- package/dist/rl/tournament.d.ts.map +0 -1
- package/dist/rl/verifiable-reward.d.ts +0 -124
- package/dist/rl/verifiable-reward.d.ts.map +0 -1
- package/dist/run-critic.d.ts +0 -23
- package/dist/run-critic.d.ts.map +0 -1
- package/dist/run-evidence.d.ts +0 -32
- package/dist/run-evidence.d.ts.map +0 -1
- package/dist/run-record.d.ts.map +0 -1
- package/dist/run-record.test.d.ts +0 -2
- package/dist/run-record.test.d.ts.map +0 -1
- package/dist/run-score.d.ts +0 -31
- package/dist/run-score.d.ts.map +0 -1
- package/dist/runtime-trajectory.d.ts +0 -47
- package/dist/runtime-trajectory.d.ts.map +0 -1
- package/dist/sandbox-harness.d.ts.map +0 -1
- package/dist/sandbox-harness.test.d.ts +0 -2
- package/dist/sandbox-harness.test.d.ts.map +0 -1
- package/dist/sandbox-pool.d.ts +0 -74
- package/dist/sandbox-pool.d.ts.map +0 -1
- package/dist/sandbox-pool.test.d.ts +0 -2
- package/dist/sandbox-pool.test.d.ts.map +0 -1
- package/dist/scorecard.d.ts +0 -133
- package/dist/scorecard.d.ts.map +0 -1
- package/dist/scorecard.test.d.ts +0 -2
- package/dist/scorecard.test.d.ts.map +0 -1
- package/dist/self-play.d.ts +0 -69
- package/dist/self-play.d.ts.map +0 -1
- package/dist/semantic-concept-judge.d.ts +0 -135
- package/dist/semantic-concept-judge.d.ts.map +0 -1
- package/dist/semantic-concept-judge.test.d.ts +0 -2
- package/dist/semantic-concept-judge.test.d.ts.map +0 -1
- package/dist/sequential.d.ts.map +0 -1
- package/dist/series-convergence.d.ts.map +0 -1
- package/dist/slo.d.ts +0 -48
- package/dist/slo.d.ts.map +0 -1
- package/dist/state-continuity.d.ts +0 -47
- package/dist/state-continuity.d.ts.map +0 -1
- package/dist/statistics.d.ts.map +0 -1
- package/dist/statistics.test.d.ts +0 -2
- package/dist/statistics.test.d.ts.map +0 -1
- package/dist/steering-optimizer.d.ts +0 -58
- package/dist/steering-optimizer.d.ts.map +0 -1
- package/dist/steering.d.ts +0 -24
- package/dist/steering.d.ts.map +0 -1
- package/dist/storyboard/code-edit.d.ts +0 -64
- package/dist/storyboard/code-edit.d.ts.map +0 -1
- package/dist/storyboard/code-edit.test.d.ts +0 -2
- package/dist/storyboard/code-edit.test.d.ts.map +0 -1
- package/dist/storyboard/index.d.ts.map +0 -1
- package/dist/storyboard/index.test.d.ts +0 -2
- package/dist/storyboard/index.test.d.ts.map +0 -1
- package/dist/summary-report.d.ts.map +0 -1
- package/dist/telemetry/client.d.ts +0 -35
- package/dist/telemetry/client.d.ts.map +0 -1
- package/dist/telemetry/index.d.ts.map +0 -1
- package/dist/telemetry/schema.d.ts +0 -61
- package/dist/telemetry/schema.d.ts.map +0 -1
- package/dist/telemetry/sink-fetch.d.ts +0 -39
- package/dist/telemetry/sink-fetch.d.ts.map +0 -1
- package/dist/telemetry/sink-file.d.ts.map +0 -1
- package/dist/test-graded-scenario.d.ts +0 -42
- package/dist/test-graded-scenario.d.ts.map +0 -1
- package/dist/testing.d.ts.map +0 -1
- package/dist/tool-use-metrics.d.ts +0 -35
- package/dist/tool-use-metrics.d.ts.map +0 -1
- package/dist/trace/capture-fetch.d.ts +0 -48
- package/dist/trace/capture-fetch.d.ts.map +0 -1
- package/dist/trace/capture-fetch.test.d.ts +0 -2
- package/dist/trace/capture-fetch.test.d.ts.map +0 -1
- package/dist/trace/emitter.d.ts.map +0 -1
- package/dist/trace/extract-usage.d.ts +0 -46
- package/dist/trace/extract-usage.d.ts.map +0 -1
- package/dist/trace/extract-usage.test.d.ts +0 -2
- package/dist/trace/extract-usage.test.d.ts.map +0 -1
- package/dist/trace/index.d.ts +0 -15
- package/dist/trace/index.d.ts.map +0 -1
- package/dist/trace/integrity.d.ts.map +0 -1
- package/dist/trace/otel-bridge.d.ts +0 -29
- package/dist/trace/otel-bridge.d.ts.map +0 -1
- package/dist/trace/otel-export.d.ts +0 -52
- package/dist/trace/otel-export.d.ts.map +0 -1
- package/dist/trace/otel.d.ts +0 -57
- package/dist/trace/otel.d.ts.map +0 -1
- package/dist/trace/otlp-attributes.d.ts +0 -17
- package/dist/trace/otlp-attributes.d.ts.map +0 -1
- package/dist/trace/query.d.ts +0 -29
- package/dist/trace/query.d.ts.map +0 -1
- package/dist/trace/query.test.d.ts +0 -2
- package/dist/trace/query.test.d.ts.map +0 -1
- package/dist/trace/raw-provider-sink.d.ts.map +0 -1
- package/dist/trace/redact.d.ts.map +0 -1
- package/dist/trace/schema.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.d.ts +0 -72
- package/dist/trace/store-to-otlp.d.ts.map +0 -1
- package/dist/trace/store-to-otlp.test.d.ts +0 -2
- package/dist/trace/store-to-otlp.test.d.ts.map +0 -1
- package/dist/trace/store.d.ts.map +0 -1
- package/dist/trace/store.test.d.ts +0 -2
- package/dist/trace/store.test.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.d.ts.map +0 -1
- package/dist/trace-analyst/analyst.test.d.ts +0 -2
- package/dist/trace-analyst/analyst.test.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.d.ts +0 -40
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +0 -1
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +0 -2
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +0 -1
- package/dist/trace-analyst/hook.d.ts +0 -55
- package/dist/trace-analyst/hook.d.ts.map +0 -1
- package/dist/trace-analyst/index.d.ts +0 -18
- package/dist/trace-analyst/index.d.ts.map +0 -1
- package/dist/trace-analyst/insights.d.ts +0 -71
- package/dist/trace-analyst/insights.d.ts.map +0 -1
- package/dist/trace-analyst/insights.test.d.ts +0 -2
- package/dist/trace-analyst/insights.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-flatten.d.ts +0 -42
- package/dist/trace-analyst/otlp-flatten.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.d.ts +0 -85
- package/dist/trace-analyst/otlp-span.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-span.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-span.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.d.ts +0 -115
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +0 -1
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/prompts.d.ts +0 -6
- package/dist/trace-analyst/prompts.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.d.ts +0 -126
- package/dist/trace-analyst/store-otlp.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.test.d.ts +0 -8
- package/dist/trace-analyst/store-otlp.test.d.ts.map +0 -1
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +0 -2
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +0 -1
- package/dist/trace-analyst/store.d.ts +0 -63
- package/dist/trace-analyst/store.d.ts.map +0 -1
- package/dist/trace-analyst/tools.d.ts +0 -44
- package/dist/trace-analyst/tools.d.ts.map +0 -1
- package/dist/trace-analyst/tools.test.d.ts +0 -10
- package/dist/trace-analyst/tools.test.d.ts.map +0 -1
- package/dist/trace-analyst/types.d.ts.map +0 -1
- package/dist/trace-contracts.d.ts +0 -180
- package/dist/trace-contracts.d.ts.map +0 -1
- package/dist/traced-analyst.d.ts +0 -26
- package/dist/traced-analyst.d.ts.map +0 -1
- package/dist/traced-judges.d.ts +0 -27
- package/dist/traced-judges.d.ts.map +0 -1
- package/dist/traces.d.ts.map +0 -1
- package/dist/trajectory.d.ts.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/ui-finding.d.ts +0 -104
- package/dist/ui-finding.d.ts.map +0 -1
- package/dist/verdict-cache.d.ts +0 -78
- package/dist/verdict-cache.d.ts.map +0 -1
- package/dist/verdict-cache.test.d.ts +0 -2
- package/dist/verdict-cache.test.d.ts.map +0 -1
- package/dist/verdict.d.ts.map +0 -1
- package/dist/visual-diff.d.ts +0 -32
- package/dist/visual-diff.d.ts.map +0 -1
- package/dist/wire/handlers.d.ts +0 -54
- package/dist/wire/handlers.d.ts.map +0 -1
- package/dist/wire/index.d.ts.map +0 -1
- package/dist/wire/openapi.d.ts +0 -3
- package/dist/wire/openapi.d.ts.map +0 -1
- package/dist/wire/rpc.d.ts +0 -21
- package/dist/wire/rpc.d.ts.map +0 -1
- package/dist/wire/rubrics.d.ts +0 -34
- package/dist/wire/rubrics.d.ts.map +0 -1
- package/dist/wire/schemas.d.ts +0 -410
- package/dist/wire/schemas.d.ts.map +0 -1
- package/dist/wire/server.d.ts +0 -60
- package/dist/wire/server.d.ts.map +0 -1
- package/dist/workflow/event-schema.d.ts +0 -5
- package/dist/workflow/event-schema.d.ts.map +0 -1
- package/dist/workflow/feedback-pack.d.ts +0 -99
- package/dist/workflow/feedback-pack.d.ts.map +0 -1
- package/dist/workflow/index.d.ts.map +0 -1
- package/dist/workflow/intelligence-export.d.ts +0 -62
- package/dist/workflow/intelligence-export.d.ts.map +0 -1
- package/dist/workflow/partner-report.d.ts +0 -49
- package/dist/workflow/partner-report.d.ts.map +0 -1
- package/dist/workflow/phase-graph.d.ts +0 -43
- package/dist/workflow/phase-graph.d.ts.map +0 -1
- package/dist/workflow/promotion-gate.d.ts +0 -61
- package/dist/workflow/promotion-gate.d.ts.map +0 -1
- package/dist/workflow/run-record.d.ts +0 -12
- package/dist/workflow/run-record.d.ts.map +0 -1
- package/dist/workflow/runtime-adapter.d.ts +0 -20
- package/dist/workflow/runtime-adapter.d.ts.map +0 -1
- package/dist/workflow/sanitize.d.ts +0 -21
- package/dist/workflow/sanitize.d.ts.map +0 -1
- package/dist/workflow/schema.d.ts +0 -5
- package/dist/workflow/schema.d.ts.map +0 -1
- package/dist/workflow/summary.d.ts +0 -43
- package/dist/workflow/summary.d.ts.map +0 -1
- package/dist/workflow/trace-event-fields.d.ts +0 -6
- package/dist/workflow/trace-event-fields.d.ts.map +0 -1
- package/dist/workflow/trajectory.d.ts +0 -15
- package/dist/workflow/trajectory.d.ts.map +0 -1
- package/dist/workflow/types.d.ts +0 -68
- package/dist/workflow/types.d.ts.map +0 -1
- package/dist/workspace-inspector.d.ts +0 -67
- package/dist/workspace-inspector.d.ts.map +0 -1
- package/dist/wrangler-deploy-runner.test.d.ts +0 -2
- package/dist/wrangler-deploy-runner.test.d.ts.map +0 -1
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Verifiable reward channel.
|
|
3
|
-
*
|
|
4
|
-
* For RL on coding / math / theorem-proving / structured-output tasks, the
|
|
5
|
-
* reward signal is *decidable* — a test passes or fails, a proof checks or
|
|
6
|
-
* doesn't, an output validates against a schema or doesn't. These rewards
|
|
7
|
-
* are dramatically more useful for RL training than LLM-judge scores
|
|
8
|
-
* because they don't drift, can't be Goodhart-gamed by the policy in the
|
|
9
|
-
* same way, and don't require a separate calibration loop.
|
|
10
|
-
*
|
|
11
|
-
* The `MultiLayerVerifier` already produces this signal — it just doesn't
|
|
12
|
-
* surface it in a shape that's clean enough for RL training. This module
|
|
13
|
-
* wraps the verifier output so consumers can:
|
|
14
|
-
*
|
|
15
|
-
* 1. Extract a clean `VerifiableReward` from a `VerificationReport`
|
|
16
|
-
* 2. Distinguish *deterministic* rewards (compile, test, schema) from
|
|
17
|
-
* *probabilistic* rewards (judge) so they can be weighted differently
|
|
18
|
-
* in the RL training step
|
|
19
|
-
* 3. Filter `RunRecord[]` to only those with a verifiable reward,
|
|
20
|
-
* producing the clean training set that DeepSeek-R1-style GRPO and
|
|
21
|
-
* AlphaProof-style search both depend on
|
|
22
|
-
*
|
|
23
|
-
* Why this matters: every credible 2025-2026 frontier RL result on coding
|
|
24
|
-
* agents leans on verifiable reward (DeepSeek-R1 GRPO on test pass-rate,
|
|
25
|
-
* o-series RL on math/code, AlphaProof on Lean kernel checking). Mixing
|
|
26
|
-
* judge scores into the reward signal poisons the gradient. This module
|
|
27
|
-
* is the seam.
|
|
28
|
-
*/
|
|
29
|
-
import type { VerificationReport } from '../multi-layer-verifier';
|
|
30
|
-
import type { RunRecord } from '../run-record';
|
|
31
|
-
export type VerifiableRewardSource = 'compile' | 'test' | 'schema' | 'sandbox' | 'judge' | 'composite';
|
|
32
|
-
export interface VerifiableReward {
|
|
33
|
-
/** Scalar in [0, 1]. The RL training signal. */
|
|
34
|
-
value: number;
|
|
35
|
-
/** What produced the reward — different sources have different determinism. */
|
|
36
|
-
source: VerifiableRewardSource;
|
|
37
|
-
/**
|
|
38
|
-
* Determinism class. `'deterministic'` rewards are repeatable byte-for-byte
|
|
39
|
-
* given the same inputs (compile, test, schema validation, sandbox exit code).
|
|
40
|
-
* `'probabilistic'` rewards depend on a stochastic component (LLM judge).
|
|
41
|
-
* Mixing these in the same training batch without separation is a known
|
|
42
|
-
* footgun in production RLHF pipelines.
|
|
43
|
-
*/
|
|
44
|
-
determinism: 'deterministic' | 'probabilistic';
|
|
45
|
-
/**
|
|
46
|
-
* Confidence in the reward value. For deterministic sources this is 1.0
|
|
47
|
-
* (the bit either flipped or didn't). For judge sources this is the
|
|
48
|
-
* judge-reported confidence or — when missing — a calibrated prior.
|
|
49
|
-
*/
|
|
50
|
-
confidence: number;
|
|
51
|
-
/** The layer / judge id that produced the signal, for provenance. */
|
|
52
|
-
origin: string;
|
|
53
|
-
/**
|
|
54
|
-
* Per-source contribution to `value`, keyed by layer/judge id. Single-source
|
|
55
|
-
* rewards carry one entry (`{ [origin]: value }`); composite rewards carry
|
|
56
|
-
* every contributing layer's score — the anti-scalar-collapse surface RL
|
|
57
|
-
* consumers weight per-source instead of trusting one blended number.
|
|
58
|
-
*/
|
|
59
|
-
components: Record<string, number>;
|
|
60
|
-
/**
|
|
61
|
-
* @deprecated Read `components` for per-source reward values. Kept for
|
|
62
|
-
* published-API compatibility: single-source rewards carry the layer's
|
|
63
|
-
* diagnostics here (e.g. `{ tests_passed: 7 }`); composite rewards carry
|
|
64
|
-
* the same per-layer scores `components` now holds.
|
|
65
|
-
*/
|
|
66
|
-
breakdown?: Record<string, number>;
|
|
67
|
-
}
|
|
68
|
-
export interface VerifiableRewardExtractionOptions {
|
|
69
|
-
/**
|
|
70
|
-
* Which layers count as deterministic-reward sources. The verifier doesn't
|
|
71
|
-
* tag layers as "this is verifiable"; the caller declares it via this list
|
|
72
|
-
* (or via the layer name → source mapping). Default treats common names
|
|
73
|
-
* (`install`, `typecheck`, `build`, `lint`, `test`, `compile`, `schema`,
|
|
74
|
-
* `sandbox`) as deterministic.
|
|
75
|
-
*/
|
|
76
|
-
deterministicLayers?: string[];
|
|
77
|
-
/**
|
|
78
|
-
* Map layer name → reward source. Defaults to a sensible string-match.
|
|
79
|
-
*/
|
|
80
|
-
sourceFor?: (layerName: string) => VerifiableRewardSource;
|
|
81
|
-
/**
|
|
82
|
-
* Whether to fall back to a probabilistic (judge) reward when no
|
|
83
|
-
* deterministic layer produced a numeric score. Default `true`. Set to
|
|
84
|
-
* `false` for "deterministic-only" training pipelines that should
|
|
85
|
-
* discard runs without a verifiable signal.
|
|
86
|
-
*/
|
|
87
|
-
fallbackToJudge?: boolean;
|
|
88
|
-
/**
|
|
89
|
-
* Default confidence for probabilistic (judge) rewards when the judge
|
|
90
|
-
* doesn't report one. Default `0.7`.
|
|
91
|
-
*/
|
|
92
|
-
judgeConfidenceFloor?: number;
|
|
93
|
-
}
|
|
94
|
-
/**
|
|
95
|
-
* Extract a `VerifiableReward` from a `VerificationReport`.
|
|
96
|
-
*
|
|
97
|
-
* Strategy: prefer the deterministic layers (in order: test → compile →
|
|
98
|
-
* schema → sandbox), fall back to the judge layer if `fallbackToJudge` is
|
|
99
|
-
* true, return `null` if no signal qualifies. When multiple deterministic
|
|
100
|
-
* layers contribute, return a `'composite'` source with a weighted blend.
|
|
101
|
-
*/
|
|
102
|
-
export declare function extractVerifiableReward(report: VerificationReport, opts?: VerifiableRewardExtractionOptions): VerifiableReward | null;
|
|
103
|
-
/**
|
|
104
|
-
* Extract verifiable rewards from `RunRecord[]` produced via the
|
|
105
|
-
* `verificationReportToRunRecord` adapter (which encodes per-layer scores
|
|
106
|
-
* in `outcome.raw['layer.<name>']`). For records that don't carry layer
|
|
107
|
-
* scores, returns `null` for that record.
|
|
108
|
-
*
|
|
109
|
-
* This is the canonical bridge from "campaign-shaped artifacts" to
|
|
110
|
-
* "RL-training-ready reward signals": every record that has a clean
|
|
111
|
-
* verifiable reward becomes a training datum, every record that doesn't
|
|
112
|
-
* gets filtered out (or kept with `'probabilistic'` determinism for
|
|
113
|
-
* separate downstream handling).
|
|
114
|
-
*/
|
|
115
|
-
export declare function extractVerifiableRewardsFromRecords(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
|
|
116
|
-
runId: string;
|
|
117
|
-
reward: VerifiableReward | null;
|
|
118
|
-
}>;
|
|
119
|
-
/** Filter `RunRecord[]` to those with deterministic verifiable rewards. */
|
|
120
|
-
export declare function filterDeterministicallyRewarded(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
|
|
121
|
-
run: RunRecord;
|
|
122
|
-
reward: VerifiableReward;
|
|
123
|
-
}>;
|
|
124
|
-
//# sourceMappingURL=verifiable-reward.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"verifiable-reward.d.ts","sourceRoot":"","sources":["../../src/rl/verifiable-reward.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,KAAK,EAAe,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAC9E,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AAE9C,MAAM,MAAM,sBAAsB,GAC9B,SAAS,GACT,MAAM,GACN,QAAQ,GACR,SAAS,GACT,OAAO,GACP,WAAW,CAAA;AAEf,MAAM,WAAW,gBAAgB;IAC/B,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAA;IACb,+EAA+E;IAC/E,MAAM,EAAE,sBAAsB,CAAA;IAC9B;;;;;;OAMG;IACH,WAAW,EAAE,eAAe,GAAG,eAAe,CAAA;IAC9C;;;;OAIG;IACH,UAAU,EAAE,MAAM,CAAA;IAClB,qEAAqE;IACrE,MAAM,EAAE,MAAM,CAAA;IACd;;;;;OAKG;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC;;;;;OAKG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,iCAAiC;IAChD;;;;;;OAMG;IACH,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC9B;;OAEG;IACH,SAAS,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,sBAAsB,CAAA;IACzD;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB;;;OAGG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAA;CAC9B;AA+BD;;;;;;;GAOG;AACH,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,kBAAkB,EAC1B,IAAI,GAAE,iCAAsC,GAC3C,gBAAgB,GAAG,IAAI,CAmEzB;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,mCAAmC,CACjD,IAAI,EAAE,SAAS,EAAE,EACjB,IAAI,GAAE,iCAAsC,GAC3C,KAAK,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CAAA;CAAE,CAAC,CA0E3D;AAED,2EAA2E;AAC3E,wBAAgB,+BAA+B,CAC7C,IAAI,EAAE,SAAS,EAAE,EACjB,IAAI,GAAE,iCAAsC,GAC3C,KAAK,CAAC;IAAE,GAAG,EAAE,SAAS,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,CAAC,CAUrD"}
|
package/dist/run-critic.d.ts
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
import { type RunScore, type RunScoreWeights } from './run-score';
|
|
2
|
-
import type { Artifact, BudgetLedgerEntry, Run, Span, TraceEvent, TraceStore } from './trace';
|
|
3
|
-
export interface RunTrace {
|
|
4
|
-
run: Run;
|
|
5
|
-
spans: Span[];
|
|
6
|
-
events: TraceEvent[];
|
|
7
|
-
artifacts: Artifact[];
|
|
8
|
-
budget: BudgetLedgerEntry[];
|
|
9
|
-
}
|
|
10
|
-
export interface RunCriticOptions {
|
|
11
|
-
weights?: Partial<RunScoreWeights>;
|
|
12
|
-
driftPatterns?: RegExp[];
|
|
13
|
-
}
|
|
14
|
-
export declare class RunCritic {
|
|
15
|
-
private readonly weights?;
|
|
16
|
-
private readonly driftPatterns;
|
|
17
|
-
constructor(options?: RunCriticOptions);
|
|
18
|
-
score(store: TraceStore, runId: string): Promise<RunScore>;
|
|
19
|
-
scoreTrace(trace: RunTrace): RunScore;
|
|
20
|
-
rank(score: RunScore): number;
|
|
21
|
-
private isDrift;
|
|
22
|
-
}
|
|
23
|
-
//# sourceMappingURL=run-critic.d.ts.map
|
package/dist/run-critic.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"run-critic.d.ts","sourceRoot":"","sources":["../src/run-critic.ts"],"names":[],"mappings":"AACA,OAAO,EAA8B,KAAK,QAAQ,EAAE,KAAK,eAAe,EAAE,MAAM,aAAa,CAAA;AAC7F,OAAO,KAAK,EAAE,QAAQ,EAAE,iBAAiB,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAE7F,MAAM,WAAW,QAAQ;IACvB,GAAG,EAAE,GAAG,CAAA;IACR,KAAK,EAAE,IAAI,EAAE,CAAA;IACb,MAAM,EAAE,UAAU,EAAE,CAAA;IACpB,SAAS,EAAE,QAAQ,EAAE,CAAA;IACrB,MAAM,EAAE,iBAAiB,EAAE,CAAA;CAC5B;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,CAAA;IAClC,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB;AAWD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAA0B;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAU;gBAE5B,OAAO,GAAE,gBAAqB;IAKpC,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC;IAYhE,UAAU,CAAC,KAAK,EAAE,QAAQ,GAAG,QAAQ;IAmHrC,IAAI,CAAC,KAAK,EAAE,QAAQ,GAAG,MAAM;IAI7B,OAAO,CAAC,OAAO;CAGhB"}
|
package/dist/run-evidence.d.ts
DELETED
|
@@ -1,32 +0,0 @@
|
|
|
1
|
-
import type { ControlEvalResult, ControlRunResult } from './control-runtime';
|
|
2
|
-
import { type RunRecord, type RunSplitTag, type RunTokenUsage } from './run-record';
|
|
3
|
-
export interface RunEvidenceMetadata {
|
|
4
|
-
experimentId: string;
|
|
5
|
-
candidateId: string;
|
|
6
|
-
seed: number;
|
|
7
|
-
model: string;
|
|
8
|
-
promptHash: string;
|
|
9
|
-
configHash: string;
|
|
10
|
-
commitSha: string;
|
|
11
|
-
splitTag: RunSplitTag;
|
|
12
|
-
tokenUsage: RunTokenUsage;
|
|
13
|
-
queueMs?: number;
|
|
14
|
-
judgeMetadata?: RunRecord['judgeMetadata'];
|
|
15
|
-
raw?: Record<string, number>;
|
|
16
|
-
}
|
|
17
|
-
export interface ControlRunToRunRecordOptions extends RunEvidenceMetadata {
|
|
18
|
-
runId?: string;
|
|
19
|
-
score?: number;
|
|
20
|
-
failureMode?: string;
|
|
21
|
-
}
|
|
22
|
-
/**
|
|
23
|
-
* Project a completed control-loop run into the strict RunRecord shape used by
|
|
24
|
-
* release gates, optimizer tables, and research reports.
|
|
25
|
-
*
|
|
26
|
-
* The control loop owns live execution evidence. The caller still supplies the
|
|
27
|
-
* experiment-cell metadata because prompt/config hashes, split assignment,
|
|
28
|
-
* model snapshot, and commit SHA are product/harness concerns.
|
|
29
|
-
*/
|
|
30
|
-
export declare function controlRunToRunRecord<TState, TAction, TActionResult, TEval extends ControlEvalResult = ControlEvalResult>(run: ControlRunResult<TState, TAction, TActionResult, TEval>, options: ControlRunToRunRecordOptions): RunRecord;
|
|
31
|
-
export declare function scoreFromEvals(evals: readonly ControlEvalResult[]): number | undefined;
|
|
32
|
-
//# sourceMappingURL=run-evidence.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"run-evidence.d.ts","sourceRoot":"","sources":["../src/run-evidence.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAA;AAC5E,OAAO,EACL,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,aAAa,EAEnB,MAAM,cAAc,CAAA;AAGrB,MAAM,WAAW,mBAAmB;IAClC,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,WAAW,CAAA;IACrB,UAAU,EAAE,aAAa,CAAA;IACzB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,aAAa,CAAC,EAAE,SAAS,CAAC,eAAe,CAAC,CAAA;IAC1C,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B;AAED,MAAM,WAAW,4BAA6B,SAAQ,mBAAmB;IACvE,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EACN,OAAO,EACP,aAAa,EACb,KAAK,SAAS,iBAAiB,GAAG,iBAAiB,EAEnD,GAAG,EAAE,gBAAgB,CAAC,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,KAAK,CAAC,EAC5D,OAAO,EAAE,4BAA4B,GACpC,SAAS,CA8BX;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,SAAS,iBAAiB,EAAE,GAAG,MAAM,GAAG,SAAS,CAMtF"}
|
package/dist/run-record.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"run-record.d.ts","sourceRoot":"","sources":["../src/run-record.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAA;AAE5D,OAAO,EAAE,eAAe,EAAE,MAAM,UAAU,CAAA;AAC1C,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAElD;sEACsE;AACtE,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,KAAK,GAAG,SAAS,CAAA;AAEtD,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAA;IACb,aAAa,EAAE,MAAM,CAAA;IACrB;mEAC+D;IAC/D,UAAU,EAAE,MAAM,CAAA;IAClB;kEAC8D;IAC9D,QAAQ,EAAE,OAAO,CAAA;CAClB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,WAAW,iBAAiB;IAChC,kGAAkG;IAClG,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;IAChD,2EAA2E;IAC3E,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC;mEAC+D;IAC/D,SAAS,EAAE,MAAM,CAAA;IACjB;;wDAEoD;IACpD,YAAY,CAAC,EAAE,MAAM,EAAE,CAAA;IACvB;iDAC6C;IAC7C,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,UAAU;IACzB;6DACyD;IACzD,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;kEAC8D;IAC9D,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;;4BAEwB;IACxB,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC3B;;;;+BAI2B;IAC3B,WAAW,CAAC,EAAE,iBAAiB,CAAA;IAC/B;;;;;kEAK8D;IAC9D,QAAQ,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAC9D;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,SAAS;IACxB,wBAAwB;IACxB,KAAK,EAAE,MAAM,CAAA;IACb;uDACmD;IACnD,YAAY,EAAE,MAAM,CAAA;IACpB;uEACmE;IACnE,WAAW,EAAE,MAAM,CAAA;IACnB;6DACyD;IACzD,IAAI,EAAE,MAAM,CAAA;IACZ,8CAA8C;IAC9C,KAAK,EAAE,MAAM,CAAA;IACb,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAA;IAClB,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAA;IAClB,wCAAwC;IACxC,SAAS,EAAE,MAAM,CAAA;IACjB,sDAAsD;IACtD,MAAM,EAAE,MAAM,CAAA;IACd,4DAA4D;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;0EACsE;IACtE,OAAO,EAAE,MAAM,CAAA;IACf,6BAA6B;IAC7B,UAAU,EAAE,aAAa,CAAA;IACzB,gDAAgD;IAChD,aAAa,CAAC,EAAE,gBAAgB,CAAA;IAChC,kCAAkC;IAClC,OAAO,EAAE,UAAU,CAAA;IACnB;;;;;yCAKqC;IACrC,YAAY,CAAC,EAAE,YAAY,CAAA;IAC3B;;8EAE0E;IAC1E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,2CAA2C;IAC3C,QAAQ,EAAE,WAAW,CAAA;IACrB;;;;;;;OAOG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;;;OAMG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAA;CAChC;AAsBD,qBAAa,wBAAyB,SAAQ,eAAe;IAC3D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;gBACT,OAAO,EAAE,MAAM,EAAE,IAAI,SAAK;CAIvC;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,SAAS,CA4I3D;AAED,4DAA4D;AAC5D,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,SAAS,CAO9D;AAED,8DAA8D;AAC9D,wBAAgB,kBAAkB,CAChC,KAAK,EAAE,OAAO,GACb;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,SAAS,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,KAAK,EAAE,wBAAwB,CAAA;CAAE,CAOjF;AAED,8EAA8E;AAC9E,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,SAAS,GAAG,SAAS,CAG/D"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"run-record.test.d.ts","sourceRoot":"","sources":["../src/run-record.test.ts"],"names":[],"mappings":""}
|
package/dist/run-score.d.ts
DELETED
|
@@ -1,31 +0,0 @@
|
|
|
1
|
-
export interface RunScore {
|
|
2
|
-
success: number;
|
|
3
|
-
goalProgress: number;
|
|
4
|
-
repoGroundedness: number;
|
|
5
|
-
driftPenalty: number;
|
|
6
|
-
toolUseQuality: number;
|
|
7
|
-
patchQuality: number;
|
|
8
|
-
testReality: number;
|
|
9
|
-
finalGate: number;
|
|
10
|
-
reviewerBlockers: number;
|
|
11
|
-
costUsd: number;
|
|
12
|
-
wallSeconds: number;
|
|
13
|
-
notes?: string[];
|
|
14
|
-
}
|
|
15
|
-
export interface RunScoreWeights {
|
|
16
|
-
success: number;
|
|
17
|
-
goalProgress: number;
|
|
18
|
-
repoGroundedness: number;
|
|
19
|
-
driftPenalty: number;
|
|
20
|
-
toolUseQuality: number;
|
|
21
|
-
patchQuality: number;
|
|
22
|
-
testReality: number;
|
|
23
|
-
finalGate: number;
|
|
24
|
-
reviewerBlockers: number;
|
|
25
|
-
costUsd: number;
|
|
26
|
-
wallSeconds: number;
|
|
27
|
-
}
|
|
28
|
-
export declare const DEFAULT_RUN_SCORE_WEIGHTS: RunScoreWeights;
|
|
29
|
-
export declare function aggregateRunScore(score: RunScore, weights?: Partial<RunScoreWeights>): number;
|
|
30
|
-
export declare function clamp01(value: number): number;
|
|
31
|
-
//# sourceMappingURL=run-score.d.ts.map
|
package/dist/run-score.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"run-score.d.ts","sourceRoot":"","sources":["../src/run-score.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,QAAQ;IACvB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,gBAAgB,EAAE,MAAM,CAAA;IACxB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,gBAAgB,EAAE,MAAM,CAAA;IACxB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,gBAAgB,EAAE,MAAM,CAAA;IACxB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,gBAAgB,EAAE,MAAM,CAAA;IACxB,OAAO,EAAE,MAAM,CAAA;IACf,WAAW,EAAE,MAAM,CAAA;CACpB;AAED,eAAO,MAAM,yBAAyB,EAAE,eAYvC,CAAA;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,GAAE,OAAO,CAAC,eAAe,CAAM,GAAG,MAAM,CAejG;AAED,wBAAgB,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAG7C"}
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import type { RunSplitTag } from './run-record';
|
|
2
|
-
export interface RuntimeTrajectoryHookEvent {
|
|
3
|
-
id: string;
|
|
4
|
-
runId: string;
|
|
5
|
-
scenarioId?: string;
|
|
6
|
-
target: string;
|
|
7
|
-
phase: string;
|
|
8
|
-
timestamp: number;
|
|
9
|
-
stepIndex?: number;
|
|
10
|
-
parentId?: string;
|
|
11
|
-
payload?: unknown;
|
|
12
|
-
metadata?: Record<string, unknown>;
|
|
13
|
-
}
|
|
14
|
-
export interface RuntimeTrajectoryRecord {
|
|
15
|
-
id?: string;
|
|
16
|
-
scenarioId?: string;
|
|
17
|
-
splitTag?: RunSplitTag;
|
|
18
|
-
runtimeEvents?: unknown;
|
|
19
|
-
[key: string]: unknown;
|
|
20
|
-
}
|
|
21
|
-
export interface RuntimeTrajectoryRunRecord {
|
|
22
|
-
runId: string;
|
|
23
|
-
scenarioId?: string;
|
|
24
|
-
splitTag: RunSplitTag;
|
|
25
|
-
}
|
|
26
|
-
export interface RuntimeTrajectoryEvidenceSummary {
|
|
27
|
-
recordCount: number;
|
|
28
|
-
recordWithRuntimeEventsCount: number;
|
|
29
|
-
runtimeRunCount: number;
|
|
30
|
-
lifecycleEventCount: number;
|
|
31
|
-
defaultedSplitCount: number;
|
|
32
|
-
}
|
|
33
|
-
export interface RuntimeTrajectoryEvidenceProjection {
|
|
34
|
-
runs: RuntimeTrajectoryRunRecord[];
|
|
35
|
-
events: RuntimeTrajectoryHookEvent[];
|
|
36
|
-
summary: RuntimeTrajectoryEvidenceSummary;
|
|
37
|
-
diagnostics: string[];
|
|
38
|
-
}
|
|
39
|
-
export interface ProjectRuntimeTrajectoryEvidenceOptions<TRecord extends RuntimeTrajectoryRecord = RuntimeTrajectoryRecord> {
|
|
40
|
-
records: TRecord[];
|
|
41
|
-
defaultSplitTag?: RunSplitTag;
|
|
42
|
-
recordIdOf?: (record: TRecord, index: number) => string | undefined;
|
|
43
|
-
scenarioIdOf?: (record: TRecord, index: number) => string | undefined;
|
|
44
|
-
}
|
|
45
|
-
export declare function projectRuntimeTrajectoryEvidence<TRecord extends RuntimeTrajectoryRecord>(options: ProjectRuntimeTrajectoryEvidenceOptions<TRecord>): RuntimeTrajectoryEvidenceProjection;
|
|
46
|
-
export declare function parseRuntimeTrajectoryHookEvent(input: unknown): RuntimeTrajectoryHookEvent | null;
|
|
47
|
-
//# sourceMappingURL=runtime-trajectory.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"runtime-trajectory.d.ts","sourceRoot":"","sources":["../src/runtime-trajectory.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAA;AAE/C,MAAM,WAAW,0BAA0B;IACzC,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,uBAAuB;IACtC,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,EAAE,WAAW,CAAA;IACtB,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACvB;AAED,MAAM,WAAW,0BAA0B;IACzC,KAAK,EAAE,MAAM,CAAA;IACb,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,WAAW,CAAA;CACtB;AAED,MAAM,WAAW,gCAAgC;IAC/C,WAAW,EAAE,MAAM,CAAA;IACnB,4BAA4B,EAAE,MAAM,CAAA;IACpC,eAAe,EAAE,MAAM,CAAA;IACvB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,mBAAmB,EAAE,MAAM,CAAA;CAC5B;AAED,MAAM,WAAW,mCAAmC;IAClD,IAAI,EAAE,0BAA0B,EAAE,CAAA;IAClC,MAAM,EAAE,0BAA0B,EAAE,CAAA;IACpC,OAAO,EAAE,gCAAgC,CAAA;IACzC,WAAW,EAAE,MAAM,EAAE,CAAA;CACtB;AAED,MAAM,WAAW,uCAAuC,CACtD,OAAO,SAAS,uBAAuB,GAAG,uBAAuB;IAEjE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClB,eAAe,CAAC,EAAE,WAAW,CAAA;IAC7B,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAA;IACnE,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,KAAK,MAAM,GAAG,SAAS,CAAA;CACtE;AAID,wBAAgB,gCAAgC,CAAC,OAAO,SAAS,uBAAuB,EACtF,OAAO,EAAE,uCAAuC,CAAC,OAAO,CAAC,GACxD,mCAAmC,CA8DrC;AAED,wBAAgB,+BAA+B,CAAC,KAAK,EAAE,OAAO,GAAG,0BAA0B,GAAG,IAAI,CAoBjG"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sandbox-harness.d.ts","sourceRoot":"","sources":["../src/sandbox-harness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAGnD,MAAM,WAAW,aAAa;IAC5B,wEAAwE;IACxE,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,uCAAuC;IACvC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,iFAAiF;IACjF,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;;;OAIG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,mCAAmC;IACnC,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,iDAAiD;IACjD,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC5B,8EAA8E;IAC9E,UAAU,CAAC,EAAE,gBAAgB,CAAA;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,CACH,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,GACf;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAA;CAC3D;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,OAAO,GAAG,KAAK,GAAG,MAAM,CAAA;IAC/B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;;;OAKG;IACH,eAAe,CAAC,EAAE,OAAO,CAAA;IACzB;;;;OAIG;IACH,eAAe,CAAC,EAAE,OAAO,CAAA;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,CACF,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,EAC7B,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,aAAa,CAAC,CAAA;CAC1B;AAID,iEAAiE;AACjE,eAAO,MAAM,gBAAgB,EAAE,gBAkB9B,CAAA;AAED,mEAAmE;AACnE,eAAO,MAAM,gBAAgB,EAAE,gBAQ9B,CAAA;AAED,8DAA8D;AAC9D,eAAO,MAAM,cAAc,EAAE,gBAO5B,CAAA;AAED,2DAA2D;AAC3D,wBAAgB,cAAc,CAAC,GAAG,OAAO,EAAE,gBAAgB,EAAE,GAAG,gBAAgB,CAW/E;AAID,MAAM,WAAW,8BAA8B;IAC7C;;;;;;;OAOG;IACH,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ;;;OAGG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC7B;AAED,qBAAa,uBAAwB,YAAW,aAAa;IAC3D,EAAE,SAAe;IACjB,OAAO,CAAC,UAAU,CAAC,CAAQ;IAC3B,OAAO,CAAC,UAAU,CAAC,CAAwB;gBAE/B,OAAO,GAAE,8BAAmC;IAKlD,IAAI,CACR,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,EAC7B,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,aAAa,CAAC;CAmI1B;AAED,qBAAa,mBAAoB,YAAW,aAAa;IACvD,EAAE,SAAW;IAEP,IAAI,CACR,KAAK,EAAE,aAAa,CAAC,OAAO,CAAC,EAC7B,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,aAAa,GACpB,OAAO,CAAC,aAAa,CAAC;CAS1B;AASD,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,OAAO,CAAA;IACf,KAAK,CAAC,EAAE,aAAa,CAAA;IACrB,GAAG,CAAC,EAAE,aAAa,CAAA;IACnB,IAAI,CAAC,EAAE,aAAa,CAAA;IACpB,WAAW,EAAE,MAAM,CAAA;IACnB,uEAAuE;IACvE,KAAK,EAAE,MAAM,CAAA;CACd;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAe;gBACjB,MAAM,GAAE,aAA6C;IAI3D,GAAG,CAAC,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,oBAAoB,CAAC;CAoEvF"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sandbox-harness.test.d.ts","sourceRoot":"","sources":["../src/sandbox-harness.test.ts"],"names":[],"mappings":""}
|
package/dist/sandbox-pool.d.ts
DELETED
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* SandboxPool — bounded checkout/release pool for mutation slots.
|
|
3
|
-
*
|
|
4
|
-
* The composite-mutator's `code` channel needs an isolated workspace per
|
|
5
|
-
* mutation attempt: a git worktree, a sandbox container, a tmpdir clone —
|
|
6
|
-
* whatever the consumer's runtime is. Without a pool, every consumer
|
|
7
|
-
* re-implements the same machinery (mint N slots, check one out per
|
|
8
|
-
* mutation, reset before reuse, drain at the end, track utilisation for
|
|
9
|
-
* the cost ledger). This primitive ships that machinery so consumers
|
|
10
|
-
* supply only a `SlotFactory`.
|
|
11
|
-
*
|
|
12
|
-
* Generic over a slot resource `T` so the same pool serves git worktrees
|
|
13
|
-
* (T = path), Tangle sandboxes (T = SandboxBox), or anything else with
|
|
14
|
-
* the create/reset/destroy lifecycle.
|
|
15
|
-
*
|
|
16
|
-
* Concurrency: FIFO via the shared `Mutex` primitive. Each `checkout()`
|
|
17
|
-
* either takes an idle slot or queues until one is released. Lifecycle
|
|
18
|
-
* is single-process — multi-process pools need external coordination
|
|
19
|
-
* (file locks, etc.) and are deliberately out of scope.
|
|
20
|
-
*/
|
|
21
|
-
export interface PoolSlot<T> {
|
|
22
|
-
/** Stable id assigned at slot creation. Use for telemetry / lineage. */
|
|
23
|
-
readonly id: string;
|
|
24
|
-
/** Consumer-defined resource. */
|
|
25
|
-
readonly resource: T;
|
|
26
|
-
}
|
|
27
|
-
export interface SlotFactory<T> {
|
|
28
|
-
/** Build a new slot. Called lazily as the pool grows up to `size`. */
|
|
29
|
-
create(slotId: string): Promise<T>;
|
|
30
|
-
/**
|
|
31
|
-
* Reset a slot to a clean state before reuse. Called BEFORE every
|
|
32
|
-
* checkout returns it (including the first — so the factory's
|
|
33
|
-
* `create` can leave the slot dirty and let `reset` normalise).
|
|
34
|
-
* Optional; default is a no-op.
|
|
35
|
-
*/
|
|
36
|
-
reset?(slot: PoolSlot<T>): Promise<void>;
|
|
37
|
-
/** Tear the slot down. Called by `drain()`. */
|
|
38
|
-
destroy(slot: PoolSlot<T>): Promise<void>;
|
|
39
|
-
}
|
|
40
|
-
export interface SandboxPool<T> {
|
|
41
|
-
/**
|
|
42
|
-
* Take a slot. If all slots are busy, the promise resolves when one
|
|
43
|
-
* is released. Always pair with the returned `release` (or wrap with
|
|
44
|
-
* `withSlot`).
|
|
45
|
-
*/
|
|
46
|
-
checkout(): Promise<{
|
|
47
|
-
slot: PoolSlot<T>;
|
|
48
|
-
release: () => void;
|
|
49
|
-
}>;
|
|
50
|
-
/**
|
|
51
|
-
* Run `fn` with a checked-out slot, releasing on completion or throw.
|
|
52
|
-
* The convenience wrapper most callers should use.
|
|
53
|
-
*/
|
|
54
|
-
withSlot<R>(fn: (slot: PoolSlot<T>) => Promise<R>): Promise<R>;
|
|
55
|
-
/** Destroy every slot. Idempotent. */
|
|
56
|
-
drain(): Promise<void>;
|
|
57
|
-
/** How many slots have been minted (≤ `size`). */
|
|
58
|
-
poolSize(): number;
|
|
59
|
-
/** How many checkouts are currently outstanding. */
|
|
60
|
-
activeCheckouts(): number;
|
|
61
|
-
/** Snapshot of busy/total durations for the cost ledger. */
|
|
62
|
-
utilization(): {
|
|
63
|
-
busyMs: number;
|
|
64
|
-
totalMs: number;
|
|
65
|
-
checkouts: number;
|
|
66
|
-
};
|
|
67
|
-
}
|
|
68
|
-
export interface CreateSandboxPoolOpts<T> {
|
|
69
|
-
/** Maximum concurrent slots. Slots are minted on first need, not eagerly. */
|
|
70
|
-
size: number;
|
|
71
|
-
factory: SlotFactory<T>;
|
|
72
|
-
}
|
|
73
|
-
export declare function createSandboxPool<T>(opts: CreateSandboxPoolOpts<T>): SandboxPool<T>;
|
|
74
|
-
//# sourceMappingURL=sandbox-pool.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sandbox-pool.d.ts","sourceRoot":"","sources":["../src/sandbox-pool.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAIH,MAAM,WAAW,QAAQ,CAAC,CAAC;IACzB,wEAAwE;IACxE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,iCAAiC;IACjC,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAA;CACrB;AAED,MAAM,WAAW,WAAW,CAAC,CAAC;IAC5B,sEAAsE;IACtE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;IAClC;;;;;OAKG;IACH,KAAK,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACxC,+CAA+C;IAC/C,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC1C;AAED,MAAM,WAAW,WAAW,CAAC,CAAC;IAC5B;;;;OAIG;IACH,QAAQ,IAAI,OAAO,CAAC;QAAE,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC;QAAC,OAAO,EAAE,MAAM,IAAI,CAAA;KAAE,CAAC,CAAA;IAC/D;;;OAGG;IACH,QAAQ,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;IAC9D,sCAAsC;IACtC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;IACtB,kDAAkD;IAClD,QAAQ,IAAI,MAAM,CAAA;IAClB,oDAAoD;IACpD,eAAe,IAAI,MAAM,CAAA;IACzB,4DAA4D;IAC5D,WAAW,IAAI;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAA;CACtE;AAED,MAAM,WAAW,qBAAqB,CAAC,CAAC;IACtC,6EAA6E;IAC7E,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,CAAA;CACxB;AAOD,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,IAAI,EAAE,qBAAqB,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,CAyMnF"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"sandbox-pool.test.d.ts","sourceRoot":"","sources":["../src/sandbox-pool.test.ts"],"names":[],"mappings":""}
|
package/dist/scorecard.d.ts
DELETED
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @stable
|
|
3
|
-
*
|
|
4
|
-
* Eval scorecard — the persistent (persona × profile) score timeline.
|
|
5
|
-
*
|
|
6
|
-
* Every benchmark run folds into per-cell entries; a cell is
|
|
7
|
-
* `(scenarioId, profileHash)` and its timeline carries one entry per commit.
|
|
8
|
-
* The scorecard answers the question a single run cannot: did THIS change
|
|
9
|
-
* regress persona P on profile F, even while the aggregate improved?
|
|
10
|
-
*
|
|
11
|
-
* Storage is an append-only JSONL log — one line per (cell, commit). Appends
|
|
12
|
-
* never read-modify-write, so concurrent campaign runs cannot clobber each
|
|
13
|
-
* other; `loadScorecard` folds the log into the queryable `Scorecard`, and a
|
|
14
|
-
* malformed line never breaks the read. `diffScorecard` compares the latest
|
|
15
|
-
* entry of each cell against its predecessor with Cohen's d + Welch's t-test.
|
|
16
|
-
*/
|
|
17
|
-
import type { AgentProfile } from './agent-profile';
|
|
18
|
-
import type { RunRecord } from './run-record';
|
|
19
|
-
/** One commit's measurement of one (scenario, profile) cell. */
|
|
20
|
-
export interface ScorecardEntry {
|
|
21
|
-
commitSha: string;
|
|
22
|
-
/** ISO timestamp the entry was recorded. */
|
|
23
|
-
timestamp: string;
|
|
24
|
-
/** Per-seed (or per-rep) scores for this cell at this commit. */
|
|
25
|
-
scores: number[];
|
|
26
|
-
/** Median of `scores` — the cell's headline score for the commit. */
|
|
27
|
-
composite: number;
|
|
28
|
-
/** Per-dimension means, when the runs carried a judge breakdown. */
|
|
29
|
-
perDimension?: Record<string, number>;
|
|
30
|
-
/** RunRecord ids folded into this entry — provenance. */
|
|
31
|
-
runIds: string[];
|
|
32
|
-
}
|
|
33
|
-
/** A (scenario, profile) cell and its commit-ordered score timeline. */
|
|
34
|
-
export interface ScorecardCell {
|
|
35
|
-
scenarioId: string;
|
|
36
|
-
profileHash: string;
|
|
37
|
-
/** Model id — denormalised from the profile for readable filtering. */
|
|
38
|
-
model: string;
|
|
39
|
-
timeline: ScorecardEntry[];
|
|
40
|
-
}
|
|
41
|
-
/** The folded scorecard: every cell, plus the profile definitions by hash. */
|
|
42
|
-
export interface Scorecard {
|
|
43
|
-
cells: ScorecardCell[];
|
|
44
|
-
/** Profile definitions seen — keeps the scorecard self-describing. */
|
|
45
|
-
profiles: Record<string, AgentProfile>;
|
|
46
|
-
}
|
|
47
|
-
/** One append-only log line — a single cell's entry for a single commit. */
|
|
48
|
-
export interface ScorecardLogLine {
|
|
49
|
-
scenarioId: string;
|
|
50
|
-
profileHash: string;
|
|
51
|
-
model: string;
|
|
52
|
-
profile: AgentProfile;
|
|
53
|
-
entry: ScorecardEntry;
|
|
54
|
-
}
|
|
55
|
-
export interface RecordRunsOptions {
|
|
56
|
-
/** The profile that produced these runs — keys the cell. */
|
|
57
|
-
profile: AgentProfile;
|
|
58
|
-
commitSha: string;
|
|
59
|
-
/** Defaults to `new Date().toISOString()`. */
|
|
60
|
-
timestamp?: string;
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* Fold a benchmark's `RunRecord[]` into per-cell scorecard log lines — one
|
|
64
|
-
* line per scenario the runs cover. All runs are attributed to the single
|
|
65
|
-
* `profile` in `opts` (the harness ran them under it); the cell key is
|
|
66
|
-
* `(scenarioId, agentProfileHash(profile))`.
|
|
67
|
-
*/
|
|
68
|
-
export declare function recordRuns(runs: RunRecord[], opts: RecordRunsOptions): ScorecardLogLine[];
|
|
69
|
-
/** Append cell entries to the JSONL scorecard log. Creates the file/dir. */
|
|
70
|
-
export declare function appendScorecard(logPath: string, lines: ScorecardLogLine[]): void;
|
|
71
|
-
/** Record runs and append them to the log in one call. Returns the lines. */
|
|
72
|
-
export declare function recordRunsToScorecard(logPath: string, runs: RunRecord[], opts: RecordRunsOptions): ScorecardLogLine[];
|
|
73
|
-
/**
|
|
74
|
-
* Fold the JSONL log into a queryable `Scorecard`. A missing file yields an
|
|
75
|
-
* empty scorecard; a malformed line is skipped — a corrupt append never
|
|
76
|
-
* breaks the read. Each cell's timeline is sorted chronologically.
|
|
77
|
-
*/
|
|
78
|
-
export declare function loadScorecard(logPath: string): Scorecard;
|
|
79
|
-
export type CellVerdict = 'improved' | 'regressed' | 'flat' | 'new';
|
|
80
|
-
export interface ScorecardCellDiff {
|
|
81
|
-
scenarioId: string;
|
|
82
|
-
profileHash: string;
|
|
83
|
-
model: string;
|
|
84
|
-
verdict: CellVerdict;
|
|
85
|
-
/** Composite of the latest entry. */
|
|
86
|
-
current: number;
|
|
87
|
-
/** Composite of the comparison entry — `null` when `verdict === 'new'`. */
|
|
88
|
-
baseline: number | null;
|
|
89
|
-
/** `current − baseline` — `null` when new. */
|
|
90
|
-
delta: number | null;
|
|
91
|
-
/** Cohen's d of current vs baseline samples — `null` when new or n < 2. */
|
|
92
|
-
cohensD: number | null;
|
|
93
|
-
/** Welch's t-test p-value — `null` when new or n < 2. */
|
|
94
|
-
pValue: number | null;
|
|
95
|
-
currentCommit: string;
|
|
96
|
-
baselineCommit: string | null;
|
|
97
|
-
}
|
|
98
|
-
export interface ScorecardDiff {
|
|
99
|
-
cells: ScorecardCellDiff[];
|
|
100
|
-
summary: {
|
|
101
|
-
improved: number;
|
|
102
|
-
regressed: number;
|
|
103
|
-
flat: number;
|
|
104
|
-
new: number;
|
|
105
|
-
};
|
|
106
|
-
}
|
|
107
|
-
export interface DiffScorecardOptions {
|
|
108
|
-
/** Compare each cell against this commit instead of its immediate predecessor. */
|
|
109
|
-
baselineCommit?: string;
|
|
110
|
-
/** |Cohen's d| at/above which a move counts as real. Default 0.5. */
|
|
111
|
-
minEffect?: number;
|
|
112
|
-
/** p-value at/below which a move is significant. Default 0.05. */
|
|
113
|
-
maxP?: number;
|
|
114
|
-
/**
|
|
115
|
-
* |delta| at/above which a move counts when statistics are unavailable
|
|
116
|
-
* (a cell with fewer than 2 samples on either side). Default 0.05.
|
|
117
|
-
*/
|
|
118
|
-
minDelta?: number;
|
|
119
|
-
}
|
|
120
|
-
/**
|
|
121
|
-
* Compare the latest entry of every cell against its predecessor (or against
|
|
122
|
-
* `baselineCommit`) and classify the move. A move is `improved`/`regressed`
|
|
123
|
-
* only when it clears both the effect-size and significance gates; otherwise
|
|
124
|
-
* `flat`. Cells with no prior entry are `new`.
|
|
125
|
-
*/
|
|
126
|
-
export declare function diffScorecard(scorecard: Scorecard, opts?: DiffScorecardOptions): ScorecardDiff;
|
|
127
|
-
/**
|
|
128
|
-
* Render a scorecard diff as a human-readable report — the block a feature
|
|
129
|
-
* PR prints. Regressions are listed first; flat cells are summarised, not
|
|
130
|
-
* enumerated.
|
|
131
|
-
*/
|
|
132
|
-
export declare function formatScorecardDiff(diff: ScorecardDiff): string;
|
|
133
|
-
//# sourceMappingURL=scorecard.d.ts.map
|
package/dist/scorecard.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scorecard.d.ts","sourceRoot":"","sources":["../src/scorecard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AAGnD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAK7C,gEAAgE;AAChE,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,4CAA4C;IAC5C,SAAS,EAAE,MAAM,CAAA;IACjB,iEAAiE;IACjE,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,qEAAqE;IACrE,SAAS,EAAE,MAAM,CAAA;IACjB,oEAAoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACrC,yDAAyD;IACzD,MAAM,EAAE,MAAM,EAAE,CAAA;CACjB;AAED,wEAAwE;AACxE,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,uEAAuE;IACvE,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,cAAc,EAAE,CAAA;CAC3B;AAED,8EAA8E;AAC9E,MAAM,WAAW,SAAS;IACxB,KAAK,EAAE,aAAa,EAAE,CAAA;IACtB,sEAAsE;IACtE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAA;CACvC;AAED,4EAA4E;AAC5E,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,YAAY,CAAA;IACrB,KAAK,EAAE,cAAc,CAAA;CACtB;AAID,MAAM,WAAW,iBAAiB;IAChC,4DAA4D;IAC5D,OAAO,EAAE,YAAY,CAAA;IACrB,SAAS,EAAE,MAAM,CAAA;IACjB,8CAA8C;IAC9C,SAAS,CAAC,EAAE,MAAM,CAAA;CACnB;AAkCD;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,iBAAiB,GAAG,gBAAgB,EAAE,CAuCzF;AAID,4EAA4E;AAC5E,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAIhF;AAED,6EAA6E;AAC7E,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,SAAS,EAAE,EACjB,IAAI,EAAE,iBAAiB,GACtB,gBAAgB,EAAE,CAIpB;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,CAkCxD;AAID,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,WAAW,GAAG,MAAM,GAAG,KAAK,CAAA;AAEnE,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,WAAW,CAAA;IACpB,qCAAqC;IACrC,OAAO,EAAE,MAAM,CAAA;IACf,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,8CAA8C;IAC9C,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,2EAA2E;IAC3E,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,yDAAyD;IACzD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,aAAa,EAAE,MAAM,CAAA;IACrB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;CAC9B;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,iBAAiB,EAAE,CAAA;IAC1B,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;CAC5E;AAED,MAAM,WAAW,oBAAoB;IACnC,kFAAkF;IAClF,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,kEAAkE;IAClE,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,SAAS,EAAE,SAAS,EACpB,IAAI,GAAE,oBAAyB,GAC9B,aAAa,CAsEf;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,aAAa,GAAG,MAAM,CAgC/D"}
|
package/dist/scorecard.test.d.ts
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"scorecard.test.d.ts","sourceRoot":"","sources":["../src/scorecard.test.ts"],"names":[],"mappings":""}
|
package/dist/self-play.d.ts
DELETED
|
@@ -1,69 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Self-play scenario evolution — agents generate adversarial scenarios
|
|
3
|
-
* against each other; survivors become part of the eval corpus.
|
|
4
|
-
*
|
|
5
|
-
* Framework-agnostic about how scenarios are generated. Caller supplies:
|
|
6
|
-
* - `propose`: asks a "proposer" agent for candidate scenarios
|
|
7
|
-
* - `scoreAgainst`: runs a target agent against a scenario and returns
|
|
8
|
-
* its score
|
|
9
|
-
*
|
|
10
|
-
* A scenario *survives* if it reveals a meaningful score difference
|
|
11
|
-
* between two target agents (or between a target agent and itself on
|
|
12
|
-
* different runs). Survivors are promoted to a Dataset; the caller
|
|
13
|
-
* decides what to do with them (hold-out, training, regression set).
|
|
14
|
-
*
|
|
15
|
-
* Guard rails: minimum absolute score delta to consider a scenario
|
|
16
|
-
* informative; floor on absolute target score so degenerate break-all
|
|
17
|
-
* scenarios (noise, gibberish) don't flood the corpus.
|
|
18
|
-
*/
|
|
19
|
-
import { Dataset } from './dataset';
|
|
20
|
-
export interface CandidateScenario {
|
|
21
|
-
id: string;
|
|
22
|
-
payload: unknown;
|
|
23
|
-
/** Free-form tags (domain, generation, parent). */
|
|
24
|
-
tags?: Record<string, string>;
|
|
25
|
-
}
|
|
26
|
-
export interface ScoredTarget {
|
|
27
|
-
targetId: string;
|
|
28
|
-
score: number;
|
|
29
|
-
}
|
|
30
|
-
export interface EvolutionRound {
|
|
31
|
-
round: number;
|
|
32
|
-
proposed: CandidateScenario[];
|
|
33
|
-
survived: CandidateScenario[];
|
|
34
|
-
rejected: Array<{
|
|
35
|
-
candidate: CandidateScenario;
|
|
36
|
-
reason: string;
|
|
37
|
-
}>;
|
|
38
|
-
scoredBreakdown: Array<{
|
|
39
|
-
candidate: CandidateScenario;
|
|
40
|
-
scores: ScoredTarget[];
|
|
41
|
-
spread: number;
|
|
42
|
-
}>;
|
|
43
|
-
}
|
|
44
|
-
export interface SelfPlayOptions {
|
|
45
|
-
/** Minimum score spread across targets for a scenario to survive. Default 0.1. */
|
|
46
|
-
minSpread?: number;
|
|
47
|
-
/** Minimum floor score across targets — keeps degenerate break-all scenarios
|
|
48
|
-
* out. Default 0.1 (if every target scores below this, discard). */
|
|
49
|
-
minAbsoluteFloor?: number;
|
|
50
|
-
/** Hard cap on survivors per round. Default 50. */
|
|
51
|
-
maxSurvivors?: number;
|
|
52
|
-
/** Rounds to run. Default 1. Each round's survivors can be fed back into
|
|
53
|
-
* `propose` to compound. */
|
|
54
|
-
rounds?: number;
|
|
55
|
-
/** Seed for scenario id generation if proposer doesn't provide one. */
|
|
56
|
-
seed?: number;
|
|
57
|
-
}
|
|
58
|
-
export interface SelfPlayProposer {
|
|
59
|
-
propose(round: number, priorSurvivors: CandidateScenario[]): Promise<CandidateScenario[]>;
|
|
60
|
-
}
|
|
61
|
-
export interface SelfPlayScorer {
|
|
62
|
-
/** Score one candidate against every target; returns parallel array. */
|
|
63
|
-
scoreCandidate(candidate: CandidateScenario, targets: string[]): Promise<ScoredTarget[]>;
|
|
64
|
-
}
|
|
65
|
-
export declare function runSelfPlay(proposer: SelfPlayProposer, scorer: SelfPlayScorer, targets: string[], options?: SelfPlayOptions): Promise<{
|
|
66
|
-
rounds: EvolutionRound[];
|
|
67
|
-
dataset: Dataset;
|
|
68
|
-
}>;
|
|
69
|
-
//# sourceMappingURL=self-play.d.ts.map
|
package/dist/self-play.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"self-play.d.ts","sourceRoot":"","sources":["../src/self-play.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,OAAO,EAAwB,MAAM,WAAW,CAAA;AAEzD,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAA;IACV,OAAO,EAAE,OAAO,CAAA;IAChB,mDAAmD;IACnD,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC9B;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,iBAAiB,EAAE,CAAA;IAC7B,QAAQ,EAAE,iBAAiB,EAAE,CAAA;IAC7B,QAAQ,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,iBAAiB,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;IACjE,eAAe,EAAE,KAAK,CAAC;QAAE,SAAS,EAAE,iBAAiB,CAAC;QAAC,MAAM,EAAE,YAAY,EAAE,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACjG;AAED,MAAM,WAAW,eAAe;IAC9B,kFAAkF;IAClF,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;yEACqE;IACrE,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,mDAAmD;IACnD,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;iCAC6B;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,uEAAuE;IACvE,IAAI,CAAC,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,cAAc,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAA;CAC1F;AAED,MAAM,WAAW,cAAc;IAC7B,wEAAwE;IACxE,cAAc,CAAC,SAAS,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;CACzF;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,gBAAgB,EAC1B,MAAM,EAAE,cAAc,EACtB,OAAO,EAAE,MAAM,EAAE,EACjB,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC;IAAE,MAAM,EAAE,cAAc,EAAE,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CAAC,CA4EzD"}
|