@tangle-network/agent-eval 0.94.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +44 -30
- package/dist/action-policy.d.ts +24 -0
- package/dist/action-policy.d.ts.map +1 -0
- package/dist/action-policy.test.d.ts +2 -0
- package/dist/action-policy.test.d.ts.map +1 -0
- package/dist/active-learning.d.ts +41 -0
- package/dist/active-learning.d.ts.map +1 -0
- package/dist/adapters/http.d.ts +15 -21
- package/dist/adapters/http.d.ts.map +1 -0
- package/dist/adapters/http.js.map +1 -1
- package/dist/adapters/langchain.d.ts +7 -13
- package/dist/adapters/langchain.d.ts.map +1 -0
- package/dist/adapters/otel.d.ts +13 -24
- package/dist/adapters/otel.d.ts.map +1 -0
- package/dist/agent-profile-cell.d.ts +101 -0
- package/dist/agent-profile-cell.d.ts.map +1 -0
- package/dist/agent-profile.d.ts +27 -0
- package/dist/agent-profile.d.ts.map +1 -0
- package/dist/agent-profile.test.d.ts +2 -0
- package/dist/agent-profile.test.d.ts.map +1 -0
- package/dist/analyst/adapters.d.ts +62 -0
- package/dist/analyst/adapters.d.ts.map +1 -0
- package/dist/analyst/analyst.test.d.ts +2 -0
- package/dist/analyst/analyst.test.d.ts.map +1 -0
- package/dist/analyst/ax-service.d.ts +27 -0
- package/dist/analyst/ax-service.d.ts.map +1 -0
- package/dist/analyst/behavioral-analyst.d.ts +28 -0
- package/dist/analyst/behavioral-analyst.d.ts.map +1 -0
- package/dist/analyst/chat-client.d.ts +91 -0
- package/dist/analyst/chat-client.d.ts.map +1 -0
- package/dist/analyst/default-registry.d.ts +27 -0
- package/dist/analyst/default-registry.d.ts.map +1 -0
- package/dist/analyst/default-registry.test.d.ts +2 -0
- package/dist/analyst/default-registry.test.d.ts.map +1 -0
- package/dist/analyst/finding-signature.d.ts +48 -0
- package/dist/analyst/finding-signature.d.ts.map +1 -0
- package/dist/analyst/finding-subject.d.ts +146 -0
- package/dist/analyst/finding-subject.d.ts.map +1 -0
- package/dist/analyst/finding-subject.test.d.ts +2 -0
- package/dist/analyst/finding-subject.test.d.ts.map +1 -0
- package/dist/analyst/findings-store.d.ts +75 -0
- package/dist/analyst/findings-store.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +28 -256
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +5 -65
- package/dist/analyst/index.js.map +1 -1
- package/dist/{kind-factory-0BhLSI27.d.ts → analyst/kind-factory.d.ts} +11 -63
- package/dist/analyst/kind-factory.d.ts.map +1 -0
- package/dist/analyst/kinds/failure-mode.d.ts +19 -0
- package/dist/analyst/kinds/failure-mode.d.ts.map +1 -0
- package/dist/analyst/kinds/improvement.d.ts +23 -0
- package/dist/analyst/kinds/improvement.d.ts.map +1 -0
- package/dist/analyst/kinds/index.d.ts +22 -0
- package/dist/analyst/kinds/index.d.ts.map +1 -0
- package/dist/analyst/kinds/kinds.test.d.ts +2 -0
- package/dist/analyst/kinds/kinds.test.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts +28 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +22 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.d.ts +84 -0
- package/dist/analyst/kinds/skill-usage.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts +2 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.d.ts +26 -0
- package/dist/analyst/parse-tolerant.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.test.d.ts +2 -0
- package/dist/analyst/parse-tolerant.test.d.ts.map +1 -0
- package/dist/analyst/registry.budget.test.d.ts +2 -0
- package/dist/analyst/registry.budget.test.d.ts.map +1 -0
- package/dist/{default-registry-6dhErQbs.d.ts → analyst/registry.d.ts} +8 -37
- package/dist/analyst/registry.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.d.ts +35 -0
- package/dist/analyst/steer-firewall.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.test.d.ts +2 -0
- package/dist/analyst/steer-firewall.test.d.ts.map +1 -0
- package/dist/analyst/structure-findings.d.ts +37 -0
- package/dist/analyst/structure-findings.d.ts.map +1 -0
- package/dist/analyst/structure-findings.test.d.ts +2 -0
- package/dist/analyst/structure-findings.test.d.ts.map +1 -0
- package/dist/analyst/tool-groups.d.ts +34 -0
- package/dist/analyst/tool-groups.d.ts.map +1 -0
- package/dist/{types-Ce17tDlG.d.ts → analyst/types.d.ts} +19 -112
- package/dist/analyst/types.d.ts.map +1 -0
- package/dist/anti-slop.d.ts +59 -0
- package/dist/anti-slop.d.ts.map +1 -0
- package/dist/artifact-validator.d.ts +74 -0
- package/dist/artifact-validator.d.ts.map +1 -0
- package/dist/attestation.d.ts +63 -0
- package/dist/attestation.d.ts.map +1 -0
- package/dist/attestation.test.d.ts +2 -0
- package/dist/attestation.test.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +15 -16
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.test.d.ts +2 -0
- package/dist/authenticity/index.test.d.ts.map +1 -0
- package/dist/auto-pr.d.ts +120 -0
- package/dist/auto-pr.d.ts.map +1 -0
- package/dist/{baseline-Bbid3WoO.d.ts → baseline.d.ts} +8 -44
- package/dist/baseline.d.ts.map +1 -0
- package/dist/behavior-dsl.d.ts +73 -0
- package/dist/behavior-dsl.d.ts.map +1 -0
- package/dist/belief-state/calibration.d.ts +10 -0
- package/dist/belief-state/calibration.d.ts.map +1 -0
- package/dist/belief-state/calibration.test.d.ts +2 -0
- package/dist/belief-state/calibration.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.d.ts +66 -0
- package/dist/belief-state/code-agent-corpus.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts +2 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.d.ts +22 -0
- package/dist/belief-state/code-agent-evidence.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts +2 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/extract.d.ts +7 -0
- package/dist/belief-state/extract.d.ts.map +1 -0
- package/dist/belief-state/extract.test.d.ts +2 -0
- package/dist/belief-state/extract.test.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +14 -604
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/ope.d.ts +17 -0
- package/dist/belief-state/ope.d.ts.map +1 -0
- package/dist/belief-state/ope.test.d.ts +2 -0
- package/dist/belief-state/ope.test.d.ts.map +1 -0
- package/dist/belief-state/phase0-measurement.d.ts +55 -0
- package/dist/belief-state/phase0-measurement.d.ts.map +1 -0
- package/dist/belief-state/report.d.ts +17 -0
- package/dist/belief-state/report.d.ts.map +1 -0
- package/dist/belief-state/report.test.d.ts +2 -0
- package/dist/belief-state/report.test.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.d.ts +23 -0
- package/dist/belief-state/research-evidence.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.test.d.ts +2 -0
- package/dist/belief-state/research-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +32 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.d.ts +87 -0
- package/dist/belief-state/runtime-hooks.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.test.d.ts +2 -0
- package/dist/belief-state/runtime-hooks.test.d.ts.map +1 -0
- package/dist/belief-state/selective.d.ts +15 -0
- package/dist/belief-state/selective.d.ts.map +1 -0
- package/dist/belief-state/selective.test.d.ts +2 -0
- package/dist/belief-state/selective.test.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.d.ts +80 -0
- package/dist/belief-state/shadow-probe.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.test.d.ts +2 -0
- package/dist/belief-state/shadow-probe.test.d.ts.map +1 -0
- package/dist/belief-state/types.d.ts +195 -0
- package/dist/belief-state/types.d.ts.map +1 -0
- package/dist/belief-state/types.test.d.ts +2 -0
- package/dist/belief-state/types.test.d.ts.map +1 -0
- package/dist/benchmark.d.ts +14 -0
- package/dist/benchmark.d.ts.map +1 -0
- package/dist/benchmarks/index.d.ts +23 -4
- package/dist/benchmarks/index.d.ts.map +1 -0
- package/dist/benchmarks/routing/dataset.d.ts +34 -0
- package/dist/benchmarks/routing/dataset.d.ts.map +1 -0
- package/dist/benchmarks/routing/index.d.ts +34 -0
- package/dist/benchmarks/routing/index.d.ts.map +1 -0
- package/dist/benchmarks/types.d.ts +49 -0
- package/dist/benchmarks/types.d.ts.map +1 -0
- package/dist/bisector.d.ts +81 -0
- package/dist/bisector.d.ts.map +1 -0
- package/dist/budget-guard.d.ts +31 -0
- package/dist/budget-guard.d.ts.map +1 -0
- package/dist/builder-eval/builder-session.d.ts +111 -0
- package/dist/builder-eval/builder-session.d.ts.map +1 -0
- package/dist/builder-eval/correlation.d.ts +32 -0
- package/dist/builder-eval/correlation.d.ts.map +1 -0
- package/dist/builder-eval/index.d.ts +5 -250
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/project-registry.d.ts +51 -0
- package/dist/builder-eval/project-registry.d.ts.map +1 -0
- package/dist/builder-eval/three-layer-eval.d.ts +55 -0
- package/dist/builder-eval/three-layer-eval.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.d.ts +108 -0
- package/dist/campaign/analyst-surface.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.test.d.ts +2 -0
- package/dist/campaign/analyst-surface.test.d.ts.map +1 -0
- package/dist/campaign/auto-pr.d.ts +46 -0
- package/dist/campaign/auto-pr.d.ts.map +1 -0
- package/dist/campaign/distillation/agreement-judge.d.ts +69 -0
- package/dist/campaign/distillation/agreement-judge.d.ts.map +1 -0
- package/dist/campaign/distillation/cli.d.ts +35 -0
- package/dist/campaign/distillation/cli.d.ts.map +1 -0
- package/dist/campaign/distillation/distillation.test.d.ts +2 -0
- package/dist/campaign/distillation/distillation.test.d.ts.map +1 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts +54 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +1 -0
- package/dist/campaign/distillation/run-distillation.d.ts +119 -0
- package/dist/campaign/distillation/run-distillation.d.ts.map +1 -0
- package/dist/campaign/gates/compose.d.ts +12 -0
- package/dist/campaign/gates/compose.d.ts.map +1 -0
- package/dist/campaign/gates/default-production-gate.d.ts +58 -0
- package/dist/campaign/gates/default-production-gate.d.ts.map +1 -0
- package/dist/campaign/gates/heldout-gate.d.ts +12 -0
- package/dist/campaign/gates/heldout-gate.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.d.ts +125 -0
- package/dist/campaign/gates/promotion-policy.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts +2 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.d.ts +146 -0
- package/dist/campaign/gates/sequential.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.test.d.ts +2 -0
- package/dist/campaign/gates/sequential.test.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.d.ts +99 -0
- package/dist/campaign/gates/statistical-heldout.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts +2 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +1 -0
- package/dist/campaign/index.d.ts +38 -1341
- package/dist/campaign/index.d.ts.map +1 -0
- package/dist/campaign/index.js +1863 -1316
- package/dist/campaign/index.js.map +1 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +59 -0
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +1 -0
- package/dist/campaign/presets/compare-proposers.d.ts +146 -0
- package/dist/campaign/presets/compare-proposers.d.ts.map +1 -0
- package/dist/campaign/presets/playback.d.ts +120 -0
- package/dist/campaign/presets/playback.d.ts.map +1 -0
- package/dist/campaign/presets/playback.test.d.ts +2 -0
- package/dist/campaign/presets/playback.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-eval.d.ts +14 -0
- package/dist/campaign/presets/run-eval.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts +63 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +2 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-optimization.d.ts +92 -0
- package/dist/campaign/presets/run-optimization.d.ts.map +1 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts +151 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +1 -0
- package/dist/campaign/presets/run-skill-opt.d.ts +96 -0
- package/dist/campaign/presets/run-skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/_findings-text.d.ts +22 -0
- package/dist/campaign/proposers/_findings-text.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.d.ts +33 -0
- package/dist/campaign/proposers/ace.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.test.d.ts +2 -0
- package/dist/campaign/proposers/ace.test.d.ts.map +1 -0
- package/dist/campaign/proposers/analysis-edit.d.ts +32 -0
- package/dist/campaign/proposers/analysis-edit.d.ts.map +1 -0
- package/dist/campaign/proposers/evolutionary.d.ts +20 -0
- package/dist/campaign/proposers/evolutionary.d.ts.map +1 -0
- package/dist/campaign/proposers/fapo.d.ts +120 -0
- package/dist/campaign/proposers/fapo.d.ts.map +1 -0
- package/dist/campaign/proposers/gepa.d.ts +86 -0
- package/dist/campaign/proposers/gepa.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.d.ts +44 -0
- package/dist/campaign/proposers/halo.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.test.d.ts +2 -0
- package/dist/campaign/proposers/halo.test.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.d.ts +47 -0
- package/dist/campaign/proposers/memory.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.test.d.ts +2 -0
- package/dist/campaign/proposers/memory.test.d.ts.map +1 -0
- package/dist/campaign/proposers/skill-opt.d.ts +88 -0
- package/dist/campaign/proposers/skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.d.ts +48 -0
- package/dist/campaign/proposers/trace-analyst.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts +2 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +1 -0
- package/dist/campaign/provenance.d.ts +185 -0
- package/dist/campaign/provenance.d.ts.map +1 -0
- package/dist/campaign/run-campaign.d.ts +90 -0
- package/dist/campaign/run-campaign.d.ts.map +1 -0
- package/dist/campaign/score-utils.d.ts +26 -0
- package/dist/campaign/score-utils.d.ts.map +1 -0
- package/dist/campaign/skill-patch.d.ts +62 -0
- package/dist/campaign/skill-patch.d.ts.map +1 -0
- package/dist/campaign/storage.d.ts +38 -0
- package/dist/campaign/storage.d.ts.map +1 -0
- package/dist/{types-BU-7W85F.d.ts → campaign/types.d.ts} +98 -99
- package/dist/campaign/types.d.ts.map +1 -0
- package/dist/campaign/worktree/index.d.ts +53 -0
- package/dist/campaign/worktree/index.d.ts.map +1 -0
- package/dist/canary.d.ts +101 -0
- package/dist/canary.d.ts.map +1 -0
- package/dist/causal-attribution.d.ts +45 -0
- package/dist/causal-attribution.d.ts.map +1 -0
- package/dist/{chunk-2K6UUZ7P.js → chunk-2T4EZACH.js} +1 -1
- package/dist/chunk-2T4EZACH.js.map +1 -0
- package/dist/{chunk-CTBHKLEU.js → chunk-77T4STFI.js} +59 -86
- package/dist/chunk-77T4STFI.js.map +1 -0
- package/dist/{chunk-EGPMSBEZ.js → chunk-7QTQKIDD.js} +178 -177
- package/dist/chunk-7QTQKIDD.js.map +1 -0
- package/dist/{chunk-MIFZUPEK.js → chunk-AQ5WQAIV.js} +21 -6
- package/dist/chunk-AQ5WQAIV.js.map +1 -0
- package/dist/chunk-DJWX3GVS.js +81 -0
- package/dist/chunk-DJWX3GVS.js.map +1 -0
- package/dist/{chunk-S6OZEZQK.js → chunk-HMA63UEO.js} +37 -9
- package/dist/{chunk-S6OZEZQK.js.map → chunk-HMA63UEO.js.map} +1 -1
- package/dist/{chunk-TBDR6PAI.js → chunk-IZCEK2HR.js} +2 -2
- package/dist/{chunk-SD2YFWQQ.js → chunk-KKWJD5E6.js} +20 -20
- package/dist/chunk-KKWJD5E6.js.map +1 -0
- package/dist/{chunk-KWRRMR3J.js → chunk-LO6IOIJ2.js} +10 -10
- package/dist/chunk-LO6IOIJ2.js.map +1 -0
- package/dist/{chunk-E4GH6USR.js → chunk-NZEQVRH5.js} +2 -2
- package/dist/chunk-NZEQVRH5.js.map +1 -0
- package/dist/{chunk-MPQWFX6Y.js → chunk-PSWWQXHF.js} +13 -88
- package/dist/chunk-PSWWQXHF.js.map +1 -0
- package/dist/{chunk-Q5LIB7BC.js → chunk-S4SYLDFX.js} +2 -2
- package/dist/chunk-S4SYLDFX.js.map +1 -0
- package/dist/{chunk-KW53MSA5.js → chunk-X74V6ESX.js} +2 -2
- package/dist/{chunk-QMUEXQJS.js → chunk-YBIGNSCZ.js} +81 -4
- package/dist/chunk-YBIGNSCZ.js.map +1 -0
- package/dist/{chunk-2KNZHH3P.js → chunk-Z6L6YSU6.js} +2 -2
- package/dist/ci-gate.d.ts +44 -0
- package/dist/ci-gate.d.ts.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/client.d.ts +77 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.test.d.ts +2 -0
- package/dist/client.test.d.ts.map +1 -0
- package/dist/command-runner.d.ts +74 -0
- package/dist/command-runner.d.ts.map +1 -0
- package/dist/command-runner.test.d.ts +2 -0
- package/dist/command-runner.test.d.ts.map +1 -0
- package/dist/completion-verifier.d.ts +147 -0
- package/dist/completion-verifier.d.ts.map +1 -0
- package/dist/completion-verifier.test.d.ts +9 -0
- package/dist/completion-verifier.test.d.ts.map +1 -0
- package/dist/concurrency.d.ts +23 -0
- package/dist/concurrency.d.ts.map +1 -0
- package/dist/contamination-guard.d.ts +81 -0
- package/dist/contamination-guard.d.ts.map +1 -0
- package/dist/{analyze-runs-B6Ljo_dI.d.ts → contract/analyze-runs.d.ts} +9 -10
- package/dist/contract/analyze-runs.d.ts.map +1 -0
- package/dist/contract/define-agent-eval.d.ts +52 -0
- package/dist/contract/define-agent-eval.d.ts.map +1 -0
- package/dist/contract/diff.d.ts +114 -0
- package/dist/contract/diff.d.ts.map +1 -0
- package/dist/contract/index.d.ts +106 -640
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +127 -17
- package/dist/contract/index.js.map +1 -1
- package/dist/{insight-report-DWl3z9tl.d.ts → contract/insight-report.d.ts} +16 -19
- package/dist/contract/insight-report.d.ts.map +1 -0
- package/dist/contract/insight-types-fwd.d.ts +7 -0
- package/dist/contract/insight-types-fwd.d.ts.map +1 -0
- package/dist/contract/intake/agent-trace.d.ts +97 -0
- package/dist/contract/intake/agent-trace.d.ts.map +1 -0
- package/dist/{code-agent-session-BO8nCnv3.d.ts → contract/intake/code-agent-session.d.ts} +15 -17
- package/dist/contract/intake/code-agent-session.d.ts.map +1 -0
- package/dist/contract/intake/feedback-table.d.ts +87 -0
- package/dist/contract/intake/feedback-table.d.ts.map +1 -0
- package/dist/contract/intake/index.d.ts +22 -0
- package/dist/contract/intake/index.d.ts.map +1 -0
- package/dist/contract/intake/otel-spans.d.ts +33 -0
- package/dist/contract/intake/otel-spans.d.ts.map +1 -0
- package/dist/contract/self-improve.d.ts +284 -0
- package/dist/contract/self-improve.d.ts.map +1 -0
- package/dist/{control-runtime-Acf9CGhw.d.ts → control-runtime.d.ts} +23 -26
- package/dist/control-runtime.d.ts.map +1 -0
- package/dist/control-runtime.test.d.ts +2 -0
- package/dist/control-runtime.test.d.ts.map +1 -0
- package/dist/control.d.ts +11 -9
- package/dist/control.d.ts.map +1 -0
- package/dist/control.js +2 -2
- package/dist/convergence.d.ts +29 -0
- package/dist/convergence.d.ts.map +1 -0
- package/dist/{cost-ledger-DuSqlw5B.d.ts → cost-ledger.d.ts} +10 -11
- package/dist/cost-ledger.d.ts.map +1 -0
- package/dist/cost-ledger.test.d.ts +2 -0
- package/dist/cost-ledger.test.d.ts.map +1 -0
- package/dist/cost-report.d.ts +43 -0
- package/dist/cost-report.d.ts.map +1 -0
- package/dist/cost-report.test.d.ts +2 -0
- package/dist/cost-report.test.d.ts.map +1 -0
- package/dist/cost-tracker.d.ts +76 -0
- package/dist/cost-tracker.d.ts.map +1 -0
- package/dist/{counterfactual-DlOz8PBx.d.ts → counterfactual.d.ts} +12 -13
- package/dist/counterfactual.d.ts.map +1 -0
- package/dist/cross-trace-diff.d.ts +56 -0
- package/dist/cross-trace-diff.d.ts.map +1 -0
- package/dist/{dataset-BbGkaN2I.d.ts → dataset.d.ts} +11 -14
- package/dist/dataset.d.ts.map +1 -0
- package/dist/deploy-gate-layer.d.ts +125 -0
- package/dist/deploy-gate-layer.d.ts.map +1 -0
- package/dist/deploy-gate-layer.test.d.ts +2 -0
- package/dist/deploy-gate-layer.test.d.ts.map +1 -0
- package/dist/description-length-gate.d.ts +119 -0
- package/dist/description-length-gate.d.ts.map +1 -0
- package/dist/detectors/edge.test.d.ts +2 -0
- package/dist/detectors/edge.test.d.ts.map +1 -0
- package/dist/detectors/index.d.ts +81 -0
- package/dist/detectors/index.d.ts.map +1 -0
- package/dist/detectors/index.test.d.ts +2 -0
- package/dist/detectors/index.test.d.ts.map +1 -0
- package/dist/diagnose/causal-sweep.d.ts +100 -0
- package/dist/diagnose/causal-sweep.d.ts.map +1 -0
- package/dist/diagnose/index.d.ts +36 -0
- package/dist/diagnose/index.d.ts.map +1 -0
- package/dist/diagnose/remediation.d.ts +68 -0
- package/dist/diagnose/remediation.d.ts.map +1 -0
- package/dist/diagnose/repair.d.ts +77 -0
- package/dist/diagnose/repair.d.ts.map +1 -0
- package/dist/diagnose.d.ts +1 -251
- package/dist/diagnose.js +1 -1
- package/dist/discover-personas.d.ts +35 -0
- package/dist/discover-personas.d.ts.map +1 -0
- package/dist/driver.d.ts +95 -0
- package/dist/driver.d.ts.map +1 -0
- package/dist/driver.test.d.ts +8 -0
- package/dist/driver.test.d.ts.map +1 -0
- package/dist/dual-agent-bench.d.ts +81 -0
- package/dist/dual-agent-bench.d.ts.map +1 -0
- package/dist/error-count-extractor.d.ts +47 -0
- package/dist/error-count-extractor.d.ts.map +1 -0
- package/dist/error-count-extractor.test.d.ts +2 -0
- package/dist/error-count-extractor.test.d.ts.map +1 -0
- package/dist/{errors-CzMUYo7b.d.ts → errors.d.ts} +10 -11
- package/dist/errors.d.ts.map +1 -0
- package/dist/{researcher-B0C2_fVO.d.ts → eval-campaign.d.ts} +34 -156
- package/dist/eval-campaign.d.ts.map +1 -0
- package/dist/eval-campaign.test.d.ts +2 -0
- package/dist/eval-campaign.test.d.ts.map +1 -0
- package/dist/eval-tools.d.ts +55 -0
- package/dist/eval-tools.d.ts.map +1 -0
- package/dist/eval-trace-store.d.ts +107 -0
- package/dist/eval-trace-store.d.ts.map +1 -0
- package/dist/eval-trace-store.test.d.ts +2 -0
- package/dist/eval-trace-store.test.d.ts.map +1 -0
- package/dist/executor.d.ts +38 -0
- package/dist/executor.d.ts.map +1 -0
- package/dist/executor.test.d.ts +10 -0
- package/dist/executor.test.d.ts.map +1 -0
- package/dist/experiment-tracker.d.ts +178 -0
- package/dist/experiment-tracker.d.ts.map +1 -0
- package/dist/experiment-tracker.test.d.ts +2 -0
- package/dist/experiment-tracker.test.d.ts.map +1 -0
- package/dist/failure-taxonomy.d.ts +38 -0
- package/dist/failure-taxonomy.d.ts.map +1 -0
- package/dist/{feedback-trajectory-BxY0cKfs.d.ts → feedback-trajectory.d.ts} +36 -38
- package/dist/feedback-trajectory.d.ts.map +1 -0
- package/dist/feedback-trajectory.test.d.ts +2 -0
- package/dist/feedback-trajectory.test.d.ts.map +1 -0
- package/dist/flow-layer.d.ts +90 -0
- package/dist/flow-layer.d.ts.map +1 -0
- package/dist/flow-layer.test.d.ts +2 -0
- package/dist/flow-layer.test.d.ts.map +1 -0
- package/dist/fuzz/capsule.d.ts +46 -0
- package/dist/fuzz/capsule.d.ts.map +1 -0
- package/dist/fuzz/cube.d.ts +36 -0
- package/dist/fuzz/cube.d.ts.map +1 -0
- package/dist/fuzz/explorer-cost.test.d.ts +2 -0
- package/dist/fuzz/explorer-cost.test.d.ts.map +1 -0
- package/dist/fuzz/explorer.d.ts +64 -0
- package/dist/fuzz/explorer.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.d.ts +16 -0
- package/dist/fuzz/fuzz-agent.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.test.d.ts +2 -0
- package/dist/fuzz/fuzz-agent.test.d.ts.map +1 -0
- package/dist/fuzz/gates.d.ts +33 -0
- package/dist/fuzz/gates.d.ts.map +1 -0
- package/dist/fuzz/index.d.ts +26 -0
- package/dist/fuzz/index.d.ts.map +1 -0
- package/dist/fuzz/policies.d.ts +28 -0
- package/dist/fuzz/policies.d.ts.map +1 -0
- package/dist/fuzz/tools.d.ts +20 -0
- package/dist/fuzz/tools.d.ts.map +1 -0
- package/dist/fuzz/types.d.ts +307 -0
- package/dist/fuzz/types.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -547
- package/dist/golden-matcher.d.ts +71 -0
- package/dist/golden-matcher.d.ts.map +1 -0
- package/dist/governance/eu-ai-act.d.ts +37 -0
- package/dist/governance/eu-ai-act.d.ts.map +1 -0
- package/dist/governance/index.d.ts +5 -135
- package/dist/governance/index.d.ts.map +1 -0
- package/dist/governance/nist-ai-rmf.d.ts +15 -0
- package/dist/governance/nist-ai-rmf.d.ts.map +1 -0
- package/dist/governance/soc2.d.ts +12 -0
- package/dist/governance/soc2.d.ts.map +1 -0
- package/dist/governance/types.d.ts +66 -0
- package/dist/governance/types.d.ts.map +1 -0
- package/dist/harness-optimizer.d.ts +82 -0
- package/dist/harness-optimizer.d.ts.map +1 -0
- package/dist/held-out-gate.d.ts +135 -0
- package/dist/held-out-gate.d.ts.map +1 -0
- package/dist/hosted/client.d.ts +73 -0
- package/dist/hosted/client.d.ts.map +1 -0
- package/dist/hosted/from-env.test.d.ts +8 -0
- package/dist/hosted/from-env.test.d.ts.map +1 -0
- package/dist/hosted/index.d.ts +10 -238
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/types.d.ts +159 -0
- package/dist/hosted/types.d.ts.map +1 -0
- package/dist/index.d.ts +277 -5665
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +66 -31
- package/dist/index.js.map +1 -1
- package/dist/integrity/backend-integrity.d.ts +71 -0
- package/dist/integrity/backend-integrity.d.ts.map +1 -0
- package/dist/integrity/preflight.d.ts +72 -0
- package/dist/integrity/preflight.d.ts.map +1 -0
- package/dist/integrity/preflight.test.d.ts +2 -0
- package/dist/integrity/preflight.test.d.ts.map +1 -0
- package/dist/integrity/single-backend.d.ts +67 -0
- package/dist/integrity/single-backend.d.ts.map +1 -0
- package/dist/intent-match-judge.d.ts +69 -0
- package/dist/intent-match-judge.d.ts.map +1 -0
- package/dist/intent-match-judge.test.d.ts +2 -0
- package/dist/intent-match-judge.test.d.ts.map +1 -0
- package/dist/{judge-calibration-0p2QcWNE.d.ts → judge-calibration.d.ts} +16 -17
- package/dist/judge-calibration.d.ts.map +1 -0
- package/dist/judge-ensemble.d.ts +66 -0
- package/dist/judge-ensemble.d.ts.map +1 -0
- package/dist/judge-ensemble.test.d.ts +8 -0
- package/dist/judge-ensemble.test.d.ts.map +1 -0
- package/dist/judge-families.d.ts +38 -0
- package/dist/judge-families.d.ts.map +1 -0
- package/dist/judge-panel.d.ts +65 -0
- package/dist/judge-panel.d.ts.map +1 -0
- package/dist/judge-retry.d.ts +70 -0
- package/dist/judge-retry.d.ts.map +1 -0
- package/dist/judge-runner.d.ts +36 -0
- package/dist/judge-runner.d.ts.map +1 -0
- package/dist/judge-runner.test.d.ts +2 -0
- package/dist/judge-runner.test.d.ts.map +1 -0
- package/dist/judges.d.ts +74 -0
- package/dist/judges.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.d.ts +89 -0
- package/dist/keyword-coverage-judge.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.test.d.ts +2 -0
- package/dist/keyword-coverage-judge.test.d.ts.map +1 -0
- package/dist/knowledge/index.d.ts +3 -103
- package/dist/knowledge/index.d.ts.map +1 -0
- package/dist/knowledge/readiness.d.ts +26 -0
- package/dist/knowledge/readiness.d.ts.map +1 -0
- package/dist/knowledge/types.d.ts +75 -0
- package/dist/knowledge/types.d.ts.map +1 -0
- package/dist/live-proof.d.ts +62 -0
- package/dist/live-proof.d.ts.map +1 -0
- package/dist/{llm-client-Bj7g0rqu.d.ts → llm-client.d.ts} +21 -23
- package/dist/llm-client.d.ts.map +1 -0
- package/dist/llm-client.test.d.ts +2 -0
- package/dist/llm-client.test.d.ts.map +1 -0
- package/dist/locked-jsonl-appender.d.ts +19 -0
- package/dist/locked-jsonl-appender.d.ts.map +1 -0
- package/dist/matrix/aggregation.d.ts +16 -0
- package/dist/matrix/aggregation.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +12 -30
- package/dist/matrix/index.d.ts.map +1 -0
- package/dist/matrix/runner.d.ts +15 -0
- package/dist/matrix/runner.d.ts.map +1 -0
- package/dist/{types-mn5Aqk7x.d.ts → matrix/types.d.ts} +11 -15
- package/dist/matrix/types.d.ts.map +1 -0
- package/dist/meta-eval/calibration.d.ts +47 -0
- package/dist/meta-eval/calibration.d.ts.map +1 -0
- package/dist/meta-eval/correlation-study.d.ts +53 -0
- package/dist/meta-eval/correlation-study.d.ts.map +1 -0
- package/dist/meta-eval/index.d.ts +6 -181
- package/dist/meta-eval/index.d.ts.map +1 -0
- package/dist/{outcome-store-rnXLEqSn.d.ts → meta-eval/outcome-store.d.ts} +7 -8
- package/dist/meta-eval/outcome-store.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → meta-eval/rubric-predictive-validity.d.ts} +8 -11
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +1 -0
- package/dist/meta-eval/sentinel.d.ts +169 -0
- package/dist/meta-eval/sentinel.d.ts.map +1 -0
- package/dist/metrics.d.ts +63 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/model-seats.d.ts +71 -0
- package/dist/model-seats.d.ts.map +1 -0
- package/dist/model-seats.test.d.ts +2 -0
- package/dist/model-seats.test.d.ts.map +1 -0
- package/dist/muffled-gate-scanner.d.ts +102 -0
- package/dist/muffled-gate-scanner.d.ts.map +1 -0
- package/dist/{multi-layer-verifier-DUZXrPDA.d.ts → multi-layer-verifier.d.ts} +12 -15
- package/dist/multi-layer-verifier.d.ts.map +1 -0
- package/dist/multi-layer-verifier.test.d.ts +2 -0
- package/dist/multi-layer-verifier.test.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.d.ts +80 -0
- package/dist/multi-toolchain-layer.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.test.d.ts +2 -0
- package/dist/multi-toolchain-layer.test.d.ts.map +1 -0
- package/dist/multishot/default-tools.d.ts +34 -0
- package/dist/multishot/default-tools.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +7 -290
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js.map +1 -1
- package/dist/multishot/judges.d.ts +32 -0
- package/dist/multishot/judges.d.ts.map +1 -0
- package/dist/multishot/matrix.d.ts +107 -0
- package/dist/multishot/matrix.d.ts.map +1 -0
- package/dist/multishot/multishot.d.ts +23 -0
- package/dist/multishot/multishot.d.ts.map +1 -0
- package/dist/multishot/router.d.ts +37 -0
- package/dist/multishot/router.d.ts.map +1 -0
- package/dist/multishot/types.d.ts +60 -0
- package/dist/multishot/types.d.ts.map +1 -0
- package/dist/observability.d.ts +71 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/oracle.d.ts +55 -0
- package/dist/oracle.d.ts.map +1 -0
- package/dist/orthogonality.d.ts +35 -0
- package/dist/orthogonality.d.ts.map +1 -0
- package/dist/otel-pipeline.d.ts +31 -0
- package/dist/otel-pipeline.d.ts.map +1 -0
- package/dist/paraphrase.d.ts +107 -0
- package/dist/paraphrase.d.ts.map +1 -0
- package/dist/{pareto-E-pembql.d.ts → pareto.d.ts} +9 -10
- package/dist/pareto.d.ts.map +1 -0
- package/dist/partition-held-out.d.ts +70 -0
- package/dist/partition-held-out.d.ts.map +1 -0
- package/dist/partition-held-out.test.d.ts +2 -0
- package/dist/partition-held-out.test.d.ts.map +1 -0
- package/dist/perf/index.d.ts +13 -119
- package/dist/perf/index.d.ts.map +1 -0
- package/dist/perf/integrity.d.ts +30 -0
- package/dist/perf/integrity.d.ts.map +1 -0
- package/dist/perf/journey.d.ts +45 -0
- package/dist/perf/journey.d.ts.map +1 -0
- package/dist/perf/ratchet.d.ts +47 -0
- package/dist/perf/ratchet.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.d.ts +31 -0
- package/dist/pipelines/budget-breach.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.test.d.ts +2 -0
- package/dist/pipelines/budget-breach.test.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.d.ts +38 -0
- package/dist/pipelines/failure-cluster.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.test.d.ts +2 -0
- package/dist/pipelines/failure-cluster.test.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.d.ts +26 -0
- package/dist/pipelines/first-divergence.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.test.d.ts +2 -0
- package/dist/pipelines/first-divergence.test.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +8 -173
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/judge-agreement.d.ts +26 -0
- package/dist/pipelines/judge-agreement.d.ts.map +1 -0
- package/dist/pipelines/judge-agreement.test.d.ts +2 -0
- package/dist/pipelines/judge-agreement.test.d.ts.map +1 -0
- package/dist/pipelines/regression.d.ts +23 -0
- package/dist/pipelines/regression.d.ts.map +1 -0
- package/dist/pipelines/regression.test.d.ts +2 -0
- package/dist/pipelines/regression.test.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.d.ts +32 -0
- package/dist/pipelines/stuck-loop.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.test.d.ts +2 -0
- package/dist/pipelines/stuck-loop.test.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.d.ts +34 -0
- package/dist/pipelines/tool-waste.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.test.d.ts +2 -0
- package/dist/pipelines/tool-waste.test.d.ts.map +1 -0
- package/dist/playbook.d.ts +16 -0
- package/dist/playbook.d.ts.map +1 -0
- package/dist/pr-review-benchmark.d.ts +88 -0
- package/dist/pr-review-benchmark.d.ts.map +1 -0
- package/dist/pr-review-benchmark.test.d.ts +2 -0
- package/dist/pr-review-benchmark.test.d.ts.map +1 -0
- package/dist/pre-registration.d.ts +125 -0
- package/dist/pre-registration.d.ts.map +1 -0
- package/dist/prm/builtin-rubrics.d.ts +33 -0
- package/dist/prm/builtin-rubrics.d.ts.map +1 -0
- package/dist/prm/index.d.ts +5 -104
- package/dist/prm/index.d.ts.map +1 -0
- package/dist/prm/inference.d.ts +29 -0
- package/dist/prm/inference.d.ts.map +1 -0
- package/dist/prm/inference.test.d.ts +2 -0
- package/dist/prm/inference.test.d.ts.map +1 -0
- package/dist/{rubric-Cc6UHvUb.d.ts → prm/rubric.d.ts} +10 -13
- package/dist/prm/rubric.d.ts.map +1 -0
- package/dist/prm/training-export.d.ts +38 -0
- package/dist/prm/training-export.d.ts.map +1 -0
- package/dist/produced-state.d.ts +63 -0
- package/dist/produced-state.d.ts.map +1 -0
- package/dist/produced-state.test.d.ts +8 -0
- package/dist/produced-state.test.d.ts.map +1 -0
- package/dist/profile/baselines.d.ts +37 -0
- package/dist/profile/baselines.d.ts.map +1 -0
- package/dist/profile/index.d.ts +105 -0
- package/dist/profile/index.d.ts.map +1 -0
- package/dist/promotion-gate.d.ts +93 -0
- package/dist/promotion-gate.d.ts.map +1 -0
- package/dist/prompt-registry.d.ts +41 -0
- package/dist/prompt-registry.d.ts.map +1 -0
- package/dist/propose-review-control.d.ts +49 -0
- package/dist/propose-review-control.d.ts.map +1 -0
- package/dist/propose-review-control.test.d.ts +2 -0
- package/dist/propose-review-control.test.d.ts.map +1 -0
- package/dist/propose-review.d.ts +155 -0
- package/dist/propose-review.d.ts.map +1 -0
- package/dist/{red-team-BWdoyleI.d.ts → red-team.d.ts} +14 -16
- package/dist/red-team.d.ts.map +1 -0
- package/dist/reference-replay-steering.d.ts +11 -0
- package/dist/reference-replay-steering.d.ts.map +1 -0
- package/dist/reference-replay.d.ts +176 -0
- package/dist/reference-replay.d.ts.map +1 -0
- package/dist/reflective-mutation.d.ts +79 -0
- package/dist/reflective-mutation.d.ts.map +1 -0
- package/dist/registry.d.ts +31 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/release-confidence.d.ts +128 -0
- package/dist/release-confidence.d.ts.map +1 -0
- package/dist/release-report.d.ts +11 -0
- package/dist/release-report.d.ts.map +1 -0
- package/dist/replay.d.ts +120 -0
- package/dist/replay.d.ts.map +1 -0
- package/dist/reporter.d.ts +14 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporting.d.ts +15 -15
- package/dist/reporting.d.ts.map +1 -0
- package/dist/researcher.d.ts +140 -0
- package/dist/researcher.d.ts.map +1 -0
- package/dist/reviewer.d.ts +118 -0
- package/dist/reviewer.d.ts.map +1 -0
- package/dist/reviewer.test.d.ts +2 -0
- package/dist/reviewer.test.d.ts.map +1 -0
- package/dist/reward-model-export.d.ts +60 -0
- package/dist/reward-model-export.d.ts.map +1 -0
- package/dist/rl/active-curriculum.d.ts +110 -0
- package/dist/rl/active-curriculum.d.ts.map +1 -0
- package/dist/rl/adaptation-eval.d.ts +109 -0
- package/dist/rl/adaptation-eval.d.ts.map +1 -0
- package/dist/{adversarial-DIVcDoI_.d.ts → rl/adversarial.d.ts} +6 -7
- package/dist/rl/adversarial.d.ts.map +1 -0
- package/dist/rl/compute-curves.d.ts +127 -0
- package/dist/rl/compute-curves.d.ts.map +1 -0
- package/dist/rl/contamination.d.ts +117 -0
- package/dist/rl/contamination.d.ts.map +1 -0
- package/dist/rl/corpus.d.ts +55 -0
- package/dist/rl/corpus.d.ts.map +1 -0
- package/dist/rl/corpus.test.d.ts +2 -0
- package/dist/rl/corpus.test.d.ts.map +1 -0
- package/dist/rl/dataset.d.ts +102 -0
- package/dist/rl/dataset.d.ts.map +1 -0
- package/dist/rl/dataset.test.d.ts +2 -0
- package/dist/rl/dataset.test.d.ts.map +1 -0
- package/dist/rl/exporters.d.ts +141 -0
- package/dist/rl/exporters.d.ts.map +1 -0
- package/dist/rl/index.d.ts +49 -0
- package/dist/rl/index.d.ts.map +1 -0
- package/dist/{off-policy-DiwuKKg7.d.ts → rl/off-policy.d.ts} +8 -9
- package/dist/rl/off-policy.d.ts.map +1 -0
- package/dist/rl/predictive-validity-researcher.d.ts +69 -0
- package/dist/rl/predictive-validity-researcher.d.ts.map +1 -0
- package/dist/rl/preferences.d.ts +141 -0
- package/dist/rl/preferences.d.ts.map +1 -0
- package/dist/rl/process-reward.d.ts +122 -0
- package/dist/rl/process-reward.d.ts.map +1 -0
- package/dist/rl/reward-hacking.d.ts +104 -0
- package/dist/rl/reward-hacking.d.ts.map +1 -0
- package/dist/rl/rl-campaign.d.ts +85 -0
- package/dist/rl/rl-campaign.d.ts.map +1 -0
- package/dist/rl/run-record-adapters.d.ts +56 -0
- package/dist/rl/run-record-adapters.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.d.ts +166 -0
- package/dist/rl/sim-fidelity.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.test.d.ts +2 -0
- package/dist/rl/sim-fidelity.test.d.ts.map +1 -0
- package/dist/rl/tournament.d.ts +115 -0
- package/dist/rl/tournament.d.ts.map +1 -0
- package/dist/rl/verifiable-reward.d.ts +124 -0
- package/dist/rl/verifiable-reward.d.ts.map +1 -0
- package/dist/rl.d.ts +1 -1192
- package/dist/rl.js +612 -612
- package/dist/rl.js.map +1 -1
- package/dist/{run-campaign-7WNXMDSN.js → run-campaign-WXY7KI67.js} +2 -2
- package/dist/run-critic.d.ts +23 -0
- package/dist/run-critic.d.ts.map +1 -0
- package/dist/run-evidence.d.ts +32 -0
- package/dist/run-evidence.d.ts.map +1 -0
- package/dist/{run-record-e7vj1uZQ.d.ts → run-record.d.ts} +16 -122
- package/dist/run-record.d.ts.map +1 -0
- package/dist/run-record.test.d.ts +2 -0
- package/dist/run-record.test.d.ts.map +1 -0
- package/dist/run-score.d.ts +31 -0
- package/dist/run-score.d.ts.map +1 -0
- package/dist/runtime-trajectory.d.ts +47 -0
- package/dist/runtime-trajectory.d.ts.map +1 -0
- package/dist/{test-graded-scenario-DeODGLra.d.ts → sandbox-harness.d.ts} +15 -60
- package/dist/sandbox-harness.d.ts.map +1 -0
- package/dist/sandbox-harness.test.d.ts +2 -0
- package/dist/sandbox-harness.test.d.ts.map +1 -0
- package/dist/sandbox-pool.d.ts +74 -0
- package/dist/sandbox-pool.d.ts.map +1 -0
- package/dist/sandbox-pool.test.d.ts +2 -0
- package/dist/sandbox-pool.test.d.ts.map +1 -0
- package/dist/scorecard.d.ts +133 -0
- package/dist/scorecard.d.ts.map +1 -0
- package/dist/scorecard.test.d.ts +2 -0
- package/dist/scorecard.test.d.ts.map +1 -0
- package/dist/self-play.d.ts +69 -0
- package/dist/self-play.d.ts.map +1 -0
- package/dist/semantic-concept-judge.d.ts +135 -0
- package/dist/semantic-concept-judge.d.ts.map +1 -0
- package/dist/semantic-concept-judge.test.d.ts +2 -0
- package/dist/semantic-concept-judge.test.d.ts.map +1 -0
- package/dist/{sequential-5iSVfzl2.d.ts → sequential.d.ts} +9 -10
- package/dist/sequential.d.ts.map +1 -0
- package/dist/{series-convergence-D5OWMBg6.d.ts → series-convergence.d.ts} +4 -5
- package/dist/series-convergence.d.ts.map +1 -0
- package/dist/slo.d.ts +48 -0
- package/dist/slo.d.ts.map +1 -0
- package/dist/state-continuity.d.ts +47 -0
- package/dist/state-continuity.d.ts.map +1 -0
- package/dist/{statistics-CCJpTGOS.d.ts → statistics.d.ts} +46 -48
- package/dist/statistics.d.ts.map +1 -0
- package/dist/statistics.test.d.ts +2 -0
- package/dist/statistics.test.d.ts.map +1 -0
- package/dist/steering-optimizer.d.ts +58 -0
- package/dist/steering-optimizer.d.ts.map +1 -0
- package/dist/steering.d.ts +24 -0
- package/dist/steering.d.ts.map +1 -0
- package/dist/storyboard/code-edit.d.ts +64 -0
- package/dist/storyboard/code-edit.d.ts.map +1 -0
- package/dist/storyboard/code-edit.test.d.ts +2 -0
- package/dist/storyboard/code-edit.test.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +16 -81
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.test.d.ts +2 -0
- package/dist/storyboard/index.test.d.ts.map +1 -0
- package/dist/{summary-report-BDOFevaT.d.ts → summary-report.d.ts} +23 -160
- package/dist/summary-report.d.ts.map +1 -0
- package/dist/telemetry/client.d.ts +35 -0
- package/dist/telemetry/client.d.ts.map +1 -0
- package/dist/telemetry/index.d.ts +17 -35
- package/dist/telemetry/index.d.ts.map +1 -0
- package/dist/telemetry/schema.d.ts +61 -0
- package/dist/telemetry/schema.d.ts.map +1 -0
- package/dist/telemetry/sink-fetch.d.ts +39 -0
- package/dist/telemetry/sink-fetch.d.ts.map +1 -0
- package/dist/telemetry/{file.d.ts → sink-file.d.ts} +5 -7
- package/dist/telemetry/sink-file.d.ts.map +1 -0
- package/dist/test-graded-scenario.d.ts +42 -0
- package/dist/test-graded-scenario.d.ts.map +1 -0
- package/dist/testing.d.ts +5 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +8 -0
- package/dist/testing.js.map +1 -0
- package/dist/tool-use-metrics.d.ts +35 -0
- package/dist/tool-use-metrics.d.ts.map +1 -0
- package/dist/trace/capture-fetch.d.ts +48 -0
- package/dist/trace/capture-fetch.d.ts.map +1 -0
- package/dist/trace/capture-fetch.test.d.ts +2 -0
- package/dist/trace/capture-fetch.test.d.ts.map +1 -0
- package/dist/{emitter-C2rqGH_l.d.ts → trace/emitter.d.ts} +9 -12
- package/dist/trace/emitter.d.ts.map +1 -0
- package/dist/trace/extract-usage.d.ts +46 -0
- package/dist/trace/extract-usage.d.ts.map +1 -0
- package/dist/trace/extract-usage.test.d.ts +2 -0
- package/dist/trace/extract-usage.test.d.ts.map +1 -0
- package/dist/trace/index.d.ts +15 -0
- package/dist/trace/index.d.ts.map +1 -0
- package/dist/{integrity-D2t12mMw.d.ts → trace/integrity.d.ts} +11 -14
- package/dist/trace/integrity.d.ts.map +1 -0
- package/dist/trace/otel-bridge.d.ts +29 -0
- package/dist/trace/otel-bridge.d.ts.map +1 -0
- package/dist/trace/otel-export.d.ts +52 -0
- package/dist/trace/otel-export.d.ts.map +1 -0
- package/dist/trace/otel.d.ts +57 -0
- package/dist/trace/otel.d.ts.map +1 -0
- package/dist/trace/otlp-attributes.d.ts +17 -0
- package/dist/trace/otlp-attributes.d.ts.map +1 -0
- package/dist/trace/query.d.ts +29 -0
- package/dist/trace/query.d.ts.map +1 -0
- package/dist/trace/query.test.d.ts +2 -0
- package/dist/trace/query.test.d.ts.map +1 -0
- package/dist/{raw-provider-sink-C46HDghv.d.ts → trace/raw-provider-sink.d.ts} +13 -14
- package/dist/trace/raw-provider-sink.d.ts.map +1 -0
- package/dist/{redact-B40YG2M_.d.ts → trace/redact.d.ts} +7 -8
- package/dist/trace/redact.d.ts.map +1 -0
- package/dist/{schema-m0gsnbt3.d.ts → trace/schema.d.ts} +29 -30
- package/dist/trace/schema.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.d.ts +72 -0
- package/dist/trace/store-to-otlp.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.test.d.ts +2 -0
- package/dist/trace/store-to-otlp.test.d.ts.map +1 -0
- package/dist/{store-BcFXE6LG.d.ts → trace/store.d.ts} +21 -12
- package/dist/trace/store.d.ts.map +1 -0
- package/dist/trace/store.test.d.ts +2 -0
- package/dist/trace/store.test.d.ts.map +1 -0
- package/dist/{analyst-C8HHvfJp.d.ts → trace-analyst/analyst.d.ts} +12 -14
- package/dist/trace-analyst/analyst.d.ts.map +1 -0
- package/dist/trace-analyst/analyst.test.d.ts +2 -0
- package/dist/trace-analyst/analyst.test.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts +40 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +2 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +1 -0
- package/dist/trace-analyst/hook.d.ts +55 -0
- package/dist/trace-analyst/hook.d.ts.map +1 -0
- package/dist/trace-analyst/index.d.ts +18 -0
- package/dist/trace-analyst/index.d.ts.map +1 -0
- package/dist/trace-analyst/insights.d.ts +71 -0
- package/dist/trace-analyst/insights.d.ts.map +1 -0
- package/dist/trace-analyst/insights.test.d.ts +2 -0
- package/dist/trace-analyst/insights.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-flatten.d.ts +42 -0
- package/dist/trace-analyst/otlp-flatten.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.d.ts +85 -0
- package/dist/trace-analyst/otlp-span.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-span.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts +115 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/prompts.d.ts +6 -0
- package/dist/trace-analyst/prompts.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.d.ts +126 -0
- package/dist/trace-analyst/store-otlp.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.test.d.ts +8 -0
- package/dist/trace-analyst/store-otlp.test.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/store.d.ts +63 -0
- package/dist/trace-analyst/store.d.ts.map +1 -0
- package/dist/trace-analyst/tools.d.ts +44 -0
- package/dist/trace-analyst/tools.d.ts.map +1 -0
- package/dist/trace-analyst/tools.test.d.ts +10 -0
- package/dist/trace-analyst/tools.test.d.ts.map +1 -0
- package/dist/{store-C1YxJDEK.d.ts → trace-analyst/types.d.ts} +18 -74
- package/dist/trace-analyst/types.d.ts.map +1 -0
- package/dist/trace-contracts.d.ts +180 -0
- package/dist/trace-contracts.d.ts.map +1 -0
- package/dist/traced-analyst.d.ts +26 -0
- package/dist/traced-analyst.d.ts.map +1 -0
- package/dist/traced-judges.d.ts +27 -0
- package/dist/traced-judges.d.ts.map +1 -0
- package/dist/traces.d.ts +4 -960
- package/dist/traces.d.ts.map +1 -0
- package/dist/traces.js +41 -11
- package/dist/{trajectory-2TkpSEVh.d.ts → trajectory.d.ts} +8 -9
- package/dist/trajectory.d.ts.map +1 -0
- package/dist/{types-C7DGg5ex.d.ts → types.d.ts} +31 -33
- package/dist/types.d.ts.map +1 -0
- package/dist/ui-finding.d.ts +104 -0
- package/dist/ui-finding.d.ts.map +1 -0
- package/dist/verdict-cache.d.ts +78 -0
- package/dist/verdict-cache.d.ts.map +1 -0
- package/dist/verdict-cache.test.d.ts +2 -0
- package/dist/verdict-cache.test.d.ts.map +1 -0
- package/dist/{verdict-C9MlYujm.d.ts → verdict.d.ts} +2 -3
- package/dist/verdict.d.ts.map +1 -0
- package/dist/visual-diff.d.ts +32 -0
- package/dist/visual-diff.d.ts.map +1 -0
- package/dist/wire/handlers.d.ts +54 -0
- package/dist/wire/handlers.d.ts.map +1 -0
- package/dist/wire/index.d.ts +13 -570
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/openapi.d.ts +3 -0
- package/dist/wire/openapi.d.ts.map +1 -0
- package/dist/wire/rpc.d.ts +21 -0
- package/dist/wire/rpc.d.ts.map +1 -0
- package/dist/wire/rubrics.d.ts +34 -0
- package/dist/wire/rubrics.d.ts.map +1 -0
- package/dist/wire/schemas.d.ts +410 -0
- package/dist/wire/schemas.d.ts.map +1 -0
- package/dist/wire/server.d.ts +60 -0
- package/dist/wire/server.d.ts.map +1 -0
- package/dist/workflow/event-schema.d.ts +5 -0
- package/dist/workflow/event-schema.d.ts.map +1 -0
- package/dist/workflow/feedback-pack.d.ts +99 -0
- package/dist/workflow/feedback-pack.d.ts.map +1 -0
- package/dist/workflow/index.d.ts +22 -495
- package/dist/workflow/index.d.ts.map +1 -0
- package/dist/workflow/index.js +1 -1
- package/dist/workflow/intelligence-export.d.ts +62 -0
- package/dist/workflow/intelligence-export.d.ts.map +1 -0
- package/dist/workflow/partner-report.d.ts +49 -0
- package/dist/workflow/partner-report.d.ts.map +1 -0
- package/dist/workflow/phase-graph.d.ts +43 -0
- package/dist/workflow/phase-graph.d.ts.map +1 -0
- package/dist/workflow/promotion-gate.d.ts +61 -0
- package/dist/workflow/promotion-gate.d.ts.map +1 -0
- package/dist/workflow/run-record.d.ts +12 -0
- package/dist/workflow/run-record.d.ts.map +1 -0
- package/dist/workflow/runtime-adapter.d.ts +20 -0
- package/dist/workflow/runtime-adapter.d.ts.map +1 -0
- package/dist/workflow/sanitize.d.ts +21 -0
- package/dist/workflow/sanitize.d.ts.map +1 -0
- package/dist/workflow/schema.d.ts +5 -0
- package/dist/workflow/schema.d.ts.map +1 -0
- package/dist/workflow/summary.d.ts +43 -0
- package/dist/workflow/summary.d.ts.map +1 -0
- package/dist/workflow/trace-event-fields.d.ts +6 -0
- package/dist/workflow/trace-event-fields.d.ts.map +1 -0
- package/dist/workflow/trajectory.d.ts +15 -0
- package/dist/workflow/trajectory.d.ts.map +1 -0
- package/dist/workflow/types.d.ts +68 -0
- package/dist/workflow/types.d.ts.map +1 -0
- package/dist/workspace-inspector.d.ts +67 -0
- package/dist/workspace-inspector.d.ts.map +1 -0
- package/dist/wrangler-deploy-runner.test.d.ts +2 -0
- package/dist/wrangler-deploy-runner.test.d.ts.map +1 -0
- package/docs/campaign-proposers.md +170 -0
- package/docs/concepts.md +8 -4
- package/docs/customer-journeys.md +15 -13
- package/docs/design/loop-taxonomy.md +34 -66
- package/docs/distributed-driver.md +14 -14
- package/docs/feature-guide.md +1 -1
- package/docs/hosted-ingest-spec.md +2 -3
- package/docs/multi-shot-optimization.md +8 -8
- package/docs/product-eval-adoption.md +1 -1
- package/docs/self-improvement-map.md +33 -29
- package/package.json +9 -15
- package/dist/calibration-BPmzuVPk.d.ts +0 -101
- package/dist/chunk-2K6UUZ7P.js.map +0 -1
- package/dist/chunk-CTBHKLEU.js.map +0 -1
- package/dist/chunk-E4GH6USR.js.map +0 -1
- package/dist/chunk-EGPMSBEZ.js.map +0 -1
- package/dist/chunk-KWRRMR3J.js.map +0 -1
- package/dist/chunk-MIFZUPEK.js.map +0 -1
- package/dist/chunk-MPQWFX6Y.js.map +0 -1
- package/dist/chunk-Q5LIB7BC.js.map +0 -1
- package/dist/chunk-QMUEXQJS.js.map +0 -1
- package/dist/chunk-SD2YFWQQ.js.map +0 -1
- package/dist/control-D6qwHXIR.d.ts +0 -259
- package/dist/corpus-B8A4BDR3.d.ts +0 -560
- package/dist/failure-cluster-DH9Flgcf.d.ts +0 -76
- package/dist/harness-optimizer-mOl9XX_O.d.ts +0 -106
- package/dist/index-Bx3gZ8xl.d.ts +0 -159
- package/dist/pre-registration-mAnCugl9.d.ts +0 -523
- package/dist/provenance-P-bCL2Fo.d.ts +0 -441
- package/dist/query-B7GGjRox.d.ts +0 -32
- package/dist/release-report-BEbWmVYj.d.ts +0 -233
- package/dist/run-critic-CmMf05uV.d.ts +0 -56
- package/dist/run-improvement-loop-DBahB8Ax.d.ts +0 -427
- package/dist/runtime-trajectory-BDgfGZSr.d.ts +0 -49
- package/dist/semantic-concept-judge-B9MgmBnM.d.ts +0 -624
- package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
- package/docs/design/external-agent-wedge.md +0 -89
- package/docs/design/phase-d-rfc.md +0 -125
- package/docs/design/phase4-consumer-migration.md +0 -70
- package/docs/design/primitives-integration-spec.md +0 -393
- package/docs/design/product-self-improvement-loop.md +0 -146
- package/docs/design/self-improvement-engine.md +0 -140
- package/docs/design/self-improvement-protocol.md +0 -223
- package/docs/design/self-improvement-roadmap.md +0 -106
- package/docs/design/substrate-gaps.md +0 -118
- package/docs/phase-b-pairing-kit.md +0 -188
- package/docs/phase-b-runbook.md +0 -176
- package/docs/pilot/README.md +0 -62
- package/docs/pilot/customer-checklist.md +0 -90
- package/docs/pilot/integration-foreign-stack.md +0 -296
- package/docs/pilot/integration-tangle-stack.md +0 -248
- package/docs/pilot/one-pager.md +0 -161
- package/docs/pilot/sample-insight-report.json +0 -172
- package/docs/quickstart-external.md +0 -229
- package/docs/research/belief-state-agent-eval-roadmap.md +0 -593
- package/docs/research/research-roadmap.md +0 -205
- package/docs/specs/driver-honest-spec.md +0 -251
- package/docs/specs/hermes-self-improvement-audit.md +0 -93
- package/docs/specs/profile-versioning.md +0 -291
- package/docs/three-package-architecture.md +0 -168
- /package/dist/{chunk-TBDR6PAI.js.map → chunk-IZCEK2HR.js.map} +0 -0
- /package/dist/{chunk-KW53MSA5.js.map → chunk-X74V6ESX.js.map} +0 -0
- /package/dist/{chunk-2KNZHH3P.js.map → chunk-Z6L6YSU6.js.map} +0 -0
- /package/dist/{run-campaign-7WNXMDSN.js.map → run-campaign-WXY7KI67.js.map} +0 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `fromAgentTrace` — provenance correlation from Cursor's Agent Trace spec
|
|
3
|
+
* (https://github.com/cursor/agent-trace, RFC v0.1.0).
|
|
4
|
+
*
|
|
5
|
+
* Agent Trace is NOT a run/quality trace — it carries no outcome, score, or
|
|
6
|
+
* cost. It records *code authorship*: for a VCS revision, which AI model /
|
|
7
|
+
* conversation authored which file ranges. It explicitly disclaims quality
|
|
8
|
+
* assessment — which is exactly what `analyzeRuns` adds.
|
|
9
|
+
*
|
|
10
|
+
* The two layers join on a key the substrate already has: a `RunRecord`
|
|
11
|
+
* carries `commitSha`, and an Agent Trace record is keyed by
|
|
12
|
+
* `vcs.revision`. So this adapter does not produce `RunRecord`s — it builds a
|
|
13
|
+
* provenance index by commit and partitions existing runs by their authoring
|
|
14
|
+
* model. Feed each cohort to `analyzeRuns` (or pass one as `baselineRuns`) to
|
|
15
|
+
* answer the question no run-only trace can: *which authoring agent's code
|
|
16
|
+
* fails / regresses / costs more.*
|
|
17
|
+
*
|
|
18
|
+
* Granularity is commit-level (the SHA join). Per-file/per-line correlation
|
|
19
|
+
* would require runs to record which files they exercised — out of scope.
|
|
20
|
+
*/
|
|
21
|
+
import type { RunRecord } from '../../run-record';
|
|
22
|
+
export type AgentTraceContributorType = 'human' | 'ai' | 'mixed' | 'unknown';
|
|
23
|
+
export interface AgentTraceContributor {
|
|
24
|
+
type: AgentTraceContributorType;
|
|
25
|
+
/** models.dev id, e.g. `anthropic/claude-opus-4-5-20251101`. */
|
|
26
|
+
model_id?: string;
|
|
27
|
+
}
|
|
28
|
+
export interface AgentTraceRange {
|
|
29
|
+
start_line: number;
|
|
30
|
+
end_line: number;
|
|
31
|
+
content_hash?: string;
|
|
32
|
+
/** Per-range contributor override (agent handoffs). Wins over the
|
|
33
|
+
* conversation-level contributor for these lines. */
|
|
34
|
+
contributor?: AgentTraceContributor;
|
|
35
|
+
}
|
|
36
|
+
export interface AgentTraceConversation {
|
|
37
|
+
url?: string;
|
|
38
|
+
contributor?: AgentTraceContributor;
|
|
39
|
+
ranges: AgentTraceRange[];
|
|
40
|
+
}
|
|
41
|
+
export interface AgentTraceFile {
|
|
42
|
+
path: string;
|
|
43
|
+
conversations: AgentTraceConversation[];
|
|
44
|
+
}
|
|
45
|
+
export interface AgentTraceRecord {
|
|
46
|
+
version: string;
|
|
47
|
+
id: string;
|
|
48
|
+
timestamp: string;
|
|
49
|
+
vcs?: {
|
|
50
|
+
type: string;
|
|
51
|
+
revision: string;
|
|
52
|
+
};
|
|
53
|
+
tool?: {
|
|
54
|
+
name?: string;
|
|
55
|
+
version?: string;
|
|
56
|
+
};
|
|
57
|
+
files: AgentTraceFile[];
|
|
58
|
+
}
|
|
59
|
+
/** Authorship provenance for one VCS revision, aggregated across the record's
|
|
60
|
+
* files/conversations/ranges. */
|
|
61
|
+
export interface AuthoringProvenance {
|
|
62
|
+
commitSha: string;
|
|
63
|
+
/** Unique AI model ids that authored code in this commit (type ai|mixed). */
|
|
64
|
+
aiModels: string[];
|
|
65
|
+
/** Tools that produced the records (e.g. `cursor`). */
|
|
66
|
+
tools: string[];
|
|
67
|
+
conversationCount: number;
|
|
68
|
+
fileCount: number;
|
|
69
|
+
/** Total attributed lines (sum of range spans). */
|
|
70
|
+
lineCount: number;
|
|
71
|
+
/** True if any range was authored (in whole or part) by a human. */
|
|
72
|
+
humanInvolved: boolean;
|
|
73
|
+
}
|
|
74
|
+
export type AgentTraceIndex = Map<string, AuthoringProvenance>;
|
|
75
|
+
/**
|
|
76
|
+
* Build a commit → provenance index from Agent Trace records. Multiple records
|
|
77
|
+
* for the same revision are merged. Records without `vcs.revision` are skipped
|
|
78
|
+
* (the SHA is the join key — without it there is nothing to correlate against).
|
|
79
|
+
*/
|
|
80
|
+
export declare function parseAgentTrace(records: AgentTraceRecord[]): AgentTraceIndex;
|
|
81
|
+
export interface PartitionByAuthoringModelResult {
|
|
82
|
+
/** Runs grouped by each AI model that authored code in the run's commit. A
|
|
83
|
+
* run whose commit had multiple authoring models appears under EACH — the
|
|
84
|
+
* cohorts overlap by construction at commit granularity. */
|
|
85
|
+
byModel: Map<string, RunRecord[]>;
|
|
86
|
+
/** Runs whose `commitSha` had no Agent Trace provenance (no record, or no
|
|
87
|
+
* AI authorship). Kept separate — never silently folded into a cohort. */
|
|
88
|
+
unattributed: RunRecord[];
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Partition runs by the AI model(s) that authored the code at each run's
|
|
92
|
+
* `commitSha`. Feed `byModel.get(modelId)` to `analyzeRuns`, or compare two
|
|
93
|
+
* model cohorts via `analyzeRuns({ runs: a, baselineRuns: b })` for a lift CI
|
|
94
|
+
* on "model A's code vs model B's code".
|
|
95
|
+
*/
|
|
96
|
+
export declare function partitionRunsByAuthoringModel(runs: RunRecord[], index: AgentTraceIndex): PartitionByAuthoringModelResult;
|
|
97
|
+
//# sourceMappingURL=agent-trace.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-trace.d.ts","sourceRoot":"","sources":["../../../src/contract/intake/agent-trace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAIjD,MAAM,MAAM,yBAAyB,GAAG,OAAO,GAAG,IAAI,GAAG,OAAO,GAAG,SAAS,CAAA;AAE5E,MAAM,WAAW,qBAAqB;IACpC,IAAI,EAAE,yBAAyB,CAAA;IAC/B,gEAAgE;IAChE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB;0DACsD;IACtD,WAAW,CAAC,EAAE,qBAAqB,CAAA;CACpC;AAED,MAAM,WAAW,sBAAsB;IACrC,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,WAAW,CAAC,EAAE,qBAAqB,CAAA;IACnC,MAAM,EAAE,eAAe,EAAE,CAAA;CAC1B;AAED,MAAM,WAAW,cAAc;IAC7B,IAAI,EAAE,MAAM,CAAA;IACZ,aAAa,EAAE,sBAAsB,EAAE,CAAA;CACxC;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,MAAM,CAAA;IACf,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,CAAA;IACjB,GAAG,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE,CAAA;IACxC,IAAI,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;IAC1C,KAAK,EAAE,cAAc,EAAE,CAAA;CACxB;AAID;kCACkC;AAClC,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,6EAA6E;IAC7E,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,uDAAuD;IACvD,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,iBAAiB,EAAE,MAAM,CAAA;IACzB,SAAS,EAAE,MAAM,CAAA;IACjB,mDAAmD;IACnD,SAAS,EAAE,MAAM,CAAA;IACjB,oEAAoE;IACpE,aAAa,EAAE,OAAO,CAAA;CACvB;AAED,MAAM,MAAM,eAAe,GAAG,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAA;AAM9D;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,eAAe,CA+D5E;AAID,MAAM,WAAW,+BAA+B;IAC9C;;iEAE6D;IAC7D,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;IACjC;+EAC2E;IAC3E,YAAY,EAAE,SAAS,EAAE,CAAA;CAC1B;AAED;;;;;GAKG;AACH,wBAAgB,6BAA6B,CAC3C,IAAI,EAAE,SAAS,EAAE,EACjB,KAAK,EAAE,eAAe,GACrB,+BAA+B,CAkBjC"}
|
|
@@ -1,11 +1,10 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
interface ParsedCodeAgentJsonl {
|
|
1
|
+
import type { RunRecord, RunSplitTag } from '../../run-record';
|
|
2
|
+
export type CodeAgentSessionSource = 'codex' | 'claude-code' | 'opencode' | 'kimi-code' | 'pi';
|
|
3
|
+
export interface ParsedCodeAgentJsonl {
|
|
5
4
|
entries: unknown[];
|
|
6
5
|
malformedLines: number;
|
|
7
6
|
}
|
|
8
|
-
interface CodeAgentSessionMetrics {
|
|
7
|
+
export interface CodeAgentSessionMetrics {
|
|
9
8
|
entries: number;
|
|
10
9
|
userMessages: number;
|
|
11
10
|
assistantMessages: number;
|
|
@@ -36,7 +35,7 @@ interface CodeAgentSessionMetrics {
|
|
|
36
35
|
wallMs: number;
|
|
37
36
|
processScore: number;
|
|
38
37
|
}
|
|
39
|
-
interface CodeAgentSessionDiagnostic {
|
|
38
|
+
export interface CodeAgentSessionDiagnostic {
|
|
40
39
|
source: CodeAgentSessionSource;
|
|
41
40
|
sessionId: string;
|
|
42
41
|
sourcePath?: string;
|
|
@@ -49,12 +48,12 @@ interface CodeAgentSessionDiagnostic {
|
|
|
49
48
|
hasCost: boolean;
|
|
50
49
|
warnings: string[];
|
|
51
50
|
}
|
|
52
|
-
interface CodeAgentSessionIntakeResult {
|
|
51
|
+
export interface CodeAgentSessionIntakeResult {
|
|
53
52
|
runs: RunRecord[];
|
|
54
53
|
diagnostics: CodeAgentSessionDiagnostic[];
|
|
55
54
|
metrics: CodeAgentSessionMetrics[];
|
|
56
55
|
}
|
|
57
|
-
interface CodeAgentSessionIntakeOptions {
|
|
56
|
+
export interface CodeAgentSessionIntakeOptions {
|
|
58
57
|
entries: unknown[];
|
|
59
58
|
malformedLines?: number;
|
|
60
59
|
sourcePath?: string;
|
|
@@ -69,12 +68,11 @@ interface CodeAgentSessionIntakeOptions {
|
|
|
69
68
|
commitSha?: string;
|
|
70
69
|
score?: number;
|
|
71
70
|
}
|
|
72
|
-
declare function parseCodeAgentJsonl(jsonl: string): ParsedCodeAgentJsonl;
|
|
73
|
-
declare function fromCodexSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
|
|
74
|
-
declare function fromClaudeCodeSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
|
|
75
|
-
declare function fromOpenCodeSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
|
|
76
|
-
declare function fromKimiCodeSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
|
|
77
|
-
declare function fromPiSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
|
|
78
|
-
declare const fromPigraphSession: typeof fromPiSession;
|
|
79
|
-
|
|
80
|
-
export { type CodeAgentSessionDiagnostic as C, type ParsedCodeAgentJsonl as P, type CodeAgentSessionIntakeOptions as a, type CodeAgentSessionIntakeResult as b, type CodeAgentSessionMetrics as c, type CodeAgentSessionSource as d, fromCodexSession as e, fromClaudeCodeSession as f, fromKimiCodeSession as g, fromOpenCodeSession as h, fromPiSession as i, fromPigraphSession as j, parseCodeAgentJsonl as p };
|
|
71
|
+
export declare function parseCodeAgentJsonl(jsonl: string): ParsedCodeAgentJsonl;
|
|
72
|
+
export declare function fromCodexSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
|
|
73
|
+
export declare function fromClaudeCodeSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
|
|
74
|
+
export declare function fromOpenCodeSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
|
|
75
|
+
export declare function fromKimiCodeSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
|
|
76
|
+
export declare function fromPiSession(options: CodeAgentSessionIntakeOptions): CodeAgentSessionIntakeResult;
|
|
77
|
+
export declare const fromPigraphSession: typeof fromPiSession;
|
|
78
|
+
//# sourceMappingURL=code-agent-session.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"code-agent-session.d.ts","sourceRoot":"","sources":["../../../src/contract/intake/code-agent-session.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAAiB,MAAM,kBAAkB,CAAA;AAE7E,MAAM,MAAM,sBAAsB,GAAG,OAAO,GAAG,aAAa,GAAG,UAAU,GAAG,WAAW,GAAG,IAAI,CAAA;AAE9F,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,OAAO,EAAE,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,uBAAuB;IACtC,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,CAAA;IACpB,iBAAiB,EAAE,MAAM,CAAA;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,MAAM,CAAA;IACrB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,OAAO,EAAE,MAAM,CAAA;IACf,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,gBAAgB,EAAE,MAAM,CAAA;IACxB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,mBAAmB,EAAE,MAAM,CAAA;IAC3B,eAAe,EAAE,MAAM,CAAA;IACvB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,eAAe,EAAE,MAAM,CAAA;IACvB,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,sBAAsB,CAAA;IAC9B,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,cAAc,EAAE,MAAM,CAAA;IACtB,aAAa,EAAE,OAAO,CAAA;IACtB,yBAAyB,EAAE,OAAO,CAAA;IAClC,eAAe,EAAE,OAAO,CAAA;IACxB,aAAa,EAAE,OAAO,CAAA;IACtB,OAAO,EAAE,OAAO,CAAA;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAA;CACnB;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,SAAS,EAAE,CAAA;IACjB,WAAW,EAAE,0BAA0B,EAAE,CAAA;IACzC,OAAO,EAAE,uBAAuB,EAAE,CAAA;CACnC;AAED,MAAM,WAAW,6BAA6B;IAC5C,OAAO,EAAE,OAAO,EAAE,CAAA;IAClB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,WAAW,CAAA;IACtB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,oBAAoB,CAavE;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,6BAA6B,GACrC,4BAA4B,CAE9B;AAED,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,6BAA6B,GACrC,4BAA4B,CAE9B;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,6BAA6B,GACrC,4BAA4B,CAE9B;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,6BAA6B,GACrC,4BAA4B,CAE9B;AAED,wBAAgB,aAAa,CAC3B,OAAO,EAAE,6BAA6B,GACrC,4BAA4B,CAE9B;AAED,eAAO,MAAM,kBAAkB,sBAAgB,CAAA"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* # `intake/feedback-table` — multi-rater approve/reject corpus → `RunRecord[]`.
|
|
3
|
+
*
|
|
4
|
+
* The generic shape behind Obsidian's `#approved` / `#rejected` tags, a
|
|
5
|
+
* Google Sheet, a Postgres `feedback` table, or any CSV with ratings.
|
|
6
|
+
*
|
|
7
|
+
* Caller supplies one row per (run, rater) tuple plus per-run metadata; the
|
|
8
|
+
* adapter rolls them up into the substrate-canonical `RunRecord` shape so
|
|
9
|
+
* `analyzeRuns({ runs, raterScores })` can produce inter-rater agreement,
|
|
10
|
+
* disagreement triage, and downstream recommendations.
|
|
11
|
+
*
|
|
12
|
+
* Per-run `RunRecord.outcome.searchScore` is the rater-mean rating
|
|
13
|
+
* (normalised to 0..1 when scale is supplied); `outcome.raw` carries the
|
|
14
|
+
* per-rater scores keyed by rater id for downstream attribution.
|
|
15
|
+
*/
|
|
16
|
+
import type { RunRecord, RunSplitTag } from '../../run-record';
|
|
17
|
+
export interface FeedbackTableRow {
|
|
18
|
+
/** Stable id for this run — the unit a rater scored. Drives pairing
|
|
19
|
+
* across analysis primitives. */
|
|
20
|
+
runId: string;
|
|
21
|
+
/** Identifier of the rater that produced this rating. */
|
|
22
|
+
rater: string;
|
|
23
|
+
/** The rating itself. Accepts boolean (approve/reject), 0..1 scalar,
|
|
24
|
+
* or any numeric scale — see `scale`. */
|
|
25
|
+
rating: number | boolean;
|
|
26
|
+
/** Optional metadata carried through to `RunRecord.outcome.raw` and the
|
|
27
|
+
* custom-shape metadata bag. */
|
|
28
|
+
metadata?: Record<string, unknown>;
|
|
29
|
+
}
|
|
30
|
+
export interface FeedbackTableMeta {
|
|
31
|
+
runId: string;
|
|
32
|
+
/** When omitted, defaults to `'feedback-corpus'`. Used to group related
|
|
33
|
+
* runs in `analyzeRuns()` lift analysis. */
|
|
34
|
+
experimentId?: string;
|
|
35
|
+
/** When omitted, defaults to `runId` — each run is its own candidate. */
|
|
36
|
+
candidateId?: string;
|
|
37
|
+
/** Cost in USD, when available. Set to 0 when unknown — the consumer's
|
|
38
|
+
* cost analysis sections will collapse gracefully. */
|
|
39
|
+
costUsd?: number;
|
|
40
|
+
/** Wall-clock ms, when available. Defaults to 0. */
|
|
41
|
+
wallMs?: number;
|
|
42
|
+
/** Model identifier including snapshot. Default `unknown@unknown`. */
|
|
43
|
+
model?: string;
|
|
44
|
+
/** Optional sha256 of the prompt; default `'sha256:unknown'`. */
|
|
45
|
+
promptHash?: string;
|
|
46
|
+
/** Default `'sha256:unknown'`. */
|
|
47
|
+
configHash?: string;
|
|
48
|
+
/** Default `'unknown'`. */
|
|
49
|
+
commitSha?: string;
|
|
50
|
+
/** Default `'holdout'` — feedback corpora are by nature the holdout
|
|
51
|
+
* signal a closed-loop improvement aims at. */
|
|
52
|
+
splitTag?: RunSplitTag;
|
|
53
|
+
/** Free-form metadata available to consumers via the cast-out path on
|
|
54
|
+
* the resulting RunRecord. */
|
|
55
|
+
extras?: Record<string, unknown>;
|
|
56
|
+
}
|
|
57
|
+
export interface FromFeedbackTableOptions {
|
|
58
|
+
/** Per-(run, rater) ratings. */
|
|
59
|
+
ratings: FeedbackTableRow[];
|
|
60
|
+
/** Per-run metadata. When a runId appears in `ratings` but not here, the
|
|
61
|
+
* adapter synthesises minimal metadata with defaults documented above. */
|
|
62
|
+
meta?: FeedbackTableMeta[];
|
|
63
|
+
/** Rating scale. Provide `{ min, max }` for non-0..1 numeric scales.
|
|
64
|
+
* Booleans are normalised: true → 1, false → 0. Default: assumes
|
|
65
|
+
* ratings are already 0..1. */
|
|
66
|
+
scale?: {
|
|
67
|
+
min: number;
|
|
68
|
+
max: number;
|
|
69
|
+
};
|
|
70
|
+
/** When true, the rater scores are emitted into `raterScores` (a sibling
|
|
71
|
+
* array `analyzeRuns()` accepts) instead of being averaged into the
|
|
72
|
+
* run's `outcome.searchScore`. Default `true` — preserves rater-level
|
|
73
|
+
* signal for inter-rater analysis. */
|
|
74
|
+
emitRaterScores?: boolean;
|
|
75
|
+
}
|
|
76
|
+
export interface FromFeedbackTableResult {
|
|
77
|
+
runs: RunRecord[];
|
|
78
|
+
/** Rater-level scores ready to pass into `analyzeRuns({ raterScores })`
|
|
79
|
+
* for inter-rater agreement + disagreement triage. */
|
|
80
|
+
raterScores: Array<{
|
|
81
|
+
runId: string;
|
|
82
|
+
rater: string;
|
|
83
|
+
score: number;
|
|
84
|
+
}>;
|
|
85
|
+
}
|
|
86
|
+
export declare function fromFeedbackTable(opts: FromFeedbackTableOptions): FromFeedbackTableResult;
|
|
87
|
+
//# sourceMappingURL=feedback-table.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"feedback-table.d.ts","sourceRoot":"","sources":["../../../src/contract/intake/feedback-table.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAiC,SAAS,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAE7F,MAAM,WAAW,gBAAgB;IAC/B;sCACkC;IAClC,KAAK,EAAE,MAAM,CAAA;IACb,yDAAyD;IACzD,KAAK,EAAE,MAAM,CAAA;IACb;8CAC0C;IAC1C,MAAM,EAAE,MAAM,GAAG,OAAO,CAAA;IACxB;qCACiC;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAA;IACb;iDAC6C;IAC7C,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,yEAAyE;IACzE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;2DACuD;IACvD,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,oDAAoD;IACpD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,sEAAsE;IACtE,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,iEAAiE;IACjE,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,kCAAkC;IAClC,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;oDACgD;IAChD,QAAQ,CAAC,EAAE,WAAW,CAAA;IACtB;mCAC+B;IAC/B,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACjC;AAED,MAAM,WAAW,wBAAwB;IACvC,gCAAgC;IAChC,OAAO,EAAE,gBAAgB,EAAE,CAAA;IAC3B;+EAC2E;IAC3E,IAAI,CAAC,EAAE,iBAAiB,EAAE,CAAA;IAC1B;;oCAEgC;IAChC,KAAK,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAA;IACpC;;;2CAGuC;IACvC,eAAe,CAAC,EAAE,OAAO,CAAA;CAC1B;AAED,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,SAAS,EAAE,CAAA;IACjB;2DACuD;IACvD,WAAW,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACpE;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,wBAAwB,GAAG,uBAAuB,CAwEzF"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* # `@tangle-network/agent-eval/contract/intake` — adapters that turn external
|
|
3
|
+
* data sources into the substrate-canonical `RunRecord` shape.
|
|
4
|
+
*
|
|
5
|
+
* Each adapter is a function `source → RunRecord[]` (or
|
|
6
|
+
* `{ runs, raterScores }` when the adapter exposes multi-rater signal).
|
|
7
|
+
* Consumers pipe the output straight into `analyzeRuns({ runs })`.
|
|
8
|
+
*
|
|
9
|
+
* Adapters:
|
|
10
|
+
* - `fromFeedbackTable` — multi-rater approve/reject corpus
|
|
11
|
+
* (Obsidian, Sheets, CSV, Postgres).
|
|
12
|
+
* - `fromOtelSpans` — OTel `TraceSpanEvent[]` from any OTel-compatible
|
|
13
|
+
* observability stack.
|
|
14
|
+
* - `fromCodexSession` / `fromClaudeCodeSession` / `fromOpenCodeSession` /
|
|
15
|
+
* `fromKimiCodeSession` / `fromPiSession` — local coding-agent and
|
|
16
|
+
* graph-shaped sessions projected into process-scored `RunRecord`s.
|
|
17
|
+
*/
|
|
18
|
+
export { type AgentTraceContributor, type AgentTraceContributorType, type AgentTraceConversation, type AgentTraceFile, type AgentTraceIndex, type AgentTraceRange, type AgentTraceRecord, type AuthoringProvenance, type PartitionByAuthoringModelResult, parseAgentTrace, partitionRunsByAuthoringModel, } from './agent-trace';
|
|
19
|
+
export { type CodeAgentSessionDiagnostic, type CodeAgentSessionIntakeOptions, type CodeAgentSessionIntakeResult, type CodeAgentSessionMetrics, type CodeAgentSessionSource, fromClaudeCodeSession, fromCodexSession, fromKimiCodeSession, fromOpenCodeSession, fromPigraphSession, fromPiSession, type ParsedCodeAgentJsonl, parseCodeAgentJsonl, } from './code-agent-session';
|
|
20
|
+
export { type FeedbackTableMeta, type FeedbackTableRow, type FromFeedbackTableOptions, type FromFeedbackTableResult, fromFeedbackTable, } from './feedback-table';
|
|
21
|
+
export { type FromOtelSpansOptions, fromOtelSpans } from './otel-spans';
|
|
22
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/contract/intake/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EACL,KAAK,qBAAqB,EAC1B,KAAK,yBAAyB,EAC9B,KAAK,sBAAsB,EAC3B,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,eAAe,EACpB,KAAK,gBAAgB,EACrB,KAAK,mBAAmB,EACxB,KAAK,+BAA+B,EACpC,eAAe,EACf,6BAA6B,GAC9B,MAAM,eAAe,CAAA;AACtB,OAAO,EACL,KAAK,0BAA0B,EAC/B,KAAK,6BAA6B,EAClC,KAAK,4BAA4B,EACjC,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,qBAAqB,EACrB,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,kBAAkB,EAClB,aAAa,EACb,KAAK,oBAAoB,EACzB,mBAAmB,GACpB,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,EAC5B,iBAAiB,GAClB,MAAM,kBAAkB,CAAA;AACzB,OAAO,EAAE,KAAK,oBAAoB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* # `intake/otel-spans` — OTel `TraceSpanEvent[]` → `RunRecord[]`.
|
|
3
|
+
*
|
|
4
|
+
* Turns an existing observability stream into the substrate-canonical
|
|
5
|
+
* `RunRecord` shape so consumers with logs but no eval discipline can
|
|
6
|
+
* call `analyzeRuns()` against their production traffic immediately.
|
|
7
|
+
*
|
|
8
|
+
* Pivot rule: spans are grouped by `tangle.runId` (the same attribute the
|
|
9
|
+
* hosted-tier wire format uses) or, when absent, by `traceId`. One group
|
|
10
|
+
* becomes one `RunRecord`. The root span (no `parentSpanId`) supplies:
|
|
11
|
+
*
|
|
12
|
+
* - `runId` (the group key)
|
|
13
|
+
* - `wallMs` from `endTimeUnixNano - startTimeUnixNano`
|
|
14
|
+
* - `model` from `gen_ai.request.model` / `llm.model` / `tangle.model`
|
|
15
|
+
* - cost from `cost.usd` / `gen_ai.usage.cost_usd` / `tangle.cost.usd`
|
|
16
|
+
* - token usage from `gen_ai.usage.{input,output}_tokens`
|
|
17
|
+
* - `outcome.searchScore` from `tangle.score` / `eval.score` when
|
|
18
|
+
* present; `outcome.raw` collects every numeric attribute.
|
|
19
|
+
*
|
|
20
|
+
* Spans that ERRORed (`status.code === 'ERROR'`) populate `failureMode`
|
|
21
|
+
* with their `name` so `analyzeRuns()`'s failure clustering sees them.
|
|
22
|
+
*/
|
|
23
|
+
import type { TraceSpanEvent } from '../../hosted/types';
|
|
24
|
+
import type { RunRecord, RunSplitTag } from '../../run-record';
|
|
25
|
+
export interface FromOtelSpansOptions {
|
|
26
|
+
spans: TraceSpanEvent[];
|
|
27
|
+
/** Default split tag for synthesized records. Defaults to `'holdout'`. */
|
|
28
|
+
defaultSplit?: RunSplitTag;
|
|
29
|
+
/** Default `experimentId` when not present on any span. */
|
|
30
|
+
experimentId?: string;
|
|
31
|
+
}
|
|
32
|
+
export declare function fromOtelSpans(opts: FromOtelSpansOptions): RunRecord[];
|
|
33
|
+
//# sourceMappingURL=otel-spans.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"otel-spans.d.ts","sourceRoot":"","sources":["../../../src/contract/intake/otel-spans.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,KAAK,EAGV,SAAS,EACT,WAAW,EAEZ,MAAM,kBAAkB,CAAA;AAezB,MAAM,WAAW,oBAAoB;IACnC,KAAK,EAAE,cAAc,EAAE,CAAA;IACvB,0EAA0E;IAC1E,YAAY,CAAC,EAAE,WAAW,CAAA;IAC1B,2DAA2D;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,oBAAoB,GAAG,SAAS,EAAE,CA0DrE"}
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* # `selfImprove()` - the one-call improvement loop.
|
|
3
|
+
*
|
|
4
|
+
* The cheapest possible call site to run a real closed-loop self-
|
|
5
|
+
* improvement over your agent. Wraps `runImprovementLoop` with smart
|
|
6
|
+
* defaults and a budget-shaped options API; every escape hatch the
|
|
7
|
+
* substrate exposes is reachable from here without losing the
|
|
8
|
+
* one-function feel.
|
|
9
|
+
*
|
|
10
|
+
* Defaults:
|
|
11
|
+
* - In-memory storage (no filesystem touch).
|
|
12
|
+
* - `gepaProposer` reflective mutation with copywriting-flavored primitives
|
|
13
|
+
* (override `proposer` or `mutationPrimitives` for any domain).
|
|
14
|
+
* - `defaultProductionGate` with `deltaThreshold: 0.05`.
|
|
15
|
+
* - Held-out split = 25% of scenarios, deterministic by id hash.
|
|
16
|
+
* - 3 generations × population 2 (raise via `budget` for more search).
|
|
17
|
+
* - `autoOnPromote: 'none'` (we don't open PRs unless you ask).
|
|
18
|
+
*
|
|
19
|
+
* Want one-click? Provide `agent` + `scenarios` + `judge`. Done.
|
|
20
|
+
* Want distributed? Pass `cellPlacement` + an `httpDispatch`-backed
|
|
21
|
+
* agent. Want a code-tier surface? Pass a `MutableSurface` + your own
|
|
22
|
+
* `proposer`. Same function.
|
|
23
|
+
*/
|
|
24
|
+
import { type RunImprovementLoopResult } from '../campaign/presets/run-improvement-loop';
|
|
25
|
+
import type { RunOptimizationOptions } from '../campaign/presets/run-optimization';
|
|
26
|
+
import { type LoopProvenanceRecord } from '../campaign/provenance';
|
|
27
|
+
import { type CampaignStorage } from '../campaign/storage';
|
|
28
|
+
import type { DispatchContext, Gate, JudgeConfig, LabeledScenarioStore, MutableSurface, Scenario, SurfaceProposer } from '../campaign/types';
|
|
29
|
+
import { type HostedTenant } from '../hosted/client';
|
|
30
|
+
import type { RunRecord } from '../run-record';
|
|
31
|
+
import type { InsightReport } from './insight-report';
|
|
32
|
+
export interface SelfImproveBudget {
|
|
33
|
+
/** Hard $ ceiling across all cells in baseline + every generation. Cells
|
|
34
|
+
* beyond the ceiling are skipped (cost-aware, not aborted). */
|
|
35
|
+
dollars?: number;
|
|
36
|
+
/** How many improvement generations to explore. Default 3. Set 0 to
|
|
37
|
+
* skip improvement entirely (selfImprove becomes a baseline-only run). */
|
|
38
|
+
generations?: number;
|
|
39
|
+
/** Candidates the proposer emits per generation. Default 2. */
|
|
40
|
+
populationSize?: number;
|
|
41
|
+
/** Max concurrent cells across the loop. Default 2. */
|
|
42
|
+
maxConcurrency?: number;
|
|
43
|
+
/** Fraction of `scenarios` held out from training, used for the gate.
|
|
44
|
+
* Default 0.25. Ignored when `holdoutScenarios` is set explicitly. */
|
|
45
|
+
holdoutFraction?: number;
|
|
46
|
+
/** Explicit held-out scenarios; overrides `holdoutFraction`. */
|
|
47
|
+
holdoutScenarios?: Scenario[];
|
|
48
|
+
/** Per-scenario replicates per cell — raises bootstrap-CI tightness. Default 1. */
|
|
49
|
+
reps?: number;
|
|
50
|
+
/** Top-scoring candidates carried into the next generation. Default 2. */
|
|
51
|
+
promoteTopK?: number;
|
|
52
|
+
}
|
|
53
|
+
export interface SelfImproveLlm {
|
|
54
|
+
/** Endpoint base URL. Default Tangle Router. */
|
|
55
|
+
baseUrl?: string;
|
|
56
|
+
/** Bearer token. Default `process.env.OPENAI_API_KEY`. */
|
|
57
|
+
apiKey?: string;
|
|
58
|
+
/** Model id used by `gepaProposer` reflection. Default
|
|
59
|
+
* `anthropic/claude-sonnet-4.6`. */
|
|
60
|
+
model?: string;
|
|
61
|
+
}
|
|
62
|
+
export type SelfImproveProgressEvent = {
|
|
63
|
+
kind: 'baseline.started';
|
|
64
|
+
scenarios: number;
|
|
65
|
+
} | {
|
|
66
|
+
kind: 'baseline.completed';
|
|
67
|
+
compositeMean: number;
|
|
68
|
+
durationMs: number;
|
|
69
|
+
} | {
|
|
70
|
+
kind: 'generation.started';
|
|
71
|
+
index: number;
|
|
72
|
+
populationSize: number;
|
|
73
|
+
} | {
|
|
74
|
+
kind: 'generation.completed';
|
|
75
|
+
index: number;
|
|
76
|
+
bestComposite: number;
|
|
77
|
+
durationMs: number;
|
|
78
|
+
} | {
|
|
79
|
+
kind: 'gate.decided';
|
|
80
|
+
decision: string;
|
|
81
|
+
lift: number;
|
|
82
|
+
};
|
|
83
|
+
export interface SelfImproveOptions<TScenario extends Scenario, TArtifact> {
|
|
84
|
+
/**
|
|
85
|
+
* Your agent — a function that takes the current `MutableSurface`
|
|
86
|
+
* (typically a system prompt the loop is optimizing) plus the
|
|
87
|
+
* scenario + cell ctx, and returns the artifact your judge scores.
|
|
88
|
+
*
|
|
89
|
+
* Same shape as `RunOptimizationOptions.dispatchWithSurface`. Wrap a
|
|
90
|
+
* plain `Dispatch` if you don't have a surface seam:
|
|
91
|
+
*
|
|
92
|
+
* agent: (_surface, scenario, ctx) => yourPlainDispatch(scenario, ctx)
|
|
93
|
+
*
|
|
94
|
+
* That mode evaluates without mutating any surface — useful as a
|
|
95
|
+
* baseline-only run (set `budget.generations = 0`).
|
|
96
|
+
*/
|
|
97
|
+
agent: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
|
|
98
|
+
/** Scenarios to evaluate against. Train/holdout split is computed from
|
|
99
|
+
* these unless `budget.holdoutScenarios` is set explicitly. */
|
|
100
|
+
scenarios: TScenario[];
|
|
101
|
+
/** Judge that scores artifacts. Bring your own; use `langchainJudge`
|
|
102
|
+
* from `/adapters/langchain` for a Runnable-shaped one. */
|
|
103
|
+
judge: JudgeConfig<TArtifact, TScenario>;
|
|
104
|
+
/** Starting surface — system prompt, JSON config, anything `MutableSurface`
|
|
105
|
+
* accepts. The proposer mutates this each generation. */
|
|
106
|
+
baselineSurface: MutableSurface;
|
|
107
|
+
/** Budget + loop shape. All fields optional. */
|
|
108
|
+
budget?: SelfImproveBudget;
|
|
109
|
+
/** Custom surface proposer. Default is `gepaProposer` configured from `llm` +
|
|
110
|
+
* `mutationPrimitives`. */
|
|
111
|
+
proposer?: SurfaceProposer;
|
|
112
|
+
/** Default-proposer overrides — used when `proposer` is unset. */
|
|
113
|
+
mutationPrimitives?: string[];
|
|
114
|
+
proposerTarget?: string;
|
|
115
|
+
/** Custom gate. Default is `defaultProductionGate` with
|
|
116
|
+
* `deltaThreshold: 0.05` on the held-out split. */
|
|
117
|
+
gate?: Gate<TArtifact, TScenario>;
|
|
118
|
+
/** LLM config consumed by the default `gepaProposer`. Ignored if you pass
|
|
119
|
+
* your own `proposer`. */
|
|
120
|
+
llm?: SelfImproveLlm;
|
|
121
|
+
/** Storage backend. Default is DURABLE: when a real (non-`mem://`) `runDir`
|
|
122
|
+
* is available, the substrate defaults to `fsCampaignStorage()` so the
|
|
123
|
+
* provenance record + OTel spans survive the call. Pass
|
|
124
|
+
* `inMemoryCampaignStorage()` explicitly to opt OUT (tests, edge runtimes).
|
|
125
|
+
* Default when `runDir` is `mem://...` (or unset): in-memory. */
|
|
126
|
+
storage?: CampaignStorage;
|
|
127
|
+
/** Run directory (logical for in-memory storage, real path for fs).
|
|
128
|
+
* Default `mem://selfImprove-<timestamp>` (in-memory, non-durable). Pass a
|
|
129
|
+
* real path to persist the provenance record + spans. */
|
|
130
|
+
runDir?: string;
|
|
131
|
+
/**
|
|
132
|
+
* Worker call records for backend provenance. The agent is opaque to the
|
|
133
|
+
* substrate (it returns an artifact, not token usage), so to capture an
|
|
134
|
+
* `assertRealBackend`-grade verdict + worker call count + model in the
|
|
135
|
+
* provenance record, the agent reports its per-call `RunRecord`s here.
|
|
136
|
+
* Called once after the loop; return the records the agent accumulated.
|
|
137
|
+
* When unset, backend provenance is derived from campaign cells (cost only;
|
|
138
|
+
* verdict will read `stub` without token usage — the honest signal that no
|
|
139
|
+
* token channel was wired).
|
|
140
|
+
*/
|
|
141
|
+
collectWorkerRecords?: () => RunRecord[];
|
|
142
|
+
/** Fires once the durable provenance record + OTel spans are emitted.
|
|
143
|
+
* Receives the structured record for inline assertions / custom routing. */
|
|
144
|
+
onProvenance?: (record: LoopProvenanceRecord) => void;
|
|
145
|
+
/** Distributed execution seam — same as `RunCampaignOptions.cellPlacement`.
|
|
146
|
+
* Returns an opaque placement key the substrate forwards to your agent
|
|
147
|
+
* as `ctx.placement`. Combined with `httpDispatch` from
|
|
148
|
+
* `/adapters/http`, fans cells across regions. */
|
|
149
|
+
cellPlacement?: (input: {
|
|
150
|
+
scenario: TScenario;
|
|
151
|
+
rep: number;
|
|
152
|
+
generation?: number;
|
|
153
|
+
}) => string | undefined;
|
|
154
|
+
/** Streaming hook — fires on baseline + each generation + gate decision.
|
|
155
|
+
* Consumer routes events wherever (UI, dashboard, logs). */
|
|
156
|
+
onProgress?: (event: SelfImproveProgressEvent) => void;
|
|
157
|
+
/** Auto-promotion behavior on a ship decision. Default `'none'` — we
|
|
158
|
+
* return the winner; you ship it however you ship. `'pr'` opens a
|
|
159
|
+
* GitHub PR via `openAutoPr`; requires `ghOwner` + `ghRepo`. */
|
|
160
|
+
autoOnPromote?: 'pr' | 'none';
|
|
161
|
+
ghOwner?: string;
|
|
162
|
+
ghRepo?: string;
|
|
163
|
+
/**
|
|
164
|
+
* Opt-in: ship eval-run events to a hosted orchestrator (ours, your
|
|
165
|
+
* self-hosted one, or any compatible implementation of the
|
|
166
|
+
* `docs/hosted-ingest-spec.md` wire format). When set, the substrate
|
|
167
|
+
* POSTs the final `EvalRunEvent` to `${endpoint}/v1/ingest/eval-runs`
|
|
168
|
+
* after the loop completes. Failures are logged but do not fail the
|
|
169
|
+
* loop — local result is always returned.
|
|
170
|
+
*
|
|
171
|
+
* For our orchestrator: `{ endpoint: 'https://orchestrator.tangle.tools/v1', apiKey, tenantId }`.
|
|
172
|
+
*
|
|
173
|
+
* For your self-hosted: any URL serving the wire format. See
|
|
174
|
+
* `examples/hosted-ingest-server/` for the reference receiver.
|
|
175
|
+
*/
|
|
176
|
+
hostedTenant?: HostedTenant;
|
|
177
|
+
/** Free-form labels attached to the hosted event (env, branch, model id,
|
|
178
|
+
* etc.). Ignored when `hostedTenant` is unset. */
|
|
179
|
+
hostedLabels?: Record<string, string>;
|
|
180
|
+
/** Capture every artifact + judge score to this store (labeled-example
|
|
181
|
+
* corpus the proposer may read for few-shot, and the dataset you ship). Pass
|
|
182
|
+
* `'off'` to disable. Default: off. */
|
|
183
|
+
labeledStore?: LabeledScenarioStore | 'off';
|
|
184
|
+
/** Capture-source tag for `labeledStore`. Default `'eval-run'`. */
|
|
185
|
+
captureSource?: 'production-trace' | 'eval-run' | 'manual' | 'red-team' | 'synthetic';
|
|
186
|
+
/**
|
|
187
|
+
* Per-cell backend-integrity expectation — the fail-loud guard. A cell that
|
|
188
|
+
* produced an artifact but reported `costUsd === 0` AND zero tokens is a
|
|
189
|
+
* stub. Modes: `'assert'` throws on the first such cell, `'warn'` logs it,
|
|
190
|
+
* `'off'` skips the check (offline/replay). Default `'assert'` — `selfImprove`
|
|
191
|
+
* is the real-run path, so a stub fails loud rather than scoring a clean 0.
|
|
192
|
+
*/
|
|
193
|
+
expectUsage?: 'assert' | 'warn' | 'off';
|
|
194
|
+
/**
|
|
195
|
+
* Per-generation findings producer. After each
|
|
196
|
+
* generation is scored, this is called; whatever it returns REPLACES the
|
|
197
|
+
* proposer's `findings` for the next generation's `propose()`. Plug a
|
|
198
|
+
* trace-analyst registry / HALO here. When absent, findings stay
|
|
199
|
+
* `opts.findings`.
|
|
200
|
+
*/
|
|
201
|
+
analyzeGeneration?: RunOptimizationOptions<TScenario, TArtifact>['analyzeGeneration'];
|
|
202
|
+
/** Static findings forwarded to the proposer's `propose()` as `ctx.findings`
|
|
203
|
+
* (a findings-grounded proposer consumes them). Default: none. */
|
|
204
|
+
findings?: unknown[];
|
|
205
|
+
}
|
|
206
|
+
export interface SelfImproveResult<TScenario extends Scenario, TArtifact> {
|
|
207
|
+
/** Composite mean across all scenarios, baseline run. */
|
|
208
|
+
baseline: {
|
|
209
|
+
compositeMean: number;
|
|
210
|
+
perScenario: Record<string, number>;
|
|
211
|
+
};
|
|
212
|
+
/** Composite mean on the held-out set, winner run. */
|
|
213
|
+
winner: {
|
|
214
|
+
compositeMean: number;
|
|
215
|
+
perScenario: Record<string, number>;
|
|
216
|
+
surface: MutableSurface;
|
|
217
|
+
/** Proposer label for the promoted change. Absent ⇒ winner == baseline or
|
|
218
|
+
* a bare-surface mutator. */
|
|
219
|
+
label?: string;
|
|
220
|
+
/** Proposer rationale — the "because Z" that motivated the promoted change.
|
|
221
|
+
* Threaded from the proposer's `ProposedCandidate` through the loop.
|
|
222
|
+
* Absent ⇒ winner == baseline. */
|
|
223
|
+
rationale?: string;
|
|
224
|
+
};
|
|
225
|
+
/** `winner.compositeMean - baselineOnHoldout.compositeMean`. Positive
|
|
226
|
+
* means the gate observed improvement. */
|
|
227
|
+
lift: number;
|
|
228
|
+
/** The explicit baseline→winner unified diff. Always present (empty string
|
|
229
|
+
* when winner == baseline). */
|
|
230
|
+
diff: string;
|
|
231
|
+
/** Durable, queryable provenance record: candidate→cell→gate→promote chain +
|
|
232
|
+
* rationale + diff + backend provenance. The artifact the hosted ingest
|
|
233
|
+
* path stores; the +lift RECOMPUTES from `record.heldOutLift`. */
|
|
234
|
+
provenance: LoopProvenanceRecord;
|
|
235
|
+
/** `defaultProductionGate.decide()` result. */
|
|
236
|
+
gateDecision: 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
|
|
237
|
+
/** Number of generations actually explored (may be less than the
|
|
238
|
+
* budget if the proposer gave up early). */
|
|
239
|
+
generationsExplored: number;
|
|
240
|
+
/** Wall-clock total. */
|
|
241
|
+
durationMs: number;
|
|
242
|
+
/** Total cost across baseline + every generation. */
|
|
243
|
+
totalCostUsd: number;
|
|
244
|
+
/**
|
|
245
|
+
* Rigor packet: distributional summary, paired-bootstrap lift CI,
|
|
246
|
+
* judge stats, contamination check, recommendations. Wired through
|
|
247
|
+
* `analyzeRuns()` on the baseline + winner cells of the campaign.
|
|
248
|
+
* Hosted-tier dashboards render this as the v3-vs-v4 decision view.
|
|
249
|
+
*/
|
|
250
|
+
insight: InsightReport;
|
|
251
|
+
/**
|
|
252
|
+
* Raw substrate result for advanced inspection — full per-generation
|
|
253
|
+
* candidates, full campaign artifacts, all judge scores. Useful for
|
|
254
|
+
* debugging or reporting beyond the summary.
|
|
255
|
+
*/
|
|
256
|
+
raw: RunImprovementLoopResult<TArtifact, TScenario>;
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* One-shot self-improvement loop. See module docstring for defaults +
|
|
260
|
+
* extension points.
|
|
261
|
+
*
|
|
262
|
+
* @example Minimum:
|
|
263
|
+
*
|
|
264
|
+
* const result = await selfImprove({
|
|
265
|
+
* agent: (surface, scenario, ctx) => myAgent(surface, scenario, ctx.signal),
|
|
266
|
+
* scenarios,
|
|
267
|
+
* judge,
|
|
268
|
+
* baselineSurface: DEFAULT_PROMPT,
|
|
269
|
+
* })
|
|
270
|
+
* console.log(`lift: ${result.lift.toFixed(3)} (${result.gateDecision})`)
|
|
271
|
+
*
|
|
272
|
+
* @example Distributed (workers in three regions):
|
|
273
|
+
*
|
|
274
|
+
* await selfImprove({
|
|
275
|
+
* agent: httpDispatch({ resolveUrl: ({ placement }) => REGION_URLS[placement!] }),
|
|
276
|
+
* scenarios,
|
|
277
|
+
* judge,
|
|
278
|
+
* baselineSurface: DEFAULT_PROMPT,
|
|
279
|
+
* cellPlacement: ({ scenario }) => scenario.region,
|
|
280
|
+
* budget: { maxConcurrency: 12 },
|
|
281
|
+
* })
|
|
282
|
+
*/
|
|
283
|
+
export declare function selfImprove<TScenario extends Scenario, TArtifact>(opts: SelfImproveOptions<TScenario, TArtifact>): Promise<SelfImproveResult<TScenario, TArtifact>>;
|
|
284
|
+
//# sourceMappingURL=self-improve.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"self-improve.d.ts","sourceRoot":"","sources":["../../src/contract/self-improve.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAIH,OAAO,EACL,KAAK,wBAAwB,EAE9B,MAAM,0CAA0C,CAAA;AACjD,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAA;AAElF,OAAO,EAEL,KAAK,oBAAoB,EAE1B,MAAM,wBAAwB,CAAA;AAC/B,OAAO,EACL,KAAK,eAAe,EAGrB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,KAAK,EAEV,eAAe,EACf,IAAI,EACJ,WAAW,EACX,oBAAoB,EACpB,cAAc,EACd,QAAQ,EACR,eAAe,EAChB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAsB,KAAK,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAExE,OAAO,KAAK,EAAqB,SAAS,EAAE,MAAM,eAAe,CAAA;AAEjE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErD,MAAM,WAAW,iBAAiB;IAChC;oEACgE;IAChE,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB;+EAC2E;IAC3E,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,+DAA+D;IAC/D,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,uDAAuD;IACvD,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB;2EACuE;IACvE,eAAe,CAAC,EAAE,MAAM,CAAA;IACxB,gEAAgE;IAChE,gBAAgB,CAAC,EAAE,QAAQ,EAAE,CAAA;IAC7B,mFAAmF;IACnF,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,MAAM,WAAW,cAAc;IAC7B,gDAAgD;IAChD,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,0DAA0D;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;yCACqC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,MAAM,wBAAwB,GAChC;IAAE,IAAI,EAAE,kBAAkB,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,GAC/C;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GACzE;IAAE,IAAI,EAAE,oBAAoB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,cAAc,EAAE,MAAM,CAAA;CAAE,GACrE;IAAE,IAAI,EAAE,sBAAsB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAC1F;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AAE5D,MAAM,WAAW,kBAAkB,CAAC,SAAS,SAAS,QAAQ,EAAE,SAAS;IACvE;;;;;;;;;;;;OAYG;IACH,KAAK,EAAE,CAAC,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,eAAe,KAAK,OAAO,CAAC,SAAS,CAAC,CAAA;IAEjG;oEACgE;IAChE,SAAS,EAAE,SAAS,EAAE,CAAA;IAEtB;gEAC4D;IAC5D,KAAK,EAAE,WAAW,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IAExC;8DAC0D;IAC1D,eAAe,EAAE,cAAc,CAAA;IAE/B,gDAAgD;IAChD,MAAM,CAAC,EAAE,iBAAiB,CAAA;IAE1B;gCAC4B;IAC5B,QAAQ,CAAC,EAAE,eAAe,CAAA;IAE1B,kEAAkE;IAClE,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC7B,cAAc,CAAC,EAAE,MAAM,CAAA;IAEvB;wDACoD;IACpD,IAAI,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IAEjC;+BAC2B;IAC3B,GAAG,CAAC,EAAE,cAAc,CAAA;IAEpB;;;;sEAIkE;IAClE,OAAO,CAAC,EAAE,eAAe,CAAA;IAEzB;;8DAE0D;IAC1D,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;;;;;;;OASG;IACH,oBAAoB,CAAC,EAAE,MAAM,SAAS,EAAE,CAAA;IAExC;iFAC6E;IAC7E,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,IAAI,CAAA;IAErD;;;uDAGmD;IACnD,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE;QACtB,QAAQ,EAAE,SAAS,CAAA;QACnB,GAAG,EAAE,MAAM,CAAA;QACX,UAAU,CAAC,EAAE,MAAM,CAAA;KACpB,KAAK,MAAM,GAAG,SAAS,CAAA;IAExB;iEAC6D;IAC7D,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,wBAAwB,KAAK,IAAI,CAAA;IAEtD;;qEAEiE;IACjE,aAAa,CAAC,EAAE,IAAI,GAAG,MAAM,CAAA;IAC7B,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IAEf;;;;;;;;;;;;OAYG;IACH,YAAY,CAAC,EAAE,YAAY,CAAA;IAE3B;uDACmD;IACnD,YAAY,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAErC;;4CAEwC;IACxC,YAAY,CAAC,EAAE,oBAAoB,GAAG,KAAK,CAAA;IAE3C,mEAAmE;IACnE,aAAa,CAAC,EAAE,kBAAkB,GAAG,UAAU,GAAG,QAAQ,GAAG,UAAU,GAAG,WAAW,CAAA;IAErF;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,QAAQ,GAAG,MAAM,GAAG,KAAK,CAAA;IAEvC;;;;;;OAMG;IACH,iBAAiB,CAAC,EAAE,sBAAsB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,mBAAmB,CAAC,CAAA;IAErF;uEACmE;IACnE,QAAQ,CAAC,EAAE,OAAO,EAAE,CAAA;CACrB;AAED,MAAM,WAAW,iBAAiB,CAAC,SAAS,SAAS,QAAQ,EAAE,SAAS;IACtE,yDAAyD;IACzD,QAAQ,EAAE;QACR,aAAa,EAAE,MAAM,CAAA;QACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KACpC,CAAA;IACD,sDAAsD;IACtD,MAAM,EAAE;QACN,aAAa,EAAE,MAAM,CAAA;QACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QACnC,OAAO,EAAE,cAAc,CAAA;QACvB;sCAC8B;QAC9B,KAAK,CAAC,EAAE,MAAM,CAAA;QACd;;2CAEmC;QACnC,SAAS,CAAC,EAAE,MAAM,CAAA;KACnB,CAAA;IACD;+CAC2C;IAC3C,IAAI,EAAE,MAAM,CAAA;IACZ;oCACgC;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ;;uEAEmE;IACnE,UAAU,EAAE,oBAAoB,CAAA;IAChC,+CAA+C;IAC/C,YAAY,EAAE,MAAM,GAAG,MAAM,GAAG,gBAAgB,GAAG,eAAe,GAAG,cAAc,CAAA;IACnF;iDAC6C;IAC7C,mBAAmB,EAAE,MAAM,CAAA;IAC3B,wBAAwB;IACxB,UAAU,EAAE,MAAM,CAAA;IAClB,qDAAqD;IACrD,YAAY,EAAE,MAAM,CAAA;IACpB;;;;;OAKG;IACH,OAAO,EAAE,aAAa,CAAA;IACtB;;;;OAIG;IACH,GAAG,EAAE,wBAAwB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;CACpD;AAkDD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,wBAAsB,WAAW,CAAC,SAAS,SAAS,QAAQ,EAAE,SAAS,EACrE,IAAI,EAAE,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,GAC7C,OAAO,CAAC,iBAAiB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CA6LlD"}
|