@tangle-network/agent-eval 0.94.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +44 -30
- package/dist/action-policy.d.ts +24 -0
- package/dist/action-policy.d.ts.map +1 -0
- package/dist/action-policy.test.d.ts +2 -0
- package/dist/action-policy.test.d.ts.map +1 -0
- package/dist/active-learning.d.ts +41 -0
- package/dist/active-learning.d.ts.map +1 -0
- package/dist/adapters/http.d.ts +15 -21
- package/dist/adapters/http.d.ts.map +1 -0
- package/dist/adapters/http.js.map +1 -1
- package/dist/adapters/langchain.d.ts +7 -13
- package/dist/adapters/langchain.d.ts.map +1 -0
- package/dist/adapters/otel.d.ts +13 -24
- package/dist/adapters/otel.d.ts.map +1 -0
- package/dist/agent-profile-cell.d.ts +101 -0
- package/dist/agent-profile-cell.d.ts.map +1 -0
- package/dist/agent-profile.d.ts +27 -0
- package/dist/agent-profile.d.ts.map +1 -0
- package/dist/agent-profile.test.d.ts +2 -0
- package/dist/agent-profile.test.d.ts.map +1 -0
- package/dist/analyst/adapters.d.ts +62 -0
- package/dist/analyst/adapters.d.ts.map +1 -0
- package/dist/analyst/analyst.test.d.ts +2 -0
- package/dist/analyst/analyst.test.d.ts.map +1 -0
- package/dist/analyst/ax-service.d.ts +27 -0
- package/dist/analyst/ax-service.d.ts.map +1 -0
- package/dist/analyst/behavioral-analyst.d.ts +28 -0
- package/dist/analyst/behavioral-analyst.d.ts.map +1 -0
- package/dist/analyst/chat-client.d.ts +91 -0
- package/dist/analyst/chat-client.d.ts.map +1 -0
- package/dist/analyst/default-registry.d.ts +27 -0
- package/dist/analyst/default-registry.d.ts.map +1 -0
- package/dist/analyst/default-registry.test.d.ts +2 -0
- package/dist/analyst/default-registry.test.d.ts.map +1 -0
- package/dist/analyst/finding-signature.d.ts +48 -0
- package/dist/analyst/finding-signature.d.ts.map +1 -0
- package/dist/analyst/finding-subject.d.ts +146 -0
- package/dist/analyst/finding-subject.d.ts.map +1 -0
- package/dist/analyst/finding-subject.test.d.ts +2 -0
- package/dist/analyst/finding-subject.test.d.ts.map +1 -0
- package/dist/analyst/findings-store.d.ts +75 -0
- package/dist/analyst/findings-store.d.ts.map +1 -0
- package/dist/analyst/index.d.ts +28 -256
- package/dist/analyst/index.d.ts.map +1 -0
- package/dist/analyst/index.js +5 -65
- package/dist/analyst/index.js.map +1 -1
- package/dist/{kind-factory-0BhLSI27.d.ts → analyst/kind-factory.d.ts} +11 -63
- package/dist/analyst/kind-factory.d.ts.map +1 -0
- package/dist/analyst/kinds/failure-mode.d.ts +19 -0
- package/dist/analyst/kinds/failure-mode.d.ts.map +1 -0
- package/dist/analyst/kinds/improvement.d.ts +23 -0
- package/dist/analyst/kinds/improvement.d.ts.map +1 -0
- package/dist/analyst/kinds/index.d.ts +22 -0
- package/dist/analyst/kinds/index.d.ts.map +1 -0
- package/dist/analyst/kinds/kinds.test.d.ts +2 -0
- package/dist/analyst/kinds/kinds.test.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts +28 -0
- package/dist/analyst/kinds/knowledge-gap.d.ts.map +1 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts +22 -0
- package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.d.ts +84 -0
- package/dist/analyst/kinds/skill-usage.d.ts.map +1 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts +2 -0
- package/dist/analyst/kinds/skill-usage.test.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.d.ts +26 -0
- package/dist/analyst/parse-tolerant.d.ts.map +1 -0
- package/dist/analyst/parse-tolerant.test.d.ts +2 -0
- package/dist/analyst/parse-tolerant.test.d.ts.map +1 -0
- package/dist/analyst/registry.budget.test.d.ts +2 -0
- package/dist/analyst/registry.budget.test.d.ts.map +1 -0
- package/dist/{default-registry-6dhErQbs.d.ts → analyst/registry.d.ts} +8 -37
- package/dist/analyst/registry.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.d.ts +35 -0
- package/dist/analyst/steer-firewall.d.ts.map +1 -0
- package/dist/analyst/steer-firewall.test.d.ts +2 -0
- package/dist/analyst/steer-firewall.test.d.ts.map +1 -0
- package/dist/analyst/structure-findings.d.ts +37 -0
- package/dist/analyst/structure-findings.d.ts.map +1 -0
- package/dist/analyst/structure-findings.test.d.ts +2 -0
- package/dist/analyst/structure-findings.test.d.ts.map +1 -0
- package/dist/analyst/tool-groups.d.ts +34 -0
- package/dist/analyst/tool-groups.d.ts.map +1 -0
- package/dist/{types-Ce17tDlG.d.ts → analyst/types.d.ts} +19 -112
- package/dist/analyst/types.d.ts.map +1 -0
- package/dist/anti-slop.d.ts +59 -0
- package/dist/anti-slop.d.ts.map +1 -0
- package/dist/artifact-validator.d.ts +74 -0
- package/dist/artifact-validator.d.ts.map +1 -0
- package/dist/attestation.d.ts +63 -0
- package/dist/attestation.d.ts.map +1 -0
- package/dist/attestation.test.d.ts +2 -0
- package/dist/attestation.test.d.ts.map +1 -0
- package/dist/authenticity/index.d.ts +15 -16
- package/dist/authenticity/index.d.ts.map +1 -0
- package/dist/authenticity/index.test.d.ts +2 -0
- package/dist/authenticity/index.test.d.ts.map +1 -0
- package/dist/auto-pr.d.ts +120 -0
- package/dist/auto-pr.d.ts.map +1 -0
- package/dist/{baseline-Bbid3WoO.d.ts → baseline.d.ts} +8 -44
- package/dist/baseline.d.ts.map +1 -0
- package/dist/behavior-dsl.d.ts +73 -0
- package/dist/behavior-dsl.d.ts.map +1 -0
- package/dist/belief-state/calibration.d.ts +10 -0
- package/dist/belief-state/calibration.d.ts.map +1 -0
- package/dist/belief-state/calibration.test.d.ts +2 -0
- package/dist/belief-state/calibration.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.d.ts +66 -0
- package/dist/belief-state/code-agent-corpus.d.ts.map +1 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts +2 -0
- package/dist/belief-state/code-agent-corpus.test.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.d.ts +22 -0
- package/dist/belief-state/code-agent-evidence.d.ts.map +1 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts +2 -0
- package/dist/belief-state/code-agent-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/extract.d.ts +7 -0
- package/dist/belief-state/extract.d.ts.map +1 -0
- package/dist/belief-state/extract.test.d.ts +2 -0
- package/dist/belief-state/extract.test.d.ts.map +1 -0
- package/dist/belief-state/index.d.ts +14 -604
- package/dist/belief-state/index.d.ts.map +1 -0
- package/dist/belief-state/ope.d.ts +17 -0
- package/dist/belief-state/ope.d.ts.map +1 -0
- package/dist/belief-state/ope.test.d.ts +2 -0
- package/dist/belief-state/ope.test.d.ts.map +1 -0
- package/dist/belief-state/phase0-measurement.d.ts +55 -0
- package/dist/belief-state/phase0-measurement.d.ts.map +1 -0
- package/dist/belief-state/report.d.ts +17 -0
- package/dist/belief-state/report.d.ts.map +1 -0
- package/dist/belief-state/report.test.d.ts +2 -0
- package/dist/belief-state/report.test.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.d.ts +23 -0
- package/dist/belief-state/research-evidence.d.ts.map +1 -0
- package/dist/belief-state/research-evidence.test.d.ts +2 -0
- package/dist/belief-state/research-evidence.test.d.ts.map +1 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts +32 -0
- package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.d.ts +87 -0
- package/dist/belief-state/runtime-hooks.d.ts.map +1 -0
- package/dist/belief-state/runtime-hooks.test.d.ts +2 -0
- package/dist/belief-state/runtime-hooks.test.d.ts.map +1 -0
- package/dist/belief-state/selective.d.ts +15 -0
- package/dist/belief-state/selective.d.ts.map +1 -0
- package/dist/belief-state/selective.test.d.ts +2 -0
- package/dist/belief-state/selective.test.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.d.ts +80 -0
- package/dist/belief-state/shadow-probe.d.ts.map +1 -0
- package/dist/belief-state/shadow-probe.test.d.ts +2 -0
- package/dist/belief-state/shadow-probe.test.d.ts.map +1 -0
- package/dist/belief-state/types.d.ts +195 -0
- package/dist/belief-state/types.d.ts.map +1 -0
- package/dist/belief-state/types.test.d.ts +2 -0
- package/dist/belief-state/types.test.d.ts.map +1 -0
- package/dist/benchmark.d.ts +14 -0
- package/dist/benchmark.d.ts.map +1 -0
- package/dist/benchmarks/index.d.ts +23 -4
- package/dist/benchmarks/index.d.ts.map +1 -0
- package/dist/benchmarks/routing/dataset.d.ts +34 -0
- package/dist/benchmarks/routing/dataset.d.ts.map +1 -0
- package/dist/benchmarks/routing/index.d.ts +34 -0
- package/dist/benchmarks/routing/index.d.ts.map +1 -0
- package/dist/benchmarks/types.d.ts +49 -0
- package/dist/benchmarks/types.d.ts.map +1 -0
- package/dist/bisector.d.ts +81 -0
- package/dist/bisector.d.ts.map +1 -0
- package/dist/budget-guard.d.ts +31 -0
- package/dist/budget-guard.d.ts.map +1 -0
- package/dist/builder-eval/builder-session.d.ts +111 -0
- package/dist/builder-eval/builder-session.d.ts.map +1 -0
- package/dist/builder-eval/correlation.d.ts +32 -0
- package/dist/builder-eval/correlation.d.ts.map +1 -0
- package/dist/builder-eval/index.d.ts +5 -250
- package/dist/builder-eval/index.d.ts.map +1 -0
- package/dist/builder-eval/project-registry.d.ts +51 -0
- package/dist/builder-eval/project-registry.d.ts.map +1 -0
- package/dist/builder-eval/three-layer-eval.d.ts +55 -0
- package/dist/builder-eval/three-layer-eval.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.d.ts +108 -0
- package/dist/campaign/analyst-surface.d.ts.map +1 -0
- package/dist/campaign/analyst-surface.test.d.ts +2 -0
- package/dist/campaign/analyst-surface.test.d.ts.map +1 -0
- package/dist/campaign/auto-pr.d.ts +46 -0
- package/dist/campaign/auto-pr.d.ts.map +1 -0
- package/dist/campaign/distillation/agreement-judge.d.ts +69 -0
- package/dist/campaign/distillation/agreement-judge.d.ts.map +1 -0
- package/dist/campaign/distillation/cli.d.ts +35 -0
- package/dist/campaign/distillation/cli.d.ts.map +1 -0
- package/dist/campaign/distillation/distillation.test.d.ts +2 -0
- package/dist/campaign/distillation/distillation.test.d.ts.map +1 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts +54 -0
- package/dist/campaign/distillation/gold-scenarios.d.ts.map +1 -0
- package/dist/campaign/distillation/run-distillation.d.ts +119 -0
- package/dist/campaign/distillation/run-distillation.d.ts.map +1 -0
- package/dist/campaign/gates/compose.d.ts +12 -0
- package/dist/campaign/gates/compose.d.ts.map +1 -0
- package/dist/campaign/gates/default-production-gate.d.ts +58 -0
- package/dist/campaign/gates/default-production-gate.d.ts.map +1 -0
- package/dist/campaign/gates/heldout-gate.d.ts +12 -0
- package/dist/campaign/gates/heldout-gate.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.d.ts +125 -0
- package/dist/campaign/gates/promotion-policy.d.ts.map +1 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts +2 -0
- package/dist/campaign/gates/promotion-policy.test.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.d.ts +146 -0
- package/dist/campaign/gates/sequential.d.ts.map +1 -0
- package/dist/campaign/gates/sequential.test.d.ts +2 -0
- package/dist/campaign/gates/sequential.test.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.d.ts +99 -0
- package/dist/campaign/gates/statistical-heldout.d.ts.map +1 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts +2 -0
- package/dist/campaign/gates/statistical-heldout.test.d.ts.map +1 -0
- package/dist/campaign/index.d.ts +38 -1341
- package/dist/campaign/index.d.ts.map +1 -0
- package/dist/campaign/index.js +1863 -1316
- package/dist/campaign/index.js.map +1 -1
- package/dist/campaign/labeled-store/fs-adapter.d.ts +59 -0
- package/dist/campaign/labeled-store/fs-adapter.d.ts.map +1 -0
- package/dist/campaign/presets/compare-proposers.d.ts +146 -0
- package/dist/campaign/presets/compare-proposers.d.ts.map +1 -0
- package/dist/campaign/presets/playback.d.ts +120 -0
- package/dist/campaign/presets/playback.d.ts.map +1 -0
- package/dist/campaign/presets/playback.test.d.ts +2 -0
- package/dist/campaign/presets/playback.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-eval.d.ts +14 -0
- package/dist/campaign/presets/run-eval.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts +63 -0
- package/dist/campaign/presets/run-improvement-loop.d.ts.map +1 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts +2 -0
- package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +1 -0
- package/dist/campaign/presets/run-optimization.d.ts +92 -0
- package/dist/campaign/presets/run-optimization.d.ts.map +1 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts +151 -0
- package/dist/campaign/presets/run-profile-matrix.d.ts.map +1 -0
- package/dist/campaign/presets/run-skill-opt.d.ts +96 -0
- package/dist/campaign/presets/run-skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/_findings-text.d.ts +22 -0
- package/dist/campaign/proposers/_findings-text.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.d.ts +33 -0
- package/dist/campaign/proposers/ace.d.ts.map +1 -0
- package/dist/campaign/proposers/ace.test.d.ts +2 -0
- package/dist/campaign/proposers/ace.test.d.ts.map +1 -0
- package/dist/campaign/proposers/analysis-edit.d.ts +32 -0
- package/dist/campaign/proposers/analysis-edit.d.ts.map +1 -0
- package/dist/campaign/proposers/evolutionary.d.ts +20 -0
- package/dist/campaign/proposers/evolutionary.d.ts.map +1 -0
- package/dist/campaign/proposers/fapo.d.ts +120 -0
- package/dist/campaign/proposers/fapo.d.ts.map +1 -0
- package/dist/campaign/proposers/gepa.d.ts +86 -0
- package/dist/campaign/proposers/gepa.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.d.ts +44 -0
- package/dist/campaign/proposers/halo.d.ts.map +1 -0
- package/dist/campaign/proposers/halo.test.d.ts +2 -0
- package/dist/campaign/proposers/halo.test.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.d.ts +47 -0
- package/dist/campaign/proposers/memory.d.ts.map +1 -0
- package/dist/campaign/proposers/memory.test.d.ts +2 -0
- package/dist/campaign/proposers/memory.test.d.ts.map +1 -0
- package/dist/campaign/proposers/skill-opt.d.ts +88 -0
- package/dist/campaign/proposers/skill-opt.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.d.ts +48 -0
- package/dist/campaign/proposers/trace-analyst.d.ts.map +1 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts +2 -0
- package/dist/campaign/proposers/trace-analyst.test.d.ts.map +1 -0
- package/dist/campaign/provenance.d.ts +185 -0
- package/dist/campaign/provenance.d.ts.map +1 -0
- package/dist/campaign/run-campaign.d.ts +90 -0
- package/dist/campaign/run-campaign.d.ts.map +1 -0
- package/dist/campaign/score-utils.d.ts +26 -0
- package/dist/campaign/score-utils.d.ts.map +1 -0
- package/dist/campaign/skill-patch.d.ts +62 -0
- package/dist/campaign/skill-patch.d.ts.map +1 -0
- package/dist/campaign/storage.d.ts +38 -0
- package/dist/campaign/storage.d.ts.map +1 -0
- package/dist/{types-BU-7W85F.d.ts → campaign/types.d.ts} +98 -99
- package/dist/campaign/types.d.ts.map +1 -0
- package/dist/campaign/worktree/index.d.ts +53 -0
- package/dist/campaign/worktree/index.d.ts.map +1 -0
- package/dist/canary.d.ts +101 -0
- package/dist/canary.d.ts.map +1 -0
- package/dist/causal-attribution.d.ts +45 -0
- package/dist/causal-attribution.d.ts.map +1 -0
- package/dist/{chunk-2K6UUZ7P.js → chunk-2T4EZACH.js} +1 -1
- package/dist/chunk-2T4EZACH.js.map +1 -0
- package/dist/{chunk-CTBHKLEU.js → chunk-77T4STFI.js} +59 -86
- package/dist/chunk-77T4STFI.js.map +1 -0
- package/dist/{chunk-EGPMSBEZ.js → chunk-7QTQKIDD.js} +178 -177
- package/dist/chunk-7QTQKIDD.js.map +1 -0
- package/dist/{chunk-MIFZUPEK.js → chunk-AQ5WQAIV.js} +21 -6
- package/dist/chunk-AQ5WQAIV.js.map +1 -0
- package/dist/chunk-DJWX3GVS.js +81 -0
- package/dist/chunk-DJWX3GVS.js.map +1 -0
- package/dist/{chunk-S6OZEZQK.js → chunk-HMA63UEO.js} +37 -9
- package/dist/{chunk-S6OZEZQK.js.map → chunk-HMA63UEO.js.map} +1 -1
- package/dist/{chunk-TBDR6PAI.js → chunk-IZCEK2HR.js} +2 -2
- package/dist/{chunk-SD2YFWQQ.js → chunk-KKWJD5E6.js} +20 -20
- package/dist/chunk-KKWJD5E6.js.map +1 -0
- package/dist/{chunk-KWRRMR3J.js → chunk-LO6IOIJ2.js} +10 -10
- package/dist/chunk-LO6IOIJ2.js.map +1 -0
- package/dist/{chunk-E4GH6USR.js → chunk-NZEQVRH5.js} +2 -2
- package/dist/chunk-NZEQVRH5.js.map +1 -0
- package/dist/{chunk-MPQWFX6Y.js → chunk-PSWWQXHF.js} +13 -88
- package/dist/chunk-PSWWQXHF.js.map +1 -0
- package/dist/{chunk-Q5LIB7BC.js → chunk-S4SYLDFX.js} +2 -2
- package/dist/chunk-S4SYLDFX.js.map +1 -0
- package/dist/{chunk-KW53MSA5.js → chunk-X74V6ESX.js} +2 -2
- package/dist/{chunk-QMUEXQJS.js → chunk-YBIGNSCZ.js} +81 -4
- package/dist/chunk-YBIGNSCZ.js.map +1 -0
- package/dist/{chunk-2KNZHH3P.js → chunk-Z6L6YSU6.js} +2 -2
- package/dist/ci-gate.d.ts +44 -0
- package/dist/ci-gate.d.ts.map +1 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/client.d.ts +77 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.test.d.ts +2 -0
- package/dist/client.test.d.ts.map +1 -0
- package/dist/command-runner.d.ts +74 -0
- package/dist/command-runner.d.ts.map +1 -0
- package/dist/command-runner.test.d.ts +2 -0
- package/dist/command-runner.test.d.ts.map +1 -0
- package/dist/completion-verifier.d.ts +147 -0
- package/dist/completion-verifier.d.ts.map +1 -0
- package/dist/completion-verifier.test.d.ts +9 -0
- package/dist/completion-verifier.test.d.ts.map +1 -0
- package/dist/concurrency.d.ts +23 -0
- package/dist/concurrency.d.ts.map +1 -0
- package/dist/contamination-guard.d.ts +81 -0
- package/dist/contamination-guard.d.ts.map +1 -0
- package/dist/{analyze-runs-B6Ljo_dI.d.ts → contract/analyze-runs.d.ts} +9 -10
- package/dist/contract/analyze-runs.d.ts.map +1 -0
- package/dist/contract/define-agent-eval.d.ts +52 -0
- package/dist/contract/define-agent-eval.d.ts.map +1 -0
- package/dist/contract/diff.d.ts +114 -0
- package/dist/contract/diff.d.ts.map +1 -0
- package/dist/contract/index.d.ts +106 -640
- package/dist/contract/index.d.ts.map +1 -0
- package/dist/contract/index.js +127 -17
- package/dist/contract/index.js.map +1 -1
- package/dist/{insight-report-DWl3z9tl.d.ts → contract/insight-report.d.ts} +16 -19
- package/dist/contract/insight-report.d.ts.map +1 -0
- package/dist/contract/insight-types-fwd.d.ts +7 -0
- package/dist/contract/insight-types-fwd.d.ts.map +1 -0
- package/dist/contract/intake/agent-trace.d.ts +97 -0
- package/dist/contract/intake/agent-trace.d.ts.map +1 -0
- package/dist/{code-agent-session-BO8nCnv3.d.ts → contract/intake/code-agent-session.d.ts} +15 -17
- package/dist/contract/intake/code-agent-session.d.ts.map +1 -0
- package/dist/contract/intake/feedback-table.d.ts +87 -0
- package/dist/contract/intake/feedback-table.d.ts.map +1 -0
- package/dist/contract/intake/index.d.ts +22 -0
- package/dist/contract/intake/index.d.ts.map +1 -0
- package/dist/contract/intake/otel-spans.d.ts +33 -0
- package/dist/contract/intake/otel-spans.d.ts.map +1 -0
- package/dist/contract/self-improve.d.ts +284 -0
- package/dist/contract/self-improve.d.ts.map +1 -0
- package/dist/{control-runtime-Acf9CGhw.d.ts → control-runtime.d.ts} +23 -26
- package/dist/control-runtime.d.ts.map +1 -0
- package/dist/control-runtime.test.d.ts +2 -0
- package/dist/control-runtime.test.d.ts.map +1 -0
- package/dist/control.d.ts +11 -9
- package/dist/control.d.ts.map +1 -0
- package/dist/control.js +2 -2
- package/dist/convergence.d.ts +29 -0
- package/dist/convergence.d.ts.map +1 -0
- package/dist/{cost-ledger-DuSqlw5B.d.ts → cost-ledger.d.ts} +10 -11
- package/dist/cost-ledger.d.ts.map +1 -0
- package/dist/cost-ledger.test.d.ts +2 -0
- package/dist/cost-ledger.test.d.ts.map +1 -0
- package/dist/cost-report.d.ts +43 -0
- package/dist/cost-report.d.ts.map +1 -0
- package/dist/cost-report.test.d.ts +2 -0
- package/dist/cost-report.test.d.ts.map +1 -0
- package/dist/cost-tracker.d.ts +76 -0
- package/dist/cost-tracker.d.ts.map +1 -0
- package/dist/{counterfactual-DlOz8PBx.d.ts → counterfactual.d.ts} +12 -13
- package/dist/counterfactual.d.ts.map +1 -0
- package/dist/cross-trace-diff.d.ts +56 -0
- package/dist/cross-trace-diff.d.ts.map +1 -0
- package/dist/{dataset-BbGkaN2I.d.ts → dataset.d.ts} +11 -14
- package/dist/dataset.d.ts.map +1 -0
- package/dist/deploy-gate-layer.d.ts +125 -0
- package/dist/deploy-gate-layer.d.ts.map +1 -0
- package/dist/deploy-gate-layer.test.d.ts +2 -0
- package/dist/deploy-gate-layer.test.d.ts.map +1 -0
- package/dist/description-length-gate.d.ts +119 -0
- package/dist/description-length-gate.d.ts.map +1 -0
- package/dist/detectors/edge.test.d.ts +2 -0
- package/dist/detectors/edge.test.d.ts.map +1 -0
- package/dist/detectors/index.d.ts +81 -0
- package/dist/detectors/index.d.ts.map +1 -0
- package/dist/detectors/index.test.d.ts +2 -0
- package/dist/detectors/index.test.d.ts.map +1 -0
- package/dist/diagnose/causal-sweep.d.ts +100 -0
- package/dist/diagnose/causal-sweep.d.ts.map +1 -0
- package/dist/diagnose/index.d.ts +36 -0
- package/dist/diagnose/index.d.ts.map +1 -0
- package/dist/diagnose/remediation.d.ts +68 -0
- package/dist/diagnose/remediation.d.ts.map +1 -0
- package/dist/diagnose/repair.d.ts +77 -0
- package/dist/diagnose/repair.d.ts.map +1 -0
- package/dist/diagnose.d.ts +1 -251
- package/dist/diagnose.js +1 -1
- package/dist/discover-personas.d.ts +35 -0
- package/dist/discover-personas.d.ts.map +1 -0
- package/dist/driver.d.ts +95 -0
- package/dist/driver.d.ts.map +1 -0
- package/dist/driver.test.d.ts +8 -0
- package/dist/driver.test.d.ts.map +1 -0
- package/dist/dual-agent-bench.d.ts +81 -0
- package/dist/dual-agent-bench.d.ts.map +1 -0
- package/dist/error-count-extractor.d.ts +47 -0
- package/dist/error-count-extractor.d.ts.map +1 -0
- package/dist/error-count-extractor.test.d.ts +2 -0
- package/dist/error-count-extractor.test.d.ts.map +1 -0
- package/dist/{errors-CzMUYo7b.d.ts → errors.d.ts} +10 -11
- package/dist/errors.d.ts.map +1 -0
- package/dist/{researcher-B0C2_fVO.d.ts → eval-campaign.d.ts} +34 -156
- package/dist/eval-campaign.d.ts.map +1 -0
- package/dist/eval-campaign.test.d.ts +2 -0
- package/dist/eval-campaign.test.d.ts.map +1 -0
- package/dist/eval-tools.d.ts +55 -0
- package/dist/eval-tools.d.ts.map +1 -0
- package/dist/eval-trace-store.d.ts +107 -0
- package/dist/eval-trace-store.d.ts.map +1 -0
- package/dist/eval-trace-store.test.d.ts +2 -0
- package/dist/eval-trace-store.test.d.ts.map +1 -0
- package/dist/executor.d.ts +38 -0
- package/dist/executor.d.ts.map +1 -0
- package/dist/executor.test.d.ts +10 -0
- package/dist/executor.test.d.ts.map +1 -0
- package/dist/experiment-tracker.d.ts +178 -0
- package/dist/experiment-tracker.d.ts.map +1 -0
- package/dist/experiment-tracker.test.d.ts +2 -0
- package/dist/experiment-tracker.test.d.ts.map +1 -0
- package/dist/failure-taxonomy.d.ts +38 -0
- package/dist/failure-taxonomy.d.ts.map +1 -0
- package/dist/{feedback-trajectory-BxY0cKfs.d.ts → feedback-trajectory.d.ts} +36 -38
- package/dist/feedback-trajectory.d.ts.map +1 -0
- package/dist/feedback-trajectory.test.d.ts +2 -0
- package/dist/feedback-trajectory.test.d.ts.map +1 -0
- package/dist/flow-layer.d.ts +90 -0
- package/dist/flow-layer.d.ts.map +1 -0
- package/dist/flow-layer.test.d.ts +2 -0
- package/dist/flow-layer.test.d.ts.map +1 -0
- package/dist/fuzz/capsule.d.ts +46 -0
- package/dist/fuzz/capsule.d.ts.map +1 -0
- package/dist/fuzz/cube.d.ts +36 -0
- package/dist/fuzz/cube.d.ts.map +1 -0
- package/dist/fuzz/explorer-cost.test.d.ts +2 -0
- package/dist/fuzz/explorer-cost.test.d.ts.map +1 -0
- package/dist/fuzz/explorer.d.ts +64 -0
- package/dist/fuzz/explorer.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.d.ts +16 -0
- package/dist/fuzz/fuzz-agent.d.ts.map +1 -0
- package/dist/fuzz/fuzz-agent.test.d.ts +2 -0
- package/dist/fuzz/fuzz-agent.test.d.ts.map +1 -0
- package/dist/fuzz/gates.d.ts +33 -0
- package/dist/fuzz/gates.d.ts.map +1 -0
- package/dist/fuzz/index.d.ts +26 -0
- package/dist/fuzz/index.d.ts.map +1 -0
- package/dist/fuzz/policies.d.ts +28 -0
- package/dist/fuzz/policies.d.ts.map +1 -0
- package/dist/fuzz/tools.d.ts +20 -0
- package/dist/fuzz/tools.d.ts.map +1 -0
- package/dist/fuzz/types.d.ts +307 -0
- package/dist/fuzz/types.d.ts.map +1 -0
- package/dist/fuzz.d.ts +1 -547
- package/dist/golden-matcher.d.ts +71 -0
- package/dist/golden-matcher.d.ts.map +1 -0
- package/dist/governance/eu-ai-act.d.ts +37 -0
- package/dist/governance/eu-ai-act.d.ts.map +1 -0
- package/dist/governance/index.d.ts +5 -135
- package/dist/governance/index.d.ts.map +1 -0
- package/dist/governance/nist-ai-rmf.d.ts +15 -0
- package/dist/governance/nist-ai-rmf.d.ts.map +1 -0
- package/dist/governance/soc2.d.ts +12 -0
- package/dist/governance/soc2.d.ts.map +1 -0
- package/dist/governance/types.d.ts +66 -0
- package/dist/governance/types.d.ts.map +1 -0
- package/dist/harness-optimizer.d.ts +82 -0
- package/dist/harness-optimizer.d.ts.map +1 -0
- package/dist/held-out-gate.d.ts +135 -0
- package/dist/held-out-gate.d.ts.map +1 -0
- package/dist/hosted/client.d.ts +73 -0
- package/dist/hosted/client.d.ts.map +1 -0
- package/dist/hosted/from-env.test.d.ts +8 -0
- package/dist/hosted/from-env.test.d.ts.map +1 -0
- package/dist/hosted/index.d.ts +10 -238
- package/dist/hosted/index.d.ts.map +1 -0
- package/dist/hosted/types.d.ts +159 -0
- package/dist/hosted/types.d.ts.map +1 -0
- package/dist/index.d.ts +277 -5665
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +66 -31
- package/dist/index.js.map +1 -1
- package/dist/integrity/backend-integrity.d.ts +71 -0
- package/dist/integrity/backend-integrity.d.ts.map +1 -0
- package/dist/integrity/preflight.d.ts +72 -0
- package/dist/integrity/preflight.d.ts.map +1 -0
- package/dist/integrity/preflight.test.d.ts +2 -0
- package/dist/integrity/preflight.test.d.ts.map +1 -0
- package/dist/integrity/single-backend.d.ts +67 -0
- package/dist/integrity/single-backend.d.ts.map +1 -0
- package/dist/intent-match-judge.d.ts +69 -0
- package/dist/intent-match-judge.d.ts.map +1 -0
- package/dist/intent-match-judge.test.d.ts +2 -0
- package/dist/intent-match-judge.test.d.ts.map +1 -0
- package/dist/{judge-calibration-0p2QcWNE.d.ts → judge-calibration.d.ts} +16 -17
- package/dist/judge-calibration.d.ts.map +1 -0
- package/dist/judge-ensemble.d.ts +66 -0
- package/dist/judge-ensemble.d.ts.map +1 -0
- package/dist/judge-ensemble.test.d.ts +8 -0
- package/dist/judge-ensemble.test.d.ts.map +1 -0
- package/dist/judge-families.d.ts +38 -0
- package/dist/judge-families.d.ts.map +1 -0
- package/dist/judge-panel.d.ts +65 -0
- package/dist/judge-panel.d.ts.map +1 -0
- package/dist/judge-retry.d.ts +70 -0
- package/dist/judge-retry.d.ts.map +1 -0
- package/dist/judge-runner.d.ts +36 -0
- package/dist/judge-runner.d.ts.map +1 -0
- package/dist/judge-runner.test.d.ts +2 -0
- package/dist/judge-runner.test.d.ts.map +1 -0
- package/dist/judges.d.ts +74 -0
- package/dist/judges.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.d.ts +89 -0
- package/dist/keyword-coverage-judge.d.ts.map +1 -0
- package/dist/keyword-coverage-judge.test.d.ts +2 -0
- package/dist/keyword-coverage-judge.test.d.ts.map +1 -0
- package/dist/knowledge/index.d.ts +3 -103
- package/dist/knowledge/index.d.ts.map +1 -0
- package/dist/knowledge/readiness.d.ts +26 -0
- package/dist/knowledge/readiness.d.ts.map +1 -0
- package/dist/knowledge/types.d.ts +75 -0
- package/dist/knowledge/types.d.ts.map +1 -0
- package/dist/live-proof.d.ts +62 -0
- package/dist/live-proof.d.ts.map +1 -0
- package/dist/{llm-client-Bj7g0rqu.d.ts → llm-client.d.ts} +21 -23
- package/dist/llm-client.d.ts.map +1 -0
- package/dist/llm-client.test.d.ts +2 -0
- package/dist/llm-client.test.d.ts.map +1 -0
- package/dist/locked-jsonl-appender.d.ts +19 -0
- package/dist/locked-jsonl-appender.d.ts.map +1 -0
- package/dist/matrix/aggregation.d.ts +16 -0
- package/dist/matrix/aggregation.d.ts.map +1 -0
- package/dist/matrix/index.d.ts +12 -30
- package/dist/matrix/index.d.ts.map +1 -0
- package/dist/matrix/runner.d.ts +15 -0
- package/dist/matrix/runner.d.ts.map +1 -0
- package/dist/{types-mn5Aqk7x.d.ts → matrix/types.d.ts} +11 -15
- package/dist/matrix/types.d.ts.map +1 -0
- package/dist/meta-eval/calibration.d.ts +47 -0
- package/dist/meta-eval/calibration.d.ts.map +1 -0
- package/dist/meta-eval/correlation-study.d.ts +53 -0
- package/dist/meta-eval/correlation-study.d.ts.map +1 -0
- package/dist/meta-eval/index.d.ts +6 -181
- package/dist/meta-eval/index.d.ts.map +1 -0
- package/dist/{outcome-store-rnXLEqSn.d.ts → meta-eval/outcome-store.d.ts} +7 -8
- package/dist/meta-eval/outcome-store.d.ts.map +1 -0
- package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → meta-eval/rubric-predictive-validity.d.ts} +8 -11
- package/dist/meta-eval/rubric-predictive-validity.d.ts.map +1 -0
- package/dist/meta-eval/sentinel.d.ts +169 -0
- package/dist/meta-eval/sentinel.d.ts.map +1 -0
- package/dist/metrics.d.ts +63 -0
- package/dist/metrics.d.ts.map +1 -0
- package/dist/model-seats.d.ts +71 -0
- package/dist/model-seats.d.ts.map +1 -0
- package/dist/model-seats.test.d.ts +2 -0
- package/dist/model-seats.test.d.ts.map +1 -0
- package/dist/muffled-gate-scanner.d.ts +102 -0
- package/dist/muffled-gate-scanner.d.ts.map +1 -0
- package/dist/{multi-layer-verifier-DUZXrPDA.d.ts → multi-layer-verifier.d.ts} +12 -15
- package/dist/multi-layer-verifier.d.ts.map +1 -0
- package/dist/multi-layer-verifier.test.d.ts +2 -0
- package/dist/multi-layer-verifier.test.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.d.ts +80 -0
- package/dist/multi-toolchain-layer.d.ts.map +1 -0
- package/dist/multi-toolchain-layer.test.d.ts +2 -0
- package/dist/multi-toolchain-layer.test.d.ts.map +1 -0
- package/dist/multishot/default-tools.d.ts +34 -0
- package/dist/multishot/default-tools.d.ts.map +1 -0
- package/dist/multishot/index.d.ts +7 -290
- package/dist/multishot/index.d.ts.map +1 -0
- package/dist/multishot/index.js.map +1 -1
- package/dist/multishot/judges.d.ts +32 -0
- package/dist/multishot/judges.d.ts.map +1 -0
- package/dist/multishot/matrix.d.ts +107 -0
- package/dist/multishot/matrix.d.ts.map +1 -0
- package/dist/multishot/multishot.d.ts +23 -0
- package/dist/multishot/multishot.d.ts.map +1 -0
- package/dist/multishot/router.d.ts +37 -0
- package/dist/multishot/router.d.ts.map +1 -0
- package/dist/multishot/types.d.ts +60 -0
- package/dist/multishot/types.d.ts.map +1 -0
- package/dist/observability.d.ts +71 -0
- package/dist/observability.d.ts.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/oracle.d.ts +55 -0
- package/dist/oracle.d.ts.map +1 -0
- package/dist/orthogonality.d.ts +35 -0
- package/dist/orthogonality.d.ts.map +1 -0
- package/dist/otel-pipeline.d.ts +31 -0
- package/dist/otel-pipeline.d.ts.map +1 -0
- package/dist/paraphrase.d.ts +107 -0
- package/dist/paraphrase.d.ts.map +1 -0
- package/dist/{pareto-E-pembql.d.ts → pareto.d.ts} +9 -10
- package/dist/pareto.d.ts.map +1 -0
- package/dist/partition-held-out.d.ts +70 -0
- package/dist/partition-held-out.d.ts.map +1 -0
- package/dist/partition-held-out.test.d.ts +2 -0
- package/dist/partition-held-out.test.d.ts.map +1 -0
- package/dist/perf/index.d.ts +13 -119
- package/dist/perf/index.d.ts.map +1 -0
- package/dist/perf/integrity.d.ts +30 -0
- package/dist/perf/integrity.d.ts.map +1 -0
- package/dist/perf/journey.d.ts +45 -0
- package/dist/perf/journey.d.ts.map +1 -0
- package/dist/perf/ratchet.d.ts +47 -0
- package/dist/perf/ratchet.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.d.ts +31 -0
- package/dist/pipelines/budget-breach.d.ts.map +1 -0
- package/dist/pipelines/budget-breach.test.d.ts +2 -0
- package/dist/pipelines/budget-breach.test.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.d.ts +38 -0
- package/dist/pipelines/failure-cluster.d.ts.map +1 -0
- package/dist/pipelines/failure-cluster.test.d.ts +2 -0
- package/dist/pipelines/failure-cluster.test.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.d.ts +26 -0
- package/dist/pipelines/first-divergence.d.ts.map +1 -0
- package/dist/pipelines/first-divergence.test.d.ts +2 -0
- package/dist/pipelines/first-divergence.test.d.ts.map +1 -0
- package/dist/pipelines/index.d.ts +8 -173
- package/dist/pipelines/index.d.ts.map +1 -0
- package/dist/pipelines/judge-agreement.d.ts +26 -0
- package/dist/pipelines/judge-agreement.d.ts.map +1 -0
- package/dist/pipelines/judge-agreement.test.d.ts +2 -0
- package/dist/pipelines/judge-agreement.test.d.ts.map +1 -0
- package/dist/pipelines/regression.d.ts +23 -0
- package/dist/pipelines/regression.d.ts.map +1 -0
- package/dist/pipelines/regression.test.d.ts +2 -0
- package/dist/pipelines/regression.test.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.d.ts +32 -0
- package/dist/pipelines/stuck-loop.d.ts.map +1 -0
- package/dist/pipelines/stuck-loop.test.d.ts +2 -0
- package/dist/pipelines/stuck-loop.test.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.d.ts +34 -0
- package/dist/pipelines/tool-waste.d.ts.map +1 -0
- package/dist/pipelines/tool-waste.test.d.ts +2 -0
- package/dist/pipelines/tool-waste.test.d.ts.map +1 -0
- package/dist/playbook.d.ts +16 -0
- package/dist/playbook.d.ts.map +1 -0
- package/dist/pr-review-benchmark.d.ts +88 -0
- package/dist/pr-review-benchmark.d.ts.map +1 -0
- package/dist/pr-review-benchmark.test.d.ts +2 -0
- package/dist/pr-review-benchmark.test.d.ts.map +1 -0
- package/dist/pre-registration.d.ts +125 -0
- package/dist/pre-registration.d.ts.map +1 -0
- package/dist/prm/builtin-rubrics.d.ts +33 -0
- package/dist/prm/builtin-rubrics.d.ts.map +1 -0
- package/dist/prm/index.d.ts +5 -104
- package/dist/prm/index.d.ts.map +1 -0
- package/dist/prm/inference.d.ts +29 -0
- package/dist/prm/inference.d.ts.map +1 -0
- package/dist/prm/inference.test.d.ts +2 -0
- package/dist/prm/inference.test.d.ts.map +1 -0
- package/dist/{rubric-Cc6UHvUb.d.ts → prm/rubric.d.ts} +10 -13
- package/dist/prm/rubric.d.ts.map +1 -0
- package/dist/prm/training-export.d.ts +38 -0
- package/dist/prm/training-export.d.ts.map +1 -0
- package/dist/produced-state.d.ts +63 -0
- package/dist/produced-state.d.ts.map +1 -0
- package/dist/produced-state.test.d.ts +8 -0
- package/dist/produced-state.test.d.ts.map +1 -0
- package/dist/profile/baselines.d.ts +37 -0
- package/dist/profile/baselines.d.ts.map +1 -0
- package/dist/profile/index.d.ts +105 -0
- package/dist/profile/index.d.ts.map +1 -0
- package/dist/promotion-gate.d.ts +93 -0
- package/dist/promotion-gate.d.ts.map +1 -0
- package/dist/prompt-registry.d.ts +41 -0
- package/dist/prompt-registry.d.ts.map +1 -0
- package/dist/propose-review-control.d.ts +49 -0
- package/dist/propose-review-control.d.ts.map +1 -0
- package/dist/propose-review-control.test.d.ts +2 -0
- package/dist/propose-review-control.test.d.ts.map +1 -0
- package/dist/propose-review.d.ts +155 -0
- package/dist/propose-review.d.ts.map +1 -0
- package/dist/{red-team-BWdoyleI.d.ts → red-team.d.ts} +14 -16
- package/dist/red-team.d.ts.map +1 -0
- package/dist/reference-replay-steering.d.ts +11 -0
- package/dist/reference-replay-steering.d.ts.map +1 -0
- package/dist/reference-replay.d.ts +176 -0
- package/dist/reference-replay.d.ts.map +1 -0
- package/dist/reflective-mutation.d.ts +79 -0
- package/dist/reflective-mutation.d.ts.map +1 -0
- package/dist/registry.d.ts +31 -0
- package/dist/registry.d.ts.map +1 -0
- package/dist/release-confidence.d.ts +128 -0
- package/dist/release-confidence.d.ts.map +1 -0
- package/dist/release-report.d.ts +11 -0
- package/dist/release-report.d.ts.map +1 -0
- package/dist/replay.d.ts +120 -0
- package/dist/replay.d.ts.map +1 -0
- package/dist/reporter.d.ts +14 -0
- package/dist/reporter.d.ts.map +1 -0
- package/dist/reporting.d.ts +15 -15
- package/dist/reporting.d.ts.map +1 -0
- package/dist/researcher.d.ts +140 -0
- package/dist/researcher.d.ts.map +1 -0
- package/dist/reviewer.d.ts +118 -0
- package/dist/reviewer.d.ts.map +1 -0
- package/dist/reviewer.test.d.ts +2 -0
- package/dist/reviewer.test.d.ts.map +1 -0
- package/dist/reward-model-export.d.ts +60 -0
- package/dist/reward-model-export.d.ts.map +1 -0
- package/dist/rl/active-curriculum.d.ts +110 -0
- package/dist/rl/active-curriculum.d.ts.map +1 -0
- package/dist/rl/adaptation-eval.d.ts +109 -0
- package/dist/rl/adaptation-eval.d.ts.map +1 -0
- package/dist/{adversarial-DIVcDoI_.d.ts → rl/adversarial.d.ts} +6 -7
- package/dist/rl/adversarial.d.ts.map +1 -0
- package/dist/rl/compute-curves.d.ts +127 -0
- package/dist/rl/compute-curves.d.ts.map +1 -0
- package/dist/rl/contamination.d.ts +117 -0
- package/dist/rl/contamination.d.ts.map +1 -0
- package/dist/rl/corpus.d.ts +55 -0
- package/dist/rl/corpus.d.ts.map +1 -0
- package/dist/rl/corpus.test.d.ts +2 -0
- package/dist/rl/corpus.test.d.ts.map +1 -0
- package/dist/rl/dataset.d.ts +102 -0
- package/dist/rl/dataset.d.ts.map +1 -0
- package/dist/rl/dataset.test.d.ts +2 -0
- package/dist/rl/dataset.test.d.ts.map +1 -0
- package/dist/rl/exporters.d.ts +141 -0
- package/dist/rl/exporters.d.ts.map +1 -0
- package/dist/rl/index.d.ts +49 -0
- package/dist/rl/index.d.ts.map +1 -0
- package/dist/{off-policy-DiwuKKg7.d.ts → rl/off-policy.d.ts} +8 -9
- package/dist/rl/off-policy.d.ts.map +1 -0
- package/dist/rl/predictive-validity-researcher.d.ts +69 -0
- package/dist/rl/predictive-validity-researcher.d.ts.map +1 -0
- package/dist/rl/preferences.d.ts +141 -0
- package/dist/rl/preferences.d.ts.map +1 -0
- package/dist/rl/process-reward.d.ts +122 -0
- package/dist/rl/process-reward.d.ts.map +1 -0
- package/dist/rl/reward-hacking.d.ts +104 -0
- package/dist/rl/reward-hacking.d.ts.map +1 -0
- package/dist/rl/rl-campaign.d.ts +85 -0
- package/dist/rl/rl-campaign.d.ts.map +1 -0
- package/dist/rl/run-record-adapters.d.ts +56 -0
- package/dist/rl/run-record-adapters.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.d.ts +166 -0
- package/dist/rl/sim-fidelity.d.ts.map +1 -0
- package/dist/rl/sim-fidelity.test.d.ts +2 -0
- package/dist/rl/sim-fidelity.test.d.ts.map +1 -0
- package/dist/rl/tournament.d.ts +115 -0
- package/dist/rl/tournament.d.ts.map +1 -0
- package/dist/rl/verifiable-reward.d.ts +124 -0
- package/dist/rl/verifiable-reward.d.ts.map +1 -0
- package/dist/rl.d.ts +1 -1192
- package/dist/rl.js +612 -612
- package/dist/rl.js.map +1 -1
- package/dist/{run-campaign-7WNXMDSN.js → run-campaign-WXY7KI67.js} +2 -2
- package/dist/run-critic.d.ts +23 -0
- package/dist/run-critic.d.ts.map +1 -0
- package/dist/run-evidence.d.ts +32 -0
- package/dist/run-evidence.d.ts.map +1 -0
- package/dist/{run-record-e7vj1uZQ.d.ts → run-record.d.ts} +16 -122
- package/dist/run-record.d.ts.map +1 -0
- package/dist/run-record.test.d.ts +2 -0
- package/dist/run-record.test.d.ts.map +1 -0
- package/dist/run-score.d.ts +31 -0
- package/dist/run-score.d.ts.map +1 -0
- package/dist/runtime-trajectory.d.ts +47 -0
- package/dist/runtime-trajectory.d.ts.map +1 -0
- package/dist/{test-graded-scenario-DeODGLra.d.ts → sandbox-harness.d.ts} +15 -60
- package/dist/sandbox-harness.d.ts.map +1 -0
- package/dist/sandbox-harness.test.d.ts +2 -0
- package/dist/sandbox-harness.test.d.ts.map +1 -0
- package/dist/sandbox-pool.d.ts +74 -0
- package/dist/sandbox-pool.d.ts.map +1 -0
- package/dist/sandbox-pool.test.d.ts +2 -0
- package/dist/sandbox-pool.test.d.ts.map +1 -0
- package/dist/scorecard.d.ts +133 -0
- package/dist/scorecard.d.ts.map +1 -0
- package/dist/scorecard.test.d.ts +2 -0
- package/dist/scorecard.test.d.ts.map +1 -0
- package/dist/self-play.d.ts +69 -0
- package/dist/self-play.d.ts.map +1 -0
- package/dist/semantic-concept-judge.d.ts +135 -0
- package/dist/semantic-concept-judge.d.ts.map +1 -0
- package/dist/semantic-concept-judge.test.d.ts +2 -0
- package/dist/semantic-concept-judge.test.d.ts.map +1 -0
- package/dist/{sequential-5iSVfzl2.d.ts → sequential.d.ts} +9 -10
- package/dist/sequential.d.ts.map +1 -0
- package/dist/{series-convergence-D5OWMBg6.d.ts → series-convergence.d.ts} +4 -5
- package/dist/series-convergence.d.ts.map +1 -0
- package/dist/slo.d.ts +48 -0
- package/dist/slo.d.ts.map +1 -0
- package/dist/state-continuity.d.ts +47 -0
- package/dist/state-continuity.d.ts.map +1 -0
- package/dist/{statistics-CCJpTGOS.d.ts → statistics.d.ts} +46 -48
- package/dist/statistics.d.ts.map +1 -0
- package/dist/statistics.test.d.ts +2 -0
- package/dist/statistics.test.d.ts.map +1 -0
- package/dist/steering-optimizer.d.ts +58 -0
- package/dist/steering-optimizer.d.ts.map +1 -0
- package/dist/steering.d.ts +24 -0
- package/dist/steering.d.ts.map +1 -0
- package/dist/storyboard/code-edit.d.ts +64 -0
- package/dist/storyboard/code-edit.d.ts.map +1 -0
- package/dist/storyboard/code-edit.test.d.ts +2 -0
- package/dist/storyboard/code-edit.test.d.ts.map +1 -0
- package/dist/storyboard/index.d.ts +16 -81
- package/dist/storyboard/index.d.ts.map +1 -0
- package/dist/storyboard/index.test.d.ts +2 -0
- package/dist/storyboard/index.test.d.ts.map +1 -0
- package/dist/{summary-report-BDOFevaT.d.ts → summary-report.d.ts} +23 -160
- package/dist/summary-report.d.ts.map +1 -0
- package/dist/telemetry/client.d.ts +35 -0
- package/dist/telemetry/client.d.ts.map +1 -0
- package/dist/telemetry/index.d.ts +17 -35
- package/dist/telemetry/index.d.ts.map +1 -0
- package/dist/telemetry/schema.d.ts +61 -0
- package/dist/telemetry/schema.d.ts.map +1 -0
- package/dist/telemetry/sink-fetch.d.ts +39 -0
- package/dist/telemetry/sink-fetch.d.ts.map +1 -0
- package/dist/telemetry/{file.d.ts → sink-file.d.ts} +5 -7
- package/dist/telemetry/sink-file.d.ts.map +1 -0
- package/dist/test-graded-scenario.d.ts +42 -0
- package/dist/test-graded-scenario.d.ts.map +1 -0
- package/dist/testing.d.ts +5 -0
- package/dist/testing.d.ts.map +1 -0
- package/dist/testing.js +8 -0
- package/dist/testing.js.map +1 -0
- package/dist/tool-use-metrics.d.ts +35 -0
- package/dist/tool-use-metrics.d.ts.map +1 -0
- package/dist/trace/capture-fetch.d.ts +48 -0
- package/dist/trace/capture-fetch.d.ts.map +1 -0
- package/dist/trace/capture-fetch.test.d.ts +2 -0
- package/dist/trace/capture-fetch.test.d.ts.map +1 -0
- package/dist/{emitter-C2rqGH_l.d.ts → trace/emitter.d.ts} +9 -12
- package/dist/trace/emitter.d.ts.map +1 -0
- package/dist/trace/extract-usage.d.ts +46 -0
- package/dist/trace/extract-usage.d.ts.map +1 -0
- package/dist/trace/extract-usage.test.d.ts +2 -0
- package/dist/trace/extract-usage.test.d.ts.map +1 -0
- package/dist/trace/index.d.ts +15 -0
- package/dist/trace/index.d.ts.map +1 -0
- package/dist/{integrity-D2t12mMw.d.ts → trace/integrity.d.ts} +11 -14
- package/dist/trace/integrity.d.ts.map +1 -0
- package/dist/trace/otel-bridge.d.ts +29 -0
- package/dist/trace/otel-bridge.d.ts.map +1 -0
- package/dist/trace/otel-export.d.ts +52 -0
- package/dist/trace/otel-export.d.ts.map +1 -0
- package/dist/trace/otel.d.ts +57 -0
- package/dist/trace/otel.d.ts.map +1 -0
- package/dist/trace/otlp-attributes.d.ts +17 -0
- package/dist/trace/otlp-attributes.d.ts.map +1 -0
- package/dist/trace/query.d.ts +29 -0
- package/dist/trace/query.d.ts.map +1 -0
- package/dist/trace/query.test.d.ts +2 -0
- package/dist/trace/query.test.d.ts.map +1 -0
- package/dist/{raw-provider-sink-C46HDghv.d.ts → trace/raw-provider-sink.d.ts} +13 -14
- package/dist/trace/raw-provider-sink.d.ts.map +1 -0
- package/dist/{redact-B40YG2M_.d.ts → trace/redact.d.ts} +7 -8
- package/dist/trace/redact.d.ts.map +1 -0
- package/dist/{schema-m0gsnbt3.d.ts → trace/schema.d.ts} +29 -30
- package/dist/trace/schema.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.d.ts +72 -0
- package/dist/trace/store-to-otlp.d.ts.map +1 -0
- package/dist/trace/store-to-otlp.test.d.ts +2 -0
- package/dist/trace/store-to-otlp.test.d.ts.map +1 -0
- package/dist/{store-BcFXE6LG.d.ts → trace/store.d.ts} +21 -12
- package/dist/trace/store.d.ts.map +1 -0
- package/dist/trace/store.test.d.ts +2 -0
- package/dist/trace/store.test.d.ts.map +1 -0
- package/dist/{analyst-C8HHvfJp.d.ts → trace-analyst/analyst.d.ts} +12 -14
- package/dist/trace-analyst/analyst.d.ts.map +1 -0
- package/dist/trace-analyst/analyst.test.d.ts +2 -0
- package/dist/trace-analyst/analyst.test.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts +40 -0
- package/dist/trace-analyst/behavioral-metrics.d.ts.map +1 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts +2 -0
- package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +1 -0
- package/dist/trace-analyst/hook.d.ts +55 -0
- package/dist/trace-analyst/hook.d.ts.map +1 -0
- package/dist/trace-analyst/index.d.ts +18 -0
- package/dist/trace-analyst/index.d.ts.map +1 -0
- package/dist/trace-analyst/insights.d.ts +71 -0
- package/dist/trace-analyst/insights.d.ts.map +1 -0
- package/dist/trace-analyst/insights.test.d.ts +2 -0
- package/dist/trace-analyst/insights.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-flatten.d.ts +42 -0
- package/dist/trace-analyst/otlp-flatten.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.d.ts +85 -0
- package/dist/trace-analyst/otlp-span.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-span.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-span.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts +115 -0
- package/dist/trace-analyst/otlp-to-run-records.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +1 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/prompts.d.ts +6 -0
- package/dist/trace-analyst/prompts.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.d.ts +126 -0
- package/dist/trace-analyst/store-otlp.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.test.d.ts +8 -0
- package/dist/trace-analyst/store-otlp.test.d.ts.map +1 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +2 -0
- package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +1 -0
- package/dist/trace-analyst/store.d.ts +63 -0
- package/dist/trace-analyst/store.d.ts.map +1 -0
- package/dist/trace-analyst/tools.d.ts +44 -0
- package/dist/trace-analyst/tools.d.ts.map +1 -0
- package/dist/trace-analyst/tools.test.d.ts +10 -0
- package/dist/trace-analyst/tools.test.d.ts.map +1 -0
- package/dist/{store-C1YxJDEK.d.ts → trace-analyst/types.d.ts} +18 -74
- package/dist/trace-analyst/types.d.ts.map +1 -0
- package/dist/trace-contracts.d.ts +180 -0
- package/dist/trace-contracts.d.ts.map +1 -0
- package/dist/traced-analyst.d.ts +26 -0
- package/dist/traced-analyst.d.ts.map +1 -0
- package/dist/traced-judges.d.ts +27 -0
- package/dist/traced-judges.d.ts.map +1 -0
- package/dist/traces.d.ts +4 -960
- package/dist/traces.d.ts.map +1 -0
- package/dist/traces.js +41 -11
- package/dist/{trajectory-2TkpSEVh.d.ts → trajectory.d.ts} +8 -9
- package/dist/trajectory.d.ts.map +1 -0
- package/dist/{types-C7DGg5ex.d.ts → types.d.ts} +31 -33
- package/dist/types.d.ts.map +1 -0
- package/dist/ui-finding.d.ts +104 -0
- package/dist/ui-finding.d.ts.map +1 -0
- package/dist/verdict-cache.d.ts +78 -0
- package/dist/verdict-cache.d.ts.map +1 -0
- package/dist/verdict-cache.test.d.ts +2 -0
- package/dist/verdict-cache.test.d.ts.map +1 -0
- package/dist/{verdict-C9MlYujm.d.ts → verdict.d.ts} +2 -3
- package/dist/verdict.d.ts.map +1 -0
- package/dist/visual-diff.d.ts +32 -0
- package/dist/visual-diff.d.ts.map +1 -0
- package/dist/wire/handlers.d.ts +54 -0
- package/dist/wire/handlers.d.ts.map +1 -0
- package/dist/wire/index.d.ts +13 -570
- package/dist/wire/index.d.ts.map +1 -0
- package/dist/wire/openapi.d.ts +3 -0
- package/dist/wire/openapi.d.ts.map +1 -0
- package/dist/wire/rpc.d.ts +21 -0
- package/dist/wire/rpc.d.ts.map +1 -0
- package/dist/wire/rubrics.d.ts +34 -0
- package/dist/wire/rubrics.d.ts.map +1 -0
- package/dist/wire/schemas.d.ts +410 -0
- package/dist/wire/schemas.d.ts.map +1 -0
- package/dist/wire/server.d.ts +60 -0
- package/dist/wire/server.d.ts.map +1 -0
- package/dist/workflow/event-schema.d.ts +5 -0
- package/dist/workflow/event-schema.d.ts.map +1 -0
- package/dist/workflow/feedback-pack.d.ts +99 -0
- package/dist/workflow/feedback-pack.d.ts.map +1 -0
- package/dist/workflow/index.d.ts +22 -495
- package/dist/workflow/index.d.ts.map +1 -0
- package/dist/workflow/index.js +1 -1
- package/dist/workflow/intelligence-export.d.ts +62 -0
- package/dist/workflow/intelligence-export.d.ts.map +1 -0
- package/dist/workflow/partner-report.d.ts +49 -0
- package/dist/workflow/partner-report.d.ts.map +1 -0
- package/dist/workflow/phase-graph.d.ts +43 -0
- package/dist/workflow/phase-graph.d.ts.map +1 -0
- package/dist/workflow/promotion-gate.d.ts +61 -0
- package/dist/workflow/promotion-gate.d.ts.map +1 -0
- package/dist/workflow/run-record.d.ts +12 -0
- package/dist/workflow/run-record.d.ts.map +1 -0
- package/dist/workflow/runtime-adapter.d.ts +20 -0
- package/dist/workflow/runtime-adapter.d.ts.map +1 -0
- package/dist/workflow/sanitize.d.ts +21 -0
- package/dist/workflow/sanitize.d.ts.map +1 -0
- package/dist/workflow/schema.d.ts +5 -0
- package/dist/workflow/schema.d.ts.map +1 -0
- package/dist/workflow/summary.d.ts +43 -0
- package/dist/workflow/summary.d.ts.map +1 -0
- package/dist/workflow/trace-event-fields.d.ts +6 -0
- package/dist/workflow/trace-event-fields.d.ts.map +1 -0
- package/dist/workflow/trajectory.d.ts +15 -0
- package/dist/workflow/trajectory.d.ts.map +1 -0
- package/dist/workflow/types.d.ts +68 -0
- package/dist/workflow/types.d.ts.map +1 -0
- package/dist/workspace-inspector.d.ts +67 -0
- package/dist/workspace-inspector.d.ts.map +1 -0
- package/dist/wrangler-deploy-runner.test.d.ts +2 -0
- package/dist/wrangler-deploy-runner.test.d.ts.map +1 -0
- package/docs/campaign-proposers.md +170 -0
- package/docs/concepts.md +8 -4
- package/docs/customer-journeys.md +15 -13
- package/docs/design/loop-taxonomy.md +34 -66
- package/docs/distributed-driver.md +14 -14
- package/docs/feature-guide.md +1 -1
- package/docs/hosted-ingest-spec.md +2 -3
- package/docs/multi-shot-optimization.md +8 -8
- package/docs/product-eval-adoption.md +1 -1
- package/docs/self-improvement-map.md +33 -29
- package/package.json +9 -15
- package/dist/calibration-BPmzuVPk.d.ts +0 -101
- package/dist/chunk-2K6UUZ7P.js.map +0 -1
- package/dist/chunk-CTBHKLEU.js.map +0 -1
- package/dist/chunk-E4GH6USR.js.map +0 -1
- package/dist/chunk-EGPMSBEZ.js.map +0 -1
- package/dist/chunk-KWRRMR3J.js.map +0 -1
- package/dist/chunk-MIFZUPEK.js.map +0 -1
- package/dist/chunk-MPQWFX6Y.js.map +0 -1
- package/dist/chunk-Q5LIB7BC.js.map +0 -1
- package/dist/chunk-QMUEXQJS.js.map +0 -1
- package/dist/chunk-SD2YFWQQ.js.map +0 -1
- package/dist/control-D6qwHXIR.d.ts +0 -259
- package/dist/corpus-B8A4BDR3.d.ts +0 -560
- package/dist/failure-cluster-DH9Flgcf.d.ts +0 -76
- package/dist/harness-optimizer-mOl9XX_O.d.ts +0 -106
- package/dist/index-Bx3gZ8xl.d.ts +0 -159
- package/dist/pre-registration-mAnCugl9.d.ts +0 -523
- package/dist/provenance-P-bCL2Fo.d.ts +0 -441
- package/dist/query-B7GGjRox.d.ts +0 -32
- package/dist/release-report-BEbWmVYj.d.ts +0 -233
- package/dist/run-critic-CmMf05uV.d.ts +0 -56
- package/dist/run-improvement-loop-DBahB8Ax.d.ts +0 -427
- package/dist/runtime-trajectory-BDgfGZSr.d.ts +0 -49
- package/dist/semantic-concept-judge-B9MgmBnM.d.ts +0 -624
- package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
- package/docs/design/external-agent-wedge.md +0 -89
- package/docs/design/phase-d-rfc.md +0 -125
- package/docs/design/phase4-consumer-migration.md +0 -70
- package/docs/design/primitives-integration-spec.md +0 -393
- package/docs/design/product-self-improvement-loop.md +0 -146
- package/docs/design/self-improvement-engine.md +0 -140
- package/docs/design/self-improvement-protocol.md +0 -223
- package/docs/design/self-improvement-roadmap.md +0 -106
- package/docs/design/substrate-gaps.md +0 -118
- package/docs/phase-b-pairing-kit.md +0 -188
- package/docs/phase-b-runbook.md +0 -176
- package/docs/pilot/README.md +0 -62
- package/docs/pilot/customer-checklist.md +0 -90
- package/docs/pilot/integration-foreign-stack.md +0 -296
- package/docs/pilot/integration-tangle-stack.md +0 -248
- package/docs/pilot/one-pager.md +0 -161
- package/docs/pilot/sample-insight-report.json +0 -172
- package/docs/quickstart-external.md +0 -229
- package/docs/research/belief-state-agent-eval-roadmap.md +0 -593
- package/docs/research/research-roadmap.md +0 -205
- package/docs/specs/driver-honest-spec.md +0 -251
- package/docs/specs/hermes-self-improvement-audit.md +0 -93
- package/docs/specs/profile-versioning.md +0 -291
- package/docs/three-package-architecture.md +0 -168
- /package/dist/{chunk-TBDR6PAI.js.map → chunk-IZCEK2HR.js.map} +0 -0
- /package/dist/{chunk-KW53MSA5.js.map → chunk-X74V6ESX.js.map} +0 -0
- /package/dist/{chunk-2KNZHH3P.js.map → chunk-Z6L6YSU6.js.map} +0 -0
- /package/dist/{run-campaign-7WNXMDSN.js.map → run-campaign-WXY7KI67.js.map} +0 -0
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Wrap a single judge LLM call with retry, optional fallback-model
|
|
3
|
+
* rotation, exponential backoff, and a typed `JudgeRetryOutcome`. Callers
|
|
4
|
+
* MUST inspect `succeeded` before using `value`; on failure the library
|
|
5
|
+
* returns `value: null` rather than substituting a default, so a judge
|
|
6
|
+
* abort cannot silently corrupt a downstream composite.
|
|
7
|
+
*
|
|
8
|
+
* Reporting contract: callers ship `TrialResult.judgeSucceeded = succeeded`
|
|
9
|
+
* and `TrialResult.judgeAttempts = attempts` so `aggregateTrialsByMode`
|
|
10
|
+
* with `mode: 'exclude-failed'` drops the trial.
|
|
11
|
+
*/
|
|
12
|
+
/** Retry policy for judge LLM calls. */
|
|
13
|
+
export interface JudgeRetryPolicy {
|
|
14
|
+
/** Max attempts per model. Default 3 (one initial + two retries). */
|
|
15
|
+
maxAttempts?: number;
|
|
16
|
+
/** Per-attempt timeout in ms. Default 300_000. */
|
|
17
|
+
timeoutMs?: number;
|
|
18
|
+
/**
|
|
19
|
+
* Models to try, in order. The first model is the primary; subsequent
|
|
20
|
+
* models are fallbacks invoked only when ALL retries on the previous
|
|
21
|
+
* model have been exhausted. Example: `['claude-code/sonnet', 'kimi-code/k2p6']`
|
|
22
|
+
* runs claude-code up to maxAttempts times, then falls back to kimi.
|
|
23
|
+
* If omitted, the caller's judge function controls model selection and
|
|
24
|
+
* the retries apply to that single model.
|
|
25
|
+
*/
|
|
26
|
+
models?: readonly string[];
|
|
27
|
+
/** Exponential backoff function, default `attempt → min(500 * 2^attempt, 16_000)`. */
|
|
28
|
+
backoffMs?: (attempt: number) => number;
|
|
29
|
+
/**
|
|
30
|
+
* Predicate deciding whether an error should trigger a retry. Defaults to
|
|
31
|
+
* `isTransientLlmError` — the package-wide classifier shared with
|
|
32
|
+
* `callLlm` — which retries aborts/timeouts, network faults, HTTP/2
|
|
33
|
+
* transport faults, and any `LlmCallError` with status in {429,502,503,504}.
|
|
34
|
+
* JSON-parse and schema-rejection errors are NOT retriable (the model
|
|
35
|
+
* needs prompt adjustment, not another shot).
|
|
36
|
+
*/
|
|
37
|
+
isRetryable?: (err: unknown) => boolean;
|
|
38
|
+
}
|
|
39
|
+
/** Outcome of a wrapped judge invocation. */
|
|
40
|
+
export interface JudgeRetryOutcome<T> {
|
|
41
|
+
/** The judge's returned value when `succeeded === true`. */
|
|
42
|
+
value: T | null;
|
|
43
|
+
/** True iff one of the attempts completed without throwing. */
|
|
44
|
+
succeeded: boolean;
|
|
45
|
+
/** Total attempts made across all models. */
|
|
46
|
+
attempts: number;
|
|
47
|
+
/** Which model the successful attempt used (when succeeded). */
|
|
48
|
+
modelUsed?: string;
|
|
49
|
+
/** Last error captured when `succeeded === false`. */
|
|
50
|
+
error?: Error;
|
|
51
|
+
/** Per-attempt error log for forensics. */
|
|
52
|
+
attemptErrors: Array<{
|
|
53
|
+
attempt: number;
|
|
54
|
+
model: string;
|
|
55
|
+
error: string;
|
|
56
|
+
}>;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Wrap a judge call with retry + fallback-model + typed outcome semantics.
|
|
60
|
+
*
|
|
61
|
+
* The `judgeFn` signature is `(model: string, signal: AbortSignal) => Promise<T>`.
|
|
62
|
+
* The signal will be aborted at `timeoutMs`. Callers should pass the signal
|
|
63
|
+
* to their underlying fetch/SDK call so the abort actually fires.
|
|
64
|
+
*
|
|
65
|
+
* Returns a typed outcome — callers MUST inspect `succeeded` before using
|
|
66
|
+
* `value`. The library refuses to default to a silent zero score because a
|
|
67
|
+
* synthetic zero is indistinguishable from a real low score downstream.
|
|
68
|
+
*/
|
|
69
|
+
export declare function withJudgeRetry<T>(judgeFn: (model: string, signal: AbortSignal) => Promise<T>, policy?: JudgeRetryPolicy): Promise<JudgeRetryOutcome<T>>;
|
|
70
|
+
//# sourceMappingURL=judge-retry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-retry.d.ts","sourceRoot":"","sources":["../src/judge-retry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,wCAAwC;AACxC,MAAM,WAAW,gBAAgB;IAC/B,qEAAqE;IACrE,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,kDAAkD;IAClD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;;;;;;OAOG;IACH,MAAM,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC1B,sFAAsF;IACtF,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,MAAM,CAAA;IACvC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,OAAO,CAAA;CACxC;AAED,6CAA6C;AAC7C,MAAM,WAAW,iBAAiB,CAAC,CAAC;IAClC,4DAA4D;IAC5D,KAAK,EAAE,CAAC,GAAG,IAAI,CAAA;IACf,+DAA+D;IAC/D,SAAS,EAAE,OAAO,CAAA;IAClB,6CAA6C;IAC7C,QAAQ,EAAE,MAAM,CAAA;IAChB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,sDAAsD;IACtD,KAAK,CAAC,EAAE,KAAK,CAAA;IACb,2CAA2C;IAC3C,aAAa,EAAE,KAAK,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACxE;AASD;;;;;;;;;;GAUG;AACH,wBAAsB,cAAc,CAAC,CAAC,EACpC,OAAO,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,EAC3D,MAAM,GAAE,gBAAqB,GAC5B,OAAO,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAiE/B"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { type HarnessConfig, type SandboxDriver, type SandboxHarnessResult } from './sandbox-harness';
|
|
2
|
+
export type SandboxJudgeKind = 'compiler' | 'test' | 'linter' | 'security';
|
|
3
|
+
export interface SandboxJudgeSpec {
|
|
4
|
+
id: string;
|
|
5
|
+
kind: SandboxJudgeKind;
|
|
6
|
+
config: HarnessConfig;
|
|
7
|
+
}
|
|
8
|
+
export interface SandboxJudgeResult {
|
|
9
|
+
id: string;
|
|
10
|
+
kind: SandboxJudgeKind;
|
|
11
|
+
passed: boolean;
|
|
12
|
+
score: number;
|
|
13
|
+
summary: string;
|
|
14
|
+
detail: SandboxHarnessResult;
|
|
15
|
+
}
|
|
16
|
+
export interface JudgeFleetOptions {
|
|
17
|
+
driver?: SandboxDriver;
|
|
18
|
+
parallel?: boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Max concurrent judge subprocesses. Each spec spawns its own subprocess
|
|
21
|
+
* via the driver, so unbounded fan-out exhausts file descriptors / PIDs.
|
|
22
|
+
* Defaults to the host CPU count. Ignored when `parallel === false`.
|
|
23
|
+
*/
|
|
24
|
+
concurrency?: number;
|
|
25
|
+
}
|
|
26
|
+
export declare class JudgeRunner {
|
|
27
|
+
private readonly driver;
|
|
28
|
+
constructor(driver?: SandboxDriver);
|
|
29
|
+
run(spec: SandboxJudgeSpec): Promise<SandboxJudgeResult>;
|
|
30
|
+
}
|
|
31
|
+
export declare function runJudgeFleet(specs: SandboxJudgeSpec[], options?: JudgeFleetOptions): Promise<SandboxJudgeResult[]>;
|
|
32
|
+
export declare function compilerJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
|
|
33
|
+
export declare function testJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
|
|
34
|
+
export declare function linterJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
|
|
35
|
+
export declare function securityJudge(id: string, config: HarnessConfig): SandboxJudgeSpec;
|
|
36
|
+
//# sourceMappingURL=judge-runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-runner.d.ts","sourceRoot":"","sources":["../src/judge-runner.ts"],"names":[],"mappings":"AACA,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,aAAa,EAElB,KAAK,oBAAoB,EAE1B,MAAM,mBAAmB,CAAA;AAI1B,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,UAAU,CAAA;AAE1E,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,gBAAgB,CAAA;IACtB,MAAM,EAAE,aAAa,CAAA;CACtB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,gBAAgB,CAAA;IACtB,MAAM,EAAE,OAAO,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,oBAAoB,CAAA;CAC7B;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,CAAC,EAAE,aAAa,CAAA;IACtB,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAe;gBAE1B,MAAM,GAAE,aAA6C;IAI3D,GAAG,CAAC,IAAI,EAAE,gBAAgB,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAoB/D;AAED,wBAAsB,aAAa,CACjC,KAAK,EAAE,gBAAgB,EAAE,EACzB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAmB/B;AAED,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,gBAAgB,CAEjF;AAED,wBAAgB,SAAS,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,gBAAgB,CAE7E;AAED,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,gBAAgB,CAE/E;AAED,wBAAgB,aAAa,CAAC,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,GAAG,gBAAgB,CAEjF"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judge-runner.test.d.ts","sourceRoot":"","sources":["../src/judge-runner.test.ts"],"names":[],"mappings":""}
|
package/dist/judges.d.ts
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { JudgeError } from './errors';
|
|
2
|
+
import type { JudgeFn } from './types';
|
|
3
|
+
/**
|
|
4
|
+
* A judge's LLM response could not be parsed into scored dimensions.
|
|
5
|
+
* Thrown instead of fabricating a `{ dimension: 'parse_error', score: 0 }`
|
|
6
|
+
* row — a synthetic zero is indistinguishable from a real low score
|
|
7
|
+
* downstream. Carries the raw response for forensics. Callers (executor,
|
|
8
|
+
* ensemble wrappers) catch this per-judge and record a failed judge.
|
|
9
|
+
*/
|
|
10
|
+
export declare class JudgeParseError extends JudgeError {
|
|
11
|
+
/** Name of the judge whose response failed to parse. */
|
|
12
|
+
readonly judgeName: string;
|
|
13
|
+
/** The raw (truncated) model response that failed to parse. */
|
|
14
|
+
readonly raw: string;
|
|
15
|
+
constructor(judgeName: string, raw: string, options?: {
|
|
16
|
+
cause?: unknown;
|
|
17
|
+
});
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Create a domain expert judge with a configurable domain.
|
|
21
|
+
*
|
|
22
|
+
* The judge evaluates professional accuracy and depth.
|
|
23
|
+
*
|
|
24
|
+
* @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.
|
|
25
|
+
* Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or
|
|
26
|
+
* multi-model panels via `ensembleJudge` (src/judge-panel.ts) — which are
|
|
27
|
+
* pluggable, fail-loud, and drive the campaign/improvement-loop engines.
|
|
28
|
+
*/
|
|
29
|
+
export declare function createDomainExpertJudge(domain: string): JudgeFn;
|
|
30
|
+
/**
|
|
31
|
+
* Code execution judge — evaluates whether code blocks are valid and runnable.
|
|
32
|
+
*
|
|
33
|
+
* @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.
|
|
34
|
+
* Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or
|
|
35
|
+
* multi-model panels via `ensembleJudge` (src/judge-panel.ts).
|
|
36
|
+
*/
|
|
37
|
+
export declare const codeExecutionJudge: JudgeFn;
|
|
38
|
+
/**
|
|
39
|
+
* Coherence judge — evaluates multi-turn consistency and progression.
|
|
40
|
+
*
|
|
41
|
+
* @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.
|
|
42
|
+
* Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or
|
|
43
|
+
* multi-model panels via `ensembleJudge` (src/judge-panel.ts).
|
|
44
|
+
*/
|
|
45
|
+
export declare const coherenceJudge: JudgeFn;
|
|
46
|
+
/**
|
|
47
|
+
* Adversarial judge — red-teams agent responses.
|
|
48
|
+
*
|
|
49
|
+
* @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.
|
|
50
|
+
* Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or
|
|
51
|
+
* multi-model panels via `ensembleJudge` (src/judge-panel.ts).
|
|
52
|
+
*/
|
|
53
|
+
export declare const adversarialJudge: JudgeFn;
|
|
54
|
+
/**
|
|
55
|
+
* Create a custom judge with a fully custom prompt.
|
|
56
|
+
*
|
|
57
|
+
* @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.
|
|
58
|
+
* Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or
|
|
59
|
+
* multi-model panels via `ensembleJudge` (src/judge-panel.ts).
|
|
60
|
+
*/
|
|
61
|
+
export declare function createCustomJudge(name: string, systemPrompt: string, opts?: {
|
|
62
|
+
model?: string;
|
|
63
|
+
temperature?: number;
|
|
64
|
+
maxTokens?: number;
|
|
65
|
+
}): JudgeFn;
|
|
66
|
+
/**
|
|
67
|
+
* Default judge set (domain must be provided for domain expert)
|
|
68
|
+
*
|
|
69
|
+
* @deprecated Legacy `JudgeFn` factory tied to the fixed gpt-4o prompt shape.
|
|
70
|
+
* Build judges as campaign `JudgeConfig`s (src/campaign/types.ts) — or
|
|
71
|
+
* multi-model panels via `ensembleJudge` (src/judge-panel.ts).
|
|
72
|
+
*/
|
|
73
|
+
export declare function defaultJudges(domain: string): JudgeFn[];
|
|
74
|
+
//# sourceMappingURL=judges.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"judges.d.ts","sourceRoot":"","sources":["../src/judges.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AACrC,OAAO,KAAK,EAAE,OAAO,EAA0B,MAAM,SAAS,CAAA;AAE9D;;;;;;GAMG;AACH,qBAAa,eAAgB,SAAQ,UAAU;IAC7C,wDAAwD;IACxD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,+DAA+D;IAC/D,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;gBAER,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE;CAK1E;AAED;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAsC/D;AAED;;;;;;GAMG;AACH,eAAO,MAAM,kBAAkB,EAAE,OA4ChC,CAAA;AAED;;;;;;GAMG;AACH,eAAO,MAAM,cAAc,EAAE,OAuC5B,CAAA;AAED;;;;;;GAMG;AACH,eAAO,MAAM,gBAAgB,EAAE,OAgC9B,CAAA;AAED;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,MAAM,EACpB,IAAI,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAAE,GAClE,OAAO,CA2BT;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,EAAE,CAEvD"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keyword-coverage judge — baseline complement to the semantic concept
|
|
3
|
+
* judge.
|
|
4
|
+
*
|
|
5
|
+
* Where {@link runSemanticConceptJudge} uses an LLM to read source code
|
|
6
|
+
* and decide whether a concept is REALLY implemented (not just
|
|
7
|
+
* keyword-mentioned), this judge does the cheap, deterministic version:
|
|
8
|
+
* fetch the served preview, concatenate every linked CSS/JS asset, and
|
|
9
|
+
* substring-match each expected concept's keywords against the
|
|
10
|
+
* concatenated haystack. Optional `requiredElement` selector adds a
|
|
11
|
+
* structural gate so "supply counter" can require an actual `<input>` or
|
|
12
|
+
* `<table>`, not just a comment containing the word.
|
|
13
|
+
*
|
|
14
|
+
* Use both judges. Keyword coverage is a fast 0-cost gate — a stub page
|
|
15
|
+
* with the right keywords passes here, fails the semantic judge. Score
|
|
16
|
+
* divergence between the two is itself a signal: high keyword coverage
|
|
17
|
+
* + low semantic = "the agent slapped the right words on the right
|
|
18
|
+
* scaffold but didn't wire any of it up."
|
|
19
|
+
*
|
|
20
|
+
* Pure functions, soft-fail on fetch error, no LLM dependency.
|
|
21
|
+
*/
|
|
22
|
+
export interface KeywordConceptSpec {
|
|
23
|
+
name: string;
|
|
24
|
+
keywords: string[];
|
|
25
|
+
/**
|
|
26
|
+
* Optional CSS selector that must match in the HTML for the concept
|
|
27
|
+
* to count as present. Tiny subset:
|
|
28
|
+
* - `tag` (e.g. `form`)
|
|
29
|
+
* - `tag[attr="value"]` (e.g. `input[type="number"]`)
|
|
30
|
+
* - `tag[attr]` (presence only)
|
|
31
|
+
* Anything more complex is rejected with `null` (treated as
|
|
32
|
+
* "unenforced", not "failed").
|
|
33
|
+
*/
|
|
34
|
+
requiredElement?: string;
|
|
35
|
+
}
|
|
36
|
+
export interface KeywordCoverageFinding {
|
|
37
|
+
concept: string;
|
|
38
|
+
found: boolean;
|
|
39
|
+
matchedKeywords: string[];
|
|
40
|
+
/** True iff the optional requiredElement selector matched; null when no selector. */
|
|
41
|
+
requiredElementPresent: boolean | null;
|
|
42
|
+
}
|
|
43
|
+
export interface KeywordCoverageResult {
|
|
44
|
+
/** 0..1 share of concepts satisfied. */
|
|
45
|
+
score: number;
|
|
46
|
+
presentCount: number;
|
|
47
|
+
totalCount: number;
|
|
48
|
+
findings: KeywordCoverageFinding[];
|
|
49
|
+
durationMs: number;
|
|
50
|
+
/** Total bytes assembled across html + linked assets. */
|
|
51
|
+
totalAssembledBytes: number;
|
|
52
|
+
/** Soft-failure reason if the audit couldn't run. */
|
|
53
|
+
error?: string;
|
|
54
|
+
}
|
|
55
|
+
export interface KeywordCoverageOptions {
|
|
56
|
+
/** Override fetch implementation — for tests. */
|
|
57
|
+
fetch?: typeof fetch;
|
|
58
|
+
/** Per-asset fetch timeout (default 3s). */
|
|
59
|
+
assetTimeoutMs?: number;
|
|
60
|
+
/** Initial-HTML fetch timeout (default 5s). */
|
|
61
|
+
htmlTimeoutMs?: number;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Element-presence check using a tiny CSS-selector subset. Returns
|
|
65
|
+
* null when the selector isn't supported — caller treats that as
|
|
66
|
+
* "unenforced" rather than "failed."
|
|
67
|
+
*/
|
|
68
|
+
export declare function htmlContainsElement(html: string, selector: string): boolean | null;
|
|
69
|
+
/**
|
|
70
|
+
* Pull every `<link rel=stylesheet href>` and `<script src>` from a
|
|
71
|
+
* raw HTML body. Returns absolute URLs resolved against `baseUrl`.
|
|
72
|
+
* Permissive regex — agent-authored markup doesn't always quote
|
|
73
|
+
* attributes the same way.
|
|
74
|
+
*/
|
|
75
|
+
export declare function extractAssetUrls(html: string, baseUrl: string): string[];
|
|
76
|
+
/**
|
|
77
|
+
* Score expected concepts against an already-fetched HTML payload + any
|
|
78
|
+
* pre-fetched CSS/JS assets. Use when the runner has the bytes in hand
|
|
79
|
+
* and doesn't want a fresh HTTP round-trip — e.g. sandbox runtime where
|
|
80
|
+
* the preview content was fetched via curl from inside the container.
|
|
81
|
+
*/
|
|
82
|
+
export declare function runKeywordCoverageJudge(html: string, expectedConcepts: ReadonlyArray<KeywordConceptSpec>, assets?: ReadonlyArray<string>): KeywordCoverageResult;
|
|
83
|
+
/**
|
|
84
|
+
* URL-fetch flavor — GET the preview, parallel-fetch every linked
|
|
85
|
+
* stylesheet + script (with bounded timeouts, soft-fail individually),
|
|
86
|
+
* then score via {@link runKeywordCoverageJudge}.
|
|
87
|
+
*/
|
|
88
|
+
export declare function runKeywordCoverageJudgeUrl(previewUrl: string, expectedConcepts: ReadonlyArray<KeywordConceptSpec>, options?: KeywordCoverageOptions): Promise<KeywordCoverageResult>;
|
|
89
|
+
//# sourceMappingURL=keyword-coverage-judge.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"keyword-coverage-judge.d.ts","sourceRoot":"","sources":["../src/keyword-coverage-judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAIH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB;;;;;;;;OAQG;IACH,eAAe,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,sBAAsB;IACrC,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,OAAO,CAAA;IACd,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,qFAAqF;IACrF,sBAAsB,EAAE,OAAO,GAAG,IAAI,CAAA;CACvC;AAED,MAAM,WAAW,qBAAqB;IACpC,wCAAwC;IACxC,KAAK,EAAE,MAAM,CAAA;IACb,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,sBAAsB,EAAE,CAAA;IAClC,UAAU,EAAE,MAAM,CAAA;IAClB,yDAAyD;IACzD,mBAAmB,EAAE,MAAM,CAAA;IAC3B,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,sBAAsB;IACrC,iDAAiD;IACjD,KAAK,CAAC,EAAE,OAAO,KAAK,CAAA;IACpB,4CAA4C;IAC5C,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,+CAA+C;IAC/C,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB;AAID;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,GAAG,IAAI,CAsBlF;AAID;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAiBxE;AAID;;;;;GAKG;AACH,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,EACZ,gBAAgB,EAAE,aAAa,CAAC,kBAAkB,CAAC,EACnD,MAAM,GAAE,aAAa,CAAC,MAAM,CAAM,GACjC,qBAAqB,CAkCvB;AAED;;;;GAIG;AACH,wBAAsB,0BAA0B,CAC9C,UAAU,EAAE,MAAM,EAClB,gBAAgB,EAAE,aAAa,CAAC,kBAAkB,CAAC,EACnD,OAAO,GAAE,sBAA2B,GACnC,OAAO,CAAC,qBAAqB,CAAC,CAgDhC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"keyword-coverage-judge.test.d.ts","sourceRoot":"","sources":["../src/keyword-coverage-judge.test.ts"],"names":[],"mappings":""}
|
|
@@ -1,103 +1,3 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
import '../store-BcFXE6LG.js';
|
|
5
|
-
|
|
6
|
-
type KnowledgeRequirementCategory = 'user_specific' | 'company_specific' | 'domain_specific' | 'codebase_specific' | 'market_specific' | 'regulatory' | 'tool_api' | 'credential_or_secret' | 'runtime_environment' | 'preference' | 'historical_context';
|
|
7
|
-
type KnowledgeAcquisitionMode = 'ask_user' | 'search_web' | 'query_connector' | 'inspect_repo' | 'run_command' | 'infer_low_confidence' | 'not_available';
|
|
8
|
-
type KnowledgeImportance = 'blocking' | 'high' | 'medium' | 'low';
|
|
9
|
-
type KnowledgeFreshness = 'static' | 'monthly' | 'weekly' | 'daily' | 'realtime';
|
|
10
|
-
type KnowledgeSensitivity = 'public' | 'private' | 'secret';
|
|
11
|
-
type KnowledgeFallbackPolicy = 'block' | 'ask' | 'continue_with_caveat' | 'use_default';
|
|
12
|
-
interface KnowledgeRequirement {
|
|
13
|
-
id: string;
|
|
14
|
-
description: string;
|
|
15
|
-
requiredFor: string[];
|
|
16
|
-
category: KnowledgeRequirementCategory;
|
|
17
|
-
acquisitionMode: KnowledgeAcquisitionMode;
|
|
18
|
-
importance: KnowledgeImportance;
|
|
19
|
-
freshness: KnowledgeFreshness;
|
|
20
|
-
sensitivity: KnowledgeSensitivity;
|
|
21
|
-
confidenceNeeded: number;
|
|
22
|
-
currentConfidence: number;
|
|
23
|
-
evidenceIds: string[];
|
|
24
|
-
fallbackPolicy: KnowledgeFallbackPolicy;
|
|
25
|
-
/**
|
|
26
|
-
* ISO timestamp after which this requirement must be treated as stale.
|
|
27
|
-
* Stale requirements score as missing even when they still have evidence.
|
|
28
|
-
*/
|
|
29
|
-
validUntil?: string;
|
|
30
|
-
/** ISO timestamp for the last source-grounding or human verification pass. */
|
|
31
|
-
lastVerifiedAt?: string;
|
|
32
|
-
metadata?: Record<string, unknown>;
|
|
33
|
-
}
|
|
34
|
-
interface KnowledgeBundle {
|
|
35
|
-
taskId: string;
|
|
36
|
-
requirements: KnowledgeRequirement[];
|
|
37
|
-
evidenceIds: string[];
|
|
38
|
-
claimIds: string[];
|
|
39
|
-
wikiPageIds: string[];
|
|
40
|
-
userAnswers: Record<string, string>;
|
|
41
|
-
missing: KnowledgeRequirement[];
|
|
42
|
-
readinessScore: number;
|
|
43
|
-
metadata?: Record<string, unknown>;
|
|
44
|
-
}
|
|
45
|
-
type KnowledgeRecommendedAction = 'run_agent' | 'ask_user' | 'collect_web_data' | 'query_connectors' | 'inspect_repo' | 'build_domain_wiki' | 'continue_with_caveat' | 'abort_or_rescope';
|
|
46
|
-
interface KnowledgeReadinessReport {
|
|
47
|
-
taskId: string;
|
|
48
|
-
readinessScore: number;
|
|
49
|
-
blockingMissingRequirements: KnowledgeRequirement[];
|
|
50
|
-
nonBlockingGaps: KnowledgeRequirement[];
|
|
51
|
-
recommendedAction: KnowledgeRecommendedAction;
|
|
52
|
-
bundle: KnowledgeBundle;
|
|
53
|
-
severity: ControlSeverity;
|
|
54
|
-
reason: string;
|
|
55
|
-
}
|
|
56
|
-
interface UserQuestion {
|
|
57
|
-
id: string;
|
|
58
|
-
question: string;
|
|
59
|
-
reason: string;
|
|
60
|
-
requirementId: string;
|
|
61
|
-
importance: KnowledgeImportance;
|
|
62
|
-
answerType: 'free_text' | 'select_one' | 'multi_select' | 'file_upload' | 'credential' | 'url';
|
|
63
|
-
defaultIfSkipped?: string;
|
|
64
|
-
impactIfUnknown: string;
|
|
65
|
-
options?: string[];
|
|
66
|
-
metadata?: Record<string, unknown>;
|
|
67
|
-
}
|
|
68
|
-
interface DataAcquisitionPlan {
|
|
69
|
-
id: string;
|
|
70
|
-
requirementIds: string[];
|
|
71
|
-
mode: Exclude<KnowledgeAcquisitionMode, 'not_available' | 'infer_low_confidence'> | 'build_domain_wiki';
|
|
72
|
-
description: string;
|
|
73
|
-
priority: KnowledgeImportance;
|
|
74
|
-
expectedEvidenceIds?: string[];
|
|
75
|
-
questions?: UserQuestion[];
|
|
76
|
-
metadata?: Record<string, unknown>;
|
|
77
|
-
}
|
|
78
|
-
type KnowledgeResponsibleSurface = 'knowledge-requirements' | 'data-acquisition' | 'retrieval-policy' | 'user-question-policy';
|
|
79
|
-
|
|
80
|
-
interface ScoreKnowledgeReadinessOptions {
|
|
81
|
-
taskId: string;
|
|
82
|
-
requirements: KnowledgeRequirement[];
|
|
83
|
-
evidenceIds?: string[];
|
|
84
|
-
claimIds?: string[];
|
|
85
|
-
wikiPageIds?: string[];
|
|
86
|
-
userAnswers?: Record<string, string>;
|
|
87
|
-
metadata?: Record<string, unknown>;
|
|
88
|
-
now?: Date;
|
|
89
|
-
}
|
|
90
|
-
declare function scoreKnowledgeReadiness(options: ScoreKnowledgeReadinessOptions): KnowledgeReadinessReport;
|
|
91
|
-
declare function blockingKnowledgeEval(report: KnowledgeReadinessReport, options?: {
|
|
92
|
-
id?: string;
|
|
93
|
-
minimumScore?: number;
|
|
94
|
-
emitter?: TraceEmitter;
|
|
95
|
-
}): ControlEvalResult;
|
|
96
|
-
declare function knowledgeReadinessTracePayload(report: KnowledgeReadinessReport, options?: {
|
|
97
|
-
passed?: boolean;
|
|
98
|
-
minimumScore?: number;
|
|
99
|
-
}): Record<string, unknown>;
|
|
100
|
-
declare function userQuestionsForKnowledgeGaps(gaps: KnowledgeRequirement[]): UserQuestion[];
|
|
101
|
-
declare function acquisitionPlansForKnowledgeGaps(gaps: KnowledgeRequirement[]): DataAcquisitionPlan[];
|
|
102
|
-
|
|
103
|
-
export { type DataAcquisitionPlan, type KnowledgeAcquisitionMode, type KnowledgeBundle, type KnowledgeFallbackPolicy, type KnowledgeFreshness, type KnowledgeImportance, type KnowledgeReadinessReport, type KnowledgeRecommendedAction, type KnowledgeRequirement, type KnowledgeRequirementCategory, type KnowledgeResponsibleSurface, type KnowledgeSensitivity, type ScoreKnowledgeReadinessOptions, type UserQuestion, acquisitionPlansForKnowledgeGaps, blockingKnowledgeEval, knowledgeReadinessTracePayload, scoreKnowledgeReadiness, userQuestionsForKnowledgeGaps };
|
|
1
|
+
export * from './readiness';
|
|
2
|
+
export * from './types';
|
|
3
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/knowledge/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAA;AAC3B,cAAc,SAAS,CAAA"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type ControlEvalResult } from '../control-runtime';
|
|
2
|
+
import type { TraceEmitter } from '../trace/emitter';
|
|
3
|
+
import type { DataAcquisitionPlan, KnowledgeReadinessReport, KnowledgeRequirement, UserQuestion } from './types';
|
|
4
|
+
export interface ScoreKnowledgeReadinessOptions {
|
|
5
|
+
taskId: string;
|
|
6
|
+
requirements: KnowledgeRequirement[];
|
|
7
|
+
evidenceIds?: string[];
|
|
8
|
+
claimIds?: string[];
|
|
9
|
+
wikiPageIds?: string[];
|
|
10
|
+
userAnswers?: Record<string, string>;
|
|
11
|
+
metadata?: Record<string, unknown>;
|
|
12
|
+
now?: Date;
|
|
13
|
+
}
|
|
14
|
+
export declare function scoreKnowledgeReadiness(options: ScoreKnowledgeReadinessOptions): KnowledgeReadinessReport;
|
|
15
|
+
export declare function blockingKnowledgeEval(report: KnowledgeReadinessReport, options?: {
|
|
16
|
+
id?: string;
|
|
17
|
+
minimumScore?: number;
|
|
18
|
+
emitter?: TraceEmitter;
|
|
19
|
+
}): ControlEvalResult;
|
|
20
|
+
export declare function knowledgeReadinessTracePayload(report: KnowledgeReadinessReport, options?: {
|
|
21
|
+
passed?: boolean;
|
|
22
|
+
minimumScore?: number;
|
|
23
|
+
}): Record<string, unknown>;
|
|
24
|
+
export declare function userQuestionsForKnowledgeGaps(gaps: KnowledgeRequirement[]): UserQuestion[];
|
|
25
|
+
export declare function acquisitionPlansForKnowledgeGaps(gaps: KnowledgeRequirement[]): DataAcquisitionPlan[];
|
|
26
|
+
//# sourceMappingURL=readiness.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"readiness.d.ts","sourceRoot":"","sources":["../../src/knowledge/readiness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,iBAAiB,EAAiB,MAAM,oBAAoB,CAAA;AAC1E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,KAAK,EACV,mBAAmB,EAInB,wBAAwB,EAExB,oBAAoB,EACpB,YAAY,EACb,MAAM,SAAS,CAAA;AAEhB,MAAM,WAAW,8BAA8B;IAC7C,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,oBAAoB,EAAE,CAAA;IACpC,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,EAAE,CAAA;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACpC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAClC,GAAG,CAAC,EAAE,IAAI,CAAA;CACX;AAED,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,8BAA8B,GACtC,wBAAwB,CA6C1B;AAED,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,wBAAwB,EAChC,OAAO,GAAE;IAAE,EAAE,CAAC,EAAE,MAAM,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,YAAY,CAAA;CAAO,GAC3E,iBAAiB,CAqBnB;AAED,wBAAgB,8BAA8B,CAC5C,MAAM,EAAE,wBAAwB,EAChC,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,OAAO,CAAC;IAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAO,GACxD,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAazB;AAED,wBAAgB,6BAA6B,CAAC,IAAI,EAAE,oBAAoB,EAAE,GAAG,YAAY,EAAE,CAY1F;AAED,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,oBAAoB,EAAE,GAC3B,mBAAmB,EAAE,CAiBvB"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type { ControlSeverity } from '../control-runtime';
|
|
2
|
+
export type KnowledgeRequirementCategory = 'user_specific' | 'company_specific' | 'domain_specific' | 'codebase_specific' | 'market_specific' | 'regulatory' | 'tool_api' | 'credential_or_secret' | 'runtime_environment' | 'preference' | 'historical_context';
|
|
3
|
+
export type KnowledgeAcquisitionMode = 'ask_user' | 'search_web' | 'query_connector' | 'inspect_repo' | 'run_command' | 'infer_low_confidence' | 'not_available';
|
|
4
|
+
export type KnowledgeImportance = 'blocking' | 'high' | 'medium' | 'low';
|
|
5
|
+
export type KnowledgeFreshness = 'static' | 'monthly' | 'weekly' | 'daily' | 'realtime';
|
|
6
|
+
export type KnowledgeSensitivity = 'public' | 'private' | 'secret';
|
|
7
|
+
export type KnowledgeFallbackPolicy = 'block' | 'ask' | 'continue_with_caveat' | 'use_default';
|
|
8
|
+
export interface KnowledgeRequirement {
|
|
9
|
+
id: string;
|
|
10
|
+
description: string;
|
|
11
|
+
requiredFor: string[];
|
|
12
|
+
category: KnowledgeRequirementCategory;
|
|
13
|
+
acquisitionMode: KnowledgeAcquisitionMode;
|
|
14
|
+
importance: KnowledgeImportance;
|
|
15
|
+
freshness: KnowledgeFreshness;
|
|
16
|
+
sensitivity: KnowledgeSensitivity;
|
|
17
|
+
confidenceNeeded: number;
|
|
18
|
+
currentConfidence: number;
|
|
19
|
+
evidenceIds: string[];
|
|
20
|
+
fallbackPolicy: KnowledgeFallbackPolicy;
|
|
21
|
+
/**
|
|
22
|
+
* ISO timestamp after which this requirement must be treated as stale.
|
|
23
|
+
* Stale requirements score as missing even when they still have evidence.
|
|
24
|
+
*/
|
|
25
|
+
validUntil?: string;
|
|
26
|
+
/** ISO timestamp for the last source-grounding or human verification pass. */
|
|
27
|
+
lastVerifiedAt?: string;
|
|
28
|
+
metadata?: Record<string, unknown>;
|
|
29
|
+
}
|
|
30
|
+
export interface KnowledgeBundle {
|
|
31
|
+
taskId: string;
|
|
32
|
+
requirements: KnowledgeRequirement[];
|
|
33
|
+
evidenceIds: string[];
|
|
34
|
+
claimIds: string[];
|
|
35
|
+
wikiPageIds: string[];
|
|
36
|
+
userAnswers: Record<string, string>;
|
|
37
|
+
missing: KnowledgeRequirement[];
|
|
38
|
+
readinessScore: number;
|
|
39
|
+
metadata?: Record<string, unknown>;
|
|
40
|
+
}
|
|
41
|
+
export type KnowledgeRecommendedAction = 'run_agent' | 'ask_user' | 'collect_web_data' | 'query_connectors' | 'inspect_repo' | 'build_domain_wiki' | 'continue_with_caveat' | 'abort_or_rescope';
|
|
42
|
+
export interface KnowledgeReadinessReport {
|
|
43
|
+
taskId: string;
|
|
44
|
+
readinessScore: number;
|
|
45
|
+
blockingMissingRequirements: KnowledgeRequirement[];
|
|
46
|
+
nonBlockingGaps: KnowledgeRequirement[];
|
|
47
|
+
recommendedAction: KnowledgeRecommendedAction;
|
|
48
|
+
bundle: KnowledgeBundle;
|
|
49
|
+
severity: ControlSeverity;
|
|
50
|
+
reason: string;
|
|
51
|
+
}
|
|
52
|
+
export interface UserQuestion {
|
|
53
|
+
id: string;
|
|
54
|
+
question: string;
|
|
55
|
+
reason: string;
|
|
56
|
+
requirementId: string;
|
|
57
|
+
importance: KnowledgeImportance;
|
|
58
|
+
answerType: 'free_text' | 'select_one' | 'multi_select' | 'file_upload' | 'credential' | 'url';
|
|
59
|
+
defaultIfSkipped?: string;
|
|
60
|
+
impactIfUnknown: string;
|
|
61
|
+
options?: string[];
|
|
62
|
+
metadata?: Record<string, unknown>;
|
|
63
|
+
}
|
|
64
|
+
export interface DataAcquisitionPlan {
|
|
65
|
+
id: string;
|
|
66
|
+
requirementIds: string[];
|
|
67
|
+
mode: Exclude<KnowledgeAcquisitionMode, 'not_available' | 'infer_low_confidence'> | 'build_domain_wiki';
|
|
68
|
+
description: string;
|
|
69
|
+
priority: KnowledgeImportance;
|
|
70
|
+
expectedEvidenceIds?: string[];
|
|
71
|
+
questions?: UserQuestion[];
|
|
72
|
+
metadata?: Record<string, unknown>;
|
|
73
|
+
}
|
|
74
|
+
export type KnowledgeResponsibleSurface = 'knowledge-requirements' | 'data-acquisition' | 'retrieval-policy' | 'user-question-policy';
|
|
75
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/knowledge/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAA;AAEzD,MAAM,MAAM,4BAA4B,GACpC,eAAe,GACf,kBAAkB,GAClB,iBAAiB,GACjB,mBAAmB,GACnB,iBAAiB,GACjB,YAAY,GACZ,UAAU,GACV,sBAAsB,GACtB,qBAAqB,GACrB,YAAY,GACZ,oBAAoB,CAAA;AAExB,MAAM,MAAM,wBAAwB,GAChC,UAAU,GACV,YAAY,GACZ,iBAAiB,GACjB,cAAc,GACd,aAAa,GACb,sBAAsB,GACtB,eAAe,CAAA;AAEnB,MAAM,MAAM,mBAAmB,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAA;AACxE,MAAM,MAAM,kBAAkB,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,GAAG,OAAO,GAAG,UAAU,CAAA;AACvF,MAAM,MAAM,oBAAoB,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,CAAA;AAClE,MAAM,MAAM,uBAAuB,GAAG,OAAO,GAAG,KAAK,GAAG,sBAAsB,GAAG,aAAa,CAAA;AAE9F,MAAM,WAAW,oBAAoB;IACnC,EAAE,EAAE,MAAM,CAAA;IACV,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,QAAQ,EAAE,4BAA4B,CAAA;IACtC,eAAe,EAAE,wBAAwB,CAAA;IACzC,UAAU,EAAE,mBAAmB,CAAA;IAC/B,SAAS,EAAE,kBAAkB,CAAA;IAC7B,WAAW,EAAE,oBAAoB,CAAA;IACjC,gBAAgB,EAAE,MAAM,CAAA;IACxB,iBAAiB,EAAE,MAAM,CAAA;IACzB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,cAAc,EAAE,uBAAuB,CAAA;IACvC;;;OAGG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,8EAA8E;IAC9E,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,oBAAoB,EAAE,CAAA;IACpC,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,WAAW,EAAE,MAAM,EAAE,CAAA;IACrB,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IACnC,OAAO,EAAE,oBAAoB,EAAE,CAAA;IAC/B,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,MAAM,0BAA0B,GAClC,WAAW,GACX,UAAU,GACV,kBAAkB,GAClB,kBAAkB,GAClB,cAAc,GACd,mBAAmB,GACnB,sBAAsB,GACtB,kBAAkB,CAAA;AAEtB,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,cAAc,EAAE,MAAM,CAAA;IACtB,2BAA2B,EAAE,oBAAoB,EAAE,CAAA;IACnD,eAAe,EAAE,oBAAoB,EAAE,CAAA;IACvC,iBAAiB,EAAE,0BAA0B,CAAA;IAC7C,MAAM,EAAE,eAAe,CAAA;IACvB,QAAQ,EAAE,eAAe,CAAA;IACzB,MAAM,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAA;IACV,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,mBAAmB,CAAA;IAC/B,UAAU,EAAE,WAAW,GAAG,YAAY,GAAG,cAAc,GAAG,aAAa,GAAG,YAAY,GAAG,KAAK,CAAA;IAC9F,gBAAgB,CAAC,EAAE,MAAM,CAAA;IACzB,eAAe,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,mBAAmB;IAClC,EAAE,EAAE,MAAM,CAAA;IACV,cAAc,EAAE,MAAM,EAAE,CAAA;IACxB,IAAI,EACA,OAAO,CAAC,wBAAwB,EAAE,eAAe,GAAG,sBAAsB,CAAC,GAC3E,mBAAmB,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,QAAQ,EAAE,mBAAmB,CAAA;IAC7B,mBAAmB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC9B,SAAS,CAAC,EAAE,YAAY,EAAE,CAAA;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,MAAM,2BAA2B,GACnC,wBAAwB,GACxB,kBAAkB,GAClB,kBAAkB,GAClB,sBAAsB,CAAA"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { type FeedbackLabel, type FeedbackTrajectory, type FeedbackTrajectoryStore } from './feedback-trajectory';
|
|
2
|
+
import type { ReleaseConfidenceScorecard, ReleaseConfidenceThresholds } from './release-confidence';
|
|
3
|
+
import type { CheckResult, TestResult } from './types';
|
|
4
|
+
export interface LiveProofArtifact {
|
|
5
|
+
kind: string;
|
|
6
|
+
id?: string;
|
|
7
|
+
path?: string;
|
|
8
|
+
url?: string;
|
|
9
|
+
metadata?: Record<string, unknown>;
|
|
10
|
+
}
|
|
11
|
+
export interface LiveProofContext {
|
|
12
|
+
projectId: string;
|
|
13
|
+
scenarioId: string;
|
|
14
|
+
task: string;
|
|
15
|
+
checks: CheckResult[];
|
|
16
|
+
artifacts: LiveProofArtifact[];
|
|
17
|
+
labels: FeedbackLabel[];
|
|
18
|
+
metadata: Record<string, unknown>;
|
|
19
|
+
transcript: {
|
|
20
|
+
role: 'user' | 'assistant' | 'system' | 'tool';
|
|
21
|
+
content: string;
|
|
22
|
+
at: string;
|
|
23
|
+
}[];
|
|
24
|
+
addCheck(check: CheckResult): void;
|
|
25
|
+
addArtifact(artifact: LiveProofArtifact): void;
|
|
26
|
+
addLabel(label: Omit<FeedbackLabel, 'createdAt'> & {
|
|
27
|
+
createdAt?: string;
|
|
28
|
+
}): void;
|
|
29
|
+
addTurn(turn: {
|
|
30
|
+
role: 'user' | 'assistant' | 'system' | 'tool';
|
|
31
|
+
content: string;
|
|
32
|
+
at?: string;
|
|
33
|
+
}): void;
|
|
34
|
+
}
|
|
35
|
+
export interface LiveProofConfig {
|
|
36
|
+
projectId: string;
|
|
37
|
+
scenarioId: string;
|
|
38
|
+
task: string;
|
|
39
|
+
drive(context: LiveProofContext): Promise<void> | void;
|
|
40
|
+
validate?(context: LiveProofContext): Promise<CheckResult[] | undefined> | CheckResult[] | undefined;
|
|
41
|
+
requiredArtifacts?: string[];
|
|
42
|
+
minPassRate?: number;
|
|
43
|
+
trajectoryStore?: FeedbackTrajectoryStore;
|
|
44
|
+
releaseConfidence?: {
|
|
45
|
+
target: string;
|
|
46
|
+
candidateId?: string;
|
|
47
|
+
baselineId?: string;
|
|
48
|
+
thresholds?: ReleaseConfidenceThresholds;
|
|
49
|
+
};
|
|
50
|
+
metadata?: Record<string, unknown>;
|
|
51
|
+
}
|
|
52
|
+
export interface LiveProofResult extends TestResult {
|
|
53
|
+
projectId: string;
|
|
54
|
+
scenarioId: string;
|
|
55
|
+
artifacts: LiveProofArtifact[];
|
|
56
|
+
labels: FeedbackLabel[];
|
|
57
|
+
transcript: LiveProofContext['transcript'];
|
|
58
|
+
trajectory: FeedbackTrajectory;
|
|
59
|
+
releaseConfidence?: ReleaseConfidenceScorecard;
|
|
60
|
+
}
|
|
61
|
+
export declare function runLiveProof(config: LiveProofConfig): Promise<LiveProofResult>;
|
|
62
|
+
//# sourceMappingURL=live-proof.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"live-proof.d.ts","sourceRoot":"","sources":["../src/live-proof.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,aAAa,EAClB,KAAK,kBAAkB,EACvB,KAAK,uBAAuB,EAC7B,MAAM,uBAAuB,CAAA;AAC9B,OAAO,KAAK,EACV,0BAA0B,EAC1B,2BAA2B,EAE5B,MAAM,sBAAsB,CAAA;AAE7B,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AAEtD,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,EAAE,CAAC,EAAE,MAAM,CAAA;IACX,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,gBAAgB;IAC/B,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,WAAW,EAAE,CAAA;IACrB,SAAS,EAAE,iBAAiB,EAAE,CAAA;IAC9B,MAAM,EAAE,aAAa,EAAE,CAAA;IACvB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACjC,UAAU,EAAE;QAAE,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;IAC7F,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,IAAI,CAAA;IAClC,WAAW,CAAC,QAAQ,EAAE,iBAAiB,GAAG,IAAI,CAAA;IAC9C,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,CAAC,GAAG;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAA;IAChF,OAAO,CAAC,IAAI,EAAE;QACZ,IAAI,EAAE,MAAM,GAAG,WAAW,GAAG,QAAQ,GAAG,MAAM,CAAA;QAC9C,OAAO,EAAE,MAAM,CAAA;QACf,EAAE,CAAC,EAAE,MAAM,CAAA;KACZ,GAAG,IAAI,CAAA;CACT;AAED,MAAM,WAAW,eAAe;IAC9B,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACtD,QAAQ,CAAC,CACP,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,WAAW,EAAE,GAAG,SAAS,CAAC,GAAG,WAAW,EAAE,GAAG,SAAS,CAAA;IACjE,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC5B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,eAAe,CAAC,EAAE,uBAAuB,CAAA;IACzC,iBAAiB,CAAC,EAAE;QAClB,MAAM,EAAE,MAAM,CAAA;QACd,WAAW,CAAC,EAAE,MAAM,CAAA;QACpB,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,UAAU,CAAC,EAAE,2BAA2B,CAAA;KACzC,CAAA;IACD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,eAAgB,SAAQ,UAAU;IACjD,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,SAAS,EAAE,iBAAiB,EAAE,CAAA;IAC9B,MAAM,EAAE,aAAa,EAAE,CAAA;IACvB,UAAU,EAAE,gBAAgB,CAAC,YAAY,CAAC,CAAA;IAC1C,UAAU,EAAE,kBAAkB,CAAA;IAC9B,iBAAiB,CAAC,EAAE,0BAA0B,CAAA;CAC/C;AAED,wBAAsB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC,CA4GpF"}
|