@tangle-network/agent-eval 0.95.0 → 0.95.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (961) hide show
  1. package/CHANGELOG.md +17 -0
  2. package/dist/adapters/http.d.ts +17 -10
  3. package/dist/adapters/langchain.d.ts +14 -7
  4. package/dist/adapters/otel.d.ts +25 -13
  5. package/dist/{rl/adversarial.d.ts → adversarial-DIVcDoI_.d.ts} +7 -6
  6. package/dist/analyst/index.d.ts +236 -28
  7. package/dist/{trace-analyst/analyst.d.ts → analyst-C8HHvfJp.d.ts} +14 -12
  8. package/dist/{contract/analyze-runs.d.ts → analyze-runs-DtT6F_6T.d.ts} +10 -9
  9. package/dist/authenticity/index.d.ts +16 -15
  10. package/dist/{baseline.d.ts → baseline-Bbid3WoO.d.ts} +44 -8
  11. package/dist/belief-state/index.d.ts +605 -14
  12. package/dist/benchmarks/index.d.ts +5 -23
  13. package/dist/builder-eval/index.d.ts +250 -5
  14. package/dist/calibration-BPmzuVPk.d.ts +101 -0
  15. package/dist/campaign/index.d.ts +1452 -38
  16. package/dist/cli.d.ts +0 -2
  17. package/dist/{contract/intake/code-agent-session.d.ts → code-agent-session-CPHRCb4-.d.ts} +17 -15
  18. package/dist/contract/index.d.ts +688 -106
  19. package/dist/control-Doncu-B_.d.ts +259 -0
  20. package/dist/{control-runtime.d.ts → control-runtime-Acf9CGhw.d.ts} +26 -23
  21. package/dist/control.d.ts +10 -11
  22. package/dist/corpus-D4YW9UoJ.d.ts +560 -0
  23. package/dist/{cost-ledger.d.ts → cost-ledger-DuSqlw5B.d.ts} +11 -10
  24. package/dist/{counterfactual.d.ts → counterfactual-DlOz8PBx.d.ts} +13 -12
  25. package/dist/{dataset.d.ts → dataset-BbGkaN2I.d.ts} +14 -11
  26. package/dist/{analyst/registry.d.ts → default-registry-GyE8X5SP.d.ts} +37 -8
  27. package/dist/diagnose.d.ts +252 -1
  28. package/dist/{trace/emitter.d.ts → emitter-C2rqGH_l.d.ts} +12 -9
  29. package/dist/{errors.d.ts → errors-CzMUYo7b.d.ts} +11 -10
  30. package/dist/failure-cluster-DH9Flgcf.d.ts +76 -0
  31. package/dist/{feedback-trajectory.d.ts → feedback-trajectory-BxY0cKfs.d.ts} +38 -36
  32. package/dist/fuzz.d.ts +547 -1
  33. package/dist/gepa-C1NCIZ9o.d.ts +414 -0
  34. package/dist/governance/index.d.ts +135 -5
  35. package/dist/harness-optimizer-mOl9XX_O.d.ts +106 -0
  36. package/dist/hosted/index.d.ts +239 -10
  37. package/dist/index-_Y4oNOOb.d.ts +159 -0
  38. package/dist/index.d.ts +5660 -277
  39. package/dist/{contract/insight-report.d.ts → insight-report-BnRjTibG.d.ts} +19 -16
  40. package/dist/{trace/integrity.d.ts → integrity-D2t12mMw.d.ts} +14 -11
  41. package/dist/{judge-calibration.d.ts → judge-calibration-0p2QcWNE.d.ts} +17 -16
  42. package/dist/{analyst/kind-factory.d.ts → kind-factory-X3eDYbKn.d.ts} +61 -10
  43. package/dist/knowledge/index.d.ts +103 -3
  44. package/dist/{llm-client.d.ts → llm-client-Bj7g0rqu.d.ts} +23 -21
  45. package/dist/matrix/index.d.ts +30 -12
  46. package/dist/meta-eval/index.d.ts +182 -6
  47. package/dist/{multi-layer-verifier.d.ts → multi-layer-verifier-DUZXrPDA.d.ts} +15 -12
  48. package/dist/multishot/index.d.ts +290 -7
  49. package/dist/{rl/off-policy.d.ts → off-policy-DiwuKKg7.d.ts} +9 -8
  50. package/dist/openapi.json +1 -1
  51. package/dist/{meta-eval/outcome-store.d.ts → outcome-store-rnXLEqSn.d.ts} +8 -7
  52. package/dist/{pareto.d.ts → pareto-E-pembql.d.ts} +10 -9
  53. package/dist/perf/index.d.ts +119 -13
  54. package/dist/pipelines/index.d.ts +173 -8
  55. package/dist/pre-registration-nfUdc9EQ.d.ts +483 -0
  56. package/dist/prm/index.d.ts +104 -5
  57. package/dist/provenance-CncDq9qE.d.ts +426 -0
  58. package/dist/query-B7GGjRox.d.ts +32 -0
  59. package/dist/{trace/raw-provider-sink.d.ts → raw-provider-sink-C46HDghv.d.ts} +14 -13
  60. package/dist/{red-team.d.ts → red-team-BWdoyleI.d.ts} +16 -14
  61. package/dist/{trace/redact.d.ts → redact-B40YG2M_.d.ts} +8 -7
  62. package/dist/release-report-pidWUMZ2.d.ts +233 -0
  63. package/dist/reporting.d.ts +16 -15
  64. package/dist/{eval-campaign.d.ts → researcher-Jr8ME1dZ.d.ts} +156 -34
  65. package/dist/rl.d.ts +1193 -1
  66. package/dist/{prm/rubric.d.ts → rubric-Cc6UHvUb.d.ts} +13 -10
  67. package/dist/{meta-eval/rubric-predictive-validity.d.ts → rubric-predictive-validity-C2hDKM8Z.d.ts} +11 -8
  68. package/dist/run-critic-CmMf05uV.d.ts +56 -0
  69. package/dist/{run-record.d.ts → run-record-CP2ObebC.d.ts} +117 -15
  70. package/dist/runtime-trajectory-BOUUjI0y.d.ts +49 -0
  71. package/dist/{trace/schema.d.ts → schema-m0gsnbt3.d.ts} +30 -29
  72. package/dist/semantic-concept-judge-DSBB2Cfp.d.ts +624 -0
  73. package/dist/{sequential.d.ts → sequential-5iSVfzl2.d.ts} +10 -9
  74. package/dist/{series-convergence.d.ts → series-convergence-D5OWMBg6.d.ts} +5 -4
  75. package/dist/sink-fetch-B1Yg4Til.d.ts +101 -0
  76. package/dist/{statistics.d.ts → statistics-CCJpTGOS.d.ts} +48 -46
  77. package/dist/{trace/store.d.ts → store-BcFXE6LG.d.ts} +12 -21
  78. package/dist/{trace-analyst/types.d.ts → store-C1YxJDEK.d.ts} +74 -18
  79. package/dist/storyboard/index.d.ts +81 -16
  80. package/dist/{summary-report.d.ts → summary-report-CInXwsza.d.ts} +160 -23
  81. package/dist/telemetry/{sink-file.d.ts → file.d.ts} +7 -5
  82. package/dist/telemetry/index.d.ts +35 -17
  83. package/dist/{sandbox-harness.d.ts → test-graded-scenario-DeODGLra.d.ts} +60 -15
  84. package/dist/{locked-jsonl-appender.d.ts → testing-C21CHsq2.d.ts} +4 -3
  85. package/dist/testing.d.ts +1 -5
  86. package/dist/traces.d.ts +976 -4
  87. package/dist/{trajectory.d.ts → trajectory-2TkpSEVh.d.ts} +9 -8
  88. package/dist/{analyst/types.d.ts → types-B5x54y6n.d.ts} +112 -19
  89. package/dist/{matrix/types.d.ts → types-BUxNaJ8c.d.ts} +11 -9
  90. package/dist/{types.d.ts → types-C7DGg5ex.d.ts} +33 -31
  91. package/dist/{campaign/types.d.ts → types-DQRY8ZT-.d.ts} +47 -44
  92. package/dist/{verdict.d.ts → verdict-C9MlYujm.d.ts} +3 -2
  93. package/dist/wire/index.d.ts +570 -13
  94. package/dist/workflow/index.d.ts +496 -22
  95. package/package.json +2 -2
  96. package/dist/action-policy.d.ts +0 -24
  97. package/dist/action-policy.d.ts.map +0 -1
  98. package/dist/action-policy.test.d.ts +0 -2
  99. package/dist/action-policy.test.d.ts.map +0 -1
  100. package/dist/active-learning.d.ts +0 -41
  101. package/dist/active-learning.d.ts.map +0 -1
  102. package/dist/adapters/http.d.ts.map +0 -1
  103. package/dist/adapters/langchain.d.ts.map +0 -1
  104. package/dist/adapters/otel.d.ts.map +0 -1
  105. package/dist/agent-profile-cell.d.ts +0 -101
  106. package/dist/agent-profile-cell.d.ts.map +0 -1
  107. package/dist/agent-profile.d.ts +0 -27
  108. package/dist/agent-profile.d.ts.map +0 -1
  109. package/dist/agent-profile.test.d.ts +0 -2
  110. package/dist/agent-profile.test.d.ts.map +0 -1
  111. package/dist/analyst/adapters.d.ts +0 -62
  112. package/dist/analyst/adapters.d.ts.map +0 -1
  113. package/dist/analyst/analyst.test.d.ts +0 -2
  114. package/dist/analyst/analyst.test.d.ts.map +0 -1
  115. package/dist/analyst/ax-service.d.ts +0 -27
  116. package/dist/analyst/ax-service.d.ts.map +0 -1
  117. package/dist/analyst/behavioral-analyst.d.ts +0 -28
  118. package/dist/analyst/behavioral-analyst.d.ts.map +0 -1
  119. package/dist/analyst/chat-client.d.ts +0 -91
  120. package/dist/analyst/chat-client.d.ts.map +0 -1
  121. package/dist/analyst/default-registry.d.ts +0 -27
  122. package/dist/analyst/default-registry.d.ts.map +0 -1
  123. package/dist/analyst/default-registry.test.d.ts +0 -2
  124. package/dist/analyst/default-registry.test.d.ts.map +0 -1
  125. package/dist/analyst/finding-signature.d.ts +0 -48
  126. package/dist/analyst/finding-signature.d.ts.map +0 -1
  127. package/dist/analyst/finding-subject.d.ts +0 -146
  128. package/dist/analyst/finding-subject.d.ts.map +0 -1
  129. package/dist/analyst/finding-subject.test.d.ts +0 -2
  130. package/dist/analyst/finding-subject.test.d.ts.map +0 -1
  131. package/dist/analyst/findings-store.d.ts +0 -75
  132. package/dist/analyst/findings-store.d.ts.map +0 -1
  133. package/dist/analyst/index.d.ts.map +0 -1
  134. package/dist/analyst/kind-factory.d.ts.map +0 -1
  135. package/dist/analyst/kinds/failure-mode.d.ts +0 -19
  136. package/dist/analyst/kinds/failure-mode.d.ts.map +0 -1
  137. package/dist/analyst/kinds/improvement.d.ts +0 -23
  138. package/dist/analyst/kinds/improvement.d.ts.map +0 -1
  139. package/dist/analyst/kinds/index.d.ts +0 -22
  140. package/dist/analyst/kinds/index.d.ts.map +0 -1
  141. package/dist/analyst/kinds/kinds.test.d.ts +0 -2
  142. package/dist/analyst/kinds/kinds.test.d.ts.map +0 -1
  143. package/dist/analyst/kinds/knowledge-gap.d.ts +0 -28
  144. package/dist/analyst/kinds/knowledge-gap.d.ts.map +0 -1
  145. package/dist/analyst/kinds/knowledge-poisoning.d.ts +0 -22
  146. package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +0 -1
  147. package/dist/analyst/kinds/skill-usage.d.ts +0 -84
  148. package/dist/analyst/kinds/skill-usage.d.ts.map +0 -1
  149. package/dist/analyst/kinds/skill-usage.test.d.ts +0 -2
  150. package/dist/analyst/kinds/skill-usage.test.d.ts.map +0 -1
  151. package/dist/analyst/parse-tolerant.d.ts +0 -26
  152. package/dist/analyst/parse-tolerant.d.ts.map +0 -1
  153. package/dist/analyst/parse-tolerant.test.d.ts +0 -2
  154. package/dist/analyst/parse-tolerant.test.d.ts.map +0 -1
  155. package/dist/analyst/registry.budget.test.d.ts +0 -2
  156. package/dist/analyst/registry.budget.test.d.ts.map +0 -1
  157. package/dist/analyst/registry.d.ts.map +0 -1
  158. package/dist/analyst/steer-firewall.d.ts +0 -35
  159. package/dist/analyst/steer-firewall.d.ts.map +0 -1
  160. package/dist/analyst/steer-firewall.test.d.ts +0 -2
  161. package/dist/analyst/steer-firewall.test.d.ts.map +0 -1
  162. package/dist/analyst/structure-findings.d.ts +0 -37
  163. package/dist/analyst/structure-findings.d.ts.map +0 -1
  164. package/dist/analyst/structure-findings.test.d.ts +0 -2
  165. package/dist/analyst/structure-findings.test.d.ts.map +0 -1
  166. package/dist/analyst/tool-groups.d.ts +0 -34
  167. package/dist/analyst/tool-groups.d.ts.map +0 -1
  168. package/dist/analyst/types.d.ts.map +0 -1
  169. package/dist/anti-slop.d.ts +0 -59
  170. package/dist/anti-slop.d.ts.map +0 -1
  171. package/dist/artifact-validator.d.ts +0 -74
  172. package/dist/artifact-validator.d.ts.map +0 -1
  173. package/dist/attestation.d.ts +0 -63
  174. package/dist/attestation.d.ts.map +0 -1
  175. package/dist/attestation.test.d.ts +0 -2
  176. package/dist/attestation.test.d.ts.map +0 -1
  177. package/dist/authenticity/index.d.ts.map +0 -1
  178. package/dist/authenticity/index.test.d.ts +0 -2
  179. package/dist/authenticity/index.test.d.ts.map +0 -1
  180. package/dist/auto-pr.d.ts +0 -120
  181. package/dist/auto-pr.d.ts.map +0 -1
  182. package/dist/baseline.d.ts.map +0 -1
  183. package/dist/behavior-dsl.d.ts +0 -73
  184. package/dist/behavior-dsl.d.ts.map +0 -1
  185. package/dist/belief-state/calibration.d.ts +0 -10
  186. package/dist/belief-state/calibration.d.ts.map +0 -1
  187. package/dist/belief-state/calibration.test.d.ts +0 -2
  188. package/dist/belief-state/calibration.test.d.ts.map +0 -1
  189. package/dist/belief-state/code-agent-corpus.d.ts +0 -66
  190. package/dist/belief-state/code-agent-corpus.d.ts.map +0 -1
  191. package/dist/belief-state/code-agent-corpus.test.d.ts +0 -2
  192. package/dist/belief-state/code-agent-corpus.test.d.ts.map +0 -1
  193. package/dist/belief-state/code-agent-evidence.d.ts +0 -22
  194. package/dist/belief-state/code-agent-evidence.d.ts.map +0 -1
  195. package/dist/belief-state/code-agent-evidence.test.d.ts +0 -2
  196. package/dist/belief-state/code-agent-evidence.test.d.ts.map +0 -1
  197. package/dist/belief-state/extract.d.ts +0 -7
  198. package/dist/belief-state/extract.d.ts.map +0 -1
  199. package/dist/belief-state/extract.test.d.ts +0 -2
  200. package/dist/belief-state/extract.test.d.ts.map +0 -1
  201. package/dist/belief-state/index.d.ts.map +0 -1
  202. package/dist/belief-state/ope.d.ts +0 -17
  203. package/dist/belief-state/ope.d.ts.map +0 -1
  204. package/dist/belief-state/ope.test.d.ts +0 -2
  205. package/dist/belief-state/ope.test.d.ts.map +0 -1
  206. package/dist/belief-state/phase0-measurement.d.ts +0 -55
  207. package/dist/belief-state/phase0-measurement.d.ts.map +0 -1
  208. package/dist/belief-state/report.d.ts +0 -17
  209. package/dist/belief-state/report.d.ts.map +0 -1
  210. package/dist/belief-state/report.test.d.ts +0 -2
  211. package/dist/belief-state/report.test.d.ts.map +0 -1
  212. package/dist/belief-state/research-evidence.d.ts +0 -23
  213. package/dist/belief-state/research-evidence.d.ts.map +0 -1
  214. package/dist/belief-state/research-evidence.test.d.ts +0 -2
  215. package/dist/belief-state/research-evidence.test.d.ts.map +0 -1
  216. package/dist/belief-state/runtime-benchmark-corpus.d.ts +0 -32
  217. package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +0 -1
  218. package/dist/belief-state/runtime-hooks.d.ts +0 -87
  219. package/dist/belief-state/runtime-hooks.d.ts.map +0 -1
  220. package/dist/belief-state/runtime-hooks.test.d.ts +0 -2
  221. package/dist/belief-state/runtime-hooks.test.d.ts.map +0 -1
  222. package/dist/belief-state/selective.d.ts +0 -15
  223. package/dist/belief-state/selective.d.ts.map +0 -1
  224. package/dist/belief-state/selective.test.d.ts +0 -2
  225. package/dist/belief-state/selective.test.d.ts.map +0 -1
  226. package/dist/belief-state/shadow-probe.d.ts +0 -80
  227. package/dist/belief-state/shadow-probe.d.ts.map +0 -1
  228. package/dist/belief-state/shadow-probe.test.d.ts +0 -2
  229. package/dist/belief-state/shadow-probe.test.d.ts.map +0 -1
  230. package/dist/belief-state/types.d.ts +0 -195
  231. package/dist/belief-state/types.d.ts.map +0 -1
  232. package/dist/belief-state/types.test.d.ts +0 -2
  233. package/dist/belief-state/types.test.d.ts.map +0 -1
  234. package/dist/benchmark.d.ts +0 -14
  235. package/dist/benchmark.d.ts.map +0 -1
  236. package/dist/benchmarks/index.d.ts.map +0 -1
  237. package/dist/benchmarks/routing/dataset.d.ts +0 -34
  238. package/dist/benchmarks/routing/dataset.d.ts.map +0 -1
  239. package/dist/benchmarks/routing/index.d.ts +0 -34
  240. package/dist/benchmarks/routing/index.d.ts.map +0 -1
  241. package/dist/benchmarks/types.d.ts +0 -49
  242. package/dist/benchmarks/types.d.ts.map +0 -1
  243. package/dist/bisector.d.ts +0 -81
  244. package/dist/bisector.d.ts.map +0 -1
  245. package/dist/budget-guard.d.ts +0 -31
  246. package/dist/budget-guard.d.ts.map +0 -1
  247. package/dist/builder-eval/builder-session.d.ts +0 -111
  248. package/dist/builder-eval/builder-session.d.ts.map +0 -1
  249. package/dist/builder-eval/correlation.d.ts +0 -32
  250. package/dist/builder-eval/correlation.d.ts.map +0 -1
  251. package/dist/builder-eval/index.d.ts.map +0 -1
  252. package/dist/builder-eval/project-registry.d.ts +0 -51
  253. package/dist/builder-eval/project-registry.d.ts.map +0 -1
  254. package/dist/builder-eval/three-layer-eval.d.ts +0 -55
  255. package/dist/builder-eval/three-layer-eval.d.ts.map +0 -1
  256. package/dist/campaign/analyst-surface.d.ts +0 -108
  257. package/dist/campaign/analyst-surface.d.ts.map +0 -1
  258. package/dist/campaign/analyst-surface.test.d.ts +0 -2
  259. package/dist/campaign/analyst-surface.test.d.ts.map +0 -1
  260. package/dist/campaign/auto-pr.d.ts +0 -46
  261. package/dist/campaign/auto-pr.d.ts.map +0 -1
  262. package/dist/campaign/distillation/agreement-judge.d.ts +0 -69
  263. package/dist/campaign/distillation/agreement-judge.d.ts.map +0 -1
  264. package/dist/campaign/distillation/cli.d.ts +0 -35
  265. package/dist/campaign/distillation/cli.d.ts.map +0 -1
  266. package/dist/campaign/distillation/distillation.test.d.ts +0 -2
  267. package/dist/campaign/distillation/distillation.test.d.ts.map +0 -1
  268. package/dist/campaign/distillation/gold-scenarios.d.ts +0 -54
  269. package/dist/campaign/distillation/gold-scenarios.d.ts.map +0 -1
  270. package/dist/campaign/distillation/run-distillation.d.ts +0 -119
  271. package/dist/campaign/distillation/run-distillation.d.ts.map +0 -1
  272. package/dist/campaign/gates/compose.d.ts +0 -12
  273. package/dist/campaign/gates/compose.d.ts.map +0 -1
  274. package/dist/campaign/gates/default-production-gate.d.ts +0 -58
  275. package/dist/campaign/gates/default-production-gate.d.ts.map +0 -1
  276. package/dist/campaign/gates/heldout-gate.d.ts +0 -12
  277. package/dist/campaign/gates/heldout-gate.d.ts.map +0 -1
  278. package/dist/campaign/gates/promotion-policy.d.ts +0 -125
  279. package/dist/campaign/gates/promotion-policy.d.ts.map +0 -1
  280. package/dist/campaign/gates/promotion-policy.test.d.ts +0 -2
  281. package/dist/campaign/gates/promotion-policy.test.d.ts.map +0 -1
  282. package/dist/campaign/gates/sequential.d.ts +0 -146
  283. package/dist/campaign/gates/sequential.d.ts.map +0 -1
  284. package/dist/campaign/gates/sequential.test.d.ts +0 -2
  285. package/dist/campaign/gates/sequential.test.d.ts.map +0 -1
  286. package/dist/campaign/gates/statistical-heldout.d.ts +0 -99
  287. package/dist/campaign/gates/statistical-heldout.d.ts.map +0 -1
  288. package/dist/campaign/gates/statistical-heldout.test.d.ts +0 -2
  289. package/dist/campaign/gates/statistical-heldout.test.d.ts.map +0 -1
  290. package/dist/campaign/index.d.ts.map +0 -1
  291. package/dist/campaign/labeled-store/fs-adapter.d.ts +0 -59
  292. package/dist/campaign/labeled-store/fs-adapter.d.ts.map +0 -1
  293. package/dist/campaign/presets/compare-proposers.d.ts +0 -146
  294. package/dist/campaign/presets/compare-proposers.d.ts.map +0 -1
  295. package/dist/campaign/presets/playback.d.ts +0 -120
  296. package/dist/campaign/presets/playback.d.ts.map +0 -1
  297. package/dist/campaign/presets/playback.test.d.ts +0 -2
  298. package/dist/campaign/presets/playback.test.d.ts.map +0 -1
  299. package/dist/campaign/presets/run-eval.d.ts +0 -14
  300. package/dist/campaign/presets/run-eval.d.ts.map +0 -1
  301. package/dist/campaign/presets/run-improvement-loop.d.ts +0 -63
  302. package/dist/campaign/presets/run-improvement-loop.d.ts.map +0 -1
  303. package/dist/campaign/presets/run-improvement-loop.test.d.ts +0 -2
  304. package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +0 -1
  305. package/dist/campaign/presets/run-optimization.d.ts +0 -92
  306. package/dist/campaign/presets/run-optimization.d.ts.map +0 -1
  307. package/dist/campaign/presets/run-profile-matrix.d.ts +0 -151
  308. package/dist/campaign/presets/run-profile-matrix.d.ts.map +0 -1
  309. package/dist/campaign/presets/run-skill-opt.d.ts +0 -96
  310. package/dist/campaign/presets/run-skill-opt.d.ts.map +0 -1
  311. package/dist/campaign/proposers/_findings-text.d.ts +0 -22
  312. package/dist/campaign/proposers/_findings-text.d.ts.map +0 -1
  313. package/dist/campaign/proposers/ace.d.ts +0 -33
  314. package/dist/campaign/proposers/ace.d.ts.map +0 -1
  315. package/dist/campaign/proposers/ace.test.d.ts +0 -2
  316. package/dist/campaign/proposers/ace.test.d.ts.map +0 -1
  317. package/dist/campaign/proposers/analysis-edit.d.ts +0 -32
  318. package/dist/campaign/proposers/analysis-edit.d.ts.map +0 -1
  319. package/dist/campaign/proposers/evolutionary.d.ts +0 -20
  320. package/dist/campaign/proposers/evolutionary.d.ts.map +0 -1
  321. package/dist/campaign/proposers/fapo.d.ts +0 -120
  322. package/dist/campaign/proposers/fapo.d.ts.map +0 -1
  323. package/dist/campaign/proposers/gepa.d.ts +0 -86
  324. package/dist/campaign/proposers/gepa.d.ts.map +0 -1
  325. package/dist/campaign/proposers/halo.d.ts +0 -44
  326. package/dist/campaign/proposers/halo.d.ts.map +0 -1
  327. package/dist/campaign/proposers/halo.test.d.ts +0 -2
  328. package/dist/campaign/proposers/halo.test.d.ts.map +0 -1
  329. package/dist/campaign/proposers/memory.d.ts +0 -47
  330. package/dist/campaign/proposers/memory.d.ts.map +0 -1
  331. package/dist/campaign/proposers/memory.test.d.ts +0 -2
  332. package/dist/campaign/proposers/memory.test.d.ts.map +0 -1
  333. package/dist/campaign/proposers/skill-opt.d.ts +0 -88
  334. package/dist/campaign/proposers/skill-opt.d.ts.map +0 -1
  335. package/dist/campaign/proposers/trace-analyst.d.ts +0 -48
  336. package/dist/campaign/proposers/trace-analyst.d.ts.map +0 -1
  337. package/dist/campaign/proposers/trace-analyst.test.d.ts +0 -2
  338. package/dist/campaign/proposers/trace-analyst.test.d.ts.map +0 -1
  339. package/dist/campaign/provenance.d.ts +0 -185
  340. package/dist/campaign/provenance.d.ts.map +0 -1
  341. package/dist/campaign/run-campaign.d.ts +0 -90
  342. package/dist/campaign/run-campaign.d.ts.map +0 -1
  343. package/dist/campaign/score-utils.d.ts +0 -26
  344. package/dist/campaign/score-utils.d.ts.map +0 -1
  345. package/dist/campaign/skill-patch.d.ts +0 -62
  346. package/dist/campaign/skill-patch.d.ts.map +0 -1
  347. package/dist/campaign/storage.d.ts +0 -38
  348. package/dist/campaign/storage.d.ts.map +0 -1
  349. package/dist/campaign/types.d.ts.map +0 -1
  350. package/dist/campaign/worktree/index.d.ts +0 -53
  351. package/dist/campaign/worktree/index.d.ts.map +0 -1
  352. package/dist/canary.d.ts +0 -101
  353. package/dist/canary.d.ts.map +0 -1
  354. package/dist/causal-attribution.d.ts +0 -45
  355. package/dist/causal-attribution.d.ts.map +0 -1
  356. package/dist/ci-gate.d.ts +0 -44
  357. package/dist/ci-gate.d.ts.map +0 -1
  358. package/dist/cli.d.ts.map +0 -1
  359. package/dist/client.d.ts +0 -77
  360. package/dist/client.d.ts.map +0 -1
  361. package/dist/client.test.d.ts +0 -2
  362. package/dist/client.test.d.ts.map +0 -1
  363. package/dist/command-runner.d.ts +0 -74
  364. package/dist/command-runner.d.ts.map +0 -1
  365. package/dist/command-runner.test.d.ts +0 -2
  366. package/dist/command-runner.test.d.ts.map +0 -1
  367. package/dist/completion-verifier.d.ts +0 -147
  368. package/dist/completion-verifier.d.ts.map +0 -1
  369. package/dist/completion-verifier.test.d.ts +0 -9
  370. package/dist/completion-verifier.test.d.ts.map +0 -1
  371. package/dist/concurrency.d.ts +0 -23
  372. package/dist/concurrency.d.ts.map +0 -1
  373. package/dist/contamination-guard.d.ts +0 -81
  374. package/dist/contamination-guard.d.ts.map +0 -1
  375. package/dist/contract/analyze-runs.d.ts.map +0 -1
  376. package/dist/contract/define-agent-eval.d.ts +0 -52
  377. package/dist/contract/define-agent-eval.d.ts.map +0 -1
  378. package/dist/contract/diff.d.ts +0 -114
  379. package/dist/contract/diff.d.ts.map +0 -1
  380. package/dist/contract/index.d.ts.map +0 -1
  381. package/dist/contract/insight-report.d.ts.map +0 -1
  382. package/dist/contract/insight-types-fwd.d.ts +0 -7
  383. package/dist/contract/insight-types-fwd.d.ts.map +0 -1
  384. package/dist/contract/intake/agent-trace.d.ts +0 -97
  385. package/dist/contract/intake/agent-trace.d.ts.map +0 -1
  386. package/dist/contract/intake/code-agent-session.d.ts.map +0 -1
  387. package/dist/contract/intake/feedback-table.d.ts +0 -87
  388. package/dist/contract/intake/feedback-table.d.ts.map +0 -1
  389. package/dist/contract/intake/index.d.ts +0 -22
  390. package/dist/contract/intake/index.d.ts.map +0 -1
  391. package/dist/contract/intake/otel-spans.d.ts +0 -33
  392. package/dist/contract/intake/otel-spans.d.ts.map +0 -1
  393. package/dist/contract/self-improve.d.ts +0 -284
  394. package/dist/contract/self-improve.d.ts.map +0 -1
  395. package/dist/control-runtime.d.ts.map +0 -1
  396. package/dist/control-runtime.test.d.ts +0 -2
  397. package/dist/control-runtime.test.d.ts.map +0 -1
  398. package/dist/control.d.ts.map +0 -1
  399. package/dist/convergence.d.ts +0 -29
  400. package/dist/convergence.d.ts.map +0 -1
  401. package/dist/cost-ledger.d.ts.map +0 -1
  402. package/dist/cost-ledger.test.d.ts +0 -2
  403. package/dist/cost-ledger.test.d.ts.map +0 -1
  404. package/dist/cost-report.d.ts +0 -43
  405. package/dist/cost-report.d.ts.map +0 -1
  406. package/dist/cost-report.test.d.ts +0 -2
  407. package/dist/cost-report.test.d.ts.map +0 -1
  408. package/dist/cost-tracker.d.ts +0 -76
  409. package/dist/cost-tracker.d.ts.map +0 -1
  410. package/dist/counterfactual.d.ts.map +0 -1
  411. package/dist/cross-trace-diff.d.ts +0 -56
  412. package/dist/cross-trace-diff.d.ts.map +0 -1
  413. package/dist/dataset.d.ts.map +0 -1
  414. package/dist/deploy-gate-layer.d.ts +0 -125
  415. package/dist/deploy-gate-layer.d.ts.map +0 -1
  416. package/dist/deploy-gate-layer.test.d.ts +0 -2
  417. package/dist/deploy-gate-layer.test.d.ts.map +0 -1
  418. package/dist/description-length-gate.d.ts +0 -119
  419. package/dist/description-length-gate.d.ts.map +0 -1
  420. package/dist/detectors/edge.test.d.ts +0 -2
  421. package/dist/detectors/edge.test.d.ts.map +0 -1
  422. package/dist/detectors/index.d.ts +0 -81
  423. package/dist/detectors/index.d.ts.map +0 -1
  424. package/dist/detectors/index.test.d.ts +0 -2
  425. package/dist/detectors/index.test.d.ts.map +0 -1
  426. package/dist/diagnose/causal-sweep.d.ts +0 -100
  427. package/dist/diagnose/causal-sweep.d.ts.map +0 -1
  428. package/dist/diagnose/index.d.ts +0 -36
  429. package/dist/diagnose/index.d.ts.map +0 -1
  430. package/dist/diagnose/remediation.d.ts +0 -68
  431. package/dist/diagnose/remediation.d.ts.map +0 -1
  432. package/dist/diagnose/repair.d.ts +0 -77
  433. package/dist/diagnose/repair.d.ts.map +0 -1
  434. package/dist/discover-personas.d.ts +0 -35
  435. package/dist/discover-personas.d.ts.map +0 -1
  436. package/dist/driver.d.ts +0 -95
  437. package/dist/driver.d.ts.map +0 -1
  438. package/dist/driver.test.d.ts +0 -8
  439. package/dist/driver.test.d.ts.map +0 -1
  440. package/dist/dual-agent-bench.d.ts +0 -81
  441. package/dist/dual-agent-bench.d.ts.map +0 -1
  442. package/dist/error-count-extractor.d.ts +0 -47
  443. package/dist/error-count-extractor.d.ts.map +0 -1
  444. package/dist/error-count-extractor.test.d.ts +0 -2
  445. package/dist/error-count-extractor.test.d.ts.map +0 -1
  446. package/dist/errors.d.ts.map +0 -1
  447. package/dist/eval-campaign.d.ts.map +0 -1
  448. package/dist/eval-campaign.test.d.ts +0 -2
  449. package/dist/eval-campaign.test.d.ts.map +0 -1
  450. package/dist/eval-tools.d.ts +0 -55
  451. package/dist/eval-tools.d.ts.map +0 -1
  452. package/dist/eval-trace-store.d.ts +0 -107
  453. package/dist/eval-trace-store.d.ts.map +0 -1
  454. package/dist/eval-trace-store.test.d.ts +0 -2
  455. package/dist/eval-trace-store.test.d.ts.map +0 -1
  456. package/dist/executor.d.ts +0 -38
  457. package/dist/executor.d.ts.map +0 -1
  458. package/dist/executor.test.d.ts +0 -10
  459. package/dist/executor.test.d.ts.map +0 -1
  460. package/dist/experiment-tracker.d.ts +0 -178
  461. package/dist/experiment-tracker.d.ts.map +0 -1
  462. package/dist/experiment-tracker.test.d.ts +0 -2
  463. package/dist/experiment-tracker.test.d.ts.map +0 -1
  464. package/dist/failure-taxonomy.d.ts +0 -38
  465. package/dist/failure-taxonomy.d.ts.map +0 -1
  466. package/dist/feedback-trajectory.d.ts.map +0 -1
  467. package/dist/feedback-trajectory.test.d.ts +0 -2
  468. package/dist/feedback-trajectory.test.d.ts.map +0 -1
  469. package/dist/flow-layer.d.ts +0 -90
  470. package/dist/flow-layer.d.ts.map +0 -1
  471. package/dist/flow-layer.test.d.ts +0 -2
  472. package/dist/flow-layer.test.d.ts.map +0 -1
  473. package/dist/fuzz/capsule.d.ts +0 -46
  474. package/dist/fuzz/capsule.d.ts.map +0 -1
  475. package/dist/fuzz/cube.d.ts +0 -36
  476. package/dist/fuzz/cube.d.ts.map +0 -1
  477. package/dist/fuzz/explorer-cost.test.d.ts +0 -2
  478. package/dist/fuzz/explorer-cost.test.d.ts.map +0 -1
  479. package/dist/fuzz/explorer.d.ts +0 -64
  480. package/dist/fuzz/explorer.d.ts.map +0 -1
  481. package/dist/fuzz/fuzz-agent.d.ts +0 -16
  482. package/dist/fuzz/fuzz-agent.d.ts.map +0 -1
  483. package/dist/fuzz/fuzz-agent.test.d.ts +0 -2
  484. package/dist/fuzz/fuzz-agent.test.d.ts.map +0 -1
  485. package/dist/fuzz/gates.d.ts +0 -33
  486. package/dist/fuzz/gates.d.ts.map +0 -1
  487. package/dist/fuzz/index.d.ts +0 -26
  488. package/dist/fuzz/index.d.ts.map +0 -1
  489. package/dist/fuzz/policies.d.ts +0 -28
  490. package/dist/fuzz/policies.d.ts.map +0 -1
  491. package/dist/fuzz/tools.d.ts +0 -20
  492. package/dist/fuzz/tools.d.ts.map +0 -1
  493. package/dist/fuzz/types.d.ts +0 -307
  494. package/dist/fuzz/types.d.ts.map +0 -1
  495. package/dist/golden-matcher.d.ts +0 -71
  496. package/dist/golden-matcher.d.ts.map +0 -1
  497. package/dist/governance/eu-ai-act.d.ts +0 -37
  498. package/dist/governance/eu-ai-act.d.ts.map +0 -1
  499. package/dist/governance/index.d.ts.map +0 -1
  500. package/dist/governance/nist-ai-rmf.d.ts +0 -15
  501. package/dist/governance/nist-ai-rmf.d.ts.map +0 -1
  502. package/dist/governance/soc2.d.ts +0 -12
  503. package/dist/governance/soc2.d.ts.map +0 -1
  504. package/dist/governance/types.d.ts +0 -66
  505. package/dist/governance/types.d.ts.map +0 -1
  506. package/dist/harness-optimizer.d.ts +0 -82
  507. package/dist/harness-optimizer.d.ts.map +0 -1
  508. package/dist/held-out-gate.d.ts +0 -135
  509. package/dist/held-out-gate.d.ts.map +0 -1
  510. package/dist/hosted/client.d.ts +0 -73
  511. package/dist/hosted/client.d.ts.map +0 -1
  512. package/dist/hosted/from-env.test.d.ts +0 -8
  513. package/dist/hosted/from-env.test.d.ts.map +0 -1
  514. package/dist/hosted/index.d.ts.map +0 -1
  515. package/dist/hosted/types.d.ts +0 -159
  516. package/dist/hosted/types.d.ts.map +0 -1
  517. package/dist/index.d.ts.map +0 -1
  518. package/dist/integrity/backend-integrity.d.ts +0 -71
  519. package/dist/integrity/backend-integrity.d.ts.map +0 -1
  520. package/dist/integrity/preflight.d.ts +0 -72
  521. package/dist/integrity/preflight.d.ts.map +0 -1
  522. package/dist/integrity/preflight.test.d.ts +0 -2
  523. package/dist/integrity/preflight.test.d.ts.map +0 -1
  524. package/dist/integrity/single-backend.d.ts +0 -67
  525. package/dist/integrity/single-backend.d.ts.map +0 -1
  526. package/dist/intent-match-judge.d.ts +0 -69
  527. package/dist/intent-match-judge.d.ts.map +0 -1
  528. package/dist/intent-match-judge.test.d.ts +0 -2
  529. package/dist/intent-match-judge.test.d.ts.map +0 -1
  530. package/dist/judge-calibration.d.ts.map +0 -1
  531. package/dist/judge-ensemble.d.ts +0 -66
  532. package/dist/judge-ensemble.d.ts.map +0 -1
  533. package/dist/judge-ensemble.test.d.ts +0 -8
  534. package/dist/judge-ensemble.test.d.ts.map +0 -1
  535. package/dist/judge-families.d.ts +0 -38
  536. package/dist/judge-families.d.ts.map +0 -1
  537. package/dist/judge-panel.d.ts +0 -65
  538. package/dist/judge-panel.d.ts.map +0 -1
  539. package/dist/judge-retry.d.ts +0 -70
  540. package/dist/judge-retry.d.ts.map +0 -1
  541. package/dist/judge-runner.d.ts +0 -36
  542. package/dist/judge-runner.d.ts.map +0 -1
  543. package/dist/judge-runner.test.d.ts +0 -2
  544. package/dist/judge-runner.test.d.ts.map +0 -1
  545. package/dist/judges.d.ts +0 -74
  546. package/dist/judges.d.ts.map +0 -1
  547. package/dist/keyword-coverage-judge.d.ts +0 -89
  548. package/dist/keyword-coverage-judge.d.ts.map +0 -1
  549. package/dist/keyword-coverage-judge.test.d.ts +0 -2
  550. package/dist/keyword-coverage-judge.test.d.ts.map +0 -1
  551. package/dist/knowledge/index.d.ts.map +0 -1
  552. package/dist/knowledge/readiness.d.ts +0 -26
  553. package/dist/knowledge/readiness.d.ts.map +0 -1
  554. package/dist/knowledge/types.d.ts +0 -75
  555. package/dist/knowledge/types.d.ts.map +0 -1
  556. package/dist/live-proof.d.ts +0 -62
  557. package/dist/live-proof.d.ts.map +0 -1
  558. package/dist/llm-client.d.ts.map +0 -1
  559. package/dist/llm-client.test.d.ts +0 -2
  560. package/dist/llm-client.test.d.ts.map +0 -1
  561. package/dist/locked-jsonl-appender.d.ts.map +0 -1
  562. package/dist/matrix/aggregation.d.ts +0 -16
  563. package/dist/matrix/aggregation.d.ts.map +0 -1
  564. package/dist/matrix/index.d.ts.map +0 -1
  565. package/dist/matrix/runner.d.ts +0 -15
  566. package/dist/matrix/runner.d.ts.map +0 -1
  567. package/dist/matrix/types.d.ts.map +0 -1
  568. package/dist/meta-eval/calibration.d.ts +0 -47
  569. package/dist/meta-eval/calibration.d.ts.map +0 -1
  570. package/dist/meta-eval/correlation-study.d.ts +0 -53
  571. package/dist/meta-eval/correlation-study.d.ts.map +0 -1
  572. package/dist/meta-eval/index.d.ts.map +0 -1
  573. package/dist/meta-eval/outcome-store.d.ts.map +0 -1
  574. package/dist/meta-eval/rubric-predictive-validity.d.ts.map +0 -1
  575. package/dist/meta-eval/sentinel.d.ts +0 -169
  576. package/dist/meta-eval/sentinel.d.ts.map +0 -1
  577. package/dist/metrics.d.ts +0 -63
  578. package/dist/metrics.d.ts.map +0 -1
  579. package/dist/model-seats.d.ts +0 -71
  580. package/dist/model-seats.d.ts.map +0 -1
  581. package/dist/model-seats.test.d.ts +0 -2
  582. package/dist/model-seats.test.d.ts.map +0 -1
  583. package/dist/muffled-gate-scanner.d.ts +0 -102
  584. package/dist/muffled-gate-scanner.d.ts.map +0 -1
  585. package/dist/multi-layer-verifier.d.ts.map +0 -1
  586. package/dist/multi-layer-verifier.test.d.ts +0 -2
  587. package/dist/multi-layer-verifier.test.d.ts.map +0 -1
  588. package/dist/multi-toolchain-layer.d.ts +0 -80
  589. package/dist/multi-toolchain-layer.d.ts.map +0 -1
  590. package/dist/multi-toolchain-layer.test.d.ts +0 -2
  591. package/dist/multi-toolchain-layer.test.d.ts.map +0 -1
  592. package/dist/multishot/default-tools.d.ts +0 -34
  593. package/dist/multishot/default-tools.d.ts.map +0 -1
  594. package/dist/multishot/index.d.ts.map +0 -1
  595. package/dist/multishot/judges.d.ts +0 -32
  596. package/dist/multishot/judges.d.ts.map +0 -1
  597. package/dist/multishot/matrix.d.ts +0 -107
  598. package/dist/multishot/matrix.d.ts.map +0 -1
  599. package/dist/multishot/multishot.d.ts +0 -23
  600. package/dist/multishot/multishot.d.ts.map +0 -1
  601. package/dist/multishot/router.d.ts +0 -37
  602. package/dist/multishot/router.d.ts.map +0 -1
  603. package/dist/multishot/types.d.ts +0 -60
  604. package/dist/multishot/types.d.ts.map +0 -1
  605. package/dist/observability.d.ts +0 -71
  606. package/dist/observability.d.ts.map +0 -1
  607. package/dist/oracle.d.ts +0 -55
  608. package/dist/oracle.d.ts.map +0 -1
  609. package/dist/orthogonality.d.ts +0 -35
  610. package/dist/orthogonality.d.ts.map +0 -1
  611. package/dist/otel-pipeline.d.ts +0 -31
  612. package/dist/otel-pipeline.d.ts.map +0 -1
  613. package/dist/paraphrase.d.ts +0 -107
  614. package/dist/paraphrase.d.ts.map +0 -1
  615. package/dist/pareto.d.ts.map +0 -1
  616. package/dist/partition-held-out.d.ts +0 -70
  617. package/dist/partition-held-out.d.ts.map +0 -1
  618. package/dist/partition-held-out.test.d.ts +0 -2
  619. package/dist/partition-held-out.test.d.ts.map +0 -1
  620. package/dist/perf/index.d.ts.map +0 -1
  621. package/dist/perf/integrity.d.ts +0 -30
  622. package/dist/perf/integrity.d.ts.map +0 -1
  623. package/dist/perf/journey.d.ts +0 -45
  624. package/dist/perf/journey.d.ts.map +0 -1
  625. package/dist/perf/ratchet.d.ts +0 -47
  626. package/dist/perf/ratchet.d.ts.map +0 -1
  627. package/dist/pipelines/budget-breach.d.ts +0 -31
  628. package/dist/pipelines/budget-breach.d.ts.map +0 -1
  629. package/dist/pipelines/budget-breach.test.d.ts +0 -2
  630. package/dist/pipelines/budget-breach.test.d.ts.map +0 -1
  631. package/dist/pipelines/failure-cluster.d.ts +0 -38
  632. package/dist/pipelines/failure-cluster.d.ts.map +0 -1
  633. package/dist/pipelines/failure-cluster.test.d.ts +0 -2
  634. package/dist/pipelines/failure-cluster.test.d.ts.map +0 -1
  635. package/dist/pipelines/first-divergence.d.ts +0 -26
  636. package/dist/pipelines/first-divergence.d.ts.map +0 -1
  637. package/dist/pipelines/first-divergence.test.d.ts +0 -2
  638. package/dist/pipelines/first-divergence.test.d.ts.map +0 -1
  639. package/dist/pipelines/index.d.ts.map +0 -1
  640. package/dist/pipelines/judge-agreement.d.ts +0 -26
  641. package/dist/pipelines/judge-agreement.d.ts.map +0 -1
  642. package/dist/pipelines/judge-agreement.test.d.ts +0 -2
  643. package/dist/pipelines/judge-agreement.test.d.ts.map +0 -1
  644. package/dist/pipelines/regression.d.ts +0 -23
  645. package/dist/pipelines/regression.d.ts.map +0 -1
  646. package/dist/pipelines/regression.test.d.ts +0 -2
  647. package/dist/pipelines/regression.test.d.ts.map +0 -1
  648. package/dist/pipelines/stuck-loop.d.ts +0 -32
  649. package/dist/pipelines/stuck-loop.d.ts.map +0 -1
  650. package/dist/pipelines/stuck-loop.test.d.ts +0 -2
  651. package/dist/pipelines/stuck-loop.test.d.ts.map +0 -1
  652. package/dist/pipelines/tool-waste.d.ts +0 -34
  653. package/dist/pipelines/tool-waste.d.ts.map +0 -1
  654. package/dist/pipelines/tool-waste.test.d.ts +0 -2
  655. package/dist/pipelines/tool-waste.test.d.ts.map +0 -1
  656. package/dist/playbook.d.ts +0 -16
  657. package/dist/playbook.d.ts.map +0 -1
  658. package/dist/pr-review-benchmark.d.ts +0 -88
  659. package/dist/pr-review-benchmark.d.ts.map +0 -1
  660. package/dist/pr-review-benchmark.test.d.ts +0 -2
  661. package/dist/pr-review-benchmark.test.d.ts.map +0 -1
  662. package/dist/pre-registration.d.ts +0 -125
  663. package/dist/pre-registration.d.ts.map +0 -1
  664. package/dist/prm/builtin-rubrics.d.ts +0 -33
  665. package/dist/prm/builtin-rubrics.d.ts.map +0 -1
  666. package/dist/prm/index.d.ts.map +0 -1
  667. package/dist/prm/inference.d.ts +0 -29
  668. package/dist/prm/inference.d.ts.map +0 -1
  669. package/dist/prm/inference.test.d.ts +0 -2
  670. package/dist/prm/inference.test.d.ts.map +0 -1
  671. package/dist/prm/rubric.d.ts.map +0 -1
  672. package/dist/prm/training-export.d.ts +0 -38
  673. package/dist/prm/training-export.d.ts.map +0 -1
  674. package/dist/produced-state.d.ts +0 -63
  675. package/dist/produced-state.d.ts.map +0 -1
  676. package/dist/produced-state.test.d.ts +0 -8
  677. package/dist/produced-state.test.d.ts.map +0 -1
  678. package/dist/profile/baselines.d.ts +0 -37
  679. package/dist/profile/baselines.d.ts.map +0 -1
  680. package/dist/profile/index.d.ts +0 -105
  681. package/dist/profile/index.d.ts.map +0 -1
  682. package/dist/promotion-gate.d.ts +0 -93
  683. package/dist/promotion-gate.d.ts.map +0 -1
  684. package/dist/prompt-registry.d.ts +0 -41
  685. package/dist/prompt-registry.d.ts.map +0 -1
  686. package/dist/propose-review-control.d.ts +0 -49
  687. package/dist/propose-review-control.d.ts.map +0 -1
  688. package/dist/propose-review-control.test.d.ts +0 -2
  689. package/dist/propose-review-control.test.d.ts.map +0 -1
  690. package/dist/propose-review.d.ts +0 -155
  691. package/dist/propose-review.d.ts.map +0 -1
  692. package/dist/red-team.d.ts.map +0 -1
  693. package/dist/reference-replay-steering.d.ts +0 -11
  694. package/dist/reference-replay-steering.d.ts.map +0 -1
  695. package/dist/reference-replay.d.ts +0 -176
  696. package/dist/reference-replay.d.ts.map +0 -1
  697. package/dist/reflective-mutation.d.ts +0 -79
  698. package/dist/reflective-mutation.d.ts.map +0 -1
  699. package/dist/registry.d.ts +0 -31
  700. package/dist/registry.d.ts.map +0 -1
  701. package/dist/release-confidence.d.ts +0 -128
  702. package/dist/release-confidence.d.ts.map +0 -1
  703. package/dist/release-report.d.ts +0 -11
  704. package/dist/release-report.d.ts.map +0 -1
  705. package/dist/replay.d.ts +0 -120
  706. package/dist/replay.d.ts.map +0 -1
  707. package/dist/reporter.d.ts +0 -14
  708. package/dist/reporter.d.ts.map +0 -1
  709. package/dist/reporting.d.ts.map +0 -1
  710. package/dist/researcher.d.ts +0 -140
  711. package/dist/researcher.d.ts.map +0 -1
  712. package/dist/reviewer.d.ts +0 -118
  713. package/dist/reviewer.d.ts.map +0 -1
  714. package/dist/reviewer.test.d.ts +0 -2
  715. package/dist/reviewer.test.d.ts.map +0 -1
  716. package/dist/reward-model-export.d.ts +0 -60
  717. package/dist/reward-model-export.d.ts.map +0 -1
  718. package/dist/rl/active-curriculum.d.ts +0 -110
  719. package/dist/rl/active-curriculum.d.ts.map +0 -1
  720. package/dist/rl/adaptation-eval.d.ts +0 -109
  721. package/dist/rl/adaptation-eval.d.ts.map +0 -1
  722. package/dist/rl/adversarial.d.ts.map +0 -1
  723. package/dist/rl/compute-curves.d.ts +0 -127
  724. package/dist/rl/compute-curves.d.ts.map +0 -1
  725. package/dist/rl/contamination.d.ts +0 -117
  726. package/dist/rl/contamination.d.ts.map +0 -1
  727. package/dist/rl/corpus.d.ts +0 -55
  728. package/dist/rl/corpus.d.ts.map +0 -1
  729. package/dist/rl/corpus.test.d.ts +0 -2
  730. package/dist/rl/corpus.test.d.ts.map +0 -1
  731. package/dist/rl/dataset.d.ts +0 -102
  732. package/dist/rl/dataset.d.ts.map +0 -1
  733. package/dist/rl/dataset.test.d.ts +0 -2
  734. package/dist/rl/dataset.test.d.ts.map +0 -1
  735. package/dist/rl/exporters.d.ts +0 -141
  736. package/dist/rl/exporters.d.ts.map +0 -1
  737. package/dist/rl/index.d.ts +0 -49
  738. package/dist/rl/index.d.ts.map +0 -1
  739. package/dist/rl/off-policy.d.ts.map +0 -1
  740. package/dist/rl/predictive-validity-researcher.d.ts +0 -69
  741. package/dist/rl/predictive-validity-researcher.d.ts.map +0 -1
  742. package/dist/rl/preferences.d.ts +0 -141
  743. package/dist/rl/preferences.d.ts.map +0 -1
  744. package/dist/rl/process-reward.d.ts +0 -122
  745. package/dist/rl/process-reward.d.ts.map +0 -1
  746. package/dist/rl/reward-hacking.d.ts +0 -104
  747. package/dist/rl/reward-hacking.d.ts.map +0 -1
  748. package/dist/rl/rl-campaign.d.ts +0 -85
  749. package/dist/rl/rl-campaign.d.ts.map +0 -1
  750. package/dist/rl/run-record-adapters.d.ts +0 -56
  751. package/dist/rl/run-record-adapters.d.ts.map +0 -1
  752. package/dist/rl/sim-fidelity.d.ts +0 -166
  753. package/dist/rl/sim-fidelity.d.ts.map +0 -1
  754. package/dist/rl/sim-fidelity.test.d.ts +0 -2
  755. package/dist/rl/sim-fidelity.test.d.ts.map +0 -1
  756. package/dist/rl/tournament.d.ts +0 -115
  757. package/dist/rl/tournament.d.ts.map +0 -1
  758. package/dist/rl/verifiable-reward.d.ts +0 -124
  759. package/dist/rl/verifiable-reward.d.ts.map +0 -1
  760. package/dist/run-critic.d.ts +0 -23
  761. package/dist/run-critic.d.ts.map +0 -1
  762. package/dist/run-evidence.d.ts +0 -32
  763. package/dist/run-evidence.d.ts.map +0 -1
  764. package/dist/run-record.d.ts.map +0 -1
  765. package/dist/run-record.test.d.ts +0 -2
  766. package/dist/run-record.test.d.ts.map +0 -1
  767. package/dist/run-score.d.ts +0 -31
  768. package/dist/run-score.d.ts.map +0 -1
  769. package/dist/runtime-trajectory.d.ts +0 -47
  770. package/dist/runtime-trajectory.d.ts.map +0 -1
  771. package/dist/sandbox-harness.d.ts.map +0 -1
  772. package/dist/sandbox-harness.test.d.ts +0 -2
  773. package/dist/sandbox-harness.test.d.ts.map +0 -1
  774. package/dist/sandbox-pool.d.ts +0 -74
  775. package/dist/sandbox-pool.d.ts.map +0 -1
  776. package/dist/sandbox-pool.test.d.ts +0 -2
  777. package/dist/sandbox-pool.test.d.ts.map +0 -1
  778. package/dist/scorecard.d.ts +0 -133
  779. package/dist/scorecard.d.ts.map +0 -1
  780. package/dist/scorecard.test.d.ts +0 -2
  781. package/dist/scorecard.test.d.ts.map +0 -1
  782. package/dist/self-play.d.ts +0 -69
  783. package/dist/self-play.d.ts.map +0 -1
  784. package/dist/semantic-concept-judge.d.ts +0 -135
  785. package/dist/semantic-concept-judge.d.ts.map +0 -1
  786. package/dist/semantic-concept-judge.test.d.ts +0 -2
  787. package/dist/semantic-concept-judge.test.d.ts.map +0 -1
  788. package/dist/sequential.d.ts.map +0 -1
  789. package/dist/series-convergence.d.ts.map +0 -1
  790. package/dist/slo.d.ts +0 -48
  791. package/dist/slo.d.ts.map +0 -1
  792. package/dist/state-continuity.d.ts +0 -47
  793. package/dist/state-continuity.d.ts.map +0 -1
  794. package/dist/statistics.d.ts.map +0 -1
  795. package/dist/statistics.test.d.ts +0 -2
  796. package/dist/statistics.test.d.ts.map +0 -1
  797. package/dist/steering-optimizer.d.ts +0 -58
  798. package/dist/steering-optimizer.d.ts.map +0 -1
  799. package/dist/steering.d.ts +0 -24
  800. package/dist/steering.d.ts.map +0 -1
  801. package/dist/storyboard/code-edit.d.ts +0 -64
  802. package/dist/storyboard/code-edit.d.ts.map +0 -1
  803. package/dist/storyboard/code-edit.test.d.ts +0 -2
  804. package/dist/storyboard/code-edit.test.d.ts.map +0 -1
  805. package/dist/storyboard/index.d.ts.map +0 -1
  806. package/dist/storyboard/index.test.d.ts +0 -2
  807. package/dist/storyboard/index.test.d.ts.map +0 -1
  808. package/dist/summary-report.d.ts.map +0 -1
  809. package/dist/telemetry/client.d.ts +0 -35
  810. package/dist/telemetry/client.d.ts.map +0 -1
  811. package/dist/telemetry/index.d.ts.map +0 -1
  812. package/dist/telemetry/schema.d.ts +0 -61
  813. package/dist/telemetry/schema.d.ts.map +0 -1
  814. package/dist/telemetry/sink-fetch.d.ts +0 -39
  815. package/dist/telemetry/sink-fetch.d.ts.map +0 -1
  816. package/dist/telemetry/sink-file.d.ts.map +0 -1
  817. package/dist/test-graded-scenario.d.ts +0 -42
  818. package/dist/test-graded-scenario.d.ts.map +0 -1
  819. package/dist/testing.d.ts.map +0 -1
  820. package/dist/tool-use-metrics.d.ts +0 -35
  821. package/dist/tool-use-metrics.d.ts.map +0 -1
  822. package/dist/trace/capture-fetch.d.ts +0 -48
  823. package/dist/trace/capture-fetch.d.ts.map +0 -1
  824. package/dist/trace/capture-fetch.test.d.ts +0 -2
  825. package/dist/trace/capture-fetch.test.d.ts.map +0 -1
  826. package/dist/trace/emitter.d.ts.map +0 -1
  827. package/dist/trace/extract-usage.d.ts +0 -46
  828. package/dist/trace/extract-usage.d.ts.map +0 -1
  829. package/dist/trace/extract-usage.test.d.ts +0 -2
  830. package/dist/trace/extract-usage.test.d.ts.map +0 -1
  831. package/dist/trace/index.d.ts +0 -15
  832. package/dist/trace/index.d.ts.map +0 -1
  833. package/dist/trace/integrity.d.ts.map +0 -1
  834. package/dist/trace/otel-bridge.d.ts +0 -29
  835. package/dist/trace/otel-bridge.d.ts.map +0 -1
  836. package/dist/trace/otel-export.d.ts +0 -52
  837. package/dist/trace/otel-export.d.ts.map +0 -1
  838. package/dist/trace/otel.d.ts +0 -57
  839. package/dist/trace/otel.d.ts.map +0 -1
  840. package/dist/trace/otlp-attributes.d.ts +0 -17
  841. package/dist/trace/otlp-attributes.d.ts.map +0 -1
  842. package/dist/trace/query.d.ts +0 -29
  843. package/dist/trace/query.d.ts.map +0 -1
  844. package/dist/trace/query.test.d.ts +0 -2
  845. package/dist/trace/query.test.d.ts.map +0 -1
  846. package/dist/trace/raw-provider-sink.d.ts.map +0 -1
  847. package/dist/trace/redact.d.ts.map +0 -1
  848. package/dist/trace/schema.d.ts.map +0 -1
  849. package/dist/trace/store-to-otlp.d.ts +0 -72
  850. package/dist/trace/store-to-otlp.d.ts.map +0 -1
  851. package/dist/trace/store-to-otlp.test.d.ts +0 -2
  852. package/dist/trace/store-to-otlp.test.d.ts.map +0 -1
  853. package/dist/trace/store.d.ts.map +0 -1
  854. package/dist/trace/store.test.d.ts +0 -2
  855. package/dist/trace/store.test.d.ts.map +0 -1
  856. package/dist/trace-analyst/analyst.d.ts.map +0 -1
  857. package/dist/trace-analyst/analyst.test.d.ts +0 -2
  858. package/dist/trace-analyst/analyst.test.d.ts.map +0 -1
  859. package/dist/trace-analyst/behavioral-metrics.d.ts +0 -40
  860. package/dist/trace-analyst/behavioral-metrics.d.ts.map +0 -1
  861. package/dist/trace-analyst/behavioral-metrics.test.d.ts +0 -2
  862. package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +0 -1
  863. package/dist/trace-analyst/hook.d.ts +0 -55
  864. package/dist/trace-analyst/hook.d.ts.map +0 -1
  865. package/dist/trace-analyst/index.d.ts +0 -18
  866. package/dist/trace-analyst/index.d.ts.map +0 -1
  867. package/dist/trace-analyst/insights.d.ts +0 -71
  868. package/dist/trace-analyst/insights.d.ts.map +0 -1
  869. package/dist/trace-analyst/insights.test.d.ts +0 -2
  870. package/dist/trace-analyst/insights.test.d.ts.map +0 -1
  871. package/dist/trace-analyst/otlp-flatten.d.ts +0 -42
  872. package/dist/trace-analyst/otlp-flatten.d.ts.map +0 -1
  873. package/dist/trace-analyst/otlp-span.d.ts +0 -85
  874. package/dist/trace-analyst/otlp-span.d.ts.map +0 -1
  875. package/dist/trace-analyst/otlp-span.test.d.ts +0 -2
  876. package/dist/trace-analyst/otlp-span.test.d.ts.map +0 -1
  877. package/dist/trace-analyst/otlp-to-run-records.d.ts +0 -115
  878. package/dist/trace-analyst/otlp-to-run-records.d.ts.map +0 -1
  879. package/dist/trace-analyst/otlp-to-run-records.test.d.ts +0 -2
  880. package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +0 -1
  881. package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +0 -2
  882. package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +0 -1
  883. package/dist/trace-analyst/prompts.d.ts +0 -6
  884. package/dist/trace-analyst/prompts.d.ts.map +0 -1
  885. package/dist/trace-analyst/store-otlp.d.ts +0 -126
  886. package/dist/trace-analyst/store-otlp.d.ts.map +0 -1
  887. package/dist/trace-analyst/store-otlp.test.d.ts +0 -8
  888. package/dist/trace-analyst/store-otlp.test.d.ts.map +0 -1
  889. package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +0 -2
  890. package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +0 -1
  891. package/dist/trace-analyst/store.d.ts +0 -63
  892. package/dist/trace-analyst/store.d.ts.map +0 -1
  893. package/dist/trace-analyst/tools.d.ts +0 -44
  894. package/dist/trace-analyst/tools.d.ts.map +0 -1
  895. package/dist/trace-analyst/tools.test.d.ts +0 -10
  896. package/dist/trace-analyst/tools.test.d.ts.map +0 -1
  897. package/dist/trace-analyst/types.d.ts.map +0 -1
  898. package/dist/trace-contracts.d.ts +0 -180
  899. package/dist/trace-contracts.d.ts.map +0 -1
  900. package/dist/traced-analyst.d.ts +0 -26
  901. package/dist/traced-analyst.d.ts.map +0 -1
  902. package/dist/traced-judges.d.ts +0 -27
  903. package/dist/traced-judges.d.ts.map +0 -1
  904. package/dist/traces.d.ts.map +0 -1
  905. package/dist/trajectory.d.ts.map +0 -1
  906. package/dist/types.d.ts.map +0 -1
  907. package/dist/ui-finding.d.ts +0 -104
  908. package/dist/ui-finding.d.ts.map +0 -1
  909. package/dist/verdict-cache.d.ts +0 -78
  910. package/dist/verdict-cache.d.ts.map +0 -1
  911. package/dist/verdict-cache.test.d.ts +0 -2
  912. package/dist/verdict-cache.test.d.ts.map +0 -1
  913. package/dist/verdict.d.ts.map +0 -1
  914. package/dist/visual-diff.d.ts +0 -32
  915. package/dist/visual-diff.d.ts.map +0 -1
  916. package/dist/wire/handlers.d.ts +0 -54
  917. package/dist/wire/handlers.d.ts.map +0 -1
  918. package/dist/wire/index.d.ts.map +0 -1
  919. package/dist/wire/openapi.d.ts +0 -3
  920. package/dist/wire/openapi.d.ts.map +0 -1
  921. package/dist/wire/rpc.d.ts +0 -21
  922. package/dist/wire/rpc.d.ts.map +0 -1
  923. package/dist/wire/rubrics.d.ts +0 -34
  924. package/dist/wire/rubrics.d.ts.map +0 -1
  925. package/dist/wire/schemas.d.ts +0 -410
  926. package/dist/wire/schemas.d.ts.map +0 -1
  927. package/dist/wire/server.d.ts +0 -60
  928. package/dist/wire/server.d.ts.map +0 -1
  929. package/dist/workflow/event-schema.d.ts +0 -5
  930. package/dist/workflow/event-schema.d.ts.map +0 -1
  931. package/dist/workflow/feedback-pack.d.ts +0 -99
  932. package/dist/workflow/feedback-pack.d.ts.map +0 -1
  933. package/dist/workflow/index.d.ts.map +0 -1
  934. package/dist/workflow/intelligence-export.d.ts +0 -62
  935. package/dist/workflow/intelligence-export.d.ts.map +0 -1
  936. package/dist/workflow/partner-report.d.ts +0 -49
  937. package/dist/workflow/partner-report.d.ts.map +0 -1
  938. package/dist/workflow/phase-graph.d.ts +0 -43
  939. package/dist/workflow/phase-graph.d.ts.map +0 -1
  940. package/dist/workflow/promotion-gate.d.ts +0 -61
  941. package/dist/workflow/promotion-gate.d.ts.map +0 -1
  942. package/dist/workflow/run-record.d.ts +0 -12
  943. package/dist/workflow/run-record.d.ts.map +0 -1
  944. package/dist/workflow/runtime-adapter.d.ts +0 -20
  945. package/dist/workflow/runtime-adapter.d.ts.map +0 -1
  946. package/dist/workflow/sanitize.d.ts +0 -21
  947. package/dist/workflow/sanitize.d.ts.map +0 -1
  948. package/dist/workflow/schema.d.ts +0 -5
  949. package/dist/workflow/schema.d.ts.map +0 -1
  950. package/dist/workflow/summary.d.ts +0 -43
  951. package/dist/workflow/summary.d.ts.map +0 -1
  952. package/dist/workflow/trace-event-fields.d.ts +0 -6
  953. package/dist/workflow/trace-event-fields.d.ts.map +0 -1
  954. package/dist/workflow/trajectory.d.ts +0 -15
  955. package/dist/workflow/trajectory.d.ts.map +0 -1
  956. package/dist/workflow/types.d.ts +0 -68
  957. package/dist/workflow/types.d.ts.map +0 -1
  958. package/dist/workspace-inspector.d.ts +0 -67
  959. package/dist/workspace-inspector.d.ts.map +0 -1
  960. package/dist/wrangler-deploy-runner.test.d.ts +0 -2
  961. package/dist/wrangler-deploy-runner.test.d.ts.map +0 -1
@@ -1,34 +0,0 @@
1
- /**
2
- * Routing benchmark — synthetic, dependency-free, ships in the
3
- * package. 16 cross-category items in `dataset.ts`. See
4
- * `routing/README.md` for the format.
5
- *
6
- * `evaluate` does case-insensitive exact match against the canonical
7
- * route plus declared synonyms. The first valid route token in the
8
- * response wins; everything else is ignored. Wrong answers also
9
- * report whether they hit a hard negative — useful when triaging
10
- * "always picks the popular route" failure modes.
11
- */
12
- import type { RunSplitTag } from '../../run-record';
13
- import type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from '../types';
14
- import { ROUTING_DATASET, type RoutingItem } from './dataset';
15
- export type { RoutingItem };
16
- export type RoutingPayload = RoutingItem;
17
- export type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
18
- declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
19
- loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
20
- evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
21
- assignSplit(itemId: string): RunSplitTag;
22
- }
23
- /**
24
- * Pull route-shaped tokens out of a model response. Routes look like
25
- * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
26
- * are not routes, but `category.action` patterns are robust to most
27
- * model wrappers (JSON output, prose explanations, code fences).
28
- */
29
- export declare function extractRouteTokens(response: string): string[];
30
- export declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
31
- export declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
32
- export declare const assignSplit: (itemId: string) => RunSplitTag;
33
- export { ROUTING_DATASET, RoutingAdapter };
34
- //# sourceMappingURL=index.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/routing/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,UAAU,CAAA;AAE3F,OAAO,EAAE,eAAe,EAAE,KAAK,WAAW,EAAE,MAAM,WAAW,CAAA;AAE7D,YAAY,EAAE,WAAW,EAAE,CAAA;AAC3B,MAAM,MAAM,cAAc,GAAG,WAAW,CAAA;AACxC,MAAM,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,cAAc,CAAC,CAAA;AAErE,cAAM,cAAe,YAAW,gBAAgB,CAAC,kBAAkB,EAAE,cAAc,CAAC;IAC5E,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC;IAM9D,QAAQ,CAAC,IAAI,EAAE,kBAAkB,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAqBxF,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW;CAGzC;AAMD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CAG7D;AAID,eAAO,MAAM,WAAW,UAjDG,WAAW,KAAG,OAAO,CAAC,kBAAkB,EAAE,CAiDT,CAAA;AAC5D,eAAO,MAAM,QAAQ,SA5CE,kBAAkB,YAAY,MAAM,KAAG,OAAO,CAAC,mBAAmB,CA4CnC,CAAA;AACtD,eAAO,MAAM,WAAW,WAxBF,MAAM,KAAG,WAwB6B,CAAA;AAC5D,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,CAAA"}
@@ -1,49 +0,0 @@
1
- /**
2
- * Shared types for the reference benchmark wrappers under
3
- * `src/benchmarks/`. Each wrapper exports the three functions in
4
- * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.
5
- */
6
- import type { RunSplitTag } from '../run-record';
7
- export interface BenchmarkDatasetItem<TPayload = unknown> {
8
- /** Stable dataset-local item id (used for split assignment + paper
9
- * references). Unique within a benchmark. */
10
- id: string;
11
- /** Free-form payload. Each benchmark defines its own shape. */
12
- payload: TPayload;
13
- }
14
- export interface BenchmarkEvaluation {
15
- /** [0, 1] score for the response on this item. Exact-match
16
- * benchmarks use 0/1; partial-credit benchmarks may return
17
- * fractional values. */
18
- score: number;
19
- /** Optional bag of raw scoring signals — e.g. parsed numeric
20
- * answer, regex match, judge sub-scores. */
21
- raw: Record<string, unknown>;
22
- }
23
- /** Common signature implemented by every adapter under `src/benchmarks/*`. */
24
- export interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown> {
25
- /** Load the dataset for the given split. May hit the network on
26
- * first call but should be cache-friendly. Adapters that don't
27
- * ship the dataset itself MUST throw a clearly-marked error
28
- * pointing the caller at the loader script. */
29
- loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
30
- /** Score a single response. Pure with respect to the inputs. */
31
- evaluate(item: BenchmarkDatasetItem<TPayload>, response: string): Promise<BenchmarkEvaluation>;
32
- /** Deterministic split assignment via item id hashing. The
33
- * fraction of items in each split is implementation-defined but
34
- * MUST be stable across processes and platforms. */
35
- assignSplit(itemId: string): RunSplitTag;
36
- }
37
- /** Split-assignment seed shared across all benchmarks. Bumping this
38
- * value reshuffles every split — do NOT do that lightly. */
39
- export declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
40
- /**
41
- * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
42
- * stable 32-bit hash of `${seed}::${id}`. Default proportions:
43
- *
44
- * search: 60% (optimization-readable)
45
- * dev: 20% (held-out for tuning, leak-on-purpose during dev)
46
- * holdout:20% (paper-grade held-out, gated reads)
47
- */
48
- export declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
49
- //# sourceMappingURL=types.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/benchmarks/types.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AAEhD,MAAM,WAAW,oBAAoB,CAAC,QAAQ,GAAG,OAAO;IACtD;kDAC8C;IAC9C,EAAE,EAAE,MAAM,CAAA;IACV,+DAA+D;IAC/D,OAAO,EAAE,QAAQ,CAAA;CAClB;AAED,MAAM,WAAW,mBAAmB;IAClC;;6BAEyB;IACzB,KAAK,EAAE,MAAM,CAAA;IACb;iDAC6C;IAC7C,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC7B;AAED,8EAA8E;AAK9E,MAAM,WAAW,gBAAgB,CAAC,MAAM,GAAG,OAAO,EAAE,QAAQ,GAAG,OAAO;IACpE;;;oDAGgD;IAChD,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,oBAAoB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;IAC1E,gEAAgE;IAChE,QAAQ,CAAC,IAAI,EAAE,oBAAoB,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAAA;IAC9F;;yDAEqD;IACrD,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAA;CACzC;AAkBD;6DAC6D;AAC7D,eAAO,MAAM,oBAAoB,kBAAkB,CAAA;AAEnD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAChC,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,MAA6B,GAClC,WAAW,CAMb"}
@@ -1,81 +0,0 @@
1
- /**
2
- * Bisector — auto-locate the change that introduced an eval regression.
3
- *
4
- * Two shapes:
5
- * - `commitBisect` — walk an ordered SHA list, binary-search for the
6
- * first commit that fails.
7
- * - `promptBisect` — given a good and bad prompt, progressively port
8
- * paragraphs from good→bad to localize the breaking change.
9
- *
10
- * Generic `bisect<T>` lets callers drive any ordered state space
11
- * (dataset versions, config files, CLI flag combinations).
12
- */
13
- export interface BisectOptions<T> {
14
- /** State known to pass. */
15
- good: T;
16
- /** State known to fail. */
17
- bad: T;
18
- /** Equality test on state values — default Object.is. */
19
- equals?: (a: T, b: T) => boolean;
20
- /** Pick the halfway state between good + bad. Return null when no further
21
- * split is possible (e.g. adjacent commits). */
22
- halfway: (good: T, bad: T) => T | null;
23
- /** Produce a verdict for a state. */
24
- runEval: (state: T) => Promise<{
25
- score: number;
26
- pass: boolean;
27
- }>;
28
- /** Hard cap on iterations (default 40 — covers ~1T ordered states). */
29
- maxIterations?: number;
30
- }
31
- export interface BisectStep<T> {
32
- state: T;
33
- score: number;
34
- pass: boolean;
35
- }
36
- export interface BisectResult<T> {
37
- /** The first bad state — typically `bad` in the final (good, bad) adjacent pair. */
38
- culprit: T;
39
- /** Ordered trace of all states evaluated. */
40
- path: BisectStep<T>[];
41
- /** True when we narrowed to an adjacent (good, bad) pair. */
42
- converged: boolean;
43
- /** True when `good` itself failed or `bad` itself passed — the caller's
44
- * premise was broken. */
45
- inputInconsistent: boolean;
46
- }
47
- export declare function bisect<T>(options: BisectOptions<T>): Promise<BisectResult<T>>;
48
- /**
49
- * Commit bisect — `commits` is an ordered SHA list, oldest to newest.
50
- * `good` and `bad` must both be present in the list.
51
- */
52
- export declare function commitBisect(options: {
53
- commits: string[];
54
- good: string;
55
- bad: string;
56
- runEval: (sha: string) => Promise<{
57
- score: number;
58
- pass: boolean;
59
- }>;
60
- maxIterations?: number;
61
- }): Promise<BisectResult<string>>;
62
- /**
63
- * Prompt bisect — splits the good and bad prompts into paragraphs, then
64
- * progressively replaces paragraphs in `good` with their counterparts
65
- * from `bad` to localize the offending change. Only works when the two
66
- * prompts have the same paragraph count (a common editorial workflow
67
- * constraint — one paragraph = one change unit).
68
- */
69
- export declare function promptBisect(options: {
70
- good: string;
71
- bad: string;
72
- runEval: (prompt: string) => Promise<{
73
- score: number;
74
- pass: boolean;
75
- }>;
76
- maxIterations?: number;
77
- paragraphSplitter?: (prompt: string) => string[];
78
- }): Promise<BisectResult<string> & {
79
- offendingParagraphIndex?: number;
80
- }>;
81
- //# sourceMappingURL=bisector.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"bisector.d.ts","sourceRoot":"","sources":["../src/bisector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,2BAA2B;IAC3B,IAAI,EAAE,CAAC,CAAA;IACP,2BAA2B;IAC3B,GAAG,EAAE,CAAC,CAAA;IACN,yDAAyD;IACzD,MAAM,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,OAAO,CAAA;IAChC;qDACiD;IACjD,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,CAAA;IACtC,qCAAqC;IACrC,OAAO,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IAChE,uEAAuE;IACvE,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,UAAU,CAAC,CAAC;IAC3B,KAAK,EAAE,CAAC,CAAA;IACR,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,OAAO,CAAA;CACd;AAED,MAAM,WAAW,YAAY,CAAC,CAAC;IAC7B,oFAAoF;IACpF,OAAO,EAAE,CAAC,CAAA;IACV,6CAA6C;IAC7C,IAAI,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,CAAA;IACrB,6DAA6D;IAC7D,SAAS,EAAE,OAAO,CAAA;IAClB;8BAC0B;IAC1B,iBAAiB,EAAE,OAAO,CAAA;CAC3B;AAED,wBAAsB,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CA8BnF;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE;IAC1C,OAAO,EAAE,MAAM,EAAE,CAAA;IACjB,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACnE,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAwBhC;AAED;;;;;;GAMG;AACH,wBAAsB,YAAY,CAAC,OAAO,EAAE;IAC1C,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;IACtE,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,MAAM,EAAE,CAAA;CACjD,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG;IAAE,uBAAuB,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CA6EvE"}
@@ -1,31 +0,0 @@
1
- /**
2
- * BudgetGuard — enforces token / wall-clock / call / $ caps, records
3
- * a ledger entry on every decrement, emits `budget_breach` + throws
4
- * `BudgetBreachError` when a cap is hit.
5
- *
6
- * Wraps a TraceEmitter. The emitter persists ledger entries + breach
7
- * events so the classifier, pipelines, and reports can all read
8
- * budget state from the trace corpus — no separate accounting.
9
- */
10
- import { AgentEvalError } from './errors';
11
- import type { TraceEmitter } from './trace/emitter';
12
- import type { BudgetSpec } from './trace/schema';
13
- export declare class BudgetBreachError extends AgentEvalError {
14
- dimension: keyof BudgetSpec;
15
- limit: number;
16
- attempted: number;
17
- constructor(dimension: keyof BudgetSpec, limit: number, attempted: number);
18
- }
19
- export declare class BudgetGuard {
20
- private consumed;
21
- private emitter;
22
- private budget;
23
- private startedAt;
24
- constructor(emitter: TraceEmitter, budget: BudgetSpec, now?: () => number);
25
- /** Record consumption. Throws `BudgetBreachError` if any dimension exceeds its cap. */
26
- charge(delta: Partial<Record<keyof BudgetSpec, number>>, spanId?: string): Promise<void>;
27
- /** Convenience: advance wall-clock budget based on elapsed wall time. */
28
- tickWall(nowMs: number, spanId?: string): Promise<void>;
29
- get state(): Record<keyof BudgetSpec, number>;
30
- }
31
- //# sourceMappingURL=budget-guard.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"budget-guard.d.ts","sourceRoot":"","sources":["../src/budget-guard.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AACzC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAA;AACnD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,qBAAa,iBAAkB,SAAQ,cAAc;IAE1C,SAAS,EAAE,MAAM,UAAU;IAC3B,KAAK,EAAE,MAAM;IACb,SAAS,EAAE,MAAM;gBAFjB,SAAS,EAAE,MAAM,UAAU,EAC3B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM;CAI3B;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAA+E;IAC/F,OAAO,CAAC,OAAO,CAAc;IAC7B,OAAO,CAAC,MAAM,CAAY;IAC1B,OAAO,CAAC,SAAS,CAAQ;gBAEb,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,GAAE,MAAM,MAAyB;IAM3F,uFAAuF;IACjF,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,UAAU,EAAE,MAAM,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA0B9F,yEAAyE;IACnE,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAO7D,IAAI,KAAK,IAAI,MAAM,CAAC,MAAM,UAAU,EAAE,MAAM,CAAC,CAE5C;CACF"}
@@ -1,111 +0,0 @@
1
- /**
2
- * BuilderSession — ties a builder-of-builders workflow together.
3
- *
4
- * Models agent-builder's shape: Project → Chat → Edit → Ship → App →
5
- * AppAgent. Each layer is a Run (linked via parentRunId). The
6
- * framework-enforced invariants:
7
- *
8
- * - One Project → many Chats; chatId scopes runs within a project.
9
- * - One Chat = one builder Run with `layer='builder'`.
10
- * - One Ship = one child Run with `layer='app-build'` + SandboxHarness.
11
- * - One AppScenario = one grandchild Run with `layer='app-runtime'`.
12
- *
13
- * Consumers obtain a BuilderSession, call `startChat`, drive the
14
- * builder agent (emitting spans), and call `ship` / `runAppScenario`
15
- * as the workflow progresses. The session reconstructs itself from
16
- * trace data via `resume(store, projectId)`.
17
- */
18
- import type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from '../sandbox-harness';
19
- import type { TestGradedRunResult, TestGradedScenario } from '../test-graded-scenario';
20
- import { TraceEmitter } from '../trace/emitter';
21
- import type { Run } from '../trace/schema';
22
- import type { TraceStore } from '../trace/store';
23
- export interface BuilderSessionInit {
24
- projectId: string;
25
- chatId?: string;
26
- /** Free-form: user's task description, project name, etc. Stored on the builder Run. */
27
- tags?: Record<string, string>;
28
- }
29
- export interface ShipOptions {
30
- harness: HarnessConfig;
31
- driver?: SandboxDriver;
32
- /** scenarioId of this app-build run. Defaults to `${projectId}/build`. */
33
- scenarioId?: string;
34
- }
35
- export interface RunAppScenarioOptions {
36
- scenario: TestGradedScenario;
37
- /** Harness driver override; defaults to the one the session was created with. */
38
- driver?: SandboxDriver;
39
- }
40
- export declare class BuilderSession {
41
- private store;
42
- private builderEmitter;
43
- readonly projectId: string;
44
- readonly chatId: string;
45
- private builderRunId?;
46
- private lastBuildRunId?;
47
- private defaultDriver?;
48
- constructor(store: TraceStore, init: BuilderSessionInit, driver?: SandboxDriver);
49
- /** Start the builder (L0) run for this chat. Returns the runId. */
50
- startChat(scenarioId?: string): Promise<string>;
51
- /** The emitter for builder-level spans (edits, LLM calls, tool invocations). */
52
- get emitter(): TraceEmitter;
53
- /**
54
- * Ship the project's generated app: run the sandbox harness as a child
55
- * Run (`layer='app-build'`). Returns the build result + runId.
56
- */
57
- ship(options: ShipOptions): Promise<{
58
- runId: string;
59
- result: SandboxHarnessResult;
60
- }>;
61
- /**
62
- * Run a domain scenario against the just-built app as a grandchild Run
63
- * (`layer='app-runtime'`). The `ship` call must precede this so the
64
- * parent is set correctly; if no build exists yet the session attaches
65
- * directly to the builder run (useful for prototypes).
66
- */
67
- runAppScenario(options: RunAppScenarioOptions): Promise<TestGradedRunResult>;
68
- /** Record an end-of-chat meta score (judge verdict on whether the builder
69
- * served the user's intent). Accepts a numeric score + optional rationale. */
70
- recordMetaScore(score: number, rationale?: string): Promise<void>;
71
- /** Close the builder Run with a final outcome. */
72
- endChat(outcome: {
73
- pass: boolean;
74
- score?: number;
75
- notes?: string;
76
- }): Promise<void>;
77
- /**
78
- * Inline app-runtime run — for cases where the "scenario" isn't a
79
- * SWE-bench-style test suite but a live agent interaction (LLM chat,
80
- * domain flow). Returns an emitter bound to a fresh Run in the
81
- * `app-runtime` layer; caller emits spans inside and calls
82
- * `.endRun()` with the final verdict.
83
- */
84
- startAppRuntime(scenarioId: string): Promise<TraceEmitter>;
85
- /**
86
- * Lightweight "ship marker" — record an app-build Run with a caller-
87
- * provided verdict. Use when there isn't a sandbox harness to run but
88
- * you still want to mark the build state at publish time.
89
- */
90
- recordShipMarker(args: {
91
- pass: boolean;
92
- score: number;
93
- scenarioId?: string;
94
- notes?: string;
95
- }): Promise<string>;
96
- get lastBuildRunIdValue(): string | undefined;
97
- get builderRunIdValue(): string | undefined;
98
- }
99
- /**
100
- * Reconstruct the most recent BuilderSession state for a given project —
101
- * returns { builderRunId, lastBuildRunId, chatRuns }. For chat-first UIs
102
- * this is how a resumed session finds its place in the edit history.
103
- */
104
- export declare function resumeBuilderSession(store: TraceStore, projectId: string): Promise<{
105
- projectId: string;
106
- chatRuns: Run[];
107
- lastBuilderRun?: Run;
108
- lastBuildRun?: Run;
109
- lastAppRuntimeRuns: Run[];
110
- }>;
111
- //# sourceMappingURL=builder-session.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"builder-session.d.ts","sourceRoot":"","sources":["../../src/builder-eval/builder-session.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAE5F,OAAO,KAAK,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAA;AAEtF,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAC/C,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,MAAM,WAAW,kBAAkB;IACjC,SAAS,EAAE,MAAM,CAAA;IACjB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,wFAAwF;IACxF,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC9B;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,aAAa,CAAA;IACtB,MAAM,CAAC,EAAE,aAAa,CAAA;IACtB,0EAA0E;IAC1E,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,iFAAiF;IACjF,MAAM,CAAC,EAAE,aAAa,CAAA;CACvB;AAED,qBAAa,cAAc;IACzB,OAAO,CAAC,KAAK,CAAY;IACzB,OAAO,CAAC,cAAc,CAAc;IACpC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,OAAO,CAAC,YAAY,CAAC,CAAQ;IAC7B,OAAO,CAAC,cAAc,CAAC,CAAQ;IAC/B,OAAO,CAAC,aAAa,CAAC,CAAe;gBAEzB,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,CAAC,EAAE,aAAa;IAQ/E,mEAAmE;IAC7D,SAAS,CAAC,UAAU,SAA2B,GAAG,OAAO,CAAC,MAAM,CAAC;IAWvE,gFAAgF;IAChF,IAAI,OAAO,IAAI,YAAY,CAG1B;IAED;;;OAGG;IACG,IAAI,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,oBAAoB,CAAA;KAAE,CAAC;IAqB1F;;;;;OAKG;IACG,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,mBAAmB,CAAC;IAmBlF;mFAC+E;IACzE,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAavE,kDAAkD;IAC5C,OAAO,CAAC,OAAO,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAQxF;;;;;;OAMG;IACG,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAiBhE;;;;OAIG;IACG,gBAAgB,CAAC,IAAI,EAAE;QAC3B,IAAI,EAAE,OAAO,CAAA;QACb,KAAK,EAAE,MAAM,CAAA;QACb,UAAU,CAAC,EAAE,MAAM,CAAA;QACnB,KAAK,CAAC,EAAE,MAAM,CAAA;KACf,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBnB,IAAI,mBAAmB,IAAI,MAAM,GAAG,SAAS,CAE5C;IACD,IAAI,iBAAiB,IAAI,MAAM,GAAG,SAAS,CAE1C;CACF;AAED;;;;GAIG;AACH,wBAAsB,oBAAoB,CACxC,KAAK,EAAE,UAAU,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC;IACT,SAAS,EAAE,MAAM,CAAA;IACjB,QAAQ,EAAE,GAAG,EAAE,CAAA;IACf,cAAc,CAAC,EAAE,GAAG,CAAA;IACpB,YAAY,CAAC,EAAE,GAAG,CAAA;IAClB,kBAAkB,EAAE,GAAG,EAAE,CAAA;CAC1B,CAAC,CAkBD"}
@@ -1,32 +0,0 @@
1
- /**
2
- * Meta-eval correlation — the highest-leverage signal in the framework.
3
- *
4
- * Given a corpus of three-layer project reports, compute how well each
5
- * pair of layers correlates. The question we care about most:
6
- *
7
- * Does `metaScore` (what the builder thinks it did) predict
8
- * `runtimeScore` (what the user actually gets)?
9
- *
10
- * If r < ~0.4, the builder's self-scoring is broken — it's optimizing
11
- * for something other than real-world success. If r > 0.7, meta_score
12
- * is a usable proxy and can drive CI gates cheaply.
13
- *
14
- * Non-parametric rank correlation (Spearman) is also reported because
15
- * meta scores are often ordinal-ish.
16
- */
17
- import type { ThreeLayerProjectReport } from './three-layer-eval';
18
- export interface LayerCorrelation {
19
- n: number;
20
- pearson: number;
21
- spearman: number;
22
- }
23
- export interface CorrelationReport {
24
- /** Pairs present in the corpus (layers with ≥ 2 matched data points). */
25
- metaVsBuild?: LayerCorrelation;
26
- metaVsRuntime?: LayerCorrelation;
27
- buildVsRuntime?: LayerCorrelation;
28
- /** Number of complete projects (all 3 scores present). */
29
- completeProjects: number;
30
- }
31
- export declare function correlateLayers(reports: ThreeLayerProjectReport[]): CorrelationReport;
32
- //# sourceMappingURL=correlation.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"correlation.d.ts","sourceRoot":"","sources":["../../src/builder-eval/correlation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAA;AAEjE,MAAM,WAAW,gBAAgB;IAC/B,CAAC,EAAE,MAAM,CAAA;IACT,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,iBAAiB;IAChC,yEAAyE;IACzE,WAAW,CAAC,EAAE,gBAAgB,CAAA;IAC9B,aAAa,CAAC,EAAE,gBAAgB,CAAA;IAChC,cAAc,CAAC,EAAE,gBAAgB,CAAA;IACjC,0DAA0D;IAC1D,gBAAgB,EAAE,MAAM,CAAA;CACzB;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,uBAAuB,EAAE,GAAG,iBAAiB,CAoBrF"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/builder-eval/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA;AACjC,cAAc,eAAe,CAAA;AAC7B,cAAc,oBAAoB,CAAA;AAClC,cAAc,oBAAoB,CAAA"}
@@ -1,51 +0,0 @@
1
- /**
2
- * ProjectRegistry — project-level aggregation over the trace corpus.
3
- *
4
- * Thin reader over TraceStore that answers the questions a chat-first,
5
- * resumable UI needs:
6
- * - listProjects() → project IDs with latest activity
7
- * - projectTimeline(id) → chats + builds + runtime runs, chronological
8
- * - projectChats(id) → chat-level summaries (turn count, outcome)
9
- *
10
- * All queries are pure reads; no state duplication.
11
- */
12
- import type { Run } from '../trace/schema';
13
- import type { TraceStore } from '../trace/store';
14
- export interface ProjectSummary {
15
- projectId: string;
16
- chatCount: number;
17
- buildCount: number;
18
- appRuntimeCount: number;
19
- lastActivityAt: number;
20
- latestChatId?: string;
21
- latestOutcome?: {
22
- pass: boolean;
23
- score?: number;
24
- };
25
- }
26
- export interface ChatSummary {
27
- chatId: string;
28
- projectId: string;
29
- builderRunId: string;
30
- startedAt: number;
31
- endedAt?: number;
32
- status: Run['status'];
33
- outcome?: Run['outcome'];
34
- /** Counts of spans emitted during the chat. */
35
- llmTurns?: number;
36
- toolCalls?: number;
37
- buildRunId?: string;
38
- appRuntimeRunIds: string[];
39
- }
40
- export interface ProjectTimelineEntry {
41
- run: Run;
42
- layerBucket: 'chat' | 'build' | 'runtime' | 'other';
43
- }
44
- export declare class ProjectRegistry {
45
- private store;
46
- constructor(store: TraceStore);
47
- listProjects(): Promise<ProjectSummary[]>;
48
- projectTimeline(projectId: string): Promise<ProjectTimelineEntry[]>;
49
- projectChats(projectId: string): Promise<ChatSummary[]>;
50
- }
51
- //# sourceMappingURL=project-registry.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"project-registry.d.ts","sourceRoot":"","sources":["../../src/builder-eval/project-registry.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,iBAAiB,CAAA;AAC1C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,eAAe,EAAE,MAAM,CAAA;IACvB,cAAc,EAAE,MAAM,CAAA;IACtB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,aAAa,CAAC,EAAE;QAAE,IAAI,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAA;CAClD;AAED,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAA;IACrB,OAAO,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,CAAA;IACxB,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,gBAAgB,EAAE,MAAM,EAAE,CAAA;CAC3B;AAED,MAAM,WAAW,oBAAoB;IACnC,GAAG,EAAE,GAAG,CAAA;IACR,WAAW,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,GAAG,OAAO,CAAA;CACpD;AAED,qBAAa,eAAe;IACd,OAAO,CAAC,KAAK;gBAAL,KAAK,EAAE,UAAU;IAE/B,YAAY,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IAgCzC,eAAe,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,EAAE,CAAC;IAgBnE,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;CAkC9D"}
@@ -1,55 +0,0 @@
1
- /**
2
- * Three-layer evaluation — the canonical scoring breakdown for
3
- * builder-of-builders workflows.
4
- *
5
- * meta_score: did the builder understand + satisfy user intent?
6
- * (judge verdict attached to the builder run)
7
- * build_score: did the generated scaffold build + pass its own tests?
8
- * (outcome.score on the app-build child run)
9
- * runtime_score: did the generated agent pass its domain scenarios?
10
- * (mean outcome.score over app-runtime grandchild runs)
11
- *
12
- * Returns a structured report per project. The cross-layer correlation
13
- * is the highest-leverage signal the framework computes — if
14
- * meta_score doesn't predict runtime_score, the builder's self-scoring
15
- * is broken.
16
- *
17
- * Scaffold-only mode: when a project has no `app-runtime` runs (e.g. a
18
- * scaffold-builder eval that grades compose + build without driving a
19
- * runtime scenario), `kind` is `'scaffold-only'` and `complete` measures
20
- * meta + build only. Consumers can tell the two apart without having to
21
- * interpret null-runtime as either "not yet computed" or "N/A for this
22
- * project shape".
23
- */
24
- import type { TraceStore } from '../trace/store';
25
- export type ProjectKind = 'full' | 'scaffold-only';
26
- export interface ThreeLayerProjectReport {
27
- projectId: string;
28
- /**
29
- * `'full'` when the project has at least one `app-runtime` run;
30
- * `'scaffold-only'` when it only has meta + build layers. Lets
31
- * downstream consumers treat a null runtime score as expected
32
- * (scaffold-only) vs. missing (full, pipeline broke).
33
- */
34
- kind: ProjectKind;
35
- builderRunId?: string;
36
- /** Judge-verdict score on the builder run (0..1 after normalization). */
37
- metaScore: number | null;
38
- buildRunId?: string;
39
- /** 0..1 from the sandbox harness (testsPassed / testsTotal). */
40
- buildScore: number | null;
41
- appRuntimeRunIds: string[];
42
- /** Mean of outcome.score over app-runtime runs, 0..1. Always null in scaffold-only mode. */
43
- runtimeScore: number | null;
44
- runtimePassRate: number | null;
45
- /**
46
- * Layer-aware completeness:
47
- * - `kind='full'`: all three layers scored
48
- * - `kind='scaffold-only'`: meta + build scored (runtime not applicable)
49
- */
50
- complete: boolean;
51
- }
52
- export declare function scoreProject(store: TraceStore, projectId: string): Promise<ThreeLayerProjectReport>;
53
- /** Aggregate scoring across every project in a corpus. */
54
- export declare function scoreAllProjects(store: TraceStore): Promise<ThreeLayerProjectReport[]>;
55
- //# sourceMappingURL=three-layer-eval.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"three-layer-eval.d.ts","sourceRoot":"","sources":["../../src/builder-eval/three-layer-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAIH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,eAAe,CAAA;AAElD,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAA;IACjB;;;;;OAKG;IACH,IAAI,EAAE,WAAW,CAAA;IACjB,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,yEAAyE;IACzE,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,gEAAgE;IAChE,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,4FAA4F;IAC5F,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B;;;;OAIG;IACH,QAAQ,EAAE,OAAO,CAAA;CAClB;AAED,wBAAsB,YAAY,CAChC,KAAK,EAAE,UAAU,EACjB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,uBAAuB,CAAC,CAoClC;AAED,0DAA0D;AAC1D,wBAAsB,gBAAgB,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAI5F"}
@@ -1,108 +0,0 @@
1
- /**
2
- * Make the trace-analyst's OWN prompt a GEPA-optimizable surface.
3
- *
4
- * The analyst that drives self-improvement is itself a prompt — and a
5
- * hand-tuned one (a hardcoded, hand-versioned `const`). This module lets the
6
- * loop optimize it: the analyst `actorDescription` becomes a `MutableSurface`
7
- * that `gepaProposer` / `haloProposer` / any `SurfaceProposer` can mutate inside
8
- * `runImprovementLoop` or `compareProposers`. That is the second-order loop —
9
- * optimizing the optimizer's eyes, not just the agent's prompt.
10
- *
11
- * Two pieces, both deliberately small (the loop engine already exists — this
12
- * only supplies the analyst-shaped dispatch + an objective scorer):
13
- *
14
- * - `buildAnalystSurfaceDispatch` — `dispatchWithSurface(surface, scenario)`
15
- * runs `analyzeTraces` with `surface` as the actorDescription over the
16
- * scenario's fixed trace corpus, returning its findings.
17
- * - `failureModeRecallJudge` — a DETERMINISTIC judge (no LLM, no opinion)
18
- * that scores those findings against the scenario's GROUND-TRUTH failure
19
- * modes. This is what keeps optimizing the analyst prompt ungameable: the
20
- * labels come from objective signal (e.g. AppWorld `world.evaluate()` tells
21
- * us which task failed and which API calls were wrong), so we reward an
22
- * analyst for surfacing the failures that really happened — not for
23
- * pleasing a judge that could be talked into anything (Goodhart).
24
- *
25
- * Wiring (the loop is unchanged; you only pass these in):
26
- *
27
- * const dispatchWithSurface = buildAnalystSurfaceDispatch({ analystOptions: { ai } })
28
- * await runImprovementLoop({
29
- * baselineSurface: TRACE_ANALYST_ACTOR_DESCRIPTION, // the prompt under optimization
30
- * scenarios: trainScenarios, // labeled trace corpora
31
- * holdoutScenarios: heldOutScenarios,
32
- * dispatchWithSurface,
33
- * judges: [failureModeRecallJudge()],
34
- * proposer: gepaProposer({ baseUrl, apiKey }),
35
- * gate: heldOutGate({ minDelta: 0.02 }),
36
- * autoOnPromote: 'none',
37
- * })
38
- */
39
- import type { AnalyzeTracesInput, AnalyzeTracesOptions, AnalyzeTracesResult } from '../trace-analyst/analyst';
40
- import type { DispatchContext, JudgeConfig, MutableSurface, Scenario } from './types';
41
- /**
42
- * A labeled trace scenario: a FIXED trace corpus plus the failure modes a
43
- * competent analyst MUST surface from it. The labels are ground truth — the
44
- * objective failures that actually occurred — which is what makes optimizing
45
- * the analyst prompt against them meaningful rather than circular.
46
- */
47
- export interface AnalystScenario extends Scenario {
48
- kind: 'analyst-surface';
49
- /** OTLP-JSONL path or an in-memory store of the traces to analyze. */
50
- source: AnalyzeTracesOptions['source'];
51
- /** The domain question handed to the analyst (framing lives here, not in
52
- * the surface under optimization). */
53
- question: string;
54
- /**
55
- * Ground-truth failure modes a good analyst must identify. A finding "hits"
56
- * a mode when it contains ANY of the mode's case-insensitive cues. Derive
57
- * these from objective signal (failed task + which step broke), never from
58
- * the analyst's own prior output.
59
- */
60
- expectedFailureModes: Array<{
61
- id: string;
62
- cues: string[];
63
- }>;
64
- /**
65
- * Cues that mark a finding as HALLUCINATED / out-of-scope for this corpus —
66
- * naming a tool, error, or failure that did not occur. Presence penalizes
67
- * precision. Optional; omit to score recall only.
68
- */
69
- forbiddenCues?: string[];
70
- }
71
- /** The analyst's output for one scenario — the artifact the judge scores. */
72
- export interface AnalystArtifact {
73
- answer: string;
74
- findings: string[];
75
- /** The hardcoded-prompt version the analyst reported (provenance only; the
76
- * optimized surface overrides the actual prompt text used). */
77
- actorPromptVersion: string;
78
- }
79
- export interface BuildAnalystSurfaceDispatchOptions {
80
- /**
81
- * Everything `analyzeTraces` needs EXCEPT `actorDescription` (supplied by the
82
- * surface under optimization) and `source` (supplied by the scenario). `ai`
83
- * (the AxAIService) is required for a live run.
84
- */
85
- analystOptions: Omit<AnalyzeTracesOptions, 'actorDescription' | 'source'>;
86
- /** Test seam: defaults to the real `analyzeTraces`. */
87
- analyze?: (input: AnalyzeTracesInput, options: AnalyzeTracesOptions) => Promise<AnalyzeTracesResult>;
88
- }
89
- /**
90
- * Build the `dispatchWithSurface(surface, scenario, ctx)` the improvement loop
91
- * calls: run the analyst with `surface` as its actorDescription over the
92
- * scenario's trace corpus and return its findings.
93
- */
94
- export declare function buildAnalystSurfaceDispatch(opts: BuildAnalystSurfaceDispatchOptions): (surface: MutableSurface, scenario: AnalystScenario, ctx: DispatchContext) => Promise<AnalystArtifact>;
95
- export interface FailureModeRecallJudgeOptions {
96
- /** Weight on recall when precision is also scored (forbiddenCues present).
97
- * Default 0.5 (equal). Recall-only when no forbiddenCues exist. */
98
- recallWeight?: number;
99
- }
100
- /**
101
- * Deterministic, ground-truth judge for analyst findings. Composite =
102
- * recall of the scenario's `expectedFailureModes` (optionally blended with a
103
- * precision term that penalizes findings tripping `forbiddenCues`). No LLM —
104
- * the score is a function of the labels, so the analyst prompt is optimized
105
- * toward surfacing real failures, not toward a judge it can flatter.
106
- */
107
- export declare function failureModeRecallJudge(opts?: FailureModeRecallJudgeOptions): JudgeConfig<AnalystArtifact, AnalystScenario>;
108
- //# sourceMappingURL=analyst-surface.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"analyst-surface.d.ts","sourceRoot":"","sources":["../../src/campaign/analyst-surface.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,KAAK,EACV,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACpB,MAAM,0BAA0B,CAAA;AAEjC,OAAO,KAAK,EAAE,eAAe,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AAErF;;;;;GAKG;AACH,MAAM,WAAW,eAAgB,SAAQ,QAAQ;IAC/C,IAAI,EAAE,iBAAiB,CAAA;IACvB,sEAAsE;IACtE,MAAM,EAAE,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IACtC;2CACuC;IACvC,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;;OAKG;IACH,oBAAoB,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAA;IAC3D;;;;OAIG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAA;CACzB;AAED,6EAA6E;AAC7E,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB;oEACgE;IAChE,kBAAkB,EAAE,MAAM,CAAA;CAC3B;AAED,MAAM,WAAW,kCAAkC;IACjD;;;;OAIG;IACH,cAAc,EAAE,IAAI,CAAC,oBAAoB,EAAE,kBAAkB,GAAG,QAAQ,CAAC,CAAA;IACzE,uDAAuD;IACvD,OAAO,CAAC,EAAE,CACR,KAAK,EAAE,kBAAkB,EACzB,OAAO,EAAE,oBAAoB,KAC1B,OAAO,CAAC,mBAAmB,CAAC,CAAA;CAClC;AAYD;;;;GAIG;AACH,wBAAgB,2BAA2B,CACzC,IAAI,EAAE,kCAAkC,GACvC,CACD,OAAO,EAAE,cAAc,EACvB,QAAQ,EAAE,eAAe,EACzB,GAAG,EAAE,eAAe,KACjB,OAAO,CAAC,eAAe,CAAC,CAc5B;AAED,MAAM,WAAW,6BAA6B;IAC5C;wEACoE;IACpE,YAAY,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;;;;;GAMG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,GAAE,6BAAkC,GACvC,WAAW,CAAC,eAAe,EAAE,eAAe,CAAC,CA6C/C"}
@@ -1,2 +0,0 @@
1
- export {};
2
- //# sourceMappingURL=analyst-surface.test.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"analyst-surface.test.d.ts","sourceRoot":"","sources":["../../src/campaign/analyst-surface.test.ts"],"names":[],"mappings":""}