@tangle-network/agent-eval 0.93.0 → 0.95.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1087) hide show
  1. package/CHANGELOG.md +15 -0
  2. package/README.md +44 -30
  3. package/dist/action-policy.d.ts +24 -0
  4. package/dist/action-policy.d.ts.map +1 -0
  5. package/dist/action-policy.test.d.ts +2 -0
  6. package/dist/action-policy.test.d.ts.map +1 -0
  7. package/dist/active-learning.d.ts +41 -0
  8. package/dist/active-learning.d.ts.map +1 -0
  9. package/dist/adapters/http.d.ts +15 -21
  10. package/dist/adapters/http.d.ts.map +1 -0
  11. package/dist/adapters/http.js.map +1 -1
  12. package/dist/adapters/langchain.d.ts +7 -13
  13. package/dist/adapters/langchain.d.ts.map +1 -0
  14. package/dist/adapters/otel.d.ts +13 -24
  15. package/dist/adapters/otel.d.ts.map +1 -0
  16. package/dist/agent-profile-cell.d.ts +101 -0
  17. package/dist/agent-profile-cell.d.ts.map +1 -0
  18. package/dist/agent-profile.d.ts +27 -0
  19. package/dist/agent-profile.d.ts.map +1 -0
  20. package/dist/agent-profile.test.d.ts +2 -0
  21. package/dist/agent-profile.test.d.ts.map +1 -0
  22. package/dist/analyst/adapters.d.ts +62 -0
  23. package/dist/analyst/adapters.d.ts.map +1 -0
  24. package/dist/analyst/analyst.test.d.ts +2 -0
  25. package/dist/analyst/analyst.test.d.ts.map +1 -0
  26. package/dist/analyst/ax-service.d.ts +27 -0
  27. package/dist/analyst/ax-service.d.ts.map +1 -0
  28. package/dist/analyst/behavioral-analyst.d.ts +28 -0
  29. package/dist/analyst/behavioral-analyst.d.ts.map +1 -0
  30. package/dist/analyst/chat-client.d.ts +91 -0
  31. package/dist/analyst/chat-client.d.ts.map +1 -0
  32. package/dist/analyst/default-registry.d.ts +27 -0
  33. package/dist/analyst/default-registry.d.ts.map +1 -0
  34. package/dist/analyst/default-registry.test.d.ts +2 -0
  35. package/dist/analyst/default-registry.test.d.ts.map +1 -0
  36. package/dist/analyst/finding-signature.d.ts +48 -0
  37. package/dist/analyst/finding-signature.d.ts.map +1 -0
  38. package/dist/analyst/finding-subject.d.ts +146 -0
  39. package/dist/analyst/finding-subject.d.ts.map +1 -0
  40. package/dist/analyst/finding-subject.test.d.ts +2 -0
  41. package/dist/analyst/finding-subject.test.d.ts.map +1 -0
  42. package/dist/analyst/findings-store.d.ts +75 -0
  43. package/dist/analyst/findings-store.d.ts.map +1 -0
  44. package/dist/analyst/index.d.ts +28 -256
  45. package/dist/analyst/index.d.ts.map +1 -0
  46. package/dist/analyst/index.js +6 -66
  47. package/dist/analyst/index.js.map +1 -1
  48. package/dist/{kind-factory-5b7xXXOr.d.ts → analyst/kind-factory.d.ts} +11 -63
  49. package/dist/analyst/kind-factory.d.ts.map +1 -0
  50. package/dist/analyst/kinds/failure-mode.d.ts +19 -0
  51. package/dist/analyst/kinds/failure-mode.d.ts.map +1 -0
  52. package/dist/analyst/kinds/improvement.d.ts +23 -0
  53. package/dist/analyst/kinds/improvement.d.ts.map +1 -0
  54. package/dist/analyst/kinds/index.d.ts +22 -0
  55. package/dist/analyst/kinds/index.d.ts.map +1 -0
  56. package/dist/analyst/kinds/kinds.test.d.ts +2 -0
  57. package/dist/analyst/kinds/kinds.test.d.ts.map +1 -0
  58. package/dist/analyst/kinds/knowledge-gap.d.ts +28 -0
  59. package/dist/analyst/kinds/knowledge-gap.d.ts.map +1 -0
  60. package/dist/analyst/kinds/knowledge-poisoning.d.ts +22 -0
  61. package/dist/analyst/kinds/knowledge-poisoning.d.ts.map +1 -0
  62. package/dist/analyst/kinds/skill-usage.d.ts +84 -0
  63. package/dist/analyst/kinds/skill-usage.d.ts.map +1 -0
  64. package/dist/analyst/kinds/skill-usage.test.d.ts +2 -0
  65. package/dist/analyst/kinds/skill-usage.test.d.ts.map +1 -0
  66. package/dist/analyst/parse-tolerant.d.ts +26 -0
  67. package/dist/analyst/parse-tolerant.d.ts.map +1 -0
  68. package/dist/analyst/parse-tolerant.test.d.ts +2 -0
  69. package/dist/analyst/parse-tolerant.test.d.ts.map +1 -0
  70. package/dist/analyst/registry.budget.test.d.ts +2 -0
  71. package/dist/analyst/registry.budget.test.d.ts.map +1 -0
  72. package/dist/{default-registry-zoGHUQEH.d.ts → analyst/registry.d.ts} +8 -37
  73. package/dist/analyst/registry.d.ts.map +1 -0
  74. package/dist/analyst/steer-firewall.d.ts +35 -0
  75. package/dist/analyst/steer-firewall.d.ts.map +1 -0
  76. package/dist/analyst/steer-firewall.test.d.ts +2 -0
  77. package/dist/analyst/steer-firewall.test.d.ts.map +1 -0
  78. package/dist/analyst/structure-findings.d.ts +37 -0
  79. package/dist/analyst/structure-findings.d.ts.map +1 -0
  80. package/dist/analyst/structure-findings.test.d.ts +2 -0
  81. package/dist/analyst/structure-findings.test.d.ts.map +1 -0
  82. package/dist/analyst/tool-groups.d.ts +34 -0
  83. package/dist/analyst/tool-groups.d.ts.map +1 -0
  84. package/dist/{types-2VVIL04s.d.ts → analyst/types.d.ts} +19 -112
  85. package/dist/analyst/types.d.ts.map +1 -0
  86. package/dist/anti-slop.d.ts +59 -0
  87. package/dist/anti-slop.d.ts.map +1 -0
  88. package/dist/artifact-validator.d.ts +74 -0
  89. package/dist/artifact-validator.d.ts.map +1 -0
  90. package/dist/attestation.d.ts +63 -0
  91. package/dist/attestation.d.ts.map +1 -0
  92. package/dist/attestation.test.d.ts +2 -0
  93. package/dist/attestation.test.d.ts.map +1 -0
  94. package/dist/authenticity/index.d.ts +15 -16
  95. package/dist/authenticity/index.d.ts.map +1 -0
  96. package/dist/authenticity/index.test.d.ts +2 -0
  97. package/dist/authenticity/index.test.d.ts.map +1 -0
  98. package/dist/auto-pr.d.ts +120 -0
  99. package/dist/auto-pr.d.ts.map +1 -0
  100. package/dist/{baseline-DE36-Np7.d.ts → baseline.d.ts} +8 -44
  101. package/dist/baseline.d.ts.map +1 -0
  102. package/dist/behavior-dsl.d.ts +73 -0
  103. package/dist/behavior-dsl.d.ts.map +1 -0
  104. package/dist/belief-state/calibration.d.ts +10 -0
  105. package/dist/belief-state/calibration.d.ts.map +1 -0
  106. package/dist/belief-state/calibration.test.d.ts +2 -0
  107. package/dist/belief-state/calibration.test.d.ts.map +1 -0
  108. package/dist/belief-state/code-agent-corpus.d.ts +66 -0
  109. package/dist/belief-state/code-agent-corpus.d.ts.map +1 -0
  110. package/dist/belief-state/code-agent-corpus.test.d.ts +2 -0
  111. package/dist/belief-state/code-agent-corpus.test.d.ts.map +1 -0
  112. package/dist/belief-state/code-agent-evidence.d.ts +22 -0
  113. package/dist/belief-state/code-agent-evidence.d.ts.map +1 -0
  114. package/dist/belief-state/code-agent-evidence.test.d.ts +2 -0
  115. package/dist/belief-state/code-agent-evidence.test.d.ts.map +1 -0
  116. package/dist/belief-state/extract.d.ts +7 -0
  117. package/dist/belief-state/extract.d.ts.map +1 -0
  118. package/dist/belief-state/extract.test.d.ts +2 -0
  119. package/dist/belief-state/extract.test.d.ts.map +1 -0
  120. package/dist/belief-state/index.d.ts +14 -604
  121. package/dist/belief-state/index.d.ts.map +1 -0
  122. package/dist/belief-state/index.js +1 -1
  123. package/dist/belief-state/ope.d.ts +17 -0
  124. package/dist/belief-state/ope.d.ts.map +1 -0
  125. package/dist/belief-state/ope.test.d.ts +2 -0
  126. package/dist/belief-state/ope.test.d.ts.map +1 -0
  127. package/dist/belief-state/phase0-measurement.d.ts +55 -0
  128. package/dist/belief-state/phase0-measurement.d.ts.map +1 -0
  129. package/dist/belief-state/report.d.ts +17 -0
  130. package/dist/belief-state/report.d.ts.map +1 -0
  131. package/dist/belief-state/report.test.d.ts +2 -0
  132. package/dist/belief-state/report.test.d.ts.map +1 -0
  133. package/dist/belief-state/research-evidence.d.ts +23 -0
  134. package/dist/belief-state/research-evidence.d.ts.map +1 -0
  135. package/dist/belief-state/research-evidence.test.d.ts +2 -0
  136. package/dist/belief-state/research-evidence.test.d.ts.map +1 -0
  137. package/dist/belief-state/runtime-benchmark-corpus.d.ts +32 -0
  138. package/dist/belief-state/runtime-benchmark-corpus.d.ts.map +1 -0
  139. package/dist/belief-state/runtime-hooks.d.ts +87 -0
  140. package/dist/belief-state/runtime-hooks.d.ts.map +1 -0
  141. package/dist/belief-state/runtime-hooks.test.d.ts +2 -0
  142. package/dist/belief-state/runtime-hooks.test.d.ts.map +1 -0
  143. package/dist/belief-state/selective.d.ts +15 -0
  144. package/dist/belief-state/selective.d.ts.map +1 -0
  145. package/dist/belief-state/selective.test.d.ts +2 -0
  146. package/dist/belief-state/selective.test.d.ts.map +1 -0
  147. package/dist/belief-state/shadow-probe.d.ts +80 -0
  148. package/dist/belief-state/shadow-probe.d.ts.map +1 -0
  149. package/dist/belief-state/shadow-probe.test.d.ts +2 -0
  150. package/dist/belief-state/shadow-probe.test.d.ts.map +1 -0
  151. package/dist/belief-state/types.d.ts +195 -0
  152. package/dist/belief-state/types.d.ts.map +1 -0
  153. package/dist/belief-state/types.test.d.ts +2 -0
  154. package/dist/belief-state/types.test.d.ts.map +1 -0
  155. package/dist/benchmark.d.ts +14 -0
  156. package/dist/benchmark.d.ts.map +1 -0
  157. package/dist/benchmarks/index.d.ts +23 -4
  158. package/dist/benchmarks/index.d.ts.map +1 -0
  159. package/dist/benchmarks/routing/dataset.d.ts +34 -0
  160. package/dist/benchmarks/routing/dataset.d.ts.map +1 -0
  161. package/dist/benchmarks/routing/index.d.ts +34 -0
  162. package/dist/benchmarks/routing/index.d.ts.map +1 -0
  163. package/dist/benchmarks/types.d.ts +49 -0
  164. package/dist/benchmarks/types.d.ts.map +1 -0
  165. package/dist/bisector.d.ts +81 -0
  166. package/dist/bisector.d.ts.map +1 -0
  167. package/dist/budget-guard.d.ts +31 -0
  168. package/dist/budget-guard.d.ts.map +1 -0
  169. package/dist/builder-eval/builder-session.d.ts +111 -0
  170. package/dist/builder-eval/builder-session.d.ts.map +1 -0
  171. package/dist/builder-eval/correlation.d.ts +32 -0
  172. package/dist/builder-eval/correlation.d.ts.map +1 -0
  173. package/dist/builder-eval/index.d.ts +5 -250
  174. package/dist/builder-eval/index.d.ts.map +1 -0
  175. package/dist/builder-eval/index.js +2 -2
  176. package/dist/builder-eval/project-registry.d.ts +51 -0
  177. package/dist/builder-eval/project-registry.d.ts.map +1 -0
  178. package/dist/builder-eval/three-layer-eval.d.ts +55 -0
  179. package/dist/builder-eval/three-layer-eval.d.ts.map +1 -0
  180. package/dist/campaign/analyst-surface.d.ts +108 -0
  181. package/dist/campaign/analyst-surface.d.ts.map +1 -0
  182. package/dist/campaign/analyst-surface.test.d.ts +2 -0
  183. package/dist/campaign/analyst-surface.test.d.ts.map +1 -0
  184. package/dist/campaign/auto-pr.d.ts +46 -0
  185. package/dist/campaign/auto-pr.d.ts.map +1 -0
  186. package/dist/campaign/distillation/agreement-judge.d.ts +69 -0
  187. package/dist/campaign/distillation/agreement-judge.d.ts.map +1 -0
  188. package/dist/campaign/distillation/cli.d.ts +35 -0
  189. package/dist/campaign/distillation/cli.d.ts.map +1 -0
  190. package/dist/campaign/distillation/distillation.test.d.ts +2 -0
  191. package/dist/campaign/distillation/distillation.test.d.ts.map +1 -0
  192. package/dist/campaign/distillation/gold-scenarios.d.ts +54 -0
  193. package/dist/campaign/distillation/gold-scenarios.d.ts.map +1 -0
  194. package/dist/campaign/distillation/run-distillation.d.ts +119 -0
  195. package/dist/campaign/distillation/run-distillation.d.ts.map +1 -0
  196. package/dist/campaign/gates/compose.d.ts +12 -0
  197. package/dist/campaign/gates/compose.d.ts.map +1 -0
  198. package/dist/campaign/gates/default-production-gate.d.ts +58 -0
  199. package/dist/campaign/gates/default-production-gate.d.ts.map +1 -0
  200. package/dist/campaign/gates/heldout-gate.d.ts +12 -0
  201. package/dist/campaign/gates/heldout-gate.d.ts.map +1 -0
  202. package/dist/campaign/gates/promotion-policy.d.ts +125 -0
  203. package/dist/campaign/gates/promotion-policy.d.ts.map +1 -0
  204. package/dist/campaign/gates/promotion-policy.test.d.ts +2 -0
  205. package/dist/campaign/gates/promotion-policy.test.d.ts.map +1 -0
  206. package/dist/campaign/gates/sequential.d.ts +146 -0
  207. package/dist/campaign/gates/sequential.d.ts.map +1 -0
  208. package/dist/campaign/gates/sequential.test.d.ts +2 -0
  209. package/dist/campaign/gates/sequential.test.d.ts.map +1 -0
  210. package/dist/campaign/gates/statistical-heldout.d.ts +99 -0
  211. package/dist/campaign/gates/statistical-heldout.d.ts.map +1 -0
  212. package/dist/campaign/gates/statistical-heldout.test.d.ts +2 -0
  213. package/dist/campaign/gates/statistical-heldout.test.d.ts.map +1 -0
  214. package/dist/campaign/index.d.ts +38 -1341
  215. package/dist/campaign/index.d.ts.map +1 -0
  216. package/dist/campaign/index.js +1865 -1318
  217. package/dist/campaign/index.js.map +1 -1
  218. package/dist/campaign/labeled-store/fs-adapter.d.ts +59 -0
  219. package/dist/campaign/labeled-store/fs-adapter.d.ts.map +1 -0
  220. package/dist/campaign/presets/compare-proposers.d.ts +146 -0
  221. package/dist/campaign/presets/compare-proposers.d.ts.map +1 -0
  222. package/dist/campaign/presets/playback.d.ts +120 -0
  223. package/dist/campaign/presets/playback.d.ts.map +1 -0
  224. package/dist/campaign/presets/playback.test.d.ts +2 -0
  225. package/dist/campaign/presets/playback.test.d.ts.map +1 -0
  226. package/dist/campaign/presets/run-eval.d.ts +14 -0
  227. package/dist/campaign/presets/run-eval.d.ts.map +1 -0
  228. package/dist/campaign/presets/run-improvement-loop.d.ts +63 -0
  229. package/dist/campaign/presets/run-improvement-loop.d.ts.map +1 -0
  230. package/dist/campaign/presets/run-improvement-loop.test.d.ts +2 -0
  231. package/dist/campaign/presets/run-improvement-loop.test.d.ts.map +1 -0
  232. package/dist/campaign/presets/run-optimization.d.ts +92 -0
  233. package/dist/campaign/presets/run-optimization.d.ts.map +1 -0
  234. package/dist/campaign/presets/run-profile-matrix.d.ts +151 -0
  235. package/dist/campaign/presets/run-profile-matrix.d.ts.map +1 -0
  236. package/dist/campaign/presets/run-skill-opt.d.ts +96 -0
  237. package/dist/campaign/presets/run-skill-opt.d.ts.map +1 -0
  238. package/dist/campaign/proposers/_findings-text.d.ts +22 -0
  239. package/dist/campaign/proposers/_findings-text.d.ts.map +1 -0
  240. package/dist/campaign/proposers/ace.d.ts +33 -0
  241. package/dist/campaign/proposers/ace.d.ts.map +1 -0
  242. package/dist/campaign/proposers/ace.test.d.ts +2 -0
  243. package/dist/campaign/proposers/ace.test.d.ts.map +1 -0
  244. package/dist/campaign/proposers/analysis-edit.d.ts +32 -0
  245. package/dist/campaign/proposers/analysis-edit.d.ts.map +1 -0
  246. package/dist/campaign/proposers/evolutionary.d.ts +20 -0
  247. package/dist/campaign/proposers/evolutionary.d.ts.map +1 -0
  248. package/dist/campaign/proposers/fapo.d.ts +120 -0
  249. package/dist/campaign/proposers/fapo.d.ts.map +1 -0
  250. package/dist/campaign/proposers/gepa.d.ts +86 -0
  251. package/dist/campaign/proposers/gepa.d.ts.map +1 -0
  252. package/dist/campaign/proposers/halo.d.ts +44 -0
  253. package/dist/campaign/proposers/halo.d.ts.map +1 -0
  254. package/dist/campaign/proposers/halo.test.d.ts +2 -0
  255. package/dist/campaign/proposers/halo.test.d.ts.map +1 -0
  256. package/dist/campaign/proposers/memory.d.ts +47 -0
  257. package/dist/campaign/proposers/memory.d.ts.map +1 -0
  258. package/dist/campaign/proposers/memory.test.d.ts +2 -0
  259. package/dist/campaign/proposers/memory.test.d.ts.map +1 -0
  260. package/dist/campaign/proposers/skill-opt.d.ts +88 -0
  261. package/dist/campaign/proposers/skill-opt.d.ts.map +1 -0
  262. package/dist/campaign/proposers/trace-analyst.d.ts +48 -0
  263. package/dist/campaign/proposers/trace-analyst.d.ts.map +1 -0
  264. package/dist/campaign/proposers/trace-analyst.test.d.ts +2 -0
  265. package/dist/campaign/proposers/trace-analyst.test.d.ts.map +1 -0
  266. package/dist/campaign/provenance.d.ts +185 -0
  267. package/dist/campaign/provenance.d.ts.map +1 -0
  268. package/dist/campaign/run-campaign.d.ts +90 -0
  269. package/dist/campaign/run-campaign.d.ts.map +1 -0
  270. package/dist/campaign/score-utils.d.ts +26 -0
  271. package/dist/campaign/score-utils.d.ts.map +1 -0
  272. package/dist/campaign/skill-patch.d.ts +62 -0
  273. package/dist/campaign/skill-patch.d.ts.map +1 -0
  274. package/dist/campaign/storage.d.ts +38 -0
  275. package/dist/campaign/storage.d.ts.map +1 -0
  276. package/dist/{types-BU-7W85F.d.ts → campaign/types.d.ts} +98 -99
  277. package/dist/campaign/types.d.ts.map +1 -0
  278. package/dist/campaign/worktree/index.d.ts +53 -0
  279. package/dist/campaign/worktree/index.d.ts.map +1 -0
  280. package/dist/canary.d.ts +101 -0
  281. package/dist/canary.d.ts.map +1 -0
  282. package/dist/causal-attribution.d.ts +45 -0
  283. package/dist/causal-attribution.d.ts.map +1 -0
  284. package/dist/{chunk-TWS7AZEY.js → chunk-2T4EZACH.js} +2 -2
  285. package/dist/chunk-2T4EZACH.js.map +1 -0
  286. package/dist/{chunk-LMZQ2Z4U.js → chunk-56GC6VYO.js} +126 -1
  287. package/dist/chunk-56GC6VYO.js.map +1 -0
  288. package/dist/{chunk-QG2OVF2D.js → chunk-77T4STFI.js} +154 -112
  289. package/dist/chunk-77T4STFI.js.map +1 -0
  290. package/dist/{chunk-ZFIBGEOL.js → chunk-7QTQKIDD.js} +179 -178
  291. package/dist/chunk-7QTQKIDD.js.map +1 -0
  292. package/dist/{chunk-UMMZHCPB.js → chunk-AQ5WQAIV.js} +26 -9
  293. package/dist/chunk-AQ5WQAIV.js.map +1 -0
  294. package/dist/{chunk-CVVHBFGN.js → chunk-CWNP4DV4.js} +53 -5
  295. package/dist/chunk-CWNP4DV4.js.map +1 -0
  296. package/dist/{chunk-QS3RBQPI.js → chunk-D5KBVULY.js} +2 -2
  297. package/dist/chunk-DJWX3GVS.js +81 -0
  298. package/dist/chunk-DJWX3GVS.js.map +1 -0
  299. package/dist/{chunk-S6OZEZQK.js → chunk-HMA63UEO.js} +37 -9
  300. package/dist/{chunk-S6OZEZQK.js.map → chunk-HMA63UEO.js.map} +1 -1
  301. package/dist/{chunk-UHMJT4T7.js → chunk-IZCEK2HR.js} +2 -2
  302. package/dist/{chunk-6SOJM3VR.js → chunk-KKWJD5E6.js} +21 -21
  303. package/dist/chunk-KKWJD5E6.js.map +1 -0
  304. package/dist/{chunk-KWRRMR3J.js → chunk-LO6IOIJ2.js} +10 -10
  305. package/dist/chunk-LO6IOIJ2.js.map +1 -0
  306. package/dist/{chunk-L3JOU6XM.js → chunk-NACM5RS7.js} +3 -3
  307. package/dist/{chunk-E4GH6USR.js → chunk-NZEQVRH5.js} +2 -2
  308. package/dist/chunk-NZEQVRH5.js.map +1 -0
  309. package/dist/{chunk-XY4DDNEG.js → chunk-PSWWQXHF.js} +14 -89
  310. package/dist/chunk-PSWWQXHF.js.map +1 -0
  311. package/dist/{chunk-4FBZZIYD.js → chunk-QTMYW64F.js} +2 -2
  312. package/dist/{chunk-D3V5B42D.js → chunk-S4SYLDFX.js} +7 -4
  313. package/dist/chunk-S4SYLDFX.js.map +1 -0
  314. package/dist/{chunk-47X6LRCE.js → chunk-UFSG7ACU.js} +10 -7
  315. package/dist/chunk-UFSG7ACU.js.map +1 -0
  316. package/dist/{chunk-CY6U5S3X.js → chunk-URWVKRCS.js} +2 -2
  317. package/dist/{chunk-GSH6QNNS.js → chunk-X74V6ESX.js} +102 -79
  318. package/dist/chunk-X74V6ESX.js.map +1 -0
  319. package/dist/{chunk-QAY5UIJO.js → chunk-YBIGNSCZ.js} +178 -59
  320. package/dist/chunk-YBIGNSCZ.js.map +1 -0
  321. package/dist/{chunk-Y47J2LJ3.js → chunk-Z6L6YSU6.js} +5 -5
  322. package/dist/{chunk-T375SUOZ.js → chunk-ZOPLCJSY.js} +86 -31
  323. package/dist/chunk-ZOPLCJSY.js.map +1 -0
  324. package/dist/ci-gate.d.ts +44 -0
  325. package/dist/ci-gate.d.ts.map +1 -0
  326. package/dist/cli.d.ts +2 -0
  327. package/dist/cli.d.ts.map +1 -0
  328. package/dist/cli.js +2 -2
  329. package/dist/client.d.ts +77 -0
  330. package/dist/client.d.ts.map +1 -0
  331. package/dist/client.test.d.ts +2 -0
  332. package/dist/client.test.d.ts.map +1 -0
  333. package/dist/command-runner.d.ts +74 -0
  334. package/dist/command-runner.d.ts.map +1 -0
  335. package/dist/command-runner.test.d.ts +2 -0
  336. package/dist/command-runner.test.d.ts.map +1 -0
  337. package/dist/completion-verifier.d.ts +147 -0
  338. package/dist/completion-verifier.d.ts.map +1 -0
  339. package/dist/completion-verifier.test.d.ts +9 -0
  340. package/dist/completion-verifier.test.d.ts.map +1 -0
  341. package/dist/concurrency.d.ts +23 -0
  342. package/dist/concurrency.d.ts.map +1 -0
  343. package/dist/contamination-guard.d.ts +81 -0
  344. package/dist/contamination-guard.d.ts.map +1 -0
  345. package/dist/{analyze-runs-DwCEkpO_.d.ts → contract/analyze-runs.d.ts} +9 -10
  346. package/dist/contract/analyze-runs.d.ts.map +1 -0
  347. package/dist/contract/define-agent-eval.d.ts +52 -0
  348. package/dist/contract/define-agent-eval.d.ts.map +1 -0
  349. package/dist/contract/diff.d.ts +114 -0
  350. package/dist/contract/diff.d.ts.map +1 -0
  351. package/dist/contract/index.d.ts +106 -640
  352. package/dist/contract/index.d.ts.map +1 -0
  353. package/dist/contract/index.js +131 -21
  354. package/dist/contract/index.js.map +1 -1
  355. package/dist/{insight-report-BBwvOh6x.d.ts → contract/insight-report.d.ts} +16 -19
  356. package/dist/contract/insight-report.d.ts.map +1 -0
  357. package/dist/contract/insight-types-fwd.d.ts +7 -0
  358. package/dist/contract/insight-types-fwd.d.ts.map +1 -0
  359. package/dist/contract/intake/agent-trace.d.ts +97 -0
  360. package/dist/contract/intake/agent-trace.d.ts.map +1 -0
  361. package/dist/{code-agent-session-BO8nCnv3.d.ts → contract/intake/code-agent-session.d.ts} +15 -17
  362. package/dist/contract/intake/code-agent-session.d.ts.map +1 -0
  363. package/dist/contract/intake/feedback-table.d.ts +87 -0
  364. package/dist/contract/intake/feedback-table.d.ts.map +1 -0
  365. package/dist/contract/intake/index.d.ts +22 -0
  366. package/dist/contract/intake/index.d.ts.map +1 -0
  367. package/dist/contract/intake/otel-spans.d.ts +33 -0
  368. package/dist/contract/intake/otel-spans.d.ts.map +1 -0
  369. package/dist/contract/self-improve.d.ts +284 -0
  370. package/dist/contract/self-improve.d.ts.map +1 -0
  371. package/dist/{control-runtime-DuFBYg7A.d.ts → control-runtime.d.ts} +23 -26
  372. package/dist/control-runtime.d.ts.map +1 -0
  373. package/dist/control-runtime.test.d.ts +2 -0
  374. package/dist/control-runtime.test.d.ts.map +1 -0
  375. package/dist/control.d.ts +11 -9
  376. package/dist/control.d.ts.map +1 -0
  377. package/dist/control.js +2 -2
  378. package/dist/convergence.d.ts +29 -0
  379. package/dist/convergence.d.ts.map +1 -0
  380. package/dist/{cost-ledger-DuSqlw5B.d.ts → cost-ledger.d.ts} +10 -11
  381. package/dist/cost-ledger.d.ts.map +1 -0
  382. package/dist/cost-ledger.test.d.ts +2 -0
  383. package/dist/cost-ledger.test.d.ts.map +1 -0
  384. package/dist/cost-report.d.ts +43 -0
  385. package/dist/cost-report.d.ts.map +1 -0
  386. package/dist/cost-report.test.d.ts +2 -0
  387. package/dist/cost-report.test.d.ts.map +1 -0
  388. package/dist/cost-tracker.d.ts +76 -0
  389. package/dist/cost-tracker.d.ts.map +1 -0
  390. package/dist/{counterfactual-Dwibr5IW.d.ts → counterfactual.d.ts} +12 -13
  391. package/dist/counterfactual.d.ts.map +1 -0
  392. package/dist/cross-trace-diff.d.ts +56 -0
  393. package/dist/cross-trace-diff.d.ts.map +1 -0
  394. package/dist/{dataset-BbGkaN2I.d.ts → dataset.d.ts} +11 -14
  395. package/dist/dataset.d.ts.map +1 -0
  396. package/dist/deploy-gate-layer.d.ts +125 -0
  397. package/dist/deploy-gate-layer.d.ts.map +1 -0
  398. package/dist/deploy-gate-layer.test.d.ts +2 -0
  399. package/dist/deploy-gate-layer.test.d.ts.map +1 -0
  400. package/dist/description-length-gate.d.ts +119 -0
  401. package/dist/description-length-gate.d.ts.map +1 -0
  402. package/dist/detectors/edge.test.d.ts +2 -0
  403. package/dist/detectors/edge.test.d.ts.map +1 -0
  404. package/dist/detectors/index.d.ts +81 -0
  405. package/dist/detectors/index.d.ts.map +1 -0
  406. package/dist/detectors/index.test.d.ts +2 -0
  407. package/dist/detectors/index.test.d.ts.map +1 -0
  408. package/dist/diagnose/causal-sweep.d.ts +100 -0
  409. package/dist/diagnose/causal-sweep.d.ts.map +1 -0
  410. package/dist/diagnose/index.d.ts +36 -0
  411. package/dist/diagnose/index.d.ts.map +1 -0
  412. package/dist/diagnose/remediation.d.ts +68 -0
  413. package/dist/diagnose/remediation.d.ts.map +1 -0
  414. package/dist/diagnose/repair.d.ts +77 -0
  415. package/dist/diagnose/repair.d.ts.map +1 -0
  416. package/dist/diagnose.d.ts +1 -251
  417. package/dist/diagnose.js +2 -2
  418. package/dist/discover-personas.d.ts +35 -0
  419. package/dist/discover-personas.d.ts.map +1 -0
  420. package/dist/driver.d.ts +95 -0
  421. package/dist/driver.d.ts.map +1 -0
  422. package/dist/driver.test.d.ts +8 -0
  423. package/dist/driver.test.d.ts.map +1 -0
  424. package/dist/dual-agent-bench.d.ts +81 -0
  425. package/dist/dual-agent-bench.d.ts.map +1 -0
  426. package/dist/error-count-extractor.d.ts +47 -0
  427. package/dist/error-count-extractor.d.ts.map +1 -0
  428. package/dist/error-count-extractor.test.d.ts +2 -0
  429. package/dist/error-count-extractor.test.d.ts.map +1 -0
  430. package/dist/{errors-CzMUYo7b.d.ts → errors.d.ts} +10 -11
  431. package/dist/errors.d.ts.map +1 -0
  432. package/dist/{researcher-DE6Gpnb4.d.ts → eval-campaign.d.ts} +34 -156
  433. package/dist/eval-campaign.d.ts.map +1 -0
  434. package/dist/eval-campaign.test.d.ts +2 -0
  435. package/dist/eval-campaign.test.d.ts.map +1 -0
  436. package/dist/eval-tools.d.ts +55 -0
  437. package/dist/eval-tools.d.ts.map +1 -0
  438. package/dist/eval-trace-store.d.ts +107 -0
  439. package/dist/eval-trace-store.d.ts.map +1 -0
  440. package/dist/eval-trace-store.test.d.ts +2 -0
  441. package/dist/eval-trace-store.test.d.ts.map +1 -0
  442. package/dist/executor.d.ts +38 -0
  443. package/dist/executor.d.ts.map +1 -0
  444. package/dist/executor.test.d.ts +10 -0
  445. package/dist/executor.test.d.ts.map +1 -0
  446. package/dist/experiment-tracker.d.ts +178 -0
  447. package/dist/experiment-tracker.d.ts.map +1 -0
  448. package/dist/experiment-tracker.test.d.ts +2 -0
  449. package/dist/experiment-tracker.test.d.ts.map +1 -0
  450. package/dist/failure-taxonomy.d.ts +38 -0
  451. package/dist/failure-taxonomy.d.ts.map +1 -0
  452. package/dist/{feedback-trajectory-D9OVLrg9.d.ts → feedback-trajectory.d.ts} +36 -38
  453. package/dist/feedback-trajectory.d.ts.map +1 -0
  454. package/dist/feedback-trajectory.test.d.ts +2 -0
  455. package/dist/feedback-trajectory.test.d.ts.map +1 -0
  456. package/dist/flow-layer.d.ts +90 -0
  457. package/dist/flow-layer.d.ts.map +1 -0
  458. package/dist/flow-layer.test.d.ts +2 -0
  459. package/dist/flow-layer.test.d.ts.map +1 -0
  460. package/dist/fuzz/capsule.d.ts +46 -0
  461. package/dist/fuzz/capsule.d.ts.map +1 -0
  462. package/dist/fuzz/cube.d.ts +36 -0
  463. package/dist/fuzz/cube.d.ts.map +1 -0
  464. package/dist/fuzz/explorer-cost.test.d.ts +2 -0
  465. package/dist/fuzz/explorer-cost.test.d.ts.map +1 -0
  466. package/dist/fuzz/explorer.d.ts +64 -0
  467. package/dist/fuzz/explorer.d.ts.map +1 -0
  468. package/dist/fuzz/fuzz-agent.d.ts +16 -0
  469. package/dist/fuzz/fuzz-agent.d.ts.map +1 -0
  470. package/dist/fuzz/fuzz-agent.test.d.ts +2 -0
  471. package/dist/fuzz/fuzz-agent.test.d.ts.map +1 -0
  472. package/dist/fuzz/gates.d.ts +33 -0
  473. package/dist/fuzz/gates.d.ts.map +1 -0
  474. package/dist/fuzz/index.d.ts +26 -0
  475. package/dist/fuzz/index.d.ts.map +1 -0
  476. package/dist/fuzz/policies.d.ts +28 -0
  477. package/dist/fuzz/policies.d.ts.map +1 -0
  478. package/dist/fuzz/tools.d.ts +20 -0
  479. package/dist/fuzz/tools.d.ts.map +1 -0
  480. package/dist/fuzz/types.d.ts +307 -0
  481. package/dist/fuzz/types.d.ts.map +1 -0
  482. package/dist/fuzz.d.ts +1 -547
  483. package/dist/golden-matcher.d.ts +71 -0
  484. package/dist/golden-matcher.d.ts.map +1 -0
  485. package/dist/governance/eu-ai-act.d.ts +37 -0
  486. package/dist/governance/eu-ai-act.d.ts.map +1 -0
  487. package/dist/governance/index.d.ts +5 -135
  488. package/dist/governance/index.d.ts.map +1 -0
  489. package/dist/governance/nist-ai-rmf.d.ts +15 -0
  490. package/dist/governance/nist-ai-rmf.d.ts.map +1 -0
  491. package/dist/governance/soc2.d.ts +12 -0
  492. package/dist/governance/soc2.d.ts.map +1 -0
  493. package/dist/governance/types.d.ts +66 -0
  494. package/dist/governance/types.d.ts.map +1 -0
  495. package/dist/harness-optimizer.d.ts +82 -0
  496. package/dist/harness-optimizer.d.ts.map +1 -0
  497. package/dist/held-out-gate.d.ts +135 -0
  498. package/dist/held-out-gate.d.ts.map +1 -0
  499. package/dist/hosted/client.d.ts +73 -0
  500. package/dist/hosted/client.d.ts.map +1 -0
  501. package/dist/hosted/from-env.test.d.ts +8 -0
  502. package/dist/hosted/from-env.test.d.ts.map +1 -0
  503. package/dist/hosted/index.d.ts +10 -238
  504. package/dist/hosted/index.d.ts.map +1 -0
  505. package/dist/hosted/types.d.ts +159 -0
  506. package/dist/hosted/types.d.ts.map +1 -0
  507. package/dist/index.d.ts +277 -5643
  508. package/dist/index.d.ts.map +1 -0
  509. package/dist/index.js +317 -108
  510. package/dist/index.js.map +1 -1
  511. package/dist/integrity/backend-integrity.d.ts +71 -0
  512. package/dist/integrity/backend-integrity.d.ts.map +1 -0
  513. package/dist/integrity/preflight.d.ts +72 -0
  514. package/dist/integrity/preflight.d.ts.map +1 -0
  515. package/dist/integrity/preflight.test.d.ts +2 -0
  516. package/dist/integrity/preflight.test.d.ts.map +1 -0
  517. package/dist/integrity/single-backend.d.ts +67 -0
  518. package/dist/integrity/single-backend.d.ts.map +1 -0
  519. package/dist/intent-match-judge.d.ts +69 -0
  520. package/dist/intent-match-judge.d.ts.map +1 -0
  521. package/dist/intent-match-judge.test.d.ts +2 -0
  522. package/dist/intent-match-judge.test.d.ts.map +1 -0
  523. package/dist/{judge-calibration-0p2QcWNE.d.ts → judge-calibration.d.ts} +16 -17
  524. package/dist/judge-calibration.d.ts.map +1 -0
  525. package/dist/judge-ensemble.d.ts +66 -0
  526. package/dist/judge-ensemble.d.ts.map +1 -0
  527. package/dist/judge-ensemble.test.d.ts +8 -0
  528. package/dist/judge-ensemble.test.d.ts.map +1 -0
  529. package/dist/judge-families.d.ts +38 -0
  530. package/dist/judge-families.d.ts.map +1 -0
  531. package/dist/judge-panel.d.ts +65 -0
  532. package/dist/judge-panel.d.ts.map +1 -0
  533. package/dist/judge-retry.d.ts +70 -0
  534. package/dist/judge-retry.d.ts.map +1 -0
  535. package/dist/judge-runner.d.ts +36 -0
  536. package/dist/judge-runner.d.ts.map +1 -0
  537. package/dist/judge-runner.test.d.ts +2 -0
  538. package/dist/judge-runner.test.d.ts.map +1 -0
  539. package/dist/judges.d.ts +74 -0
  540. package/dist/judges.d.ts.map +1 -0
  541. package/dist/keyword-coverage-judge.d.ts +89 -0
  542. package/dist/keyword-coverage-judge.d.ts.map +1 -0
  543. package/dist/keyword-coverage-judge.test.d.ts +2 -0
  544. package/dist/keyword-coverage-judge.test.d.ts.map +1 -0
  545. package/dist/knowledge/index.d.ts +3 -103
  546. package/dist/knowledge/index.d.ts.map +1 -0
  547. package/dist/knowledge/readiness.d.ts +26 -0
  548. package/dist/knowledge/readiness.d.ts.map +1 -0
  549. package/dist/knowledge/types.d.ts +75 -0
  550. package/dist/knowledge/types.d.ts.map +1 -0
  551. package/dist/live-proof.d.ts +62 -0
  552. package/dist/live-proof.d.ts.map +1 -0
  553. package/dist/{llm-client-BeEcAokY.d.ts → llm-client.d.ts} +52 -23
  554. package/dist/llm-client.d.ts.map +1 -0
  555. package/dist/llm-client.test.d.ts +2 -0
  556. package/dist/llm-client.test.d.ts.map +1 -0
  557. package/dist/locked-jsonl-appender.d.ts +19 -0
  558. package/dist/locked-jsonl-appender.d.ts.map +1 -0
  559. package/dist/matrix/aggregation.d.ts +16 -0
  560. package/dist/matrix/aggregation.d.ts.map +1 -0
  561. package/dist/matrix/index.d.ts +12 -30
  562. package/dist/matrix/index.d.ts.map +1 -0
  563. package/dist/matrix/runner.d.ts +15 -0
  564. package/dist/matrix/runner.d.ts.map +1 -0
  565. package/dist/{types-mn5Aqk7x.d.ts → matrix/types.d.ts} +11 -15
  566. package/dist/matrix/types.d.ts.map +1 -0
  567. package/dist/meta-eval/calibration.d.ts +47 -0
  568. package/dist/meta-eval/calibration.d.ts.map +1 -0
  569. package/dist/meta-eval/correlation-study.d.ts +53 -0
  570. package/dist/meta-eval/correlation-study.d.ts.map +1 -0
  571. package/dist/meta-eval/index.d.ts +6 -181
  572. package/dist/meta-eval/index.d.ts.map +1 -0
  573. package/dist/meta-eval/index.js +1 -1
  574. package/dist/{outcome-store-rnXLEqSn.d.ts → meta-eval/outcome-store.d.ts} +7 -8
  575. package/dist/meta-eval/outcome-store.d.ts.map +1 -0
  576. package/dist/{rubric-predictive-validity-Cy_W-hWZ.d.ts → meta-eval/rubric-predictive-validity.d.ts} +8 -11
  577. package/dist/meta-eval/rubric-predictive-validity.d.ts.map +1 -0
  578. package/dist/meta-eval/sentinel.d.ts +169 -0
  579. package/dist/meta-eval/sentinel.d.ts.map +1 -0
  580. package/dist/metrics.d.ts +63 -0
  581. package/dist/metrics.d.ts.map +1 -0
  582. package/dist/model-seats.d.ts +71 -0
  583. package/dist/model-seats.d.ts.map +1 -0
  584. package/dist/model-seats.test.d.ts +2 -0
  585. package/dist/model-seats.test.d.ts.map +1 -0
  586. package/dist/muffled-gate-scanner.d.ts +102 -0
  587. package/dist/muffled-gate-scanner.d.ts.map +1 -0
  588. package/dist/{multi-layer-verifier-DUZXrPDA.d.ts → multi-layer-verifier.d.ts} +12 -15
  589. package/dist/multi-layer-verifier.d.ts.map +1 -0
  590. package/dist/multi-layer-verifier.test.d.ts +2 -0
  591. package/dist/multi-layer-verifier.test.d.ts.map +1 -0
  592. package/dist/multi-toolchain-layer.d.ts +80 -0
  593. package/dist/multi-toolchain-layer.d.ts.map +1 -0
  594. package/dist/multi-toolchain-layer.test.d.ts +2 -0
  595. package/dist/multi-toolchain-layer.test.d.ts.map +1 -0
  596. package/dist/multishot/default-tools.d.ts +34 -0
  597. package/dist/multishot/default-tools.d.ts.map +1 -0
  598. package/dist/multishot/index.d.ts +7 -290
  599. package/dist/multishot/index.d.ts.map +1 -0
  600. package/dist/multishot/index.js.map +1 -1
  601. package/dist/multishot/judges.d.ts +32 -0
  602. package/dist/multishot/judges.d.ts.map +1 -0
  603. package/dist/multishot/matrix.d.ts +107 -0
  604. package/dist/multishot/matrix.d.ts.map +1 -0
  605. package/dist/multishot/multishot.d.ts +23 -0
  606. package/dist/multishot/multishot.d.ts.map +1 -0
  607. package/dist/multishot/router.d.ts +37 -0
  608. package/dist/multishot/router.d.ts.map +1 -0
  609. package/dist/multishot/types.d.ts +60 -0
  610. package/dist/multishot/types.d.ts.map +1 -0
  611. package/dist/observability.d.ts +71 -0
  612. package/dist/observability.d.ts.map +1 -0
  613. package/dist/openapi.json +1 -1
  614. package/dist/oracle.d.ts +55 -0
  615. package/dist/oracle.d.ts.map +1 -0
  616. package/dist/orthogonality.d.ts +35 -0
  617. package/dist/orthogonality.d.ts.map +1 -0
  618. package/dist/otel-pipeline.d.ts +31 -0
  619. package/dist/otel-pipeline.d.ts.map +1 -0
  620. package/dist/paraphrase.d.ts +107 -0
  621. package/dist/paraphrase.d.ts.map +1 -0
  622. package/dist/{pareto-E-pembql.d.ts → pareto.d.ts} +9 -10
  623. package/dist/pareto.d.ts.map +1 -0
  624. package/dist/partition-held-out.d.ts +70 -0
  625. package/dist/partition-held-out.d.ts.map +1 -0
  626. package/dist/partition-held-out.test.d.ts +2 -0
  627. package/dist/partition-held-out.test.d.ts.map +1 -0
  628. package/dist/perf/index.d.ts +13 -119
  629. package/dist/perf/index.d.ts.map +1 -0
  630. package/dist/perf/integrity.d.ts +30 -0
  631. package/dist/perf/integrity.d.ts.map +1 -0
  632. package/dist/perf/journey.d.ts +45 -0
  633. package/dist/perf/journey.d.ts.map +1 -0
  634. package/dist/perf/ratchet.d.ts +47 -0
  635. package/dist/perf/ratchet.d.ts.map +1 -0
  636. package/dist/pipelines/budget-breach.d.ts +31 -0
  637. package/dist/pipelines/budget-breach.d.ts.map +1 -0
  638. package/dist/pipelines/budget-breach.test.d.ts +2 -0
  639. package/dist/pipelines/budget-breach.test.d.ts.map +1 -0
  640. package/dist/pipelines/failure-cluster.d.ts +38 -0
  641. package/dist/pipelines/failure-cluster.d.ts.map +1 -0
  642. package/dist/pipelines/failure-cluster.test.d.ts +2 -0
  643. package/dist/pipelines/failure-cluster.test.d.ts.map +1 -0
  644. package/dist/pipelines/first-divergence.d.ts +26 -0
  645. package/dist/pipelines/first-divergence.d.ts.map +1 -0
  646. package/dist/pipelines/first-divergence.test.d.ts +2 -0
  647. package/dist/pipelines/first-divergence.test.d.ts.map +1 -0
  648. package/dist/pipelines/index.d.ts +8 -173
  649. package/dist/pipelines/index.d.ts.map +1 -0
  650. package/dist/pipelines/index.js +40 -15
  651. package/dist/pipelines/index.js.map +1 -1
  652. package/dist/pipelines/judge-agreement.d.ts +26 -0
  653. package/dist/pipelines/judge-agreement.d.ts.map +1 -0
  654. package/dist/pipelines/judge-agreement.test.d.ts +2 -0
  655. package/dist/pipelines/judge-agreement.test.d.ts.map +1 -0
  656. package/dist/pipelines/regression.d.ts +23 -0
  657. package/dist/pipelines/regression.d.ts.map +1 -0
  658. package/dist/pipelines/regression.test.d.ts +2 -0
  659. package/dist/pipelines/regression.test.d.ts.map +1 -0
  660. package/dist/pipelines/stuck-loop.d.ts +32 -0
  661. package/dist/pipelines/stuck-loop.d.ts.map +1 -0
  662. package/dist/pipelines/stuck-loop.test.d.ts +2 -0
  663. package/dist/pipelines/stuck-loop.test.d.ts.map +1 -0
  664. package/dist/pipelines/tool-waste.d.ts +34 -0
  665. package/dist/pipelines/tool-waste.d.ts.map +1 -0
  666. package/dist/pipelines/tool-waste.test.d.ts +2 -0
  667. package/dist/pipelines/tool-waste.test.d.ts.map +1 -0
  668. package/dist/playbook.d.ts +16 -0
  669. package/dist/playbook.d.ts.map +1 -0
  670. package/dist/pr-review-benchmark.d.ts +88 -0
  671. package/dist/pr-review-benchmark.d.ts.map +1 -0
  672. package/dist/pr-review-benchmark.test.d.ts +2 -0
  673. package/dist/pr-review-benchmark.test.d.ts.map +1 -0
  674. package/dist/pre-registration.d.ts +125 -0
  675. package/dist/pre-registration.d.ts.map +1 -0
  676. package/dist/prm/builtin-rubrics.d.ts +33 -0
  677. package/dist/prm/builtin-rubrics.d.ts.map +1 -0
  678. package/dist/prm/index.d.ts +5 -100
  679. package/dist/prm/index.d.ts.map +1 -0
  680. package/dist/prm/index.js +55 -12
  681. package/dist/prm/index.js.map +1 -1
  682. package/dist/prm/inference.d.ts +29 -0
  683. package/dist/prm/inference.d.ts.map +1 -0
  684. package/dist/prm/inference.test.d.ts +2 -0
  685. package/dist/prm/inference.test.d.ts.map +1 -0
  686. package/dist/{rubric-BOfxn4ja.d.ts → prm/rubric.d.ts} +10 -13
  687. package/dist/prm/rubric.d.ts.map +1 -0
  688. package/dist/prm/training-export.d.ts +38 -0
  689. package/dist/prm/training-export.d.ts.map +1 -0
  690. package/dist/produced-state.d.ts +63 -0
  691. package/dist/produced-state.d.ts.map +1 -0
  692. package/dist/produced-state.test.d.ts +8 -0
  693. package/dist/produced-state.test.d.ts.map +1 -0
  694. package/dist/profile/baselines.d.ts +37 -0
  695. package/dist/profile/baselines.d.ts.map +1 -0
  696. package/dist/profile/index.d.ts +105 -0
  697. package/dist/profile/index.d.ts.map +1 -0
  698. package/dist/promotion-gate.d.ts +93 -0
  699. package/dist/promotion-gate.d.ts.map +1 -0
  700. package/dist/prompt-registry.d.ts +41 -0
  701. package/dist/prompt-registry.d.ts.map +1 -0
  702. package/dist/propose-review-control.d.ts +49 -0
  703. package/dist/propose-review-control.d.ts.map +1 -0
  704. package/dist/propose-review-control.test.d.ts +2 -0
  705. package/dist/propose-review-control.test.d.ts.map +1 -0
  706. package/dist/propose-review.d.ts +155 -0
  707. package/dist/propose-review.d.ts.map +1 -0
  708. package/dist/{red-team-BXHil6c8.d.ts → red-team.d.ts} +14 -16
  709. package/dist/red-team.d.ts.map +1 -0
  710. package/dist/reference-replay-steering.d.ts +11 -0
  711. package/dist/reference-replay-steering.d.ts.map +1 -0
  712. package/dist/reference-replay.d.ts +176 -0
  713. package/dist/reference-replay.d.ts.map +1 -0
  714. package/dist/reflective-mutation.d.ts +79 -0
  715. package/dist/reflective-mutation.d.ts.map +1 -0
  716. package/dist/registry.d.ts +31 -0
  717. package/dist/registry.d.ts.map +1 -0
  718. package/dist/release-confidence.d.ts +128 -0
  719. package/dist/release-confidence.d.ts.map +1 -0
  720. package/dist/release-report.d.ts +11 -0
  721. package/dist/release-report.d.ts.map +1 -0
  722. package/dist/replay.d.ts +120 -0
  723. package/dist/replay.d.ts.map +1 -0
  724. package/dist/reporter.d.ts +14 -0
  725. package/dist/reporter.d.ts.map +1 -0
  726. package/dist/reporting.d.ts +15 -15
  727. package/dist/reporting.d.ts.map +1 -0
  728. package/dist/reporting.js +3 -3
  729. package/dist/researcher.d.ts +140 -0
  730. package/dist/researcher.d.ts.map +1 -0
  731. package/dist/reviewer.d.ts +118 -0
  732. package/dist/reviewer.d.ts.map +1 -0
  733. package/dist/reviewer.test.d.ts +2 -0
  734. package/dist/reviewer.test.d.ts.map +1 -0
  735. package/dist/reward-model-export.d.ts +60 -0
  736. package/dist/reward-model-export.d.ts.map +1 -0
  737. package/dist/rl/active-curriculum.d.ts +110 -0
  738. package/dist/rl/active-curriculum.d.ts.map +1 -0
  739. package/dist/rl/adaptation-eval.d.ts +109 -0
  740. package/dist/rl/adaptation-eval.d.ts.map +1 -0
  741. package/dist/{adversarial-DIVcDoI_.d.ts → rl/adversarial.d.ts} +6 -7
  742. package/dist/rl/adversarial.d.ts.map +1 -0
  743. package/dist/rl/compute-curves.d.ts +127 -0
  744. package/dist/rl/compute-curves.d.ts.map +1 -0
  745. package/dist/rl/contamination.d.ts +117 -0
  746. package/dist/rl/contamination.d.ts.map +1 -0
  747. package/dist/rl/corpus.d.ts +55 -0
  748. package/dist/rl/corpus.d.ts.map +1 -0
  749. package/dist/rl/corpus.test.d.ts +2 -0
  750. package/dist/rl/corpus.test.d.ts.map +1 -0
  751. package/dist/rl/dataset.d.ts +102 -0
  752. package/dist/rl/dataset.d.ts.map +1 -0
  753. package/dist/rl/dataset.test.d.ts +2 -0
  754. package/dist/rl/dataset.test.d.ts.map +1 -0
  755. package/dist/rl/exporters.d.ts +141 -0
  756. package/dist/rl/exporters.d.ts.map +1 -0
  757. package/dist/rl/index.d.ts +49 -0
  758. package/dist/rl/index.d.ts.map +1 -0
  759. package/dist/{off-policy-DiwuKKg7.d.ts → rl/off-policy.d.ts} +8 -9
  760. package/dist/rl/off-policy.d.ts.map +1 -0
  761. package/dist/rl/predictive-validity-researcher.d.ts +69 -0
  762. package/dist/rl/predictive-validity-researcher.d.ts.map +1 -0
  763. package/dist/rl/preferences.d.ts +141 -0
  764. package/dist/rl/preferences.d.ts.map +1 -0
  765. package/dist/rl/process-reward.d.ts +122 -0
  766. package/dist/rl/process-reward.d.ts.map +1 -0
  767. package/dist/rl/reward-hacking.d.ts +104 -0
  768. package/dist/rl/reward-hacking.d.ts.map +1 -0
  769. package/dist/rl/rl-campaign.d.ts +85 -0
  770. package/dist/rl/rl-campaign.d.ts.map +1 -0
  771. package/dist/rl/run-record-adapters.d.ts +56 -0
  772. package/dist/rl/run-record-adapters.d.ts.map +1 -0
  773. package/dist/rl/sim-fidelity.d.ts +166 -0
  774. package/dist/rl/sim-fidelity.d.ts.map +1 -0
  775. package/dist/rl/sim-fidelity.test.d.ts +2 -0
  776. package/dist/rl/sim-fidelity.test.d.ts.map +1 -0
  777. package/dist/rl/tournament.d.ts +115 -0
  778. package/dist/rl/tournament.d.ts.map +1 -0
  779. package/dist/rl/verifiable-reward.d.ts +124 -0
  780. package/dist/rl/verifiable-reward.d.ts.map +1 -0
  781. package/dist/rl.d.ts +1 -1192
  782. package/dist/rl.js +615 -615
  783. package/dist/rl.js.map +1 -1
  784. package/dist/{run-campaign-RDGAM5KJ.js → run-campaign-WXY7KI67.js} +3 -3
  785. package/dist/run-critic.d.ts +23 -0
  786. package/dist/run-critic.d.ts.map +1 -0
  787. package/dist/run-evidence.d.ts +32 -0
  788. package/dist/run-evidence.d.ts.map +1 -0
  789. package/dist/{run-record-e7vj1uZQ.d.ts → run-record.d.ts} +16 -122
  790. package/dist/run-record.d.ts.map +1 -0
  791. package/dist/run-record.test.d.ts +2 -0
  792. package/dist/run-record.test.d.ts.map +1 -0
  793. package/dist/run-score.d.ts +31 -0
  794. package/dist/run-score.d.ts.map +1 -0
  795. package/dist/runtime-trajectory.d.ts +47 -0
  796. package/dist/runtime-trajectory.d.ts.map +1 -0
  797. package/dist/{test-graded-scenario-BdVaPyHT.d.ts → sandbox-harness.d.ts} +34 -60
  798. package/dist/sandbox-harness.d.ts.map +1 -0
  799. package/dist/sandbox-harness.test.d.ts +2 -0
  800. package/dist/sandbox-harness.test.d.ts.map +1 -0
  801. package/dist/sandbox-pool.d.ts +74 -0
  802. package/dist/sandbox-pool.d.ts.map +1 -0
  803. package/dist/sandbox-pool.test.d.ts +2 -0
  804. package/dist/sandbox-pool.test.d.ts.map +1 -0
  805. package/dist/scorecard.d.ts +133 -0
  806. package/dist/scorecard.d.ts.map +1 -0
  807. package/dist/scorecard.test.d.ts +2 -0
  808. package/dist/scorecard.test.d.ts.map +1 -0
  809. package/dist/self-play.d.ts +69 -0
  810. package/dist/self-play.d.ts.map +1 -0
  811. package/dist/semantic-concept-judge.d.ts +135 -0
  812. package/dist/semantic-concept-judge.d.ts.map +1 -0
  813. package/dist/semantic-concept-judge.test.d.ts +2 -0
  814. package/dist/semantic-concept-judge.test.d.ts.map +1 -0
  815. package/dist/{sequential-5iSVfzl2.d.ts → sequential.d.ts} +9 -10
  816. package/dist/sequential.d.ts.map +1 -0
  817. package/dist/{series-convergence-D5OWMBg6.d.ts → series-convergence.d.ts} +4 -5
  818. package/dist/series-convergence.d.ts.map +1 -0
  819. package/dist/slo.d.ts +48 -0
  820. package/dist/slo.d.ts.map +1 -0
  821. package/dist/state-continuity.d.ts +47 -0
  822. package/dist/state-continuity.d.ts.map +1 -0
  823. package/dist/{statistics-C7PozGrZ.d.ts → statistics.d.ts} +152 -39
  824. package/dist/statistics.d.ts.map +1 -0
  825. package/dist/statistics.test.d.ts +2 -0
  826. package/dist/statistics.test.d.ts.map +1 -0
  827. package/dist/steering-optimizer.d.ts +58 -0
  828. package/dist/steering-optimizer.d.ts.map +1 -0
  829. package/dist/steering.d.ts +24 -0
  830. package/dist/steering.d.ts.map +1 -0
  831. package/dist/storyboard/code-edit.d.ts +64 -0
  832. package/dist/storyboard/code-edit.d.ts.map +1 -0
  833. package/dist/storyboard/code-edit.test.d.ts +2 -0
  834. package/dist/storyboard/code-edit.test.d.ts.map +1 -0
  835. package/dist/storyboard/index.d.ts +16 -81
  836. package/dist/storyboard/index.d.ts.map +1 -0
  837. package/dist/storyboard/index.test.d.ts +2 -0
  838. package/dist/storyboard/index.test.d.ts.map +1 -0
  839. package/dist/{summary-report-DGmUucwQ.d.ts → summary-report.d.ts} +23 -160
  840. package/dist/summary-report.d.ts.map +1 -0
  841. package/dist/telemetry/client.d.ts +35 -0
  842. package/dist/telemetry/client.d.ts.map +1 -0
  843. package/dist/telemetry/index.d.ts +17 -35
  844. package/dist/telemetry/index.d.ts.map +1 -0
  845. package/dist/telemetry/schema.d.ts +61 -0
  846. package/dist/telemetry/schema.d.ts.map +1 -0
  847. package/dist/telemetry/sink-fetch.d.ts +39 -0
  848. package/dist/telemetry/sink-fetch.d.ts.map +1 -0
  849. package/dist/telemetry/{file.d.ts → sink-file.d.ts} +5 -7
  850. package/dist/telemetry/sink-file.d.ts.map +1 -0
  851. package/dist/test-graded-scenario.d.ts +42 -0
  852. package/dist/test-graded-scenario.d.ts.map +1 -0
  853. package/dist/testing.d.ts +5 -0
  854. package/dist/testing.d.ts.map +1 -0
  855. package/dist/testing.js +8 -0
  856. package/dist/testing.js.map +1 -0
  857. package/dist/tool-use-metrics.d.ts +35 -0
  858. package/dist/tool-use-metrics.d.ts.map +1 -0
  859. package/dist/trace/capture-fetch.d.ts +48 -0
  860. package/dist/trace/capture-fetch.d.ts.map +1 -0
  861. package/dist/trace/capture-fetch.test.d.ts +2 -0
  862. package/dist/trace/capture-fetch.test.d.ts.map +1 -0
  863. package/dist/{emitter-DEZwY14K.d.ts → trace/emitter.d.ts} +9 -12
  864. package/dist/trace/emitter.d.ts.map +1 -0
  865. package/dist/trace/extract-usage.d.ts +46 -0
  866. package/dist/trace/extract-usage.d.ts.map +1 -0
  867. package/dist/trace/extract-usage.test.d.ts +2 -0
  868. package/dist/trace/extract-usage.test.d.ts.map +1 -0
  869. package/dist/trace/index.d.ts +15 -0
  870. package/dist/trace/index.d.ts.map +1 -0
  871. package/dist/{integrity-VJ9A7aST.d.ts → trace/integrity.d.ts} +11 -14
  872. package/dist/trace/integrity.d.ts.map +1 -0
  873. package/dist/trace/otel-bridge.d.ts +29 -0
  874. package/dist/trace/otel-bridge.d.ts.map +1 -0
  875. package/dist/trace/otel-export.d.ts +52 -0
  876. package/dist/trace/otel-export.d.ts.map +1 -0
  877. package/dist/trace/otel.d.ts +57 -0
  878. package/dist/trace/otel.d.ts.map +1 -0
  879. package/dist/trace/otlp-attributes.d.ts +17 -0
  880. package/dist/trace/otlp-attributes.d.ts.map +1 -0
  881. package/dist/trace/query.d.ts +29 -0
  882. package/dist/trace/query.d.ts.map +1 -0
  883. package/dist/trace/query.test.d.ts +2 -0
  884. package/dist/trace/query.test.d.ts.map +1 -0
  885. package/dist/{raw-provider-sink-C46HDghv.d.ts → trace/raw-provider-sink.d.ts} +13 -14
  886. package/dist/trace/raw-provider-sink.d.ts.map +1 -0
  887. package/dist/{redact-B40YG2M_.d.ts → trace/redact.d.ts} +7 -8
  888. package/dist/trace/redact.d.ts.map +1 -0
  889. package/dist/{schema-m0gsnbt3.d.ts → trace/schema.d.ts} +29 -30
  890. package/dist/trace/schema.d.ts.map +1 -0
  891. package/dist/trace/store-to-otlp.d.ts +72 -0
  892. package/dist/trace/store-to-otlp.d.ts.map +1 -0
  893. package/dist/trace/store-to-otlp.test.d.ts +2 -0
  894. package/dist/trace/store-to-otlp.test.d.ts.map +1 -0
  895. package/dist/{store-CKUAgsJz.d.ts → trace/store.d.ts} +37 -13
  896. package/dist/trace/store.d.ts.map +1 -0
  897. package/dist/trace/store.test.d.ts +2 -0
  898. package/dist/trace/store.test.d.ts.map +1 -0
  899. package/dist/{analyst-C8HHvfJp.d.ts → trace-analyst/analyst.d.ts} +12 -14
  900. package/dist/trace-analyst/analyst.d.ts.map +1 -0
  901. package/dist/trace-analyst/analyst.test.d.ts +2 -0
  902. package/dist/trace-analyst/analyst.test.d.ts.map +1 -0
  903. package/dist/trace-analyst/behavioral-metrics.d.ts +40 -0
  904. package/dist/trace-analyst/behavioral-metrics.d.ts.map +1 -0
  905. package/dist/trace-analyst/behavioral-metrics.test.d.ts +2 -0
  906. package/dist/trace-analyst/behavioral-metrics.test.d.ts.map +1 -0
  907. package/dist/trace-analyst/hook.d.ts +55 -0
  908. package/dist/trace-analyst/hook.d.ts.map +1 -0
  909. package/dist/trace-analyst/index.d.ts +18 -0
  910. package/dist/trace-analyst/index.d.ts.map +1 -0
  911. package/dist/trace-analyst/insights.d.ts +71 -0
  912. package/dist/trace-analyst/insights.d.ts.map +1 -0
  913. package/dist/trace-analyst/insights.test.d.ts +2 -0
  914. package/dist/trace-analyst/insights.test.d.ts.map +1 -0
  915. package/dist/trace-analyst/otlp-flatten.d.ts +42 -0
  916. package/dist/trace-analyst/otlp-flatten.d.ts.map +1 -0
  917. package/dist/trace-analyst/otlp-span.d.ts +85 -0
  918. package/dist/trace-analyst/otlp-span.d.ts.map +1 -0
  919. package/dist/trace-analyst/otlp-span.test.d.ts +2 -0
  920. package/dist/trace-analyst/otlp-span.test.d.ts.map +1 -0
  921. package/dist/trace-analyst/otlp-to-run-records.d.ts +115 -0
  922. package/dist/trace-analyst/otlp-to-run-records.d.ts.map +1 -0
  923. package/dist/trace-analyst/otlp-to-run-records.test.d.ts +2 -0
  924. package/dist/trace-analyst/otlp-to-run-records.test.d.ts.map +1 -0
  925. package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts +2 -0
  926. package/dist/trace-analyst/otlp-to-run-records.timestamps.test.d.ts.map +1 -0
  927. package/dist/trace-analyst/prompts.d.ts +6 -0
  928. package/dist/trace-analyst/prompts.d.ts.map +1 -0
  929. package/dist/trace-analyst/store-otlp.d.ts +126 -0
  930. package/dist/trace-analyst/store-otlp.d.ts.map +1 -0
  931. package/dist/trace-analyst/store-otlp.test.d.ts +8 -0
  932. package/dist/trace-analyst/store-otlp.test.d.ts.map +1 -0
  933. package/dist/trace-analyst/store-otlp.timestamps.test.d.ts +2 -0
  934. package/dist/trace-analyst/store-otlp.timestamps.test.d.ts.map +1 -0
  935. package/dist/trace-analyst/store.d.ts +63 -0
  936. package/dist/trace-analyst/store.d.ts.map +1 -0
  937. package/dist/trace-analyst/tools.d.ts +44 -0
  938. package/dist/trace-analyst/tools.d.ts.map +1 -0
  939. package/dist/trace-analyst/tools.test.d.ts +10 -0
  940. package/dist/trace-analyst/tools.test.d.ts.map +1 -0
  941. package/dist/{store-C1YxJDEK.d.ts → trace-analyst/types.d.ts} +18 -74
  942. package/dist/trace-analyst/types.d.ts.map +1 -0
  943. package/dist/trace-contracts.d.ts +180 -0
  944. package/dist/trace-contracts.d.ts.map +1 -0
  945. package/dist/traced-analyst.d.ts +26 -0
  946. package/dist/traced-analyst.d.ts.map +1 -0
  947. package/dist/traced-judges.d.ts +27 -0
  948. package/dist/traced-judges.d.ts.map +1 -0
  949. package/dist/traces.d.ts +4 -947
  950. package/dist/traces.d.ts.map +1 -0
  951. package/dist/traces.js +42 -12
  952. package/dist/{trajectory-GEdXJCL5.d.ts → trajectory.d.ts} +8 -9
  953. package/dist/trajectory.d.ts.map +1 -0
  954. package/dist/{types-Croy5h7V.d.ts → types.d.ts} +33 -33
  955. package/dist/types.d.ts.map +1 -0
  956. package/dist/ui-finding.d.ts +104 -0
  957. package/dist/ui-finding.d.ts.map +1 -0
  958. package/dist/verdict-cache.d.ts +78 -0
  959. package/dist/verdict-cache.d.ts.map +1 -0
  960. package/dist/verdict-cache.test.d.ts +2 -0
  961. package/dist/verdict-cache.test.d.ts.map +1 -0
  962. package/dist/{verdict-C9MlYujm.d.ts → verdict.d.ts} +2 -3
  963. package/dist/verdict.d.ts.map +1 -0
  964. package/dist/visual-diff.d.ts +32 -0
  965. package/dist/visual-diff.d.ts.map +1 -0
  966. package/dist/wire/handlers.d.ts +54 -0
  967. package/dist/wire/handlers.d.ts.map +1 -0
  968. package/dist/wire/index.d.ts +13 -570
  969. package/dist/wire/index.d.ts.map +1 -0
  970. package/dist/wire/index.js +2 -2
  971. package/dist/wire/openapi.d.ts +3 -0
  972. package/dist/wire/openapi.d.ts.map +1 -0
  973. package/dist/wire/rpc.d.ts +21 -0
  974. package/dist/wire/rpc.d.ts.map +1 -0
  975. package/dist/wire/rubrics.d.ts +34 -0
  976. package/dist/wire/rubrics.d.ts.map +1 -0
  977. package/dist/wire/schemas.d.ts +410 -0
  978. package/dist/wire/schemas.d.ts.map +1 -0
  979. package/dist/wire/server.d.ts +60 -0
  980. package/dist/wire/server.d.ts.map +1 -0
  981. package/dist/workflow/event-schema.d.ts +5 -0
  982. package/dist/workflow/event-schema.d.ts.map +1 -0
  983. package/dist/workflow/feedback-pack.d.ts +99 -0
  984. package/dist/workflow/feedback-pack.d.ts.map +1 -0
  985. package/dist/workflow/index.d.ts +22 -495
  986. package/dist/workflow/index.d.ts.map +1 -0
  987. package/dist/workflow/index.js +2 -2
  988. package/dist/workflow/intelligence-export.d.ts +62 -0
  989. package/dist/workflow/intelligence-export.d.ts.map +1 -0
  990. package/dist/workflow/partner-report.d.ts +49 -0
  991. package/dist/workflow/partner-report.d.ts.map +1 -0
  992. package/dist/workflow/phase-graph.d.ts +43 -0
  993. package/dist/workflow/phase-graph.d.ts.map +1 -0
  994. package/dist/workflow/promotion-gate.d.ts +61 -0
  995. package/dist/workflow/promotion-gate.d.ts.map +1 -0
  996. package/dist/workflow/run-record.d.ts +12 -0
  997. package/dist/workflow/run-record.d.ts.map +1 -0
  998. package/dist/workflow/runtime-adapter.d.ts +20 -0
  999. package/dist/workflow/runtime-adapter.d.ts.map +1 -0
  1000. package/dist/workflow/sanitize.d.ts +21 -0
  1001. package/dist/workflow/sanitize.d.ts.map +1 -0
  1002. package/dist/workflow/schema.d.ts +5 -0
  1003. package/dist/workflow/schema.d.ts.map +1 -0
  1004. package/dist/workflow/summary.d.ts +43 -0
  1005. package/dist/workflow/summary.d.ts.map +1 -0
  1006. package/dist/workflow/trace-event-fields.d.ts +6 -0
  1007. package/dist/workflow/trace-event-fields.d.ts.map +1 -0
  1008. package/dist/workflow/trajectory.d.ts +15 -0
  1009. package/dist/workflow/trajectory.d.ts.map +1 -0
  1010. package/dist/workflow/types.d.ts +68 -0
  1011. package/dist/workflow/types.d.ts.map +1 -0
  1012. package/dist/workspace-inspector.d.ts +67 -0
  1013. package/dist/workspace-inspector.d.ts.map +1 -0
  1014. package/dist/wrangler-deploy-runner.test.d.ts +2 -0
  1015. package/dist/wrangler-deploy-runner.test.d.ts.map +1 -0
  1016. package/docs/campaign-proposers.md +170 -0
  1017. package/docs/concepts.md +8 -4
  1018. package/docs/customer-journeys.md +15 -13
  1019. package/docs/design/loop-taxonomy.md +34 -66
  1020. package/docs/distributed-driver.md +14 -14
  1021. package/docs/feature-guide.md +1 -1
  1022. package/docs/hosted-ingest-spec.md +2 -3
  1023. package/docs/multi-shot-optimization.md +8 -8
  1024. package/docs/product-eval-adoption.md +1 -1
  1025. package/docs/self-improvement-map.md +33 -29
  1026. package/package.json +9 -15
  1027. package/dist/calibration-Cpr3WaX3.d.ts +0 -101
  1028. package/dist/chunk-47X6LRCE.js.map +0 -1
  1029. package/dist/chunk-6SOJM3VR.js.map +0 -1
  1030. package/dist/chunk-CVVHBFGN.js.map +0 -1
  1031. package/dist/chunk-D3V5B42D.js.map +0 -1
  1032. package/dist/chunk-E4GH6USR.js.map +0 -1
  1033. package/dist/chunk-GSH6QNNS.js.map +0 -1
  1034. package/dist/chunk-KWRRMR3J.js.map +0 -1
  1035. package/dist/chunk-LMZQ2Z4U.js.map +0 -1
  1036. package/dist/chunk-QAY5UIJO.js.map +0 -1
  1037. package/dist/chunk-QG2OVF2D.js.map +0 -1
  1038. package/dist/chunk-T375SUOZ.js.map +0 -1
  1039. package/dist/chunk-TWS7AZEY.js.map +0 -1
  1040. package/dist/chunk-UMMZHCPB.js.map +0 -1
  1041. package/dist/chunk-XY4DDNEG.js.map +0 -1
  1042. package/dist/chunk-ZFIBGEOL.js.map +0 -1
  1043. package/dist/control-_Qb7skHX.d.ts +0 -259
  1044. package/dist/corpus-BoR-041R.d.ts +0 -560
  1045. package/dist/failure-cluster-CL7IVgkJ.d.ts +0 -76
  1046. package/dist/harness-optimizer-EnEnQPsr.d.ts +0 -106
  1047. package/dist/index-Bx3gZ8xl.d.ts +0 -159
  1048. package/dist/pre-registration-mAnCugl9.d.ts +0 -523
  1049. package/dist/provenance-LnqRT0sS.d.ts +0 -441
  1050. package/dist/query-CqTxMwDw.d.ts +0 -31
  1051. package/dist/release-report-euXIV_Sk.d.ts +0 -233
  1052. package/dist/run-critic-BAIjX99r.d.ts +0 -56
  1053. package/dist/run-improvement-loop-5z_l5zDz.d.ts +0 -427
  1054. package/dist/runtime-trajectory-BDgfGZSr.d.ts +0 -49
  1055. package/dist/semantic-concept-judge-Dn8Z6KEG.d.ts +0 -624
  1056. package/dist/sink-fetch-B1Yg4Til.d.ts +0 -101
  1057. package/docs/design/external-agent-wedge.md +0 -89
  1058. package/docs/design/phase-d-rfc.md +0 -125
  1059. package/docs/design/phase4-consumer-migration.md +0 -70
  1060. package/docs/design/primitives-integration-spec.md +0 -393
  1061. package/docs/design/product-self-improvement-loop.md +0 -146
  1062. package/docs/design/self-improvement-engine.md +0 -140
  1063. package/docs/design/self-improvement-protocol.md +0 -223
  1064. package/docs/design/self-improvement-roadmap.md +0 -106
  1065. package/docs/design/substrate-gaps.md +0 -118
  1066. package/docs/phase-b-pairing-kit.md +0 -188
  1067. package/docs/phase-b-runbook.md +0 -176
  1068. package/docs/pilot/README.md +0 -62
  1069. package/docs/pilot/customer-checklist.md +0 -90
  1070. package/docs/pilot/integration-foreign-stack.md +0 -296
  1071. package/docs/pilot/integration-tangle-stack.md +0 -248
  1072. package/docs/pilot/one-pager.md +0 -161
  1073. package/docs/pilot/sample-insight-report.json +0 -172
  1074. package/docs/quickstart-external.md +0 -229
  1075. package/docs/research/belief-state-agent-eval-roadmap.md +0 -593
  1076. package/docs/research/research-roadmap.md +0 -205
  1077. package/docs/specs/driver-honest-spec.md +0 -251
  1078. package/docs/specs/hermes-self-improvement-audit.md +0 -93
  1079. package/docs/specs/profile-versioning.md +0 -291
  1080. package/docs/three-package-architecture.md +0 -168
  1081. /package/dist/{chunk-QS3RBQPI.js.map → chunk-D5KBVULY.js.map} +0 -0
  1082. /package/dist/{chunk-UHMJT4T7.js.map → chunk-IZCEK2HR.js.map} +0 -0
  1083. /package/dist/{chunk-L3JOU6XM.js.map → chunk-NACM5RS7.js.map} +0 -0
  1084. /package/dist/{chunk-4FBZZIYD.js.map → chunk-QTMYW64F.js.map} +0 -0
  1085. /package/dist/{chunk-CY6U5S3X.js.map → chunk-URWVKRCS.js.map} +0 -0
  1086. /package/dist/{chunk-Y47J2LJ3.js.map → chunk-Z6L6YSU6.js.map} +0 -0
  1087. /package/dist/{run-campaign-RDGAM5KJ.js.map → run-campaign-WXY7KI67.js.map} +0 -0
@@ -1,1342 +1,39 @@
1
- import { A as AnalyzeTracesOptions, a as AnalyzeTracesInput, b as AnalyzeTracesResult } from '../analyst-C8HHvfJp.js';
2
- import { S as Scenario, M as MutableSurface, b as DispatchContext, a as JudgeConfig, I as ImprovementDriver, q as ProposeContext, m as GenerationRecord, G as Gate, J as JudgeScore, L as LabeledScenarioStore, r as LabeledScenarioWrite, s as LabeledScenarioSampleArgs, t as LabeledScenarioRecord, u as LabelTrust, v as LabeledScenarioSource, g as CampaignResult, i as CodeSurface } from '../types-BU-7W85F.js';
3
- export { C as CampaignAggregates, d as CampaignArtifactWriter, e as CampaignCellResult, f as CampaignCostMeter, w as CampaignTokenUsage, h as CampaignTraceWriter, D as DispatchFn, j as GateContext, c as GateDecision, k as GateResult, l as GenerationCandidate, x as JudgeAggregate, n as JudgeDimension, o as Mutator, O as OptimizerConfig, P as ParetoParent, y as ProposedCandidate, R as RedactionStatus, z as ScenarioAggregate, p as SessionScript, T as TraceSpan, A as isProposedCandidate, B as labelTrustRank } from '../types-BU-7W85F.js';
4
- import { b as RunCampaignOptions, c as RunImprovementLoopOptions, C as CampaignStorage } from '../run-improvement-loop-5z_l5zDz.js';
5
- export { e as GepaDriverConstraints, G as GepaDriverOptions, O as OpenAutoPrOptions, h as OpenAutoPrResult, a as RunImprovementLoopResult, R as RunOptimizationOptions, j as RunOptimizationResult, k as countSentenceEdits, l as defaultRenderDiff, m as extractH2Sections, f as fsCampaignStorage, g as gepaDriver, i as inMemoryCampaignStorage, o as openAutoPr, r as runCampaign, d as runImprovementLoop, n as runOptimization, s as surfaceHash } from '../run-improvement-loop-5z_l5zDz.js';
6
- export { A as AxisEvidence, a as AxisVerdict, B as BuildEvidenceVectorOptions, k as BuildLoopProvenanceArgs, D as DefaultProductionGateOptions, l as EmitLoopProvenanceArgs, m as EmitLoopProvenanceResult, E as EvidenceVector, b as EvolutionaryDriverOptions, H as HeldOutGateOptions, n as LoopProvenanceBackend, o as LoopProvenanceCandidate, L as LoopProvenanceRecord, O as ObjectiveSource, P as ParetoSignificanceGateOptions, c as PromotionObjective, d as PromotionPolicy, R as RunEvalOptions, e as buildEvidenceVector, q as buildLoopProvenanceRecord, f as composeGate, g as defaultProductionGate, s as emitLoopProvenance, h as evolutionaryDriver, i as heldOutGate, t as loopProvenanceSpans, p as paretoPolicy, j as paretoSignificanceGate, u as provenanceRecordPath, v as provenanceSpansPath, r as runEval, w as surfaceContentHash } from '../provenance-LnqRT0sS.js';
7
- import { L as LlmClientOptions } from '../llm-client-BeEcAokY.js';
8
- import { T as TraceAnalystKindSpec } from '../kind-factory-5b7xXXOr.js';
9
- import { c as AnalystFinding } from '../types-2VVIL04s.js';
10
- import { S as SignedManifest, A as AgentProfile, B as BackendIntegrityReport, C as CompletionRequirement, R as RuntimeEventLike, a as CompletionVerdict, P as ProducedState, b as CorrectnessChecker } from '../pre-registration-mAnCugl9.js';
11
- import { E as EProcessState, a as PairedBootstrapResult } from '../statistics-C7PozGrZ.js';
12
- import { A as AgentEvalError } from '../errors-CzMUYo7b.js';
13
- import { a as RunSplitTag, R as RunRecord } from '../run-record-e7vj1uZQ.js';
14
- import '@ax-llm/ax';
15
- import '../store-C1YxJDEK.js';
16
- import '../red-team-BXHil6c8.js';
17
- import '../dataset-BbGkaN2I.js';
18
- import '../store-CKUAgsJz.js';
19
- import '../schema-m0gsnbt3.js';
20
- import '../pareto-E-pembql.js';
21
- import '../hosted/index.js';
22
- import '../insight-report-BBwvOh6x.js';
23
- import '../summary-report-DGmUucwQ.js';
24
- import '../failure-cluster-CL7IVgkJ.js';
25
- import '../judge-calibration-0p2QcWNE.js';
26
- import '../raw-provider-sink-C46HDghv.js';
27
- import 'zod';
28
- import '../types-Croy5h7V.js';
29
- import '@tangle-network/tcloud';
30
- import '../verdict-C9MlYujm.js';
31
-
32
1
  /**
33
- * @experimental
34
- *
35
- * Make the trace-analyst's OWN prompt a GEPA-optimizable surface.
36
- *
37
- * The analyst that drives self-improvement is itself a prompt — and a
38
- * hand-tuned one (a hardcoded, hand-versioned `const`). This module lets the
39
- * loop optimize it: the analyst `actorDescription` becomes a `MutableSurface`
40
- * that `gepaDriver` / `haloDriver` / any `ImprovementDriver` can mutate inside
41
- * `runImprovementLoop` or `compareDrivers`. That is the second-order loop —
42
- * optimizing the optimizer's eyes, not just the agent's prompt.
43
- *
44
- * Two pieces, both deliberately small (the loop engine already exists this
45
- * only supplies the analyst-shaped dispatch + an objective scorer):
46
- *
47
- * - `buildAnalystSurfaceDispatch` `dispatchWithSurface(surface, scenario)`
48
- * runs `analyzeTraces` with `surface` as the actorDescription over the
49
- * scenario's fixed trace corpus, returning its findings.
50
- * - `failureModeRecallJudge` a DETERMINISTIC judge (no LLM, no opinion)
51
- * that scores those findings against the scenario's GROUND-TRUTH failure
52
- * modes. This is what keeps optimizing the analyst prompt ungameable: the
53
- * labels come from objective signal (e.g. AppWorld `world.evaluate()` tells
54
- * us which task failed and which API calls were wrong), so we reward an
55
- * analyst for surfacing the failures that really happened — not for
56
- * pleasing a judge that could be talked into anything (Goodhart).
57
- *
58
- * Wiring (the loop is unchanged; you only pass these in):
59
- *
60
- * const dispatchWithSurface = buildAnalystSurfaceDispatch({ analystOptions: { ai } })
61
- * await runImprovementLoop({
62
- * baselineSurface: TRACE_ANALYST_ACTOR_DESCRIPTION, // the prompt under optimization
63
- * scenarios: trainScenarios, // labeled trace corpora
64
- * holdoutScenarios: heldOutScenarios,
65
- * dispatchWithSurface,
66
- * judges: [failureModeRecallJudge()],
67
- * driver: gepaDriver({ baseUrl, apiKey }),
68
- * gate: heldOutGate({ minDelta: 0.02 }),
69
- * autoOnPromote: 'none',
70
- * })
71
- */
72
-
73
- /**
74
- * A labeled trace scenario: a FIXED trace corpus plus the failure modes a
75
- * competent analyst MUST surface from it. The labels are ground truth — the
76
- * objective failures that actually occurred — which is what makes optimizing
77
- * the analyst prompt against them meaningful rather than circular.
78
- */
79
- interface AnalystScenario extends Scenario {
80
- kind: 'analyst-surface';
81
- /** OTLP-JSONL path or an in-memory store of the traces to analyze. */
82
- source: AnalyzeTracesOptions['source'];
83
- /** The domain question handed to the analyst (framing lives here, not in
84
- * the surface under optimization). */
85
- question: string;
86
- /**
87
- * Ground-truth failure modes a good analyst must identify. A finding "hits"
88
- * a mode when it contains ANY of the mode's case-insensitive cues. Derive
89
- * these from objective signal (failed task + which step broke), never from
90
- * the analyst's own prior output.
91
- */
92
- expectedFailureModes: Array<{
93
- id: string;
94
- cues: string[];
95
- }>;
96
- /**
97
- * Cues that mark a finding as HALLUCINATED / out-of-scope for this corpus —
98
- * naming a tool, error, or failure that did not occur. Presence penalizes
99
- * precision. Optional; omit to score recall only.
100
- */
101
- forbiddenCues?: string[];
102
- }
103
- /** The analyst's output for one scenario — the artifact the judge scores. */
104
- interface AnalystArtifact {
105
- answer: string;
106
- findings: string[];
107
- /** The hardcoded-prompt version the analyst reported (provenance only; the
108
- * optimized surface overrides the actual prompt text used). */
109
- actorPromptVersion: string;
110
- }
111
- interface BuildAnalystSurfaceDispatchOptions {
112
- /**
113
- * Everything `analyzeTraces` needs EXCEPT `actorDescription` (supplied by the
114
- * surface under optimization) and `source` (supplied by the scenario). `ai`
115
- * (the AxAIService) is required for a live run.
116
- */
117
- analystOptions: Omit<AnalyzeTracesOptions, 'actorDescription' | 'source'>;
118
- /** Test seam: defaults to the real `analyzeTraces`. */
119
- analyze?: (input: AnalyzeTracesInput, options: AnalyzeTracesOptions) => Promise<AnalyzeTracesResult>;
120
- }
121
- /**
122
- * Build the `dispatchWithSurface(surface, scenario, ctx)` the improvement loop
123
- * calls: run the analyst with `surface` as its actorDescription over the
124
- * scenario's trace corpus and return its findings.
125
- */
126
- declare function buildAnalystSurfaceDispatch(opts: BuildAnalystSurfaceDispatchOptions): (surface: MutableSurface, scenario: AnalystScenario, ctx: DispatchContext) => Promise<AnalystArtifact>;
127
- interface FailureModeRecallJudgeOptions {
128
- /** Weight on recall when precision is also scored (forbiddenCues present).
129
- * Default 0.5 (equal). Recall-only when no forbiddenCues exist. */
130
- recallWeight?: number;
131
- }
132
- /**
133
- * Deterministic, ground-truth judge for analyst findings. Composite =
134
- * recall of the scenario's `expectedFailureModes` (optionally blended with a
135
- * precision term that penalizes findings tripping `forbiddenCues`). No LLM —
136
- * the score is a function of the labels, so the analyst prompt is optimized
137
- * toward surfacing real failures, not toward a judge it can flatter.
138
- */
139
- declare function failureModeRecallJudge(opts?: FailureModeRecallJudgeOptions): JudgeConfig<AnalystArtifact, AnalystScenario>;
140
-
141
- /**
142
- * @experimental
143
- *
144
- * `aceDriver` — Agentic Context Engineering: an APPEND-MOSTLY curator, the
145
- * deliberate contrast to `memoryCurationDriver`'s dedup-and-replace. ACE's
146
- * thesis (arXiv:2510.04618) is that aggressively deduping/rewriting a context
147
- * causes "context collapse" — hard-won specific lessons get summarized away. So
148
- * the playbook GROWS by appending each generation's new lessons as provenance-
149
- * tagged delta bullets; existing bullets are preserved verbatim, never merged.
150
- *
151
- * Each generation it:
152
- * 1. reads the playbook block already in the parent surface (verbatim);
153
- * 2. turns this generation's `findings` into lessons, keeping only the ones not
154
- * already present (idempotency — a recurring finding is not re-appended, but
155
- * a genuinely NEW lesson always is, even if similar to an old one);
156
- * 3. appends the new lessons as `- [gN] <lesson>` deltas and re-emits the block.
157
- *
158
- * Bounded WITHOUT collapse: when the playbook exceeds `maxEntries`, the OLDEST
159
- * deltas are evicted (FIFO) — recency is kept, but no two distinct lessons are
160
- * ever merged into one. Deterministic (no LLM) so a lift is attributable to the
161
- * accumulated lessons, not a rewrite's model noise.
162
- *
163
- * Fail-loud: with no new lesson this generation it returns NO candidate (the
164
- * playbook is unchanged — nothing to propose), never a fabricated bullet.
165
- */
166
-
167
- interface AceDriverOptions {
168
- /** Max delta bullets retained in the playbook. On overflow the OLDEST are
169
- * evicted (FIFO) — never merged. Default 50 (ACE keeps a long context). */
170
- maxEntries?: number;
171
- /** Heading rendered above the bullets inside the block. */
172
- sectionHeading?: string;
173
- }
174
- declare function aceDriver(opts?: AceDriverOptions): ImprovementDriver;
175
-
176
- /**
177
- * @experimental
178
- *
179
- * `haloDriver` — wraps the REAL halo-engine (Inference.net's hierarchical
180
- * agentic trace analyzer, `pip install halo-engine`, repo context-labs/halo)
181
- * as an agent-eval `ImprovementDriver`, so HALO competes head-to-head with
182
- * `gepaDriver` — and with our own `traceAnalystDriver` — inside `compareDrivers`
183
- * on identical traces / scenarios / held-out scoring.
184
- *
185
- * It PRESERVES halo's actual working usage — `analyze` shells out to the
186
- * published CLI (`halo <traces.jsonl> -p <prompt> -m <model>`) and uses its real
187
- * RLM findings verbatim. We do NOT reimplement its analysis; that would make the
188
- * benchmark meaningless. The materialize/apply pipeline is the shared
189
- * `analysisEditDriver` — identical to `traceAnalystDriver`, which is what makes
190
- * the comparison apples-to-apples.
191
- *
192
- * Fail-loud: no traces → throw; halo errors → throw; empty findings → throw.
193
- */
194
-
195
- interface HaloDriverOptions {
196
- /** OpenAI-compatible base URL for BOTH halo's RLM analysis and the apply
197
- * step (e.g. the Tangle router `https://router.tangle.tools/v1`). */
198
- baseUrl: string;
199
- /** Bearer key (else relies on OPENAI_API_KEY in the env halo inherits). */
200
- apiKey?: string;
201
- /** Model for halo's `--model` (its RLM). Default 'gpt-5.4-mini' (halo's own default). */
202
- model?: string;
203
- /** Model used to APPLY halo's findings to the prompt surface. Default = `model`. */
204
- applyModel?: string;
205
- /** The real halo binary. Default 'halo' (from `pip install halo-engine`). */
206
- haloBin?: string;
207
- /** Resolve the OTLP traces (JSONL string) halo should analyze for THIS
208
- * generation. Returning empty throws (halo has nothing to analyze). */
209
- resolveTraces: (ctx: ProposeContext) => string | Promise<string>;
210
- /** halo's analysis prompt (`-p`). Default targets the failure taxonomy. */
211
- analysisPrompt?: string;
212
- /** halo `--max-depth` / `--max-turns` passthrough. */
213
- maxDepth?: number;
214
- maxTurns?: number;
215
- /** Test seam: inject a fetch for the apply-step callLlm (no network in unit tests). */
216
- fetchImpl?: LlmClientOptions['fetch'];
217
- }
218
- /** Wrap the real halo-engine CLI as an ImprovementDriver (prompt-tier). */
219
- declare function haloDriver(opts: HaloDriverOptions): ImprovementDriver;
220
-
221
- /**
222
- * @experimental
223
- *
224
- * `memoryCurationDriver` — a CURATOR `ImprovementDriver`, the complement to the
225
- * OPTIMIZER drivers (`gepaDriver` rewrites the prompt; this one BUILDS a
226
- * searchable memory of what prior trajectories taught and grafts the most
227
- * relevant lessons onto the surface).
228
- *
229
- * Each generation it:
230
- * 1. collects lessons — this generation's trace-analyst `findings` PLUS the
231
- * memory already carried in the parent surface (so memory accumulates
232
- * across generations instead of resetting);
233
- * 2. curates them — normalizes, deduplicates near-identical lessons, and ranks
234
- * by recurrence (a lesson seen across many findings outranks a one-off);
235
- * 3. retrieves the top-K and writes them back as a single delimited memory
236
- * block in the surface (idempotent — the block is replaced, never stacked,
237
- * so the prompt does not grow without bound).
238
- *
239
- * This is the substrate behind the "knowledge base of working trajectories" the
240
- * agent searches: the curated block IS the retrieved memory the next run reads.
241
- * Curation is DETERMINISTIC (no LLM) so a lift it produces is attributable to
242
- * the lessons, not to model noise in a rewrite. An optional `distill` LLM step
243
- * can compress raw findings into crisp imperatives; default is verbatim.
244
- *
245
- * Fail-loud: never fabricates a lesson. With no findings and no prior memory it
246
- * returns no candidate (nothing learned yet — gen 0). It does not throw on an
247
- * empty generation because early generations legitimately have no findings.
248
- */
249
-
250
- interface MemoryCurationDriverOptions {
251
- /** Top-K lessons retained in the surface memory block. Default 12. */
252
- maxEntries?: number;
253
- /** Heading rendered above the lessons inside the block. Default below. */
254
- sectionHeading?: string;
255
- /**
256
- * Optional LLM distillation: compress raw findings into crisp, generalizable
257
- * one-line imperatives before curating. Omit for verbatim (deterministic).
258
- */
259
- distill?: {
260
- baseUrl: string;
261
- apiKey?: string;
262
- model: string;
263
- fetchImpl?: LlmClientOptions['fetch'];
264
- };
265
- }
266
- /** Build the CURATOR driver. */
267
- declare function memoryCurationDriver(opts?: MemoryCurationDriverOptions): ImprovementDriver;
268
-
269
- /**
270
- * @experimental
271
- *
272
- * SkillOpt patch primitives (Microsoft, arXiv:2605.23904 — "Executive
273
- * Strategy for Self-Evolving Agent Skills"). Where GEPA regenerates a surface
274
- * by reflection, SkillOpt emits BOUNDED, anchored edits to ONE skill document
275
- * — add / delete / replace — and accepts an edit only if it strictly improves
276
- * a held-out score. Bounded edits are the "textual learning rate": small,
277
- * reversible, and cheap to accept/reject, so a good rule introduced earlier is
278
- * not overwritten by a later sweeping rewrite.
279
- *
280
- * This module applies a patch deterministically and reports, per op, what
281
- * applied and what could not (a missing anchor is a rejected op, never a
282
- * silently dropped one). Pure, no I/O.
283
- */
284
- /** A single bounded edit against a skill surface.
285
- * - `add` — insert `text` after the first line containing `after`
286
- * (append to the end when `after` is absent/empty).
287
- * - `delete` — remove the first line containing `anchor`.
288
- * - `replace` — replace the first line containing `anchor` with `text`.
289
- * `text` may be multi-line; it is spliced in as multiple lines. Anchors match
290
- * the FIRST line that contains the substring (deterministic; SkillOpt is
291
- * expected to anchor on unique text). */
292
- type SkillPatchOp = {
293
- op: 'add';
294
- after?: string;
295
- text: string;
296
- } | {
297
- op: 'delete';
298
- anchor: string;
299
- } | {
300
- op: 'replace';
301
- anchor: string;
302
- text: string;
303
- };
304
- /** A named, attributable bundle of ops the optimizer proposes as one edit. */
305
- interface SkillPatch {
306
- label: string;
307
- rationale: string;
308
- ops: SkillPatchOp[];
309
- }
310
- interface SkillPatchRejection {
311
- op: SkillPatchOp;
312
- reason: string;
313
- }
314
- interface ApplySkillPatchResult {
315
- surface: string;
316
- /** Count of ops that mutated the surface. */
317
- applied: number;
318
- /** Ops that could not apply (unanchored / empty), with the reason. The
319
- * surface still reflects every APPLIED op — partial application is honest,
320
- * and the caller decides whether a partial patch is worth scoring. */
321
- rejected: SkillPatchRejection[];
322
- }
323
- /**
324
- * Apply a SkillOpt patch to a text surface. Ops apply in array order against
325
- * the evolving line buffer (an `add after X` followed by a `delete X` sees the
326
- * inserted lines). A missing anchor rejects only that op; the rest still apply.
327
- */
328
- declare function applySkillPatch(surface: string, patch: SkillPatch): ApplySkillPatchResult;
329
- /** Total ops in a patch — the edit-budget axis (SkillOpt's "textual learning
330
- * rate" caps this per epoch). */
331
- declare function patchEditCount(patch: SkillPatch): number;
332
-
333
- /**
334
- * @experimental
335
- *
336
- * `skillOptDriver` — a patch-mode `ImprovementDriver` implementing SkillOpt
337
- * (Microsoft, arXiv:2605.23904). Where `gepaDriver` regenerates the whole
338
- * surface by reflection, SkillOpt proposes BOUNDED, anchored edits
339
- * (add/delete/replace) to ONE skill document, so a good rule introduced
340
- * earlier is not clobbered by a later sweeping rewrite. The edit budget is the
341
- * paper's "textual learning rate"; a rejected-edit buffer + a slow-update
342
- * meta-note steer the optimizer away from dead ends.
343
- *
344
- * This module is the PROPOSER — the LLM call that turns evidence into
345
- * structured patches. The accept-only-if-held-out-improves loop, the budget
346
- * annealing, and the rejected buffer live in the `runSkillOpt` preset, which
347
- * owns the epoch hill-climb. The driver also conforms to `ImprovementDriver`
348
- * (`propose` applies its patches to the current surface and returns the
349
- * candidate surfaces) so it is a drop-in for `runOptimization` and a fair
350
- * entrant in `compareDrivers`.
351
- */
352
-
353
- /** Evidence the optimizer reflects on: where the current surface is weakest.
354
- * Computed by the caller (the preset uses a TRAIN campaign so proposals never
355
- * see the held-out split; the generic loop derives it from history). */
356
- interface SkillOptEvidence {
357
- /** Lowest-scoring scenarios (drives WHICH behavior to patch). */
358
- weakScenarios: Array<{
359
- scenarioId: string;
360
- composite: number;
361
- }>;
362
- /** Lowest-scoring judge dimensions (drives WHAT to patch for). */
363
- weakDimensions: Array<{
364
- dimension: string;
365
- score: number;
366
- }>;
367
- }
368
- /** A patch that was tried and not accepted — fed back to the model so it does
369
- * not re-propose a dead end (SkillOpt's rejected-edit buffer). */
370
- interface RejectedEdit {
371
- label: string;
372
- rationale: string;
373
- reason: string;
374
- }
375
- interface ProposePatchesArgs {
376
- surface: string;
377
- evidence: SkillOptEvidence;
378
- /** Max ops per patch this round (the annealed textual learning rate). */
379
- editBudget: number;
380
- rejectedBuffer: RejectedEdit[];
381
- /** Slow-update meta guidance accumulated across epochs. */
382
- metaNote?: string;
383
- /** Analyst findings + research report rendered as a prompt block (the
384
- * EYES→HANDS wire) so a patch targets a NAMED diagnosed root cause. Built by
385
- * the driver from `ctx.findings`/`ctx.report`; the patch-native `runSkillOpt`
386
- * path may also supply it. */
387
- findingsNote?: string;
388
- /** How many candidate patches to propose. */
389
- count: number;
390
- signal: AbortSignal;
391
- }
392
- interface SkillOptDriverOptions {
393
- llm: LlmClientOptions;
394
- model: string;
395
- /** What the skill document governs — orients the prompt. */
396
- target: string;
397
- /** Default ops-per-patch cap when used as a bare `ImprovementDriver`. The
398
- * `runSkillOpt` preset overrides this per epoch as it anneals. Default 3. */
399
- editBudget?: number;
400
- temperature?: number;
401
- maxTokens?: number;
402
- /** Top-K weak scenarios/dimensions surfaced as evidence. Default 3. */
403
- evidenceK?: number;
404
- }
405
- interface SkillOptDriver extends ImprovementDriver {
406
- /** Patch-native path used by `runSkillOpt` (the SkillOpt epoch loop owns
407
- * acceptance/budget/buffer). Returns structured patches, NOT surfaces. */
408
- proposePatches(args: ProposePatchesArgs): Promise<SkillPatch[]>;
409
- }
410
- declare function skillOptDriver(opts: SkillOptDriverOptions): SkillOptDriver;
411
- /** Parse + validate the patch response. Throws `SkillPatchParseError` when the
412
- * response is not valid JSON at all (a router/model failure the caller must
413
- * see — never a silent no-op epoch). Returns `[]` only for the legitimate
414
- * "valid JSON, zero usable patches" case. Malformed ops within a patch are
415
- * dropped (not silently mutated); each patch is truncated to the edit budget. */
416
- declare class SkillPatchParseError extends Error {
417
- constructor(message: string);
418
- }
419
- declare function parseSkillPatchResponse(raw: string, maxPatches: number, editBudget: number): SkillPatch[];
420
-
421
- /**
422
- * @experimental
423
- *
424
- * `traceAnalystDriver` — wraps agent-eval's OWN trace-analyst engine
425
- * (`AnalystRegistry` over the agentic OTLP reader) as an `ImprovementDriver`.
426
- * It is the symmetric opponent to `haloDriver`: both run the SAME shared
427
- * `analysisEditDriver` pipeline (materialize identical traces → apply via one
428
- * identical LLM edit), so a `compareDrivers` lift delta isolates a single
429
- * variable — ANALYSIS QUALITY. The benchmark answers "is our HALO clone as good
430
- * as the real HALO?" as a held-out lift CI, not a vibe.
431
- *
432
- * Findings come from the REGISTRY (structured `AnalystFinding[]` carrying
433
- * area / severity / recommended_action), rendered into the report the shared
434
- * apply step consumes.
435
- *
436
- * Fail-loud: no traces → throw; analyst run errors → throw; zero findings →
437
- * throw. Never fabricate a candidate.
438
- */
439
-
440
- interface TraceAnalystDriverOptions {
441
- /** OpenAI-compatible base URL for BOTH the analyst's agentic reads and the
442
- * apply step (e.g. `https://api.deepseek.com/v1` or the Tangle router). */
443
- baseUrl: string;
444
- /** Bearer key. Required — the Ax AI service has no env fallback here. */
445
- apiKey: string;
446
- /** Model the analyst kinds use for their agentic trace reads. */
447
- model: string;
448
- /** Model used to APPLY findings to the prompt surface. Default = `model`.
449
- * Keep this EQUAL to haloDriver's `applyModel` for an apples-to-apples run. */
450
- applyModel?: string;
451
- /** Ax provider name. Default 'openai' — works for any OpenAI-compatible base
452
- * via `apiURL`. Use 'deepseek' to hit DeepSeek's native provider. */
453
- provider?: string;
454
- /** Which analyst kinds to run. Default = the full shipped suite. */
455
- kinds?: readonly TraceAnalystKindSpec[];
456
- /** Resolve the OTLP traces (JSONL string) the analyst should read for THIS
457
- * generation — identical contract to `haloDriver.resolveTraces`. */
458
- resolveTraces: (ctx: ProposeContext) => string | Promise<string>;
459
- /** Override the findings producer. Default: the shipped `AnalystRegistry`
460
- * over `kinds`. The unit suite injects canned findings here. */
461
- analyze?: (tracePath: string, ctx: ProposeContext) => Promise<ReadonlyArray<AnalystFinding>>;
462
- /** Test seam: inject a fetch for the apply-step `callLlm`. */
463
- fetchImpl?: LlmClientOptions['fetch'];
464
- }
465
- /** Wrap agent-eval's trace-analyst registry as an ImprovementDriver (prompt-tier). */
466
- declare function traceAnalystDriver(opts: TraceAnalystDriverOptions): ImprovementDriver;
467
-
468
- /**
469
- * @experimental
470
- *
471
- * Anytime-valid sequential promotion gate — an e-process (betting
472
- * test-martingale, see `eProcess` in `statistics.ts`) over paired
473
- * per-scenario deltas, so a campaign stops the MOMENT evidence decides
474
- * instead of burning a fixed-n budget. Decisions remain valid at any
475
- * data-dependent stopping time (Ville's inequality), which is exactly what
476
- * fixed-n machinery cannot offer: peeking at a bootstrap CI after every
477
- * observation and stopping on the first significant peek inflates type-I
478
- * error far beyond alpha.
479
- *
480
- * REPLACES, never layers on, a fixed-n gate. Running `heldoutSignificance`
481
- * or `paretoSignificanceGate` repeatedly on a growing sample and stopping
482
- * early is optional stopping no matter how it is dressed up; this gate is
483
- * the valid way to stop early. Use one or the other per evidence stream.
484
- *
485
- * Pre-registration binding: anytime validity holds only for the
486
- * PRE-REGISTERED statistic. When a `SignedManifest` is bound, the gate takes
487
- * alpha from `manifest.alpha`, the observation budget from
488
- * `manifest.preRegisteredN`, orients deltas by `manifest.direction`, and
489
- * shifts the null boundary by `manifest.minEffect` — re-deciding the same
490
- * stream under different parameters after seeing data would reopen optional
491
- * stopping under a fancier name. The manifest's content hash is verified at
492
- * construction (sync, same `sha256-content` scheme as `signManifest`).
493
- *
494
- * Non-iid caveat (stated honestly): the supermartingale guarantee needs each
495
- * delta's conditional mean under H0 to stay ≤ the null boundary given the
496
- * past — exchangeable scenario deltas suffice. Scenario streams ordered by
497
- * difficulty or by scenario family violate this; `decide(ctx)` therefore
498
- * shuffles the paired deltas with a SEEDED permutation by default (the
499
- * permutation is data-independent, so bet predictability is preserved).
500
- * Stratified betting (per-stratum λ) is future work, not implemented here.
501
- */
502
-
503
- type SequentialDecision = 'promote' | 'continue' | 'undecided-at-maxN';
504
- interface SequentialObservation {
505
- decision: SequentialDecision;
506
- /** Current e-value (the betting wealth) against H0. */
507
- eValue: number;
508
- /** Paired deltas consumed so far. */
509
- n: number;
510
- /** Names the decision basis. For 'undecided-at-maxN' it states explicitly
511
- * that exhausting the budget is NOT evidence of no effect. */
512
- reason: string;
513
- }
514
- interface SequentialPairedGateOptions {
515
- /** Type-I budget. With `preRegistration` bound this MUST match
516
- * `manifest.alpha` (conflict throws). Default 0.05. */
517
- alpha?: number;
518
- /** Minimum paired deltas before a promote may fire. The stopping rule is
519
- * "first n ≥ minN with e-value ≥ 1/alpha" — still a valid stopping time.
520
- * Default 5. */
521
- minN?: number;
522
- /** Pre-registered observation budget. Required unless `preRegistration`
523
- * supplies it via `preRegisteredN` (conflict throws). */
524
- maxN?: number;
525
- /** Bet truncation forwarded to `eProcess`. Default 0.5. */
526
- maxBet?: number;
527
- /** Bound on |delta| in the judge's native scale; deltas are mapped to
528
- * x = (d/scale + 1)/2 ∈ [0,1]. A delta outside ±scale throws (use
529
- * `detectScale` to pick 1 vs 100 BEFORE streaming). Default 1. */
530
- scale?: number;
531
- /** Seed for the data-independent shuffle of paired deltas in `decide(ctx)`
532
- * (exchangeability guard). Default 1337. */
533
- shuffleSeed?: number;
534
- /** Bind the pre-registered hypothesis. Verified (content hash) at
535
- * construction; alpha/maxN/direction/minEffect come FROM the manifest. */
536
- preRegistration?: SignedManifest;
537
- /** Override the gate name in reports. */
538
- name?: string;
539
- }
540
- interface SequentialPairedGate<TArtifact = unknown, TScenario extends Scenario = Scenario> extends Gate<TArtifact, TScenario> {
541
- /** Streaming entry point: feed one paired per-scenario delta
542
- * (candidate − baseline, native scale). Each gate instance carries ONE
543
- * observe-stream; `decide(ctx)` runs on its own fresh stream and never
544
- * consumes or advances this one. 'promote' is sticky; observing past the
545
- * pre-registered maxN throws (extending a finished stream after seeing
546
- * the result reopens optional stopping — start a NEW pre-registered
547
- * test). */
548
- observe(delta: number): SequentialObservation;
549
- /** Read-only snapshot of the observe-stream. */
550
- state(): EProcessState & {
551
- decision: SequentialDecision;
552
- };
553
- }
554
- /**
555
- * Anytime-valid sequential paired gate. Conforms to the existing `Gate`
556
- * contract (`decide(ctx)` consumes candidate vs baseline judge scores via
557
- * `pairHoldout` — same pairing granularity as the fixed-n gates: full cellId,
558
- * never scenarioId) and adds a streaming `observe(delta)` entry for campaigns
559
- * that score cells incrementally and want to stop mid-stream.
560
- *
561
- * Decision mapping onto the substrate's five-valued `GateDecision`:
562
- * - 'promote' → 'ship'
563
- * - 'continue' → 'need_more_work' (stream ended before maxN with
564
- * the e-value undecided — more reps could decide)
565
- * - 'undecided-at-maxN' → 'hold', with the reason stating it is NOT
566
- * evidence of no effect (never a silent default)
567
- */
568
- declare function sequentialPairedGate<TArtifact = unknown, TScenario extends Scenario = Scenario>(options: SequentialPairedGateOptions): SequentialPairedGate<TArtifact, TScenario>;
569
- interface SequentialDecideOptions {
570
- /** Type-I budget for the early-stop evidence. Default 0.05. */
571
- alpha?: number;
572
- /** Minimum paired deltas before a stop may fire. Default 5. */
573
- minN?: number;
574
- /** Bet truncation forwarded to `eProcess`. Default 0.5. */
575
- maxBet?: number;
576
- /** Bound on |per-scenario composite delta|. Default 1. */
577
- scale?: number;
578
- }
579
- interface SequentialDecideFn {
580
- (args: {
581
- history: GenerationRecord[];
582
- }): {
583
- stop: boolean;
584
- reason?: string;
585
- };
586
- /** Read-only snapshot of the accumulated e-process (observability + tests). */
587
- state(): EProcessState;
588
- }
589
- /**
590
- * `ImprovementDriver.decide` adapter — stops the optimization loop the moment
591
- * the e-process decides the loop has produced a real improvement, instead of
592
- * always running `maxGenerations`.
593
- *
594
- * Stream: for each generation g ≥ 1, the per-scenario composite deltas of
595
- * generation g's top candidate vs the generation-0 top candidate (the
596
- * incumbent the loop set out to beat), paired by scenarioId. H0: no proposed
597
- * surface improves any scenario's expected composite over the incumbent —
598
- * under it every delta has conditional mean ≤ 0 and the e-process is valid.
599
- * Once wealth ≥ 1/alpha the loop stops and hands the winner to the promotion
600
- * gate (which re-scores on HELD-OUT data — this adapter only spends the
601
- * exploration budget, it never promotes).
602
- *
603
- * Honesty caveats: (1) the incumbent's scores are measured once and shared
604
- * across all generations' deltas, so type-I control is exact only insofar as
605
- * those scores approximate the incumbent's true per-scenario means (more reps
606
- * → tighter); (2) an UNDECIDED process never stops the loop — absence of a
607
- * crossing is NOT evidence of no effect, so the loop simply runs its normal
608
- * course. Calling the adapter repeatedly with a growing history consumes each
609
- * generation exactly once (re-feeding an already-seen record would double-count
610
- * evidence).
611
- */
612
- declare function sequentialDecide(options?: SequentialDecideOptions): SequentialDecideFn;
613
-
614
- /**
615
- * @experimental
616
- *
617
- * Statistical held-out promotion machinery — the trustworthy core the
618
- * point-estimate `heldout-delta` gate lacked.
619
- *
620
- * The shipped false positive it prevents: a winner re-scored against the
621
- * baseline on the holdout read run-to-run model NOISE (e.g. 91 vs 95) as a
622
- * "+4 lift" and shipped, because the gate compared point estimates with no
623
- * confidence interval. Here we pair candidate vs baseline holdout observations
624
- * and bootstrap a CI on the paired delta — a candidate ships only when the CI
625
- * lower bound clears the effect-size threshold (the gain is real at the
626
- * confidence level, not noise), and is blocked when a critical dimension
627
- * (e.g. `hallucination_free` for a legal agent) significantly regresses even if
628
- * the net composite rose (anti-Goodhart).
629
- *
630
- * Two traps this module is built around (both produce a NEW false positive if
631
- * gotten wrong):
632
- * 1. PAIRING GRANULARITY — pairs by FULL `cellId` (`scenario:rep`), never by
633
- * `scenarioId` (which averages reps away and destroys the within-pair
634
- * variance reduction that makes a paired bootstrap tighter than unpaired).
635
- * One paired observation per cell ⇒ reps multiply n.
636
- * 2. SCALE — a judge may emit composites/dimensions on [0,1] or 0-100. The
637
- * threshold + tolerance are interpreted in the judge's NATIVE scale; the
638
- * per-dimension tolerance auto-scales off the observed baseline magnitudes
639
- * so `-0.10` on [0,1] doesn't silently become a no-op on a 0-100 dimension.
640
- */
641
-
642
- interface PairedHoldout {
643
- /** Baseline scalar per paired cell (same order as `after`/`cellIds`). */
644
- before: number[];
645
- /** Candidate scalar per paired cell. */
646
- after: number[];
647
- /** The full cellIds (`scenario:rep`) that paired, in order. */
648
- cellIds: string[];
649
- }
650
- /**
651
- * Pair candidate vs baseline holdout observations by FULL cellId. `select`
652
- * pulls the scalar from a cell's judge reports (composite, or a named
653
- * dimension); a cell contributes the mean of `select` across its judges. Cells
654
- * whose scenario is not in `scenarioIds`, or where `select` is undefined for
655
- * every judge on either side, are skipped on BOTH sides so the arrays stay
656
- * paired. Throws when the two maps disagree on which holdout cells exist — a
657
- * load-bearing invariant: the baseline + winner holdout campaigns run the same
658
- * scenarios with the same seed base, so their cellIds MUST align; a mismatch
659
- * means a silent pairing bug, not a soft fallback.
660
- */
661
- declare function pairHoldout(candidate: Map<string, Record<string, JudgeScore>>, baseline: Map<string, Record<string, JudgeScore>>, scenarioIds: Set<string>, select: (s: JudgeScore) => number | undefined): PairedHoldout;
662
- interface HeldoutSignificance {
663
- paired: PairedHoldout;
664
- bootstrap: PairedBootstrapResult;
665
- /** n paired observations. */
666
- n: number;
667
- /** True iff n >= minProductiveRuns AND the CI lower bound clears the threshold. */
668
- significant: boolean;
669
- /** Set when n < minProductiveRuns — too little evidence to claim significance. */
670
- fewRuns: boolean;
671
- }
672
- interface HeldoutSignificanceOptions {
673
- deltaThreshold?: number;
674
- minProductiveRuns?: number;
675
- confidence?: number;
676
- resamples?: number;
677
- /** Fixed by default for a deterministic, reproducible gate verdict. */
678
- seed?: number;
679
- statistic?: 'mean' | 'median';
680
- }
681
- /** Significance of the held-out composite lift: ship only when the paired
682
- * bootstrap CI lower bound on (candidate − baseline) exceeds `deltaThreshold`
683
- * (default 0 ⇒ "confidently positive"). Below `minProductiveRuns` paired
684
- * observations there is not enough evidence to claim significance → not
685
- * significant (`fewRuns`). Interpret `deltaThreshold` in the judge's native
686
- * composite scale. */
687
- declare function heldoutSignificance(paired: PairedHoldout, opts?: HeldoutSignificanceOptions): HeldoutSignificance;
688
- interface DimensionRegression {
689
- dimension: string;
690
- bootstrap: PairedBootstrapResult;
691
- /** True iff the CI lower bound on (candidate − baseline) is below −tolerance:
692
- * the candidate may have regressed this dimension by more than tolerance. */
693
- regressed: boolean;
694
- tolerance: number;
695
- n: number;
696
- }
697
- /** Detect the native scale of a set of scores: 0-100 when any magnitude clears
698
- * 1.5, else [0,1]. Used to auto-scale the regression tolerance so a default
699
- * expressed for [0,1] is not silently a no-op on a 0-100 dimension. */
700
- declare function detectScale(values: number[]): 1 | 100;
701
- /** Per-critical-dimension regression guard. For each dimension, pair the
702
- * candidate vs baseline values by full cellId and bootstrap the paired delta;
703
- * a dimension is "regressed" when the CI lower bound < −tolerance (conservative
704
- * — blocks if the credible worst case exceeds tolerance, which is the right
705
- * posture for safety dimensions like `hallucination_free`). When `tolerance`
706
- * is omitted it auto-scales: 0.05 on [0,1], 5 on 0-100. */
707
- declare function dimensionRegressions(candidate: Map<string, Record<string, JudgeScore>>, baseline: Map<string, Record<string, JudgeScore>>, scenarioIds: Set<string>, criticalDimensions: string[], opts?: {
708
- tolerance?: number;
709
- confidence?: number;
710
- resamples?: number;
711
- seed?: number;
712
- }): DimensionRegression[];
713
-
714
- /**
715
- * @experimental
716
- *
717
- * Filesystem `LabeledScenarioStore` adapter. The default capture sink for
718
- * traces + eval artifacts. Production deployments typically swap for a
719
- * Turso/SQLite adapter (same interface).
720
- *
721
- * Records land as one JSONL file per source under `<root>/<source>.jsonl`.
722
- * Each line is a `LabeledScenarioRecord`. Append-only — no in-place edits.
723
- *
724
- * Safety properties enforced at write-time:
725
- *
726
- * - **Provenance required**: writes without `source`, `sourceVersionHash`,
727
- * `capturedAt`, `redactionStatus` are rejected. Closes the alignment
728
- * reviewer's data-poisoning gap.
729
- * - **Per-source rate limits**: optional `rateLimitBucket` + `maxWritesPerMinute`
730
- * stops a single tenant/source from flooding the store.
731
- *
732
- * Safety properties enforced at sample-time:
733
- *
734
- * - **Required split + capturedBefore**: substrate refuses to sample without
735
- * an explicit `split` ('train' | 'test') AND a temporal cutoff. Eliminates
736
- * accidental train/test contamination.
737
- * - **Default training-source filter**: when the store is sampled with
738
- * `split: 'train'`, production-trace records are EXCLUDED unless the
739
- * caller passes `filter.source: 'production-trace'` explicitly. Closes
740
- * the contamination-by-default gap flagged by the senior eval engineer.
741
- */
742
-
743
- interface FsLabeledScenarioStoreOptions {
744
- /** Root directory for JSONL files. Created if missing. */
745
- root: string;
746
- /** Per-source rate limit. When set, writes exceeding the cap are rejected
747
- * with a typed error. Default: no limit. */
748
- maxWritesPerMinutePerBucket?: number;
749
- /** Test seam — override `Date.now()` for deterministic tests. */
750
- now?: () => number;
751
- }
752
- declare class LabeledScenarioStoreError extends Error {
753
- readonly code: string;
754
- constructor(code: string, message: string);
755
- }
756
- declare class FsLabeledScenarioStore implements LabeledScenarioStore {
757
- private readonly options;
758
- private readonly now;
759
- private readonly rateLimits;
760
- constructor(options: FsLabeledScenarioStoreOptions);
761
- observe(write: LabeledScenarioWrite): Promise<void>;
762
- sample(args: LabeledScenarioSampleArgs): Promise<LabeledScenarioRecord[]>;
763
- size(): Promise<{
764
- train: number;
765
- test: number;
766
- bySource: Record<string, number>;
767
- byTrust: Record<LabelTrust, number>;
768
- }>;
769
- private assertProvenance;
770
- private assertRateLimit;
771
- private toRecord;
772
- private pathForSource;
773
- }
774
-
775
- /**
776
- * @experimental
777
- *
778
- * `compareDrivers` — a head-to-head lift benchmark across optimizer drivers on
779
- * ONE corpus. This is the forcing function: optimizer quality (GEPA reflection
780
- * vs GEPA+Pareto vs SkillOpt) becomes a NUMBER with a confidence interval, so a
781
- * driver regression — or shipping a simplified driver and calling it the real
782
- * one — turns a build red instead of going measurement-invisible.
783
- *
784
- * Every entrant is scored the SAME way: each driver returns the surface it
785
- * promoted, then `compareDrivers` scores the baseline + every winner on the
786
- * SAME held-out scenarios with the SAME judges. Apples-to-apples by
787
- * construction — the comparison never depends on how a driver measured itself.
788
- * The per-scenario held-out composites feed a paired bootstrap (`statistics.ts`)
789
- * for each driver's lift CI and for the pairwise "which driver wins" CI.
790
- */
791
-
792
- /** What an optimizer produced: the surface it promoted + what it cost to get
793
- * there. `compareDrivers` does the held-out scoring itself, so an entry only
794
- * needs to run its loop and hand back the winner. */
795
- interface DriverEntry {
796
- name: string;
797
- optimize: () => Promise<{
798
- winnerSurface: MutableSurface;
799
- costUsd: number;
800
- durationMs?: number;
801
- }>;
802
- }
803
- interface DriverScore {
804
- name: string;
805
- /** Mean held-out composite of the baseline (identical across drivers). */
806
- baselineComposite: number;
807
- /** Mean held-out composite of this driver's promoted surface. */
808
- winnerComposite: number;
809
- /** Mean per-scenario held-out lift (winner − baseline). */
810
- lift: number;
811
- /** Paired-bootstrap CI of the per-scenario lift. `low > 0` ⇒ a real gain. */
812
- liftCi: {
813
- low: number;
814
- high: number;
815
- };
816
- costUsd: number;
817
- durationMs?: number;
818
- winnerSurface: MutableSurface;
819
- /** 1-based, by descending lift. */
820
- rank: number;
821
- }
822
- interface DriverPairwise {
823
- /** Higher-ranked driver. */
824
- a: string;
825
- b: string;
826
- /** Mean per-scenario held-out delta (a − b). */
827
- deltaMean: number;
828
- low: number;
829
- high: number;
830
- /** `a` if the CI clears 0, `b` if it is entirely negative, else `'tie'`. */
831
- favored: string;
832
- }
833
- interface DriverComparison {
834
- /** Sorted by descending lift; `rank` set accordingly. */
835
- scores: DriverScore[];
836
- best: DriverScore;
837
- /** Best vs each other driver, paired-bootstrap on the held-out winners. */
838
- pairwise: DriverPairwise[];
839
- holdoutScenarioIds: string[];
840
- }
841
- interface CompareDriversOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'dispatch' | 'scenarios'> {
842
- drivers: DriverEntry[];
843
- baselineSurface: MutableSurface;
844
- /** The held-out scenarios every winner is scored on. */
845
- holdoutScenarios: TScenario[];
846
- /** Scores a surface on a scenario — the same dispatcher the drivers used. */
847
- dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
848
- /** Bootstrap resamples for the lift CIs. Default 2000. */
849
- resamples?: number;
850
- /** CI confidence. Default 0.95. */
851
- confidence?: number;
852
- }
853
- declare function compareDrivers<TScenario extends Scenario, TArtifact>(opts: CompareDriversOptions<TScenario, TArtifact>): Promise<DriverComparison>;
854
- /** Shared corpus + transport for the three built-in optimizer entries. */
855
- interface OptimizerEntryConfig<TScenario extends Scenario, TArtifact> {
856
- baselineSurface: string;
857
- /** Training scenarios the drivers reflect on. */
858
- trainScenarios: TScenario[];
859
- /** Held-out scenarios (the gate axis + the benchmark scoring axis). */
860
- holdoutScenarios: TScenario[];
861
- dispatchWithSurface: (surface: MutableSurface, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
862
- judges: JudgeConfig<TArtifact, TScenario>[];
863
- llm: LlmClientOptions;
864
- model: string;
865
- target: string;
866
- runDir: string;
867
- seed?: number;
868
- /** GEPA population per generation. Default 2. */
869
- populationSize?: number;
870
- /** GEPA generations. Default 3. */
871
- maxGenerations?: number;
872
- /** SkillOpt epochs. Default 6. */
873
- maxEpochs?: number;
874
- mutationPrimitives?: string[];
875
- /** Static findings seed forwarded to each GEPA driver's `propose()` as
876
- * `ctx.findings` (the EYES→HANDS wire). Forwarded by `gepaReflectionEntry` /
877
- * `gepaParetoEntry`; `skillOptEntry` runs findings-BLIND (see its doc). */
878
- findings?: unknown[];
879
- /** Per-generation findings producer (EYES→HANDS loop closure): after each
880
- * generation scores, this re-diagnoses and REPLACES `ctx.findings` for the
881
- * next generation's `propose()`. Reuses the `runOptimization` field type so
882
- * it cannot drift. GEPA entries only. */
883
- analyzeGeneration?: RunImprovementLoopOptions<TScenario, TArtifact>['analyzeGeneration'];
884
- /** Phase-2 research report forwarded to `propose()` as `ctx.report`. */
885
- report?: unknown;
886
- }
887
- /** GEPA, reflection-only (single-parent, no Pareto combine). */
888
- declare function gepaReflectionEntry<TScenario extends Scenario, TArtifact>(config: OptimizerEntryConfig<TScenario, TArtifact>, name?: string): DriverEntry;
889
- /** GEPA with the Pareto frontier + combine-complementary-lessons. */
890
- declare function gepaParetoEntry<TScenario extends Scenario, TArtifact>(config: OptimizerEntryConfig<TScenario, TArtifact>, name?: string): DriverEntry;
891
- /** SkillOpt patch-mode hill-climb. Runs findings-BLIND: `runSkillOpt` owns its
892
- * own epoch acceptance/budget loop and does not thread `analyzeGeneration`, so
893
- * `config.findings` is intentionally NOT forwarded here. In a findings-fed
894
- * comparison this entry is the blind control — do not read its result as
895
- * findings-fed. (Threading findings into the SkillOpt epoch loop is a separate
896
- * refactor, deferred not faked.) */
897
- declare function skillOptEntry<TScenario extends Scenario, TArtifact>(config: OptimizerEntryConfig<TScenario, TArtifact>, name?: string): DriverEntry;
898
-
899
- /**
900
- * @experimental
901
- *
902
- * `runProfileMatrix` — the missing keystone between `runAgentMatrix` and the
903
- * backend-integrity guard.
904
- *
905
- * The gap it closes: `runAgentMatrix` is a topology-opaque scheduler whose
906
- * cells return a bare `{ output, verdict, costUsd }` — no `tokenUsage`, not a
907
- * `RunRecord`. `assertRealBackend` / `summarizeBackendIntegrity` key on
908
- * `RunRecord.tokenUsage`, so they cannot run on a raw matrix result. Every
909
- * consumer therefore hand-writes the same bridge: fan a profile × scenario
910
- * cartesian, call dispatch, fabricate a `RunRecord` with token usage, thread it
911
- * back, run the integrity guard. That hand-rolled bridge is exactly the pile of
912
- * bespoke `eval:*` scripts the adoption skills keep trying (and failing) to
913
- * forbid.
914
- *
915
- * `runProfileMatrix` IS that bridge, once:
916
- *
917
- * - axis 3 (PROFILE) = `profiles: AgentProfile[]`
918
- * - axis 1 (PERSONA/SCENARIO) = `scenarios: Scenario[]` (each scenario carries
919
- * its persona; `personaOf` groups them for the `byPersona` pivot)
920
- * - the scoring axis = `judges`
921
- *
922
- * It runs `runCampaign` once per profile (reusing its seeds, reps, bootstrap
923
- * CIs, resumability, and the `LabeledScenarioStore` capture flywheel), maps
924
- * every cell to a validated `RunRecord` carrying the real `tokenUsage` the
925
- * dispatch reported via `ctx.cost.observeTokens`, and runs `assertRealBackend`
926
- * BY CONSTRUCTION before returning — so a stub-backend run fails loudly instead
927
- * of reporting a clean 0/N leaderboard.
928
- *
929
- * Dispatch contract: a dispatch that calls an LLM MUST report usage via
930
- * `ctx.cost.observeTokens({ input, output })` (and cost via `ctx.cost.observe`).
931
- * A dispatch that reports zero tokens is indistinguishable from a stub and the
932
- * integrity guard treats it as one.
933
- */
934
-
935
- /** Thrown when the matrix is misconfigured (no profiles, a profile whose model
936
- * lacks a snapshot version, etc.). Distinct from `BackendIntegrityError`,
937
- * which signals a stub backend at run time. */
938
- declare class ProfileMatrixError extends AgentEvalError {
939
- constructor(message: string);
940
- }
941
- /** Dispatch for one cell: render `profile` against `scenario`, returning the
942
- * artifact the judges score. Report LLM usage via `ctx.cost.observeTokens`
943
- * and `ctx.cost.observe` — the integrity guard depends on it. */
944
- type ProfileDispatchFn<TScenario extends Scenario, TArtifact> = (profile: AgentProfile, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
945
- interface RunProfileMatrixOptions<TScenario extends Scenario, TArtifact> {
946
- /** Axis 3 — the agent-under-test configurations. Each is one column. */
947
- profiles: AgentProfile[];
948
- /** Axis 1 — the persona/scenario corpus, run against every profile. */
949
- scenarios: TScenario[];
950
- /** Renders one (profile, scenario) cell. */
951
- dispatch: ProfileDispatchFn<TScenario, TArtifact>;
952
- /** The scoring axis. */
953
- judges?: JudgeConfig<TArtifact, TScenario>[];
954
- /** Where each profile's campaign writes artifacts/traces. One subdir per
955
- * profile. */
956
- runDir: string;
957
- /** Git SHA the harness ran from — stamped onto every RunRecord (mandatory
958
- * for paper-grade records). */
959
- commitSha: string;
960
- /** Logical experiment id shared across the whole matrix so the promotion
961
- * gate can pair profiles on matched scenarios. Default: a hash of the
962
- * profile + scenario ids. */
963
- experimentId?: string;
964
- /** Which split these runs belong to. Default `'search'`. */
965
- splitTag?: RunSplitTag;
966
- /** Replicates per (profile, scenario) cell for CI bands. Default 1. */
967
- reps?: number;
968
- /** Campaign seed (per profile). Default 42. */
969
- seed?: number;
970
- /**
971
- * Backend-integrity posture, enforced AFTER the matrix completes:
972
- * - `'assert'` (default) — throw `BackendIntegrityError` if the run was a
973
- * stub (and, with `allowMixed:false`, if it was mixed).
974
- * - `'warn'` — log the verdict but never throw.
975
- * - `'off'` — skip the guard entirely (only for offline/replay analysis).
976
- */
977
- integrity?: 'assert' | 'warn' | 'off';
978
- /** Forwarded to `assertRealBackend`. Default true (tolerate partial 429
979
- * cascades); set false for strict CI gates. */
980
- allowMixed?: boolean;
981
- /** Max concurrent cells WITHIN each profile's campaign. Default 2.
982
- * Profiles run sequentially so the cost ceiling is honored deterministically. */
983
- maxConcurrency?: number;
984
- /** Cumulative USD cap per profile campaign. */
985
- costCeiling?: number;
986
- /** Capture flywheel — forwarded to each campaign. */
987
- labeledStore?: LabeledScenarioStore | 'off';
988
- captureSource?: LabeledScenarioSource;
989
- /** Storage backend. Default `fsCampaignStorage`. Pass
990
- * `inMemoryCampaignStorage()` for edge/CF-Worker/test runs. */
991
- storage?: CampaignStorage;
992
- /** Test seam — override the wall clock. */
993
- now?: () => Date;
994
- /** Optional persona key per scenario — drives the `byPersona` pivot. When
995
- * unset, `byPersona` is omitted. */
996
- personaOf?: (scenario: TScenario) => string;
997
- /** Validate every produced RunRecord with `validateRunRecord` (fail-loud).
998
- * Default true — catches bad model snapshots and non-finite judge dims at
999
- * the boundary instead of letting them poison downstream analysis. */
1000
- validate?: boolean;
1001
- /** Corpus-by-default: derive the trajectory text (`prompt` + `completion`)
1002
- * for each cell from its artifact + scenario. When set, every produced
1003
- * record carries `prompt`/`completion` (a `CorpusRecord`) so the run's
1004
- * graded trajectories can be appended to the durable RL corpus with no
1005
- * side-channel — `appendToCorpus(result.records, path)`. Fail-soft: a
1006
- * throwing or undefined-returning extractor just omits the text. */
1007
- corpusText?: (artifact: TArtifact, scenario: TScenario) => {
1008
- prompt: string;
1009
- completion: string;
1010
- } | undefined;
1011
- }
1012
- interface ProfileSummary {
1013
- profileId: string;
1014
- profileHash: string;
1015
- model: string;
1016
- /** RunRecords produced for this profile (= scenarios × reps). */
1017
- records: number;
1018
- /** Mean composite across this profile's records. */
1019
- meanComposite: number;
1020
- totalCostUsd: number;
1021
- /** Per-profile integrity verdict — surfaces a single profile that ran stub
1022
- * even when the matrix as a whole looks real. */
1023
- integrity: BackendIntegrityReport;
1024
- }
1025
- interface ScenarioRollup {
1026
- meanComposite: number;
1027
- n: number;
1028
- }
1029
- interface RunProfileMatrixResult<TArtifact, TScenario extends Scenario> {
1030
- matrixId: string;
1031
- experimentId: string;
1032
- /** One RunRecord per (profile, scenario, rep) cell — the integrity-checked,
1033
- * paper-grade output. Feed straight into `analyzeRuns`, `HeldOutGate`,
1034
- * scorecards, the hosted wire format. */
1035
- records: RunRecord[];
1036
- byProfile: Record<string, ProfileSummary>;
1037
- byScenario: Record<string, ScenarioRollup>;
1038
- /** Present only when `personaOf` was supplied. */
1039
- byPersona?: Record<string, ScenarioRollup>;
1040
- /** Whole-matrix integrity report (the one `integrity:'assert'` enforces). */
1041
- integrity: BackendIntegrityReport;
1042
- /** The raw per-profile campaign results, keyed by profile id. */
1043
- campaigns: Record<string, CampaignResult<TArtifact, TScenario>>;
1044
- }
1045
- declare function runProfileMatrix<TScenario extends Scenario, TArtifact>(opts: RunProfileMatrixOptions<TScenario, TArtifact>): Promise<RunProfileMatrixResult<TArtifact, TScenario>>;
1046
-
1047
- /**
1048
- * Product-flow playback — drive the REAL product through a user story and
1049
- * score the produced state per requirement (the launch "Jira tick-off").
1050
- *
1051
- * This is the substrate adapter + contract only. It plugs a `PlaybackDriver`
1052
- * into the existing `runProfileMatrix` dispatch seam: a driver drives the real
1053
- * product (a Playwright UI session or a sandbox workspace) and returns the
1054
- * runtime event stream; `extractProducedState` + `verifyCompletion` then score
1055
- * each requirement PASS/FAIL. The concrete drivers live in consumers — they
1056
- * depend on browser / runtime infra the substrate must not import — so
1057
- * agent-eval owns the seam, the `UserStory` contract, and the scoreboard.
1058
- */
1059
-
1060
- /** One step of a user story — what the user does. The driver interprets
1061
- * `payload` (a Playwright selector + action, or a sandbox chat turn). */
1062
- interface PlaybackStep {
1063
- /** Human-readable action, captured verbatim in the UX narrative. */
1064
- action: string;
1065
- /** Driver-specific payload (e.g. `{ selector, fill }` or `{ turn }`). */
1066
- payload?: Record<string, unknown>;
1067
- }
1068
- /**
1069
- * A user story = a runnable product journey plus the requirements that define
1070
- * "this story works". Each requirement is one Jira ticket line. Extends
1071
- * `Scenario` so a catalog drops straight into `runProfileMatrix({ scenarios })`.
1072
- */
1073
- interface UserStory extends Scenario {
1074
- /** Human-readable story title (the ticket headline). */
1075
- title: string;
1076
- /** Ordered steps the driver executes. */
1077
- steps: PlaybackStep[];
1078
- /** What must hold in the produced state for the story to pass. */
1079
- requirements: CompletionRequirement[];
1080
- }
1081
- /** Dispatch context plus the profile under test (which cheap model, etc.). */
1082
- interface PlaybackContext extends DispatchContext {
1083
- profile: AgentProfile;
1084
- }
1085
- /**
1086
- * Drives the real product through a story and returns the runtime event stream
1087
- * `extractProducedState` consumes. Implemented by CONSUMERS —
1088
- * `SandboxPlaybackDriver` (real API / sandbox workspace) and
1089
- * `PlaywrightPlaybackDriver` (real UI) — because they depend on runtime /
1090
- * browser infra the substrate must not import. The driver MUST report LLM
1091
- * usage via `ctx.cost.observeTokens` so the backend-integrity guard sees real
1092
- * tokens (a run that never reports tokens reads as a stub).
1093
- */
1094
- interface PlaybackDriver<TStory extends UserStory = UserStory> {
1095
- run(story: TStory, ctx: PlaybackContext): Promise<readonly RuntimeEventLike[]>;
1096
- }
1097
- /**
1098
- * Adapt a `PlaybackDriver` into a `runProfileMatrix` dispatch. The artifact the
1099
- * matrix scores is the `ProducedState` extracted from the driver's event
1100
- * stream — grade it with `scoreUserStory` (or a judge wrapping it).
1101
- */
1102
- declare function makePlaybackDispatch<TStory extends UserStory>(driver: PlaybackDriver<TStory>): ProfileDispatchFn<TStory, ProducedState>;
1103
- /** A scored user story — the completion verdict plus its human title. */
1104
- interface UserStoryVerdict extends CompletionVerdict {
1105
- title: string;
1106
- }
1107
- /**
1108
- * Score one story's produced state against its requirements. Thin wrapper over
1109
- * `verifyCompletion` that builds the gold from the story and returns a
1110
- * per-requirement PASS/FAIL verdict. `checkCorrectness` is injected — a
1111
- * deterministic stub in tests, `createLlmCorrectnessChecker` in production.
1112
- */
1113
- declare function scoreUserStory(story: UserStory, state: ProducedState, checkCorrectness: CorrectnessChecker): Promise<UserStoryVerdict>;
1114
- /** One row of the launch scoreboard — story × requirement → PASS/FAIL. */
1115
- interface ScoreboardRow {
1116
- storyId: string;
1117
- storyTitle: string;
1118
- reqId: string;
1119
- reqTitle: string;
1120
- status: 'PASS' | 'FAIL';
1121
- evidence: string[];
1122
- }
1123
- /**
1124
- * Flatten story verdicts into the per-requirement scoreboard — the literal
1125
- * Jira tick-off: one row per (story, requirement) with PASS/FAIL and the
1126
- * evidence behind the verdict.
1127
- */
1128
- declare function userStoryScoreboard(verdicts: readonly UserStoryVerdict[]): ScoreboardRow[];
1129
- /** Launch-readiness headline counts rolled up from the per-requirement rows. */
1130
- interface ScoreboardSummary {
1131
- /** Distinct user stories on the board. */
1132
- stories: number;
1133
- /** Stories whose every requirement passed. */
1134
- storiesFullyComplete: number;
1135
- /** Total (story, requirement) rows. */
1136
- requirements: number;
1137
- /** Rows with status PASS. */
1138
- passed: number;
1139
- /** Rows with status FAIL. */
1140
- failed: number;
1141
- /** passed / requirements; 0 when there are no rows. */
1142
- passRate: number;
1143
- }
1144
- /** Roll the per-requirement rows up into the launch headline counts. */
1145
- declare function scoreboardSummary(rows: readonly ScoreboardRow[]): ScoreboardSummary;
1146
- interface ScoreboardRenderOptions {
1147
- /** Document H1. Defaults to a generic playback title. */
1148
- title?: string;
1149
- /** Key/value run metadata rendered under the headline (runId, backend, model, date). */
1150
- meta?: Record<string, string>;
1151
- /** Max chars of joined evidence shown per row. Default 160. */
1152
- maxEvidenceChars?: number;
1153
- }
1154
- /**
1155
- * Render the scoreboard as a launch-readiness Markdown document — the literal
1156
- * "tick off every user story" artifact: a headline roll-up, the open tickets
1157
- * (FAIL rows) up top as the launch blockers, then a per-story table of
1158
- * requirement → PASS/FAIL with the evidence behind each verdict. Pure: same
1159
- * rows in, same bytes out (no clock/random), so it is safe to snapshot.
1160
- */
1161
- declare function renderScoreboardMarkdown(rows: readonly ScoreboardRow[], opts?: ScoreboardRenderOptions): string;
1162
-
1163
- /**
1164
- * @experimental
1165
- *
1166
- * `runSkillOpt` — the SkillOpt epoch hill-climb (Microsoft, arXiv:2605.23904).
1167
- * Unlike `runOptimization`'s population/promote-top-K search, SkillOpt is a
1168
- * sequential, held-out-gated hill-climb on ONE skill document:
1169
- *
1170
- * each epoch:
1171
- * 1. reflect on the CURRENT surface's weakest TRAIN scenarios/dimensions
1172
- * (never the held-out split — proposals must not see the acceptance axis)
1173
- * 2. propose ≤ `patchesPerEpoch` bounded patches (≤ `editBudget` ops each)
1174
- * 3. apply each; score the candidate on the HELD-OUT split
1175
- * 4. ACCEPT the first patch that STRICTLY improves the held-out composite;
1176
- * otherwise push it to the rejected-edit buffer (fed back so the model
1177
- * does not re-propose dead ends)
1178
- * 5. anneal the edit budget down after consecutive rejections (the
1179
- * "textual learning rate" decay); refresh the slow-update meta note
1180
- * 6. stop at `maxEpochs` or after `patience` epochs with no acceptance
1181
- *
1182
- * The accept-only-if-held-out-improves rule is the same discipline as
1183
- * `HeldOutGate`/`defaultProductionGate`, applied per edit instead of once at
1184
- * the end — which is why the held-out composite is monotonically
1185
- * non-decreasing and a regression can never ship. `runCampaign` is the
1186
- * measurement; `applySkillPatch` applies the edits; `skillOptDriver` proposes
1187
- * them.
1188
- */
1189
-
1190
- interface RunSkillOptOptions<TScenario extends Scenario, TArtifact> extends Omit<RunCampaignOptions<TScenario, TArtifact>, 'dispatch' | 'scenarios'> {
1191
- /** The skill document being optimized. */
1192
- baselineSurface: string;
1193
- /** Dispatcher taking the CURRENT skill surface + scenario → artifact. */
1194
- dispatchWithSurface: (surface: string, scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
1195
- driver: SkillOptDriver;
1196
- /** Scenarios the optimizer reflects on for evidence. MUST be disjoint from
1197
- * `holdoutScenarios` — proposals never see the acceptance axis. */
1198
- trainScenarios: TScenario[];
1199
- /** Held-out scenarios. An edit is accepted ONLY if it strictly improves the
1200
- * mean composite here. */
1201
- holdoutScenarios: TScenario[];
1202
- maxEpochs: number;
1203
- /** Candidate patches proposed per epoch. Default 2. */
1204
- patchesPerEpoch?: number;
1205
- /** Initial ops-per-patch cap (the textual learning rate). Default 3. */
1206
- editBudget?: number;
1207
- /** Strict acceptance margin: accept iff the held-out composite improves by
1208
- * MORE than this. Default 0 (any strict improvement). */
1209
- minImprovement?: number;
1210
- /** Stop after this many consecutive epochs with no acceptance. Default =
1211
- * `maxEpochs` (never early-stop). */
1212
- patience?: number;
1213
- /** Shrink the edit budget by 1 after 2 consecutive rejected epochs (min 1).
1214
- * Default true. */
1215
- budgetAnneal?: boolean;
1216
- /** Cap on the rejected-edit buffer (most-recent kept). Default 12. */
1217
- rejectedBufferSize?: number;
1218
- /** Refresh the slow-update meta note every N epochs. Default 2. 0 disables. */
1219
- slowMetaEvery?: number;
1220
- /** Top-K weak scenarios/dimensions surfaced as evidence each epoch. Default 3. */
1221
- evidenceK?: number;
1222
- /** Abort signal forwarded to the patch-proposing LLM calls. */
1223
- signal?: AbortSignal;
1224
- }
1225
- interface AcceptedEdit {
1226
- epoch: number;
1227
- label: string;
1228
- rationale: string;
1229
- /** Held-out composite improvement vs the surface before this edit. */
1230
- holdoutDelta: number;
1231
- }
1232
- interface SkillOptEpochRecord {
1233
- epoch: number;
1234
- editBudget: number;
1235
- proposed: number;
1236
- /** The accepted edit this epoch, or null if every proposal was rejected. */
1237
- accepted: AcceptedEdit | null;
1238
- rejected: RejectedEdit[];
1239
- /** Held-out composite of the CURRENT surface at the END of the epoch. */
1240
- holdoutComposite: number;
1241
- }
1242
- interface RunSkillOptResult {
1243
- winnerSurface: string;
1244
- baselineHoldoutComposite: number;
1245
- winnerHoldoutComposite: number;
1246
- /** `winnerHoldoutComposite - baselineHoldoutComposite` — monotonically ≥ 0
1247
- * by construction (only strictly-improving edits are accepted). */
1248
- lift: number;
1249
- acceptedEdits: AcceptedEdit[];
1250
- rejectedEdits: RejectedEdit[];
1251
- epochsRun: number;
1252
- history: SkillOptEpochRecord[];
1253
- /** Total cost across every scoring campaign (train evidence + holdout
1254
- * acceptance) the hill-climb ran. */
1255
- totalCostUsd: number;
1256
- }
1257
- declare function runSkillOpt<TScenario extends Scenario, TArtifact>(opts: RunSkillOptOptions<TScenario, TArtifact>): Promise<RunSkillOptResult>;
1258
-
1259
- /**
1260
- * @experimental
1261
- *
1262
- * Shared campaign-score reductions used by every optimizer preset
1263
- * (`runOptimization`, `runSkillOpt`, `compareDrivers`). ONE definition of
1264
- * "composite of a campaign" and "per-scenario / per-dimension breakdown" so
1265
- * the optimizers cannot drift on how a surface's score is computed.
1266
- */
1267
-
1268
- /** Mean composite across a campaign: per cell, the mean of its judges'
1269
- * composites; then the mean across cells. Cells with no judge scores are
1270
- * skipped. Empty ⇒ 0. */
1271
- declare function campaignMeanComposite<TArtifact, TScenario extends Scenario>(campaign: CampaignResult<TArtifact, TScenario>): number;
1272
- interface CampaignBreakdown {
1273
- /** Mean score per judge dimension across all cells. */
1274
- dimensions: Record<string, number>;
1275
- /** Per-scenario composite (mean over reps + judges) + the judge's free-form
1276
- * `notes` for that scenario (the "why" a reflective driver grounds on). */
1277
- scenarios: Array<{
1278
- scenarioId: string;
1279
- composite: number;
1280
- notes?: string;
1281
- }>;
1282
- }
1283
- /** Per-candidate evidence a reflective/patch driver grounds its next proposal
1284
- * on: mean score per judge dimension + per-scenario composite. */
1285
- declare function campaignBreakdown<TArtifact, TScenario extends Scenario>(campaign: CampaignResult<TArtifact, TScenario>): CampaignBreakdown;
1286
-
1287
- /**
1288
- * @experimental
1289
- *
1290
- * VCS-pluggable worktree adapter. One improvement = one worktree, PR-like
1291
- * (multiple commits allowed). A code-tier driver's `propose()` creates a
1292
- * worktree, an agent commits the change into it, and `finalize()` returns a
1293
- * `CodeSurface{ worktreeRef }` the measurement checks out to run the worker
1294
- * against the changed code. On promotion the worktree becomes the PR branch.
1295
- *
1296
- * The interface is VCS-agnostic so a future `jj` ([jj-vcs](https://github.com/jj-vcs/jj))
1297
- * adapter can slot in without touching driver code. Only the git adapter
1298
- * ships today. See `docs/design/self-improvement-engine.md`.
1299
- */
1300
-
1301
- interface Worktree {
1302
- /** Absolute path to the checked-out worktree directory. */
1303
- path: string;
1304
- /** The branch the worktree is on (becomes the PR branch on promotion). */
1305
- branch: string;
1306
- /** The ref the worktree was forked from. */
1307
- baseRef: string;
1308
- }
1309
- interface WorktreeAdapter {
1310
- /** Create an isolated worktree on a fresh branch off `baseRef`. */
1311
- create(opts: {
1312
- baseRef: string;
1313
- label: string;
1314
- }): Promise<Worktree>;
1315
- /** Commit any pending changes in the worktree, then return a CodeSurface
1316
- * pointing at it. The agent has already written its change into
1317
- * `worktree.path` by the time this is called. */
1318
- finalize(worktree: Worktree, summary: string): Promise<CodeSurface>;
1319
- /** Remove the worktree (and its branch) — called for losing candidates. */
1320
- discard(worktree: Worktree): Promise<void>;
1321
- }
1322
- declare class WorktreeAdapterError extends Error {
1323
- readonly cause?: unknown | undefined;
1324
- constructor(message: string, cause?: unknown | undefined);
1325
- }
1326
- interface GitWorktreeAdapterOptions {
1327
- /** Repo root the worktrees fork from. */
1328
- repoRoot: string;
1329
- /** Directory worktrees are created under. Default: `<repoRoot>/.worktrees`. */
1330
- worktreeDir?: string;
1331
- /** Branch-name prefix. Default: `improve`. */
1332
- branchPrefix?: string;
1333
- /** Test seam — defaults to a real `git` runner. */
1334
- git?: (args: string[], cwd: string) => string;
1335
- }
1336
- declare function gitWorktreeAdapter(opts: GitWorktreeAdapterOptions): WorktreeAdapter;
1337
- /** Resolve a `CodeSurface`'s worktreeRef to a directory the measurement can
1338
- * run the worker in. A path ref is returned as-is; anything else is treated
1339
- * as a ref under the adapter's worktree dir. */
1340
- declare function resolveWorktreePath(surface: CodeSurface, worktreeDir?: string): string;
1341
-
1342
- export { type AcceptedEdit, type AceDriverOptions, type AnalystArtifact, type AnalystScenario, type ApplySkillPatchResult, type BuildAnalystSurfaceDispatchOptions, type CampaignBreakdown, CampaignResult, CampaignStorage, CodeSurface, type CompareDriversOptions, type DimensionRegression, DispatchContext, type DriverComparison, type DriverEntry, type DriverPairwise, type DriverScore, type FailureModeRecallJudgeOptions, FsLabeledScenarioStore, type FsLabeledScenarioStoreOptions, Gate, GenerationRecord, type GitWorktreeAdapterOptions, type HaloDriverOptions, type HeldoutSignificance, type HeldoutSignificanceOptions, ImprovementDriver, JudgeConfig, JudgeScore, LabelTrust, LabeledScenarioRecord, LabeledScenarioSampleArgs, LabeledScenarioSource, LabeledScenarioStore, LabeledScenarioStoreError, LabeledScenarioWrite, type MemoryCurationDriverOptions, MutableSurface, type OptimizerEntryConfig, type PairedHoldout, type PlaybackContext, type PlaybackDriver, type PlaybackStep, type ProfileDispatchFn, ProfileMatrixError, type ProfileSummary, ProposeContext, type ProposePatchesArgs, type RejectedEdit, RunCampaignOptions, RunImprovementLoopOptions, type RunProfileMatrixOptions, type RunProfileMatrixResult, type RunSkillOptOptions, type RunSkillOptResult, Scenario, type ScenarioRollup, type ScoreboardRenderOptions, type ScoreboardRow, type ScoreboardSummary, type SequentialDecideFn, type SequentialDecideOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, type SkillOptDriver, type SkillOptDriverOptions, type SkillOptEpochRecord, type SkillOptEvidence, type SkillPatch, type SkillPatchOp, SkillPatchParseError, type SkillPatchRejection, type TraceAnalystDriverOptions, type UserStory, type UserStoryVerdict, type Worktree, type WorktreeAdapter, WorktreeAdapterError, aceDriver, applySkillPatch, buildAnalystSurfaceDispatch, campaignBreakdown, campaignMeanComposite, compareDrivers, detectScale, dimensionRegressions, failureModeRecallJudge, gepaParetoEntry, gepaReflectionEntry, gitWorktreeAdapter, haloDriver, heldoutSignificance, makePlaybackDispatch, memoryCurationDriver, pairHoldout, parseSkillPatchResponse, patchEditCount, renderScoreboardMarkdown, resolveWorktreePath, runProfileMatrix, runSkillOpt, scoreUserStory, scoreboardSummary, sequentialDecide, sequentialPairedGate, skillOptDriver, skillOptEntry, traceAnalystDriver, userStoryScoreboard };
2
+ * `@tangle-network/agent-eval/campaign` — measurement + improvement loop.
3
+ *
4
+ * `runCampaign` is the measurement primitive (a surface scored over scenarios);
5
+ * `runImprovementLoop` is the proposer-agnostic improvement loop on top of it.
6
+ */
7
+ export { type AnalystArtifact, type AnalystScenario, type BuildAnalystSurfaceDispatchOptions, buildAnalystSurfaceDispatch, type FailureModeRecallJudgeOptions, failureModeRecallJudge, } from './analyst-surface';
8
+ export { type OpenAutoPrOptions, type OpenAutoPrResult, openAutoPr, } from './auto-pr';
9
+ export { composeGate } from './gates/compose';
10
+ export { type DefaultProductionGateOptions, defaultProductionGate, } from './gates/default-production-gate';
11
+ export { type HeldOutGateOptions, heldOutGate } from './gates/heldout-gate';
12
+ export { type AxisEvidence, type AxisVerdict, type BuildEvidenceVectorOptions, buildEvidenceVector, type EvidenceVector, type ObjectiveSource, type ParetoSignificanceGateOptions, type PromotionObjective, type PromotionPolicy, paretoPolicy, paretoSignificanceGate, } from './gates/promotion-policy';
13
+ export { type SequentialDecideFn, type SequentialDecideOptions, type SequentialDecision, type SequentialObservation, type SequentialPairedGate, type SequentialPairedGateOptions, sequentialDecide, sequentialPairedGate, } from './gates/sequential';
14
+ export { type DimensionRegression, detectScale, dimensionRegressions, type HeldoutSignificance, type HeldoutSignificanceOptions, heldoutSignificance, type PairedHoldout, pairHoldout, } from './gates/statistical-heldout';
15
+ export { FsLabeledScenarioStore, type FsLabeledScenarioStoreOptions, LabeledScenarioStoreError, } from './labeled-store/fs-adapter';
16
+ export { type CompareProposersOptions, compareProposers, type FapoEntryConfig, fapoEscalationEntry, gepaParetoEntry, gepaReflectionEntry, type OptimizerEntryConfig, type ProposerComparison, type ProposerEntry, type ProposerPairwise, type ProposerScore, skillOptEntry, } from './presets/compare-proposers';
17
+ export { makePlaybackDispatch, type PlaybackContext, type PlaybackDriver, type PlaybackStep, renderScoreboardMarkdown, type ScoreboardRenderOptions, type ScoreboardRow, type ScoreboardSummary, scoreboardSummary, scoreUserStory, type UserStory, type UserStoryVerdict, userStoryScoreboard, } from './presets/playback';
18
+ export { type RunEvalOptions, runEval } from './presets/run-eval';
19
+ export { defaultRenderDiff, type RunImprovementLoopOptions, type RunImprovementLoopResult, runImprovementLoop, } from './presets/run-improvement-loop';
20
+ export { type RunOptimizationOptions, type RunOptimizationResult, runOptimization, surfaceHash, } from './presets/run-optimization';
21
+ export { type ProfileDispatchFn, ProfileMatrixError, type ProfileSummary, type RunProfileMatrixOptions, type RunProfileMatrixResult, runProfileMatrix, type ScenarioRollup, } from './presets/run-profile-matrix';
22
+ export { type AcceptedEdit, type RunSkillOptOptions, type RunSkillOptResult, runSkillOpt, type SkillOptEpochRecord, } from './presets/run-skill-opt';
23
+ export { type AceProposerOptions, aceProposer } from './proposers/ace';
24
+ export { type EvolutionaryProposerOptions, evolutionaryProposer } from './proposers/evolutionary';
25
+ export { extractFapoAttributionSignals, type FapoAttributionSignals, type FapoFailureCluster, type FapoOptimizationLevel, type FapoProposerOptions, type FapoReviewInput, type FapoReviewIssue, type FapoReviewResult, type FapoScopeContract, fapoProposer, type JsonPrimitive, type JsonValue, type ParameterCandidate, type ParameterChange, type ParameterSweepProposerOptions, parameterSweepProposer, } from './proposers/fapo';
26
+ export { countSentenceEdits, extractH2Sections, type GepaProposerConstraints, type GepaProposerOptions, gepaProposer, } from './proposers/gepa';
27
+ export { type HaloProposerOptions, haloProposer } from './proposers/halo';
28
+ export { type MemoryCurationProposerOptions, memoryCurationProposer } from './proposers/memory';
29
+ export { type ProposePatchesArgs, parseSkillPatchResponse, type RejectedEdit, type SkillOptEvidence, type SkillOptProposer, type SkillOptProposerOptions, SkillPatchParseError, skillOptProposer, } from './proposers/skill-opt';
30
+ export { type TraceAnalystProposerOptions, traceAnalystProposer } from './proposers/trace-analyst';
31
+ export { type BuildLoopProvenanceArgs, buildLoopProvenanceRecord, type EmitLoopProvenanceArgs, type EmitLoopProvenanceResult, emitLoopProvenance, type LoopProvenanceBackend, type LoopProvenanceCandidate, type LoopProvenanceRecord, loopProvenanceSpans, provenanceRecordPath, provenanceSpansPath, surfaceContentHash, } from './provenance';
32
+ export { type RunCampaignOptions, runCampaign } from './run-campaign';
33
+ export { type CampaignBreakdown, campaignBreakdown, campaignMeanComposite } from './score-utils';
34
+ export { type ApplySkillPatchResult, applySkillPatch, patchEditCount, type SkillPatch, type SkillPatchOp, type SkillPatchRejection, } from './skill-patch';
35
+ export { type CampaignStorage, fsCampaignStorage, inMemoryCampaignStorage } from './storage';
36
+ export type { CampaignAggregates, CampaignArtifactWriter, CampaignCellResult, CampaignCostMeter, CampaignResult, CampaignTokenUsage, CampaignTraceWriter, CodeSurface, DispatchContext, DispatchFn, Gate, GateContext, GateDecision, GateResult, GenerationCandidate, GenerationRecord, JudgeAggregate, JudgeConfig, JudgeDimension, JudgeScore, LabeledScenarioRecord, LabeledScenarioSampleArgs, LabeledScenarioSource, LabeledScenarioStore, LabeledScenarioWrite, LabelTrust, MutableSurface, Mutator, OptimizationProposer, OptimizerConfig, ParetoParent, ProposeContext, ProposedCandidate, RedactionStatus, Scenario, ScenarioAggregate, SessionScript, SurfaceProposer, TraceSpan, } from './types';
37
+ export { isProposedCandidate, labelTrustRank } from './types';
38
+ export { type GitWorktreeAdapterOptions, gitWorktreeAdapter, resolveWorktreePath, type Worktree, type WorktreeAdapter, WorktreeAdapterError, } from './worktree';
39
+ //# sourceMappingURL=index.d.ts.map