@stigmer/cli 0.1.0-dev.20261017020314.g644422724c27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1074) hide show
  1. package/LICENSE +194 -0
  2. package/README.md +45 -0
  3. package/auth/auth0.d.ts +19 -0
  4. package/auth/auth0.d.ts.map +1 -0
  5. package/auth/auth0.js +39 -0
  6. package/auth/auth0.js.map +1 -0
  7. package/auth/callback-server.d.ts +18 -0
  8. package/auth/callback-server.d.ts.map +1 -0
  9. package/auth/callback-server.js +77 -0
  10. package/auth/callback-server.js.map +1 -0
  11. package/auth/index.d.ts +5 -0
  12. package/auth/index.d.ts.map +1 -0
  13. package/auth/index.js +6 -0
  14. package/auth/index.js.map +1 -0
  15. package/auth/login.d.ts +2 -0
  16. package/auth/login.d.ts.map +1 -0
  17. package/auth/login.js +81 -0
  18. package/auth/login.js.map +1 -0
  19. package/auth/pages.d.ts +7 -0
  20. package/auth/pages.d.ts.map +1 -0
  21. package/auth/pages.js +21 -0
  22. package/auth/pages.js.map +1 -0
  23. package/auth/pkce.d.ts +7 -0
  24. package/auth/pkce.d.ts.map +1 -0
  25. package/auth/pkce.js +22 -0
  26. package/auth/pkce.js.map +1 -0
  27. package/auth/token.d.ts +38 -0
  28. package/auth/token.d.ts.map +1 -0
  29. package/auth/token.js +120 -0
  30. package/auth/token.js.map +1 -0
  31. package/backend.d.ts +5 -0
  32. package/backend.d.ts.map +1 -0
  33. package/backend.js +14 -0
  34. package/backend.js.map +1 -0
  35. package/cli/stigmer.d.ts +3 -0
  36. package/cli/stigmer.d.ts.map +1 -0
  37. package/cli/stigmer.js +14 -0
  38. package/cli/stigmer.js.map +1 -0
  39. package/client/client.d.ts +44 -0
  40. package/client/client.d.ts.map +1 -0
  41. package/client/client.js +82 -0
  42. package/client/client.js.map +1 -0
  43. package/client/index.d.ts +2 -0
  44. package/client/index.d.ts.map +1 -0
  45. package/client/index.js +3 -0
  46. package/client/index.js.map +1 -0
  47. package/client/organizations.d.ts +41 -0
  48. package/client/organizations.d.ts.map +1 -0
  49. package/client/organizations.js +86 -0
  50. package/client/organizations.js.map +1 -0
  51. package/client/single-org.d.ts +16 -0
  52. package/client/single-org.d.ts.map +1 -0
  53. package/client/single-org.js +35 -0
  54. package/client/single-org.js.map +1 -0
  55. package/commands/access-history/index.d.ts +3 -0
  56. package/commands/access-history/index.d.ts.map +1 -0
  57. package/commands/access-history/index.js +99 -0
  58. package/commands/access-history/index.js.map +1 -0
  59. package/commands/agent-exec-flags.d.ts +30 -0
  60. package/commands/agent-exec-flags.d.ts.map +1 -0
  61. package/commands/agent-exec-flags.js +49 -0
  62. package/commands/agent-exec-flags.js.map +1 -0
  63. package/commands/agent.d.ts +3 -0
  64. package/commands/agent.d.ts.map +1 -0
  65. package/commands/agent.js +43 -0
  66. package/commands/agent.js.map +1 -0
  67. package/commands/apikey/duration.d.ts +2 -0
  68. package/commands/apikey/duration.d.ts.map +1 -0
  69. package/commands/apikey/duration.js +30 -0
  70. package/commands/apikey/duration.js.map +1 -0
  71. package/commands/apikey/index.d.ts +18 -0
  72. package/commands/apikey/index.d.ts.map +1 -0
  73. package/commands/apikey/index.js +198 -0
  74. package/commands/apikey/index.js.map +1 -0
  75. package/commands/apply.d.ts +3 -0
  76. package/commands/apply.d.ts.map +1 -0
  77. package/commands/apply.js +146 -0
  78. package/commands/apply.js.map +1 -0
  79. package/commands/auth/index.d.ts +3 -0
  80. package/commands/auth/index.d.ts.map +1 -0
  81. package/commands/auth/index.js +58 -0
  82. package/commands/auth/index.js.map +1 -0
  83. package/commands/auth/login.d.ts +13 -0
  84. package/commands/auth/login.d.ts.map +1 -0
  85. package/commands/auth/login.js +58 -0
  86. package/commands/auth/login.js.map +1 -0
  87. package/commands/auth/whoami.d.ts +22 -0
  88. package/commands/auth/whoami.d.ts.map +1 -0
  89. package/commands/auth/whoami.js +123 -0
  90. package/commands/auth/whoami.js.map +1 -0
  91. package/commands/channel.d.ts +3 -0
  92. package/commands/channel.d.ts.map +1 -0
  93. package/commands/channel.js +114 -0
  94. package/commands/channel.js.map +1 -0
  95. package/commands/completion.d.ts +3 -0
  96. package/commands/completion.d.ts.map +1 -0
  97. package/commands/completion.js +80 -0
  98. package/commands/completion.js.map +1 -0
  99. package/commands/config/backend.d.ts +3 -0
  100. package/commands/config/backend.d.ts.map +1 -0
  101. package/commands/config/backend.js +171 -0
  102. package/commands/config/backend.js.map +1 -0
  103. package/commands/config/context.d.ts +3 -0
  104. package/commands/config/context.d.ts.map +1 -0
  105. package/commands/config/context.js +88 -0
  106. package/commands/config/context.js.map +1 -0
  107. package/commands/config/index.d.ts +3 -0
  108. package/commands/config/index.d.ts.map +1 -0
  109. package/commands/config/index.js +13 -0
  110. package/commands/config/index.js.map +1 -0
  111. package/commands/config/kv.d.ts +3 -0
  112. package/commands/config/kv.d.ts.map +1 -0
  113. package/commands/config/kv.js +39 -0
  114. package/commands/config/kv.js.map +1 -0
  115. package/commands/connect.d.ts +3 -0
  116. package/commands/connect.d.ts.map +1 -0
  117. package/commands/connect.js +74 -0
  118. package/commands/connect.js.map +1 -0
  119. package/commands/delete.d.ts +3 -0
  120. package/commands/delete.d.ts.map +1 -0
  121. package/commands/delete.js +44 -0
  122. package/commands/delete.js.map +1 -0
  123. package/commands/down.d.ts +3 -0
  124. package/commands/down.d.ts.map +1 -0
  125. package/commands/down.js +23 -0
  126. package/commands/down.js.map +1 -0
  127. package/commands/download.d.ts +3 -0
  128. package/commands/download.d.ts.map +1 -0
  129. package/commands/download.js +79 -0
  130. package/commands/download.js.map +1 -0
  131. package/commands/eval.d.ts +3 -0
  132. package/commands/eval.d.ts.map +1 -0
  133. package/commands/eval.js +115 -0
  134. package/commands/eval.js.map +1 -0
  135. package/commands/get.d.ts +3 -0
  136. package/commands/get.d.ts.map +1 -0
  137. package/commands/get.js +113 -0
  138. package/commands/get.js.map +1 -0
  139. package/commands/install.d.ts +3 -0
  140. package/commands/install.d.ts.map +1 -0
  141. package/commands/install.js +68 -0
  142. package/commands/install.js.map +1 -0
  143. package/commands/internal-daemon.d.ts +3 -0
  144. package/commands/internal-daemon.d.ts.map +1 -0
  145. package/commands/internal-daemon.js +15 -0
  146. package/commands/internal-daemon.js.map +1 -0
  147. package/commands/list.d.ts +10 -0
  148. package/commands/list.d.ts.map +1 -0
  149. package/commands/list.js +131 -0
  150. package/commands/list.js.map +1 -0
  151. package/commands/logs.d.ts +3 -0
  152. package/commands/logs.d.ts.map +1 -0
  153. package/commands/logs.js +59 -0
  154. package/commands/logs.js.map +1 -0
  155. package/commands/marketplace.d.ts +3 -0
  156. package/commands/marketplace.d.ts.map +1 -0
  157. package/commands/marketplace.js +94 -0
  158. package/commands/marketplace.js.map +1 -0
  159. package/commands/mcp-server.d.ts +22 -0
  160. package/commands/mcp-server.d.ts.map +1 -0
  161. package/commands/mcp-server.js +105 -0
  162. package/commands/mcp-server.js.map +1 -0
  163. package/commands/mcp.d.ts +3 -0
  164. package/commands/mcp.d.ts.map +1 -0
  165. package/commands/mcp.js +58 -0
  166. package/commands/mcp.js.map +1 -0
  167. package/commands/push.d.ts +27 -0
  168. package/commands/push.d.ts.map +1 -0
  169. package/commands/push.js +401 -0
  170. package/commands/push.js.map +1 -0
  171. package/commands/reset.d.ts +3 -0
  172. package/commands/reset.d.ts.map +1 -0
  173. package/commands/reset.js +45 -0
  174. package/commands/reset.js.map +1 -0
  175. package/commands/resume.d.ts +3 -0
  176. package/commands/resume.d.ts.map +1 -0
  177. package/commands/resume.js +90 -0
  178. package/commands/resume.js.map +1 -0
  179. package/commands/run.d.ts +3 -0
  180. package/commands/run.d.ts.map +1 -0
  181. package/commands/run.js +158 -0
  182. package/commands/run.js.map +1 -0
  183. package/commands/runs/approve.d.ts +3 -0
  184. package/commands/runs/approve.d.ts.map +1 -0
  185. package/commands/runs/approve.js +37 -0
  186. package/commands/runs/approve.js.map +1 -0
  187. package/commands/runs/control.d.ts +3 -0
  188. package/commands/runs/control.d.ts.map +1 -0
  189. package/commands/runs/control.js +71 -0
  190. package/commands/runs/control.js.map +1 -0
  191. package/commands/runs/index.d.ts +3 -0
  192. package/commands/runs/index.d.ts.map +1 -0
  193. package/commands/runs/index.js +28 -0
  194. package/commands/runs/index.js.map +1 -0
  195. package/commands/runs/logs.d.ts +3 -0
  196. package/commands/runs/logs.d.ts.map +1 -0
  197. package/commands/runs/logs.js +33 -0
  198. package/commands/runs/logs.js.map +1 -0
  199. package/commands/runs/scores.d.ts +3 -0
  200. package/commands/runs/scores.d.ts.map +1 -0
  201. package/commands/runs/scores.js +26 -0
  202. package/commands/runs/scores.js.map +1 -0
  203. package/commands/runs/to-eval-case.d.ts +3 -0
  204. package/commands/runs/to-eval-case.d.ts.map +1 -0
  205. package/commands/runs/to-eval-case.js +35 -0
  206. package/commands/runs/to-eval-case.js.map +1 -0
  207. package/commands/runs/trace.d.ts +3 -0
  208. package/commands/runs/trace.d.ts.map +1 -0
  209. package/commands/runs/trace.js +24 -0
  210. package/commands/runs/trace.js.map +1 -0
  211. package/commands/schedule.d.ts +3 -0
  212. package/commands/schedule.d.ts.map +1 -0
  213. package/commands/schedule.js +57 -0
  214. package/commands/schedule.js.map +1 -0
  215. package/commands/search.d.ts +3 -0
  216. package/commands/search.d.ts.map +1 -0
  217. package/commands/search.js +76 -0
  218. package/commands/search.js.map +1 -0
  219. package/commands/service-account/index.d.ts +7 -0
  220. package/commands/service-account/index.d.ts.map +1 -0
  221. package/commands/service-account/index.js +144 -0
  222. package/commands/service-account/index.js.map +1 -0
  223. package/commands/setup.d.ts +3 -0
  224. package/commands/setup.d.ts.map +1 -0
  225. package/commands/setup.js +76 -0
  226. package/commands/setup.js.map +1 -0
  227. package/commands/shared.d.ts +15 -0
  228. package/commands/shared.d.ts.map +1 -0
  229. package/commands/shared.js +34 -0
  230. package/commands/shared.js.map +1 -0
  231. package/commands/status.d.ts +3 -0
  232. package/commands/status.d.ts.map +1 -0
  233. package/commands/status.js +18 -0
  234. package/commands/status.js.map +1 -0
  235. package/commands/tag.d.ts +3 -0
  236. package/commands/tag.d.ts.map +1 -0
  237. package/commands/tag.js +24 -0
  238. package/commands/tag.js.map +1 -0
  239. package/commands/up.d.ts +3 -0
  240. package/commands/up.d.ts.map +1 -0
  241. package/commands/up.js +84 -0
  242. package/commands/up.js.map +1 -0
  243. package/commands/usage.d.ts +5 -0
  244. package/commands/usage.d.ts.map +1 -0
  245. package/commands/usage.js +92 -0
  246. package/commands/usage.js.map +1 -0
  247. package/commands/validate.d.ts +3 -0
  248. package/commands/validate.d.ts.map +1 -0
  249. package/commands/validate.js +95 -0
  250. package/commands/validate.js.map +1 -0
  251. package/commands/vault.d.ts +3 -0
  252. package/commands/vault.d.ts.map +1 -0
  253. package/commands/vault.js +143 -0
  254. package/commands/vault.js.map +1 -0
  255. package/commands/version.d.ts +3 -0
  256. package/commands/version.d.ts.map +1 -0
  257. package/commands/version.js +11 -0
  258. package/commands/version.js.map +1 -0
  259. package/config/config.d.ts +110 -0
  260. package/config/config.d.ts.map +1 -0
  261. package/config/config.js +199 -0
  262. package/config/config.js.map +1 -0
  263. package/config/index.d.ts +5 -0
  264. package/config/index.d.ts.map +1 -0
  265. package/config/index.js +6 -0
  266. package/config/index.js.map +1 -0
  267. package/config/keys.d.ts +6 -0
  268. package/config/keys.d.ts.map +1 -0
  269. package/config/keys.js +80 -0
  270. package/config/keys.js.map +1 -0
  271. package/config/paths.d.ts +10 -0
  272. package/config/paths.d.ts.map +1 -0
  273. package/config/paths.js +21 -0
  274. package/config/paths.js.map +1 -0
  275. package/config/resolve.d.ts +65 -0
  276. package/config/resolve.d.ts.map +1 -0
  277. package/config/resolve.js +160 -0
  278. package/config/resolve.js.map +1 -0
  279. package/errors/classify.d.ts +17 -0
  280. package/errors/classify.d.ts.map +1 -0
  281. package/errors/classify.js +204 -0
  282. package/errors/classify.js.map +1 -0
  283. package/errors/cli-exit-error.d.ts +6 -0
  284. package/errors/cli-exit-error.d.ts.map +1 -0
  285. package/errors/cli-exit-error.js +15 -0
  286. package/errors/cli-exit-error.js.map +1 -0
  287. package/errors/exit-codes.d.ts +10 -0
  288. package/errors/exit-codes.d.ts.map +1 -0
  289. package/errors/exit-codes.js +15 -0
  290. package/errors/exit-codes.js.map +1 -0
  291. package/errors/handle.d.ts +14 -0
  292. package/errors/handle.d.ts.map +1 -0
  293. package/errors/handle.js +46 -0
  294. package/errors/handle.js.map +1 -0
  295. package/errors/index.d.ts +6 -0
  296. package/errors/index.d.ts.map +1 -0
  297. package/errors/index.js +7 -0
  298. package/errors/index.js.map +1 -0
  299. package/errors/usage-error.d.ts +5 -0
  300. package/errors/usage-error.d.ts.map +1 -0
  301. package/errors/usage-error.js +13 -0
  302. package/errors/usage-error.js.map +1 -0
  303. package/index.d.ts +3 -0
  304. package/index.d.ts.map +1 -0
  305. package/index.js +8 -0
  306. package/index.js.map +1 -0
  307. package/local/artifact.d.ts +77 -0
  308. package/local/artifact.d.ts.map +1 -0
  309. package/local/artifact.js +268 -0
  310. package/local/artifact.js.map +1 -0
  311. package/local/constants.d.ts +41 -0
  312. package/local/constants.d.ts.map +1 -0
  313. package/local/constants.js +48 -0
  314. package/local/constants.js.map +1 -0
  315. package/local/daemon/components.d.ts +20 -0
  316. package/local/daemon/components.d.ts.map +1 -0
  317. package/local/daemon/components.js +140 -0
  318. package/local/daemon/components.js.map +1 -0
  319. package/local/daemon/env.d.ts +77 -0
  320. package/local/daemon/env.d.ts.map +1 -0
  321. package/local/daemon/env.js +107 -0
  322. package/local/daemon/env.js.map +1 -0
  323. package/local/daemon/host.d.ts +15 -0
  324. package/local/daemon/host.d.ts.map +1 -0
  325. package/local/daemon/host.js +115 -0
  326. package/local/daemon/host.js.map +1 -0
  327. package/local/daemon/index.d.ts +8 -0
  328. package/local/daemon/index.d.ts.map +1 -0
  329. package/local/daemon/index.js +8 -0
  330. package/local/daemon/index.js.map +1 -0
  331. package/local/daemon/launch.d.ts +38 -0
  332. package/local/daemon/launch.d.ts.map +1 -0
  333. package/local/daemon/launch.js +261 -0
  334. package/local/daemon/launch.js.map +1 -0
  335. package/local/daemon/process.d.ts +42 -0
  336. package/local/daemon/process.d.ts.map +1 -0
  337. package/local/daemon/process.js +166 -0
  338. package/local/daemon/process.js.map +1 -0
  339. package/local/daemon/supervisor.d.ts +56 -0
  340. package/local/daemon/supervisor.d.ts.map +1 -0
  341. package/local/daemon/supervisor.js +211 -0
  342. package/local/daemon/supervisor.js.map +1 -0
  343. package/local/daemon/types.d.ts +65 -0
  344. package/local/daemon/types.d.ts.map +1 -0
  345. package/local/daemon/types.js +9 -0
  346. package/local/daemon/types.js.map +1 -0
  347. package/local/llm-config.d.ts +24 -0
  348. package/local/llm-config.d.ts.map +1 -0
  349. package/local/llm-config.js +74 -0
  350. package/local/llm-config.js.map +1 -0
  351. package/local/logs/tail.d.ts +19 -0
  352. package/local/logs/tail.d.ts.map +1 -0
  353. package/local/logs/tail.js +86 -0
  354. package/local/logs/tail.js.map +1 -0
  355. package/local/logs/view.d.ts +20 -0
  356. package/local/logs/view.d.ts.map +1 -0
  357. package/local/logs/view.js +50 -0
  358. package/local/logs/view.js.map +1 -0
  359. package/local/net/tcp.d.ts +23 -0
  360. package/local/net/tcp.d.ts.map +1 -0
  361. package/local/net/tcp.js +47 -0
  362. package/local/net/tcp.js.map +1 -0
  363. package/local/operator-config.d.ts +28 -0
  364. package/local/operator-config.d.ts.map +1 -0
  365. package/local/operator-config.js +77 -0
  366. package/local/operator-config.js.map +1 -0
  367. package/local/paths.d.ts +47 -0
  368. package/local/paths.d.ts.map +1 -0
  369. package/local/paths.js +83 -0
  370. package/local/paths.js.map +1 -0
  371. package/local/reset.d.ts +14 -0
  372. package/local/reset.d.ts.map +1 -0
  373. package/local/reset.js +57 -0
  374. package/local/reset.js.map +1 -0
  375. package/local/runtime/index.d.ts +5 -0
  376. package/local/runtime/index.d.ts.map +1 -0
  377. package/local/runtime/index.js +8 -0
  378. package/local/runtime/index.js.map +1 -0
  379. package/local/runtime/node.d.ts +17 -0
  380. package/local/runtime/node.d.ts.map +1 -0
  381. package/local/runtime/node.js +139 -0
  382. package/local/runtime/node.js.map +1 -0
  383. package/local/runtime/runner.d.ts +45 -0
  384. package/local/runtime/runner.d.ts.map +1 -0
  385. package/local/runtime/runner.js +131 -0
  386. package/local/runtime/runner.js.map +1 -0
  387. package/local/runtime/runtimes-install.d.ts +13 -0
  388. package/local/runtime/runtimes-install.d.ts.map +1 -0
  389. package/local/runtime/runtimes-install.js +63 -0
  390. package/local/runtime/runtimes-install.js.map +1 -0
  391. package/local/runtime/server.d.ts +40 -0
  392. package/local/runtime/server.d.ts.map +1 -0
  393. package/local/runtime/server.js +144 -0
  394. package/local/runtime/server.js.map +1 -0
  395. package/local/runtime/which.d.ts +3 -0
  396. package/local/runtime/which.d.ts.map +1 -0
  397. package/local/runtime/which.js +30 -0
  398. package/local/runtime/which.js.map +1 -0
  399. package/local/setup/prompt.d.ts +8 -0
  400. package/local/setup/prompt.d.ts.map +1 -0
  401. package/local/setup/prompt.js +63 -0
  402. package/local/setup/prompt.js.map +1 -0
  403. package/local/setup/wizard.d.ts +43 -0
  404. package/local/setup/wizard.d.ts.map +1 -0
  405. package/local/setup/wizard.js +119 -0
  406. package/local/setup/wizard.js.map +1 -0
  407. package/local/state/health-state.d.ts +26 -0
  408. package/local/state/health-state.d.ts.map +1 -0
  409. package/local/state/health-state.js +34 -0
  410. package/local/state/health-state.js.map +1 -0
  411. package/local/state/index.d.ts +7 -0
  412. package/local/state/index.d.ts.map +1 -0
  413. package/local/state/index.js +8 -0
  414. package/local/state/index.js.map +1 -0
  415. package/local/state/lock.d.ts +13 -0
  416. package/local/state/lock.d.ts.map +1 -0
  417. package/local/state/lock.js +84 -0
  418. package/local/state/lock.js.map +1 -0
  419. package/local/state/log-rotation.d.ts +17 -0
  420. package/local/state/log-rotation.d.ts.map +1 -0
  421. package/local/state/log-rotation.js +101 -0
  422. package/local/state/log-rotation.js.map +1 -0
  423. package/local/state/pidfile.d.ts +11 -0
  424. package/local/state/pidfile.d.ts.map +1 -0
  425. package/local/state/pidfile.js +35 -0
  426. package/local/state/pidfile.js.map +1 -0
  427. package/local/state/proc.d.ts +29 -0
  428. package/local/state/proc.d.ts.map +1 -0
  429. package/local/state/proc.js +84 -0
  430. package/local/state/proc.js.map +1 -0
  431. package/local/state/startup-config.d.ts +24 -0
  432. package/local/state/startup-config.d.ts.map +1 -0
  433. package/local/state/startup-config.js +36 -0
  434. package/local/state/startup-config.js.map +1 -0
  435. package/local/status.d.ts +9 -0
  436. package/local/status.d.ts.map +1 -0
  437. package/local/status.js +170 -0
  438. package/local/status.js.map +1 -0
  439. package/local/temporal/download.d.ts +42 -0
  440. package/local/temporal/download.d.ts.map +1 -0
  441. package/local/temporal/download.js +55 -0
  442. package/local/temporal/download.js.map +1 -0
  443. package/local/temporal/index.d.ts +5 -0
  444. package/local/temporal/index.d.ts.map +1 -0
  445. package/local/temporal/index.js +6 -0
  446. package/local/temporal/index.js.map +1 -0
  447. package/local/temporal/inspect.d.ts +15 -0
  448. package/local/temporal/inspect.d.ts.map +1 -0
  449. package/local/temporal/inspect.js +72 -0
  450. package/local/temporal/inspect.js.map +1 -0
  451. package/local/temporal/manager.d.ts +62 -0
  452. package/local/temporal/manager.d.ts.map +1 -0
  453. package/local/temporal/manager.js +221 -0
  454. package/local/temporal/manager.js.map +1 -0
  455. package/local/temporal/supervisor.d.ts +34 -0
  456. package/local/temporal/supervisor.d.ts.map +1 -0
  457. package/local/temporal/supervisor.js +83 -0
  458. package/local/temporal/supervisor.js.map +1 -0
  459. package/local/webconsole/index.d.ts +3 -0
  460. package/local/webconsole/index.d.ts.map +1 -0
  461. package/local/webconsole/index.js +27 -0
  462. package/local/webconsole/index.js.map +1 -0
  463. package/logger.d.ts +13 -0
  464. package/logger.d.ts.map +1 -0
  465. package/logger.js +35 -0
  466. package/logger.js.map +1 -0
  467. package/marketplace/config.d.ts +61 -0
  468. package/marketplace/config.d.ts.map +1 -0
  469. package/marketplace/config.js +179 -0
  470. package/marketplace/config.js.map +1 -0
  471. package/marketplace/github.d.ts +68 -0
  472. package/marketplace/github.d.ts.map +1 -0
  473. package/marketplace/github.js +272 -0
  474. package/marketplace/github.js.map +1 -0
  475. package/marketplace/index.d.ts +8 -0
  476. package/marketplace/index.d.ts.map +1 -0
  477. package/marketplace/index.js +12 -0
  478. package/marketplace/index.js.map +1 -0
  479. package/marketplace/install.d.ts +35 -0
  480. package/marketplace/install.d.ts.map +1 -0
  481. package/marketplace/install.js +156 -0
  482. package/marketplace/install.js.map +1 -0
  483. package/marketplace/materialize.d.ts +19 -0
  484. package/marketplace/materialize.d.ts.map +1 -0
  485. package/marketplace/materialize.js +53 -0
  486. package/marketplace/materialize.js.map +1 -0
  487. package/marketplace/official.d.ts +28 -0
  488. package/marketplace/official.d.ts.map +1 -0
  489. package/marketplace/official.js +87 -0
  490. package/marketplace/official.js.map +1 -0
  491. package/marketplace/read.d.ts +17 -0
  492. package/marketplace/read.d.ts.map +1 -0
  493. package/marketplace/read.js +44 -0
  494. package/marketplace/read.js.map +1 -0
  495. package/marketplace/ref.d.ts +13 -0
  496. package/marketplace/ref.d.ts.map +1 -0
  497. package/marketplace/ref.js +54 -0
  498. package/marketplace/ref.js.map +1 -0
  499. package/marketplace/render.d.ts +22 -0
  500. package/marketplace/render.d.ts.map +1 -0
  501. package/marketplace/render.js +115 -0
  502. package/marketplace/render.js.map +1 -0
  503. package/output/command-result.d.ts +55 -0
  504. package/output/command-result.d.ts.map +1 -0
  505. package/output/command-result.js +147 -0
  506. package/output/command-result.js.map +1 -0
  507. package/output/confirm.d.ts +22 -0
  508. package/output/confirm.d.ts.map +1 -0
  509. package/output/confirm.js +43 -0
  510. package/output/confirm.js.map +1 -0
  511. package/output/format.d.ts +22 -0
  512. package/output/format.d.ts.map +1 -0
  513. package/output/format.js +72 -0
  514. package/output/format.js.map +1 -0
  515. package/output/index.d.ts +7 -0
  516. package/output/index.d.ts.map +1 -0
  517. package/output/index.js +8 -0
  518. package/output/index.js.map +1 -0
  519. package/output/ndjson.d.ts +7 -0
  520. package/output/ndjson.d.ts.map +1 -0
  521. package/output/ndjson.js +13 -0
  522. package/output/ndjson.js.map +1 -0
  523. package/output/proto.d.ts +12 -0
  524. package/output/proto.d.ts.map +1 -0
  525. package/output/proto.js +34 -0
  526. package/output/proto.js.map +1 -0
  527. package/output/style.d.ts +19 -0
  528. package/output/style.d.ts.map +1 -0
  529. package/output/style.js +44 -0
  530. package/output/style.js.map +1 -0
  531. package/output/table.d.ts +5 -0
  532. package/output/table.d.ts.map +1 -0
  533. package/output/table.js +43 -0
  534. package/output/table.js.map +1 -0
  535. package/package.json +50 -0
  536. package/program.d.ts +17 -0
  537. package/program.d.ts.map +1 -0
  538. package/program.js +195 -0
  539. package/program.js.map +1 -0
  540. package/registry/aliases.d.ts +21 -0
  541. package/registry/aliases.d.ts.map +1 -0
  542. package/registry/aliases.js +99 -0
  543. package/registry/aliases.js.map +1 -0
  544. package/registry/index.d.ts +6 -0
  545. package/registry/index.d.ts.map +1 -0
  546. package/registry/index.js +7 -0
  547. package/registry/index.js.map +1 -0
  548. package/registry/metadata.d.ts +27 -0
  549. package/registry/metadata.d.ts.map +1 -0
  550. package/registry/metadata.js +152 -0
  551. package/registry/metadata.js.map +1 -0
  552. package/registry/registry.d.ts +56 -0
  553. package/registry/registry.d.ts.map +1 -0
  554. package/registry/registry.js +85 -0
  555. package/registry/registry.js.map +1 -0
  556. package/registry/verb-support.d.ts +5 -0
  557. package/registry/verb-support.d.ts.map +1 -0
  558. package/registry/verb-support.js +112 -0
  559. package/registry/verb-support.js.map +1 -0
  560. package/registry/verbs.d.ts +16 -0
  561. package/registry/verbs.d.ts.map +1 -0
  562. package/registry/verbs.js +35 -0
  563. package/registry/verbs.js.map +1 -0
  564. package/resources/access-history.d.ts +28 -0
  565. package/resources/access-history.d.ts.map +1 -0
  566. package/resources/access-history.js +124 -0
  567. package/resources/access-history.js.map +1 -0
  568. package/resources/agent/quality.d.ts +40 -0
  569. package/resources/agent/quality.d.ts.map +1 -0
  570. package/resources/agent/quality.js +338 -0
  571. package/resources/agent/quality.js.map +1 -0
  572. package/resources/agent/resolve.d.ts +5 -0
  573. package/resources/agent/resolve.d.ts.map +1 -0
  574. package/resources/agent/resolve.js +23 -0
  575. package/resources/agent/resolve.js.map +1 -0
  576. package/resources/apply/apply.d.ts +62 -0
  577. package/resources/apply/apply.d.ts.map +1 -0
  578. package/resources/apply/apply.js +371 -0
  579. package/resources/apply/apply.js.map +1 -0
  580. package/resources/apply/handlers.d.ts +54 -0
  581. package/resources/apply/handlers.d.ts.map +1 -0
  582. package/resources/apply/handlers.js +56 -0
  583. package/resources/apply/handlers.js.map +1 -0
  584. package/resources/channel.d.ts +52 -0
  585. package/resources/channel.d.ts.map +1 -0
  586. package/resources/channel.js +404 -0
  587. package/resources/channel.js.map +1 -0
  588. package/resources/connect/address.d.ts +5 -0
  589. package/resources/connect/address.d.ts.map +1 -0
  590. package/resources/connect/address.js +49 -0
  591. package/resources/connect/address.js.map +1 -0
  592. package/resources/connect/connect.d.ts +30 -0
  593. package/resources/connect/connect.d.ts.map +1 -0
  594. package/resources/connect/connect.js +163 -0
  595. package/resources/connect/connect.js.map +1 -0
  596. package/resources/connect/discover.d.ts +7 -0
  597. package/resources/connect/discover.d.ts.map +1 -0
  598. package/resources/connect/discover.js +122 -0
  599. package/resources/connect/discover.js.map +1 -0
  600. package/resources/connect/display.d.ts +10 -0
  601. package/resources/connect/display.d.ts.map +1 -0
  602. package/resources/connect/display.js +50 -0
  603. package/resources/connect/display.js.map +1 -0
  604. package/resources/connect/sign-in.d.ts +48 -0
  605. package/resources/connect/sign-in.d.ts.map +1 -0
  606. package/resources/connect/sign-in.js +158 -0
  607. package/resources/connect/sign-in.js.map +1 -0
  608. package/resources/cursor-pages.d.ts +9 -0
  609. package/resources/cursor-pages.d.ts.map +1 -0
  610. package/resources/cursor-pages.js +25 -0
  611. package/resources/cursor-pages.js.map +1 -0
  612. package/resources/delete.d.ts +34 -0
  613. package/resources/delete.d.ts.map +1 -0
  614. package/resources/delete.js +260 -0
  615. package/resources/delete.js.map +1 -0
  616. package/resources/documents.d.ts +16 -0
  617. package/resources/documents.d.ts.map +1 -0
  618. package/resources/documents.js +89 -0
  619. package/resources/documents.js.map +1 -0
  620. package/resources/download.d.ts +16 -0
  621. package/resources/download.d.ts.map +1 -0
  622. package/resources/download.js +101 -0
  623. package/resources/download.js.map +1 -0
  624. package/resources/eval-run/follow.d.ts +67 -0
  625. package/resources/eval-run/follow.d.ts.map +1 -0
  626. package/resources/eval-run/follow.js +285 -0
  627. package/resources/eval-run/follow.js.map +1 -0
  628. package/resources/eval-run/node-io.d.ts +4 -0
  629. package/resources/eval-run/node-io.d.ts.map +1 -0
  630. package/resources/eval-run/node-io.js +39 -0
  631. package/resources/eval-run/node-io.js.map +1 -0
  632. package/resources/eval-run/options.d.ts +116 -0
  633. package/resources/eval-run/options.d.ts.map +1 -0
  634. package/resources/eval-run/options.js +265 -0
  635. package/resources/eval-run/options.js.map +1 -0
  636. package/resources/eval-run/report.d.ts +64 -0
  637. package/resources/eval-run/report.d.ts.map +1 -0
  638. package/resources/eval-run/report.js +434 -0
  639. package/resources/eval-run/report.js.map +1 -0
  640. package/resources/eval-run/run.d.ts +10 -0
  641. package/resources/eval-run/run.d.ts.map +1 -0
  642. package/resources/eval-run/run.js +158 -0
  643. package/resources/eval-run/run.js.map +1 -0
  644. package/resources/eval.d.ts +69 -0
  645. package/resources/eval.d.ts.map +1 -0
  646. package/resources/eval.js +225 -0
  647. package/resources/eval.js.map +1 -0
  648. package/resources/get-bindings.d.ts +13 -0
  649. package/resources/get-bindings.d.ts.map +1 -0
  650. package/resources/get-bindings.js +93 -0
  651. package/resources/get-bindings.js.map +1 -0
  652. package/resources/get.d.ts +6 -0
  653. package/resources/get.d.ts.map +1 -0
  654. package/resources/get.js +43 -0
  655. package/resources/get.js.map +1 -0
  656. package/resources/git.d.ts +15 -0
  657. package/resources/git.d.ts.map +1 -0
  658. package/resources/git.js +69 -0
  659. package/resources/git.js.map +1 -0
  660. package/resources/ignore/index.d.ts +3 -0
  661. package/resources/ignore/index.d.ts.map +1 -0
  662. package/resources/ignore/index.js +6 -0
  663. package/resources/ignore/index.js.map +1 -0
  664. package/resources/ignore/matcher.d.ts +11 -0
  665. package/resources/ignore/matcher.d.ts.map +1 -0
  666. package/resources/ignore/matcher.js +41 -0
  667. package/resources/ignore/matcher.js.map +1 -0
  668. package/resources/list.d.ts +16 -0
  669. package/resources/list.d.ts.map +1 -0
  670. package/resources/list.js +453 -0
  671. package/resources/list.js.map +1 -0
  672. package/resources/mcp/add.d.ts +18 -0
  673. package/resources/mcp/add.d.ts.map +1 -0
  674. package/resources/mcp/add.js +95 -0
  675. package/resources/mcp/add.js.map +1 -0
  676. package/resources/mcp/placeholder-resolver.d.ts +19 -0
  677. package/resources/mcp/placeholder-resolver.d.ts.map +1 -0
  678. package/resources/mcp/placeholder-resolver.js +52 -0
  679. package/resources/mcp/placeholder-resolver.js.map +1 -0
  680. package/resources/picker/ink.d.ts +45 -0
  681. package/resources/picker/ink.d.ts.map +1 -0
  682. package/resources/picker/ink.js +45 -0
  683. package/resources/picker/ink.js.map +1 -0
  684. package/resources/picker/tty.d.ts +19 -0
  685. package/resources/picker/tty.d.ts.map +1 -0
  686. package/resources/picker/tty.js +21 -0
  687. package/resources/picker/tty.js.map +1 -0
  688. package/resources/plugin.d.ts +131 -0
  689. package/resources/plugin.d.ts.map +1 -0
  690. package/resources/plugin.js +382 -0
  691. package/resources/plugin.js.map +1 -0
  692. package/resources/reference.d.ts +44 -0
  693. package/resources/reference.d.ts.map +1 -0
  694. package/resources/reference.js +152 -0
  695. package/resources/reference.js.map +1 -0
  696. package/resources/render.d.ts +52 -0
  697. package/resources/render.d.ts.map +1 -0
  698. package/resources/render.js +135 -0
  699. package/resources/render.js.map +1 -0
  700. package/resources/run/agent-exec.d.ts +16 -0
  701. package/resources/run/agent-exec.d.ts.map +1 -0
  702. package/resources/run/agent-exec.js +89 -0
  703. package/resources/run/agent-exec.js.map +1 -0
  704. package/resources/run/attachments.d.ts +24 -0
  705. package/resources/run/attachments.d.ts.map +1 -0
  706. package/resources/run/attachments.js +237 -0
  707. package/resources/run/attachments.js.map +1 -0
  708. package/resources/run/create.d.ts +57 -0
  709. package/resources/run/create.d.ts.map +1 -0
  710. package/resources/run/create.js +112 -0
  711. package/resources/run/create.js.map +1 -0
  712. package/resources/run/epilogue.d.ts +9 -0
  713. package/resources/run/epilogue.d.ts.map +1 -0
  714. package/resources/run/epilogue.js +172 -0
  715. package/resources/run/epilogue.js.map +1 -0
  716. package/resources/run/header.d.ts +31 -0
  717. package/resources/run/header.d.ts.map +1 -0
  718. package/resources/run/header.js +54 -0
  719. package/resources/run/header.js.map +1 -0
  720. package/resources/run/plugins.d.ts +11 -0
  721. package/resources/run/plugins.d.ts.map +1 -0
  722. package/resources/run/plugins.js +18 -0
  723. package/resources/run/plugins.js.map +1 -0
  724. package/resources/run/prepare.d.ts +149 -0
  725. package/resources/run/prepare.d.ts.map +1 -0
  726. package/resources/run/prepare.js +180 -0
  727. package/resources/run/prepare.js.map +1 -0
  728. package/resources/run/references.d.ts +22 -0
  729. package/resources/run/references.d.ts.map +1 -0
  730. package/resources/run/references.js +56 -0
  731. package/resources/run/references.js.map +1 -0
  732. package/resources/run/resolve.d.ts +5 -0
  733. package/resources/run/resolve.d.ts.map +1 -0
  734. package/resources/run/resolve.js +16 -0
  735. package/resources/run/resolve.js.map +1 -0
  736. package/resources/run/resume.d.ts +12 -0
  737. package/resources/run/resume.d.ts.map +1 -0
  738. package/resources/run/resume.js +97 -0
  739. package/resources/run/resume.js.map +1 -0
  740. package/resources/run/stream.d.ts +24 -0
  741. package/resources/run/stream.d.ts.map +1 -0
  742. package/resources/run/stream.js +74 -0
  743. package/resources/run/stream.js.map +1 -0
  744. package/resources/run/vaults.d.ts +11 -0
  745. package/resources/run/vaults.d.ts.map +1 -0
  746. package/resources/run/vaults.js +19 -0
  747. package/resources/run/vaults.js.map +1 -0
  748. package/resources/run/workspace.d.ts +17 -0
  749. package/resources/run/workspace.d.ts.map +1 -0
  750. package/resources/run/workspace.js +147 -0
  751. package/resources/run/workspace.js.map +1 -0
  752. package/resources/run-approve.d.ts +18 -0
  753. package/resources/run-approve.d.ts.map +1 -0
  754. package/resources/run-approve.js +72 -0
  755. package/resources/run-approve.js.map +1 -0
  756. package/resources/run-control.d.ts +19 -0
  757. package/resources/run-control.d.ts.map +1 -0
  758. package/resources/run-control.js +37 -0
  759. package/resources/run-control.js.map +1 -0
  760. package/resources/run-format.d.ts +15 -0
  761. package/resources/run-format.d.ts.map +1 -0
  762. package/resources/run-format.js +49 -0
  763. package/resources/run-format.js.map +1 -0
  764. package/resources/run-logs.d.ts +17 -0
  765. package/resources/run-logs.d.ts.map +1 -0
  766. package/resources/run-logs.js +105 -0
  767. package/resources/run-logs.js.map +1 -0
  768. package/resources/run-scores.d.ts +20 -0
  769. package/resources/run-scores.d.ts.map +1 -0
  770. package/resources/run-scores.js +132 -0
  771. package/resources/run-scores.js.map +1 -0
  772. package/resources/run-to-eval-case.d.ts +24 -0
  773. package/resources/run-to-eval-case.d.ts.map +1 -0
  774. package/resources/run-to-eval-case.js +113 -0
  775. package/resources/run-to-eval-case.js.map +1 -0
  776. package/resources/run-trace.d.ts +10 -0
  777. package/resources/run-trace.d.ts.map +1 -0
  778. package/resources/run-trace.js +81 -0
  779. package/resources/run-trace.js.map +1 -0
  780. package/resources/runs.d.ts +45 -0
  781. package/resources/runs.d.ts.map +1 -0
  782. package/resources/runs.js +123 -0
  783. package/resources/runs.js.map +1 -0
  784. package/resources/schedule.d.ts +22 -0
  785. package/resources/schedule.d.ts.map +1 -0
  786. package/resources/schedule.js +116 -0
  787. package/resources/schedule.js.map +1 -0
  788. package/resources/search.d.ts +21 -0
  789. package/resources/search.d.ts.map +1 -0
  790. package/resources/search.js +25 -0
  791. package/resources/search.js.map +1 -0
  792. package/resources/service-account.d.ts +37 -0
  793. package/resources/service-account.d.ts.map +1 -0
  794. package/resources/service-account.js +139 -0
  795. package/resources/service-account.js.map +1 -0
  796. package/resources/session.d.ts +20 -0
  797. package/resources/session.d.ts.map +1 -0
  798. package/resources/session.js +51 -0
  799. package/resources/session.js.map +1 -0
  800. package/resources/skill.d.ts +123 -0
  801. package/resources/skill.d.ts.map +1 -0
  802. package/resources/skill.js +441 -0
  803. package/resources/skill.js.map +1 -0
  804. package/resources/stream/answer.d.ts +34 -0
  805. package/resources/stream/answer.d.ts.map +1 -0
  806. package/resources/stream/answer.js +88 -0
  807. package/resources/stream/answer.js.map +1 -0
  808. package/resources/stream/headless.d.ts +45 -0
  809. package/resources/stream/headless.d.ts.map +1 -0
  810. package/resources/stream/headless.js +100 -0
  811. package/resources/stream/headless.js.map +1 -0
  812. package/resources/stream/ink.d.ts +22 -0
  813. package/resources/stream/ink.d.ts.map +1 -0
  814. package/resources/stream/ink.js +24 -0
  815. package/resources/stream/ink.js.map +1 -0
  816. package/resources/stream/render-ndjson.d.ts +23 -0
  817. package/resources/stream/render-ndjson.d.ts.map +1 -0
  818. package/resources/stream/render-ndjson.js +28 -0
  819. package/resources/stream/render-ndjson.js.map +1 -0
  820. package/resources/stream/render-plaintext.d.ts +27 -0
  821. package/resources/stream/render-plaintext.d.ts.map +1 -0
  822. package/resources/stream/render-plaintext.js +142 -0
  823. package/resources/stream/render-plaintext.js.map +1 -0
  824. package/resources/stream/submit.d.ts +25 -0
  825. package/resources/stream/submit.d.ts.map +1 -0
  826. package/resources/stream/submit.js +132 -0
  827. package/resources/stream/submit.js.map +1 -0
  828. package/resources/stream/tty.d.ts +8 -0
  829. package/resources/stream/tty.d.ts.map +1 -0
  830. package/resources/stream/tty.js +15 -0
  831. package/resources/stream/tty.js.map +1 -0
  832. package/resources/tag.d.ts +4 -0
  833. package/resources/tag.d.ts.map +1 -0
  834. package/resources/tag.js +37 -0
  835. package/resources/tag.js.map +1 -0
  836. package/resources/usage.d.ts +47 -0
  837. package/resources/usage.d.ts.map +1 -0
  838. package/resources/usage.js +376 -0
  839. package/resources/usage.js.map +1 -0
  840. package/resources/validate.d.ts +7 -0
  841. package/resources/validate.d.ts.map +1 -0
  842. package/resources/validate.js +36 -0
  843. package/resources/validate.js.map +1 -0
  844. package/resources/vault.d.ts +79 -0
  845. package/resources/vault.d.ts.map +1 -0
  846. package/resources/vault.js +277 -0
  847. package/resources/vault.js.map +1 -0
  848. package/resources/version.d.ts +11 -0
  849. package/resources/version.d.ts.map +1 -0
  850. package/resources/version.js +80 -0
  851. package/resources/version.js.map +1 -0
  852. package/runtime.d.ts +5 -0
  853. package/runtime.d.ts.map +1 -0
  854. package/runtime.js +19 -0
  855. package/runtime.js.map +1 -0
  856. package/src/auth/auth0.ts +43 -0
  857. package/src/auth/callback-server.ts +104 -0
  858. package/src/auth/index.ts +6 -0
  859. package/src/auth/login.ts +106 -0
  860. package/src/auth/pages.ts +31 -0
  861. package/src/auth/pkce.ts +26 -0
  862. package/src/auth/token.ts +159 -0
  863. package/src/backend.ts +15 -0
  864. package/src/cli/stigmer.ts +16 -0
  865. package/src/client/client.ts +157 -0
  866. package/src/client/index.ts +3 -0
  867. package/src/client/organizations.ts +118 -0
  868. package/src/client/single-org.ts +62 -0
  869. package/src/commands/access-history/index.ts +124 -0
  870. package/src/commands/agent-exec-flags.ts +76 -0
  871. package/src/commands/agent.ts +57 -0
  872. package/src/commands/apikey/duration.ts +35 -0
  873. package/src/commands/apikey/index.ts +273 -0
  874. package/src/commands/apply.ts +200 -0
  875. package/src/commands/auth/index.ts +76 -0
  876. package/src/commands/auth/login.ts +70 -0
  877. package/src/commands/auth/whoami.ts +157 -0
  878. package/src/commands/channel.ts +178 -0
  879. package/src/commands/completion.ts +94 -0
  880. package/src/commands/config/backend.ts +239 -0
  881. package/src/commands/config/context.ts +110 -0
  882. package/src/commands/config/index.ts +18 -0
  883. package/src/commands/config/kv.ts +44 -0
  884. package/src/commands/connect.ts +115 -0
  885. package/src/commands/delete.ts +59 -0
  886. package/src/commands/down.ts +27 -0
  887. package/src/commands/download.ts +113 -0
  888. package/src/commands/eval.ts +148 -0
  889. package/src/commands/get.ts +138 -0
  890. package/src/commands/install.ts +108 -0
  891. package/src/commands/internal-daemon.ts +17 -0
  892. package/src/commands/list.ts +164 -0
  893. package/src/commands/logs.ts +73 -0
  894. package/src/commands/marketplace.ts +133 -0
  895. package/src/commands/mcp-server.ts +125 -0
  896. package/src/commands/mcp.ts +92 -0
  897. package/src/commands/push.ts +630 -0
  898. package/src/commands/reset.ts +57 -0
  899. package/src/commands/resume.ts +122 -0
  900. package/src/commands/run.ts +228 -0
  901. package/src/commands/runs/approve.ts +57 -0
  902. package/src/commands/runs/control.ts +96 -0
  903. package/src/commands/runs/index.ts +31 -0
  904. package/src/commands/runs/logs.ts +45 -0
  905. package/src/commands/runs/scores.ts +34 -0
  906. package/src/commands/runs/to-eval-case.ts +50 -0
  907. package/src/commands/runs/trace.ts +32 -0
  908. package/src/commands/schedule.ts +70 -0
  909. package/src/commands/search.ts +95 -0
  910. package/src/commands/service-account/index.ts +180 -0
  911. package/src/commands/setup.ts +97 -0
  912. package/src/commands/shared.ts +41 -0
  913. package/src/commands/status.ts +20 -0
  914. package/src/commands/tag.ts +31 -0
  915. package/src/commands/up.ts +113 -0
  916. package/src/commands/usage.ts +116 -0
  917. package/src/commands/validate.ts +127 -0
  918. package/src/commands/vault.ts +245 -0
  919. package/src/commands/version.ts +13 -0
  920. package/src/config/config.ts +308 -0
  921. package/src/config/index.ts +33 -0
  922. package/src/config/keys.ts +105 -0
  923. package/src/config/paths.ts +25 -0
  924. package/src/config/resolve.ts +174 -0
  925. package/src/errors/classify.ts +249 -0
  926. package/src/errors/cli-exit-error.ts +16 -0
  927. package/src/errors/exit-codes.ts +17 -0
  928. package/src/errors/handle.ts +53 -0
  929. package/src/errors/index.ts +7 -0
  930. package/src/errors/usage-error.ts +14 -0
  931. package/src/index.ts +8 -0
  932. package/src/local/artifact.ts +351 -0
  933. package/src/local/constants.ts +55 -0
  934. package/src/local/daemon/components.ts +145 -0
  935. package/src/local/daemon/env.ts +171 -0
  936. package/src/local/daemon/host.ts +128 -0
  937. package/src/local/daemon/index.ts +18 -0
  938. package/src/local/daemon/launch.ts +326 -0
  939. package/src/local/daemon/process.ts +213 -0
  940. package/src/local/daemon/supervisor.ts +249 -0
  941. package/src/local/daemon/types.ts +78 -0
  942. package/src/local/llm-config.ts +86 -0
  943. package/src/local/logs/tail.ts +100 -0
  944. package/src/local/logs/view.ts +65 -0
  945. package/src/local/net/tcp.ts +65 -0
  946. package/src/local/operator-config.ts +95 -0
  947. package/src/local/paths.ts +95 -0
  948. package/src/local/reset.ts +90 -0
  949. package/src/local/runtime/index.ts +19 -0
  950. package/src/local/runtime/node.ts +173 -0
  951. package/src/local/runtime/runner.ts +159 -0
  952. package/src/local/runtime/runtimes-install.ts +75 -0
  953. package/src/local/runtime/server.ts +197 -0
  954. package/src/local/runtime/which.ts +30 -0
  955. package/src/local/setup/prompt.ts +70 -0
  956. package/src/local/setup/wizard.ts +140 -0
  957. package/src/local/state/health-state.ts +55 -0
  958. package/src/local/state/index.ts +8 -0
  959. package/src/local/state/lock.ts +92 -0
  960. package/src/local/state/log-rotation.ts +112 -0
  961. package/src/local/state/pidfile.ts +37 -0
  962. package/src/local/state/proc.ts +82 -0
  963. package/src/local/state/startup-config.ts +55 -0
  964. package/src/local/status.ts +189 -0
  965. package/src/local/temporal/download.ts +84 -0
  966. package/src/local/temporal/index.ts +14 -0
  967. package/src/local/temporal/inspect.ts +80 -0
  968. package/src/local/temporal/manager.ts +261 -0
  969. package/src/local/temporal/supervisor.ts +97 -0
  970. package/src/local/webconsole/index.ts +30 -0
  971. package/src/logger.ts +39 -0
  972. package/src/marketplace/config.ts +255 -0
  973. package/src/marketplace/github.ts +410 -0
  974. package/src/marketplace/index.ts +54 -0
  975. package/src/marketplace/install.ts +217 -0
  976. package/src/marketplace/materialize.ts +80 -0
  977. package/src/marketplace/official.ts +126 -0
  978. package/src/marketplace/read.ts +76 -0
  979. package/src/marketplace/ref.ts +74 -0
  980. package/src/marketplace/render.ts +151 -0
  981. package/src/output/command-result.ts +199 -0
  982. package/src/output/confirm.ts +54 -0
  983. package/src/output/format.ts +101 -0
  984. package/src/output/index.ts +30 -0
  985. package/src/output/ndjson.ts +14 -0
  986. package/src/output/proto.ts +55 -0
  987. package/src/output/style.ts +57 -0
  988. package/src/output/table.ts +52 -0
  989. package/src/program.ts +209 -0
  990. package/src/registry/aliases.ts +113 -0
  991. package/src/registry/index.ts +30 -0
  992. package/src/registry/metadata.ts +170 -0
  993. package/src/registry/registry.ts +135 -0
  994. package/src/registry/verb-support.ts +117 -0
  995. package/src/registry/verbs.ts +42 -0
  996. package/src/resources/access-history.ts +167 -0
  997. package/src/resources/agent/quality.ts +424 -0
  998. package/src/resources/agent/resolve.ts +31 -0
  999. package/src/resources/apply/apply.ts +472 -0
  1000. package/src/resources/apply/handlers.ts +115 -0
  1001. package/src/resources/channel.ts +548 -0
  1002. package/src/resources/connect/address.ts +47 -0
  1003. package/src/resources/connect/connect.ts +219 -0
  1004. package/src/resources/connect/discover.ts +145 -0
  1005. package/src/resources/connect/display.ts +67 -0
  1006. package/src/resources/connect/sign-in.ts +210 -0
  1007. package/src/resources/cursor-pages.ts +35 -0
  1008. package/src/resources/delete.ts +381 -0
  1009. package/src/resources/documents.ts +102 -0
  1010. package/src/resources/download.ts +147 -0
  1011. package/src/resources/eval-run/follow.ts +390 -0
  1012. package/src/resources/eval-run/node-io.ts +42 -0
  1013. package/src/resources/eval-run/options.ts +391 -0
  1014. package/src/resources/eval-run/report.ts +529 -0
  1015. package/src/resources/eval-run/run.ts +230 -0
  1016. package/src/resources/eval.ts +278 -0
  1017. package/src/resources/get-bindings.ts +137 -0
  1018. package/src/resources/get.ts +51 -0
  1019. package/src/resources/git.ts +71 -0
  1020. package/src/resources/ignore/index.ts +20 -0
  1021. package/src/resources/ignore/matcher.ts +55 -0
  1022. package/src/resources/list.ts +552 -0
  1023. package/src/resources/mcp/add.ts +105 -0
  1024. package/src/resources/mcp/placeholder-resolver.ts +63 -0
  1025. package/src/resources/picker/ink.tsx +123 -0
  1026. package/src/resources/picker/tty.ts +30 -0
  1027. package/src/resources/plugin.ts +560 -0
  1028. package/src/resources/reference.ts +170 -0
  1029. package/src/resources/render.ts +198 -0
  1030. package/src/resources/run/agent-exec.ts +115 -0
  1031. package/src/resources/run/attachments.ts +300 -0
  1032. package/src/resources/run/create.ts +181 -0
  1033. package/src/resources/run/epilogue.ts +200 -0
  1034. package/src/resources/run/header.ts +72 -0
  1035. package/src/resources/run/plugins.ts +32 -0
  1036. package/src/resources/run/prepare.ts +350 -0
  1037. package/src/resources/run/references.ts +77 -0
  1038. package/src/resources/run/resolve.ts +18 -0
  1039. package/src/resources/run/resume.ts +122 -0
  1040. package/src/resources/run/stream.ts +96 -0
  1041. package/src/resources/run/vaults.ts +33 -0
  1042. package/src/resources/run/workspace.ts +171 -0
  1043. package/src/resources/run-approve.ts +94 -0
  1044. package/src/resources/run-control.ts +59 -0
  1045. package/src/resources/run-format.ts +44 -0
  1046. package/src/resources/run-logs.ts +131 -0
  1047. package/src/resources/run-scores.ts +164 -0
  1048. package/src/resources/run-to-eval-case.ts +160 -0
  1049. package/src/resources/run-trace.ts +102 -0
  1050. package/src/resources/runs.ts +155 -0
  1051. package/src/resources/schedule.ts +132 -0
  1052. package/src/resources/search.ts +62 -0
  1053. package/src/resources/service-account.ts +195 -0
  1054. package/src/resources/session.ts +64 -0
  1055. package/src/resources/skill.ts +577 -0
  1056. package/src/resources/stream/answer.ts +125 -0
  1057. package/src/resources/stream/headless.ts +150 -0
  1058. package/src/resources/stream/ink.tsx +56 -0
  1059. package/src/resources/stream/render-ndjson.ts +44 -0
  1060. package/src/resources/stream/render-plaintext.ts +153 -0
  1061. package/src/resources/stream/submit.ts +145 -0
  1062. package/src/resources/stream/tty.ts +14 -0
  1063. package/src/resources/tag.ts +51 -0
  1064. package/src/resources/usage.ts +443 -0
  1065. package/src/resources/validate.ts +37 -0
  1066. package/src/resources/vault.ts +372 -0
  1067. package/src/resources/version.ts +103 -0
  1068. package/src/runtime.ts +23 -0
  1069. package/src/types/marked-terminal.d.ts +31 -0
  1070. package/src/version.ts +46 -0
  1071. package/version.d.ts +2 -0
  1072. package/version.d.ts.map +1 -0
  1073. package/version.js +45 -0
  1074. package/version.js.map +1 -0
@@ -0,0 +1,434 @@
1
+ // What `eval run` prints and the code it exits with, read from the eval
2
+ // run. Pure: every line is a function of the eval run, so the views are
3
+ // unit-tested.
4
+ //
5
+ // The shapes are Claude Code's `claude plugin eval` output, so its readers
6
+ // read Stigmer's: a case's `mocked:` line before its first finished try
7
+ // (each mocked server with its tools, each other server "not started: no
8
+ // mock"), a progress line as each try finishes (case, target, arm, try, and
9
+ // its score or why it was not graded, and which mock stopped it when one
10
+ // did), then the summary table (`CASE WITH W/OUT Δ PASS^k RUNS COST NOTES`
11
+ // when the eval run compares a plugin, else `SCORE PASS%`), then the line
12
+ // `N case(s) · mean Δ +0.67 · 74s · $0.41`. Stigmer adds the `PASS^k`
13
+ // column (every try passed) and lists the cases it did not run with the
14
+ // feature named.
15
+ //
16
+ // An eval run's cases are grouped by the plugin they come from, so the
17
+ // table is one block per plugin and target, headed by what ran: the
18
+ // agent's slug and short version hash on an eval run about an agent, the
19
+ // plugin's slug and short digest (marked `compared` when it is the plugin
20
+ // each case ran with and without), and the target, so two versions'
21
+ // outputs read side by side. A plugin none of whose cases ran (not
22
+ // selected, or no cases at the version a try loads) is a block of its own
23
+ // heading and its reason. An Eval whose cases ran on the agent follows the
24
+ // plugins, one block of its own per target, headed `eval <slug>@<digest>`.
25
+ // On an eval run about an agent a progress line names its case as
26
+ // `<plugin>/<case>` or `<eval>/<case>`, since two sources may each have a
27
+ // case of one name; about a plugin, by its name alone, as the format does.
28
+ //
29
+ // The exit code is the format's: 0 when every case that ran reached the
30
+ // threshold and every case file loaded; 1 for a case below it, a load
31
+ // finding, no case at all, or a case whose with-plugin tries on a target
32
+ // were all not graded (a case that never ran is not a pass); 2 for a
33
+ // partial run (the spending limit, or a reason or phase a newer server
34
+ // added); 130 when the eval run was cancelled, by this command's Ctrl+C or
35
+ // any other cancel (the console's Cancel, `eval cancel`). A connection or
36
+ // sign-in failure keeps the CLI's own 3 or 4, outside this file. A case
37
+ // Stigmer lists as not run (a feature it does not run yet) is named, not
38
+ // failed: it never ran, and `--case` or `--tag` leaves it out.
39
+ //
40
+ // Every text that reaches the terminal from the eval run (a case name, a
41
+ // target's model name as the suite's case.yaml wrote it, a reason, an
42
+ // error) goes through `oneLine`, so a suite cannot write a newline or an
43
+ // escape sequence there.
44
+ import { evalRunTargetLabel } from "@stigmer/client";
45
+ import { EvalRunAblation } from "@stigmer/apis/ai/stigmer/agentic/evalrun/v1/spec_pb";
46
+ import { EvalRunPartialReason, EvalRunPhase, EvalRunTryState, } from "@stigmer/apis/ai/stigmer/agentic/evalrun/v1/status_pb";
47
+ import { renderTable } from "../../output/index.js";
48
+ /** The exit codes of `eval run`, as Claude Code's `claude plugin eval` has them. */
49
+ export const EvalExit = {
50
+ Passed: 0,
51
+ Failed: 1,
52
+ Partial: 2,
53
+ Interrupted: 130,
54
+ };
55
+ /** The longest NOTES cell, so one long error does not widen every row. */
56
+ const MAX_NOTE = 60;
57
+ /** How many characters of a version hash or a digest a heading shows. */
58
+ const SHORT_HASH = 12;
59
+ /** Every finished try of every plugin and Eval, with its progress line, in the eval run's order. */
60
+ export function finishedTries(evalRun) {
61
+ return groupsOf(evalRun).flatMap((group) => finishedTriesOf(group.cases, group.scope));
62
+ }
63
+ /** Every finished try of `cases`, with its progress line, in their order. */
64
+ function finishedTriesOf(cases, scope) {
65
+ const finished = [];
66
+ for (const evalCase of cases) {
67
+ let mockedShown = evalCase.mockedServers.length === 0;
68
+ const caseName = `${scope.name}${oneLine(evalCase.caseName)}`;
69
+ for (const result of evalCase.targets) {
70
+ const target = targetLabel(result.target);
71
+ for (const [arm, tries] of armsOf(result)) {
72
+ for (const attempt of tries) {
73
+ if (!isFinished(attempt))
74
+ continue;
75
+ if (!mockedShown) {
76
+ mockedShown = true;
77
+ finished.push({ key: `${scope.key}${evalCase.path}|mocked`, line: mockedLine(caseName, evalCase) });
78
+ }
79
+ finished.push({
80
+ key: `${scope.key}${evalCase.path}|${target}|${arm}|${attempt.index}`,
81
+ line: progressLine(caseName, target, arm, attempt, tries.length),
82
+ });
83
+ }
84
+ }
85
+ }
86
+ }
87
+ return finished;
88
+ }
89
+ /**
90
+ * The table blocks, plugin by plugin in the eval run's order: one per
91
+ * target a plugin's cases ran on, or one line of why none of them ran.
92
+ * `subjectSlug` names an eval run's agent; unused on one about a plugin.
93
+ */
94
+ export function renderEvalTables(evalRun, subjectSlug) {
95
+ const agent = isAboutAgent(evalRun) ? headingOf(oneLine(subjectSlug), versionHashOf(evalRun)) : "";
96
+ const oneArm = !isComparing(evalRun);
97
+ const blocks = [];
98
+ for (const group of groupsOf(evalRun)) {
99
+ const head = agent === "" ? group.heading : `${agent} · ${group.heading}`;
100
+ if (group.notRunReason !== "") {
101
+ blocks.push(`${head}\n not run: ${oneLine(group.notRunReason)}\n`);
102
+ continue;
103
+ }
104
+ blocks.push(...renderCaseBlocks(group.cases, evalRun.spec?.targets ?? [], oneArm, (target) => `${head} · ${target}`));
105
+ }
106
+ return blocks.join("\n");
107
+ }
108
+ /**
109
+ * The table blocks of `cases`, one per target any of them ran on (the
110
+ * spec's targets first), each under `heading(target)`; cases not run are
111
+ * left out, and a target nothing ran on has no block.
112
+ */
113
+ function renderCaseBlocks(allCases, specTargets, oneArm, heading) {
114
+ const cases = allCases.filter((evalCase) => evalCase.notRunReason === "");
115
+ const blocks = [];
116
+ for (const label of targetLabels(specTargets, allCases)) {
117
+ const rows = cases.flatMap((evalCase) => {
118
+ const result = evalCase.targets.find((candidate) => targetLabel(candidate.target) === label);
119
+ if (result === undefined || result.notRunReason !== "")
120
+ return [];
121
+ return [oneArm ? oneArmRow(evalCase, result) : twoArmRow(evalCase, result)];
122
+ });
123
+ if (rows.length === 0)
124
+ continue;
125
+ const headers = oneArm
126
+ ? ["CASE", "SCORE", "PASS%", "PASS^k", "RUNS", "COST", "NOTES"]
127
+ : ["CASE", "WITH", "W/OUT", "Δ", "PASS^k", "RUNS", "COST", "NOTES"];
128
+ blocks.push(`${heading(label)}\n${renderTable(headers, rows)}`);
129
+ }
130
+ return blocks;
131
+ }
132
+ /** `N case(s) · mean Δ +0.67 · 74s · $0.41`. */
133
+ export function renderSummaryLine(evalRun, durationSeconds) {
134
+ const status = evalRun.status;
135
+ const parts = [`${status?.aggregates?.casesTotal ?? 0} case(s)`];
136
+ const meanDelta = status?.aggregates?.meanDelta;
137
+ if (meanDelta !== undefined) {
138
+ parts.push(`mean Δ ${signed(meanDelta)}`);
139
+ }
140
+ parts.push(`${Math.round(durationSeconds)}s`, usd(status?.costUsd ?? 0));
141
+ return parts.join(" · ");
142
+ }
143
+ /**
144
+ * The text for stdout: the tables, the summary, why the eval run stopped
145
+ * (or `stillRunning` in its place, when not empty), and the cases not run.
146
+ */
147
+ export function renderEvalReport(evalRun, opts) {
148
+ const tables = renderEvalTables(evalRun, opts.subjectSlug);
149
+ const lines = [];
150
+ if (tables !== "")
151
+ lines.push(tables);
152
+ lines.push(renderSummaryLine(evalRun, opts.durationSeconds));
153
+ const partial = opts.stillRunning === "" ? partialLine(evalRun, opts.interrupted) : opts.stillRunning;
154
+ if (partial !== "")
155
+ lines.push(partial);
156
+ const notRun = renderNotRun(evalRun);
157
+ return `\n${lines.join("\n")}\n${notRun === "" ? "" : `\n${notRun}`}`;
158
+ }
159
+ /** The cases, or cases on one target, the eval run listed and did not run, one line each. */
160
+ export function renderNotRun(evalRun) {
161
+ return notRunSection(groupsOf(evalRun).flatMap((group) => notRunLinesOf(group.cases, group.scope)));
162
+ }
163
+ /** The `Not run:` section for `lines`; empty when there are none. */
164
+ function notRunSection(lines) {
165
+ return lines.length === 0 ? "" : `Not run:\n${lines.join("\n")}\n`;
166
+ }
167
+ /** One indented line per case of `cases`, or case on one target, listed and not run. */
168
+ function notRunLinesOf(cases, scope) {
169
+ const lines = [];
170
+ for (const evalCase of cases) {
171
+ const caseName = `${scope.name}${oneLine(evalCase.caseName)}`;
172
+ if (evalCase.notRunReason !== "") {
173
+ lines.push(` ${caseName}: ${oneLine(evalCase.notRunReason)}`);
174
+ continue;
175
+ }
176
+ for (const result of evalCase.targets) {
177
+ if (result.notRunReason !== "") {
178
+ lines.push(` ${caseName} on ${targetLabel(result.target)}: ${oneLine(result.notRunReason)}`);
179
+ }
180
+ }
181
+ }
182
+ return lines;
183
+ }
184
+ /** Why the eval stopped early, in words; empty when it did not. */
185
+ export function partialLine(evaluation, interrupted) {
186
+ const status = evaluation.status;
187
+ const reason = status?.partialReason ?? EvalRunPartialReason.unspecified;
188
+ const ran = `${status?.triesFinished ?? 0} of ${status?.triesTotal ?? 0} tries ran`;
189
+ switch (reason) {
190
+ case EvalRunPartialReason.cost_ceiling:
191
+ return `Stopped at the spending limit: ${ran}.`;
192
+ case EvalRunPartialReason.cancelled:
193
+ return `Cancelled: ${ran}.`;
194
+ case EvalRunPartialReason.unspecified:
195
+ return interrupted ? `Cancelled: ${ran}.` : "";
196
+ default: {
197
+ // A reason a newer server added: the run still stopped early.
198
+ const unknown = reason;
199
+ return `Stopped early (reason ${String(unknown)}): ${ran}.`;
200
+ }
201
+ }
202
+ }
203
+ /** The exit code a finished (or cancelled) eval run earns over every plugin's cases (the module header). */
204
+ export function evalExitCode(evalRun, opts) {
205
+ const status = evalRun.status;
206
+ const cases = groupsOf(evalRun).flatMap((group) => group.cases);
207
+ const reason = status?.partialReason ?? EvalRunPartialReason.unspecified;
208
+ if (opts.interrupted || reason === EvalRunPartialReason.cancelled) {
209
+ return EvalExit.Interrupted;
210
+ }
211
+ if (reason !== EvalRunPartialReason.unspecified) {
212
+ // The spending limit, or a reason a newer server added: the
213
+ // eval stopped early, so its aggregates cover only part of the suite.
214
+ return EvalExit.Partial;
215
+ }
216
+ const phase = status?.phase ?? EvalRunPhase.unspecified;
217
+ switch (phase) {
218
+ case EvalRunPhase.failed:
219
+ return EvalExit.Failed;
220
+ case EvalRunPhase.partial:
221
+ return EvalExit.Partial;
222
+ case EvalRunPhase.unspecified:
223
+ case EvalRunPhase.pending:
224
+ case EvalRunPhase.running:
225
+ case EvalRunPhase.completed:
226
+ break;
227
+ default: {
228
+ // A phase a newer server added: not one this command knows as a full
229
+ // result, so never a pass.
230
+ const unknown = phase;
231
+ return typeof unknown === "number" ? EvalExit.Partial : EvalExit.Failed;
232
+ }
233
+ }
234
+ const aggregates = status?.aggregates;
235
+ const total = aggregates?.casesTotal ?? 0;
236
+ if (total === 0 ||
237
+ opts.loadFindings > 0 ||
238
+ (aggregates?.casesPassed ?? 0) < total ||
239
+ cases.some(neverGradedWithPlugin)) {
240
+ return EvalExit.Failed;
241
+ }
242
+ return EvalExit.Passed;
243
+ }
244
+ /**
245
+ * Whether a case that ran left a target with-plugin tries none of which
246
+ * was graded: the server leaves such a case out of its counts, but it
247
+ * never ran there, so it is not a pass.
248
+ */
249
+ function neverGradedWithPlugin(evalCase) {
250
+ if (evalCase.notRunReason !== "")
251
+ return false;
252
+ return evalCase.targets.some((result) => {
253
+ const tries = result.withPlugin?.tries ?? [];
254
+ return (result.notRunReason === "" &&
255
+ tries.length > 0 &&
256
+ tries.every((attempt) => attempt.state !== EvalRunTryState.graded));
257
+ });
258
+ }
259
+ /** Whether the eval has stopped changing. */
260
+ export function isSettled(evaluation) {
261
+ const phase = evaluation.status?.phase ?? EvalRunPhase.unspecified;
262
+ switch (phase) {
263
+ case EvalRunPhase.completed:
264
+ case EvalRunPhase.partial:
265
+ case EvalRunPhase.failed:
266
+ return true;
267
+ case EvalRunPhase.unspecified:
268
+ case EvalRunPhase.pending:
269
+ case EvalRunPhase.running:
270
+ return false;
271
+ default: {
272
+ // A phase a newer server added: stop following rather than poll forever.
273
+ const unknown = phase;
274
+ return typeof unknown === "number";
275
+ }
276
+ }
277
+ }
278
+ /**
279
+ * The plugins of the eval run, in its order, then its Evals: each headed
280
+ * by its slug and short version (a plugin marked `compared` on an eval run
281
+ * about an agent, an Eval as `eval <slug>`), its cases named
282
+ * `<source>/<case>` on an eval run about an agent and keyed by its id.
283
+ */
284
+ function groupsOf(evalRun) {
285
+ const onAgent = isAboutAgent(evalRun);
286
+ const plugins = (evalRun.status?.plugins ?? []).map((plugin) => ({
287
+ heading: `${headingOf(oneLine(plugin.pluginSlug), plugin.pluginDigest)}${plugin.compared && onAgent ? " (compared)" : ""}`,
288
+ notRunReason: plugin.notRunReason,
289
+ cases: plugin.cases,
290
+ scope: { key: `${plugin.pluginId}|`, name: onAgent ? `${oneLine(plugin.pluginSlug)}/` : "" },
291
+ }));
292
+ const evals = (evalRun.status?.evals ?? []).map((evalBlock) => ({
293
+ heading: `eval ${headingOf(oneLine(evalBlock.evalSlug), evalBlock.evalDigest)}`,
294
+ notRunReason: evalBlock.notRunReason,
295
+ cases: evalBlock.cases,
296
+ scope: { key: `${evalBlock.evalId}|`, name: `${oneLine(evalBlock.evalSlug)}/` },
297
+ }));
298
+ return [...plugins, ...evals];
299
+ }
300
+ /** Whether the eval run is about an agent. */
301
+ function isAboutAgent(evalRun) {
302
+ return evalRun.spec?.subject.case === "agentId";
303
+ }
304
+ /** Whether each case runs with and without a plugin: one is compared, and the ablation is not none. */
305
+ function isComparing(evalRun) {
306
+ const spec = evalRun.spec;
307
+ const compared = spec?.subject.case === "pluginId" || (spec?.comparedPluginId ?? "") !== "";
308
+ return compared && spec?.ablation !== EvalRunAblation.none;
309
+ }
310
+ /** `<name>@<short hash>`, or the name alone when there is no hash yet. */
311
+ function headingOf(name, hash) {
312
+ const short = oneLine(hash).slice(0, SHORT_HASH);
313
+ return short === "" ? name : `${name}@${short}`;
314
+ }
315
+ /** The agent version the tries ran: the status's stamp, or the spec's before the status carries one. */
316
+ function versionHashOf(evalRun) {
317
+ const stamped = evalRun.status?.agentVersionHash ?? "";
318
+ return stamped !== "" ? stamped : (evalRun.spec?.agentVersion ?? "");
319
+ }
320
+ /** A target's label as one line: its model name is the suite author's text. */
321
+ function targetLabel(target) {
322
+ return oneLine(evalRunTargetLabel(target));
323
+ }
324
+ function targetLabels(specTargets, cases) {
325
+ const labels = [];
326
+ const add = (label) => {
327
+ if (!labels.includes(label))
328
+ labels.push(label);
329
+ };
330
+ for (const target of specTargets)
331
+ add(targetLabel(target));
332
+ for (const evalCase of cases) {
333
+ for (const result of evalCase.targets)
334
+ add(targetLabel(result.target));
335
+ }
336
+ return labels;
337
+ }
338
+ function twoArmRow(evalCase, result) {
339
+ return [
340
+ oneLine(evalCase.caseName),
341
+ scoreCell(result.withPlugin),
342
+ scoreCell(result.withoutPlugin),
343
+ result.delta === undefined ? "—" : signed(result.delta),
344
+ result.passK ? "yes" : "no",
345
+ String(triesOf(result).length),
346
+ usd(costOf(result)),
347
+ notesOf(evalCase, result),
348
+ ];
349
+ }
350
+ function oneArmRow(evalCase, result) {
351
+ const arm = result.withPlugin;
352
+ const graded = arm?.gradedTries ?? 0;
353
+ return [
354
+ oneLine(evalCase.caseName),
355
+ scoreCell(arm),
356
+ graded === 0 ? "—" : `${Math.round(((arm?.perfectRuns ?? 0) / graded) * 100)}%`,
357
+ result.passK ? "yes" : "no",
358
+ String(triesOf(result).length),
359
+ usd(costOf(result)),
360
+ notesOf(evalCase, result),
361
+ ];
362
+ }
363
+ /** `<case> · mocked: plugin_kit_tracker(create_issue, close_issue), plugin_kit_wiki[not started: no mock]`. */
364
+ function mockedLine(caseName, evalCase) {
365
+ const servers = evalCase.mockedServers.map((server) => server.notStartedReason !== ""
366
+ ? `${oneLine(server.name)}[${oneLine(server.notStartedReason)}]`
367
+ : `${oneLine(server.name)}(${server.tools.map(oneLine).join(", ")})`);
368
+ return `${caseName} · mocked: ${servers.join(", ")}`;
369
+ }
370
+ /** How a mock stopped a try, as the format reports it. */
371
+ function abortedText(aborted) {
372
+ return `aborted by ${aborted.server}.${aborted.tool}: ${aborted.reason}`;
373
+ }
374
+ /** The with-arm's first error, abort or not-graded reason, else the case's own notes. */
375
+ function notesOf(evalCase, result) {
376
+ const tries = result.withPlugin?.tries ?? [];
377
+ const error = tries.find((attempt) => attempt.error !== "")?.error;
378
+ const aborted = tries.find((attempt) => attempt.aborted !== undefined)?.aborted;
379
+ const notGraded = tries.find((attempt) => attempt.notGradedReason !== "")?.notGradedReason;
380
+ const note = error ??
381
+ (aborted !== undefined
382
+ ? abortedText(aborted)
383
+ : notGraded === undefined
384
+ ? evalCase.notes.join("; ")
385
+ : `not graded: ${notGraded}`);
386
+ const line = oneLine(note);
387
+ return line.length > MAX_NOTE ? `${line.slice(0, MAX_NOTE - 1)}…` : line;
388
+ }
389
+ function armsOf(result) {
390
+ const arms = [["with", result.withPlugin?.tries ?? []]];
391
+ if (result.withoutPlugin !== undefined) {
392
+ arms.push(["without", result.withoutPlugin.tries]);
393
+ }
394
+ return arms;
395
+ }
396
+ function triesOf(result) {
397
+ return [...(result.withPlugin?.tries ?? []), ...(result.withoutPlugin?.tries ?? [])];
398
+ }
399
+ function costOf(result) {
400
+ return triesOf(result).reduce((sum, attempt) => sum + attempt.costUsd, 0);
401
+ }
402
+ function isFinished(attempt) {
403
+ return attempt.state === EvalRunTryState.graded || attempt.state === EvalRunTryState.not_graded;
404
+ }
405
+ function progressLine(caseName, target, arm, attempt, of) {
406
+ const outcome = attempt.state === EvalRunTryState.graded
407
+ ? attempt.score.toFixed(2)
408
+ : `not graded: ${oneLine(attempt.notGradedReason) || "no reason given"}`;
409
+ const error = attempt.error !== ""
410
+ ? ` (${oneLine(attempt.error)})`
411
+ : attempt.aborted !== undefined
412
+ ? ` (${oneLine(abortedText(attempt.aborted))})`
413
+ : "";
414
+ return `${caseName} · ${target} · ${arm} · try ${attempt.index}/${of} · ${outcome}${error}`;
415
+ }
416
+ function scoreCell(arm) {
417
+ return arm?.score === undefined ? "—" : arm.score.toFixed(2);
418
+ }
419
+ function signed(value) {
420
+ const fixed = value.toFixed(2);
421
+ return value > 0 && fixed !== "0.00" ? `+${fixed}` : fixed === "-0.00" ? "0.00" : fixed;
422
+ }
423
+ function usd(amount) {
424
+ return `$${amount.toFixed(2)}`;
425
+ }
426
+ /**
427
+ * Text the server relays from a run (an error, a reason) as one line: a
428
+ * run's error can quote a model's output, so a newline or an escape
429
+ * sequence never reaches the terminal.
430
+ */
431
+ export function oneLine(text) {
432
+ return text.replace(/[\u0000-\u001f\u007f-\u009f]+/g, " ").trim();
433
+ }
434
+ //# sourceMappingURL=report.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/resources/eval-run/report.ts"],"names":[],"mappings":"AAAA,wEAAwE;AACxE,wEAAwE;AACxE,eAAe;AACf,EAAE;AACF,2EAA2E;AAC3E,wEAAwE;AACxE,yEAAyE;AACzE,4EAA4E;AAC5E,yEAAyE;AACzE,2EAA2E;AAC3E,0EAA0E;AAC1E,sEAAsE;AACtE,wEAAwE;AACxE,iBAAiB;AACjB,EAAE;AACF,uEAAuE;AACvE,oEAAoE;AACpE,yEAAyE;AACzE,0EAA0E;AAC1E,oEAAoE;AACpE,mEAAmE;AACnE,0EAA0E;AAC1E,2EAA2E;AAC3E,2EAA2E;AAC3E,kEAAkE;AAClE,0EAA0E;AAC1E,2EAA2E;AAC3E,EAAE;AACF,wEAAwE;AACxE,sEAAsE;AACtE,yEAAyE;AACzE,qEAAqE;AACrE,uEAAuE;AACvE,2EAA2E;AAC3E,0EAA0E;AAC1E,wEAAwE;AACxE,yEAAyE;AACzE,+DAA+D;AAC/D,EAAE;AACF,yEAAyE;AACzE,sEAAsE;AACtE,yEAAyE;AACzE,yBAAyB;AAEzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAErD,OAAO,EAAE,eAAe,EAAE,MAAM,qDAAqD,CAAC;AAEtF,OAAO,EACL,oBAAoB,EACpB,YAAY,EACZ,eAAe,GAKhB,MAAM,uDAAuD,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAEpD,oFAAoF;AACpF,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,MAAM,EAAE,CAAC;IACT,MAAM,EAAE,CAAC;IACT,OAAO,EAAE,CAAC;IACV,WAAW,EAAE,GAAG;CACR,CAAC;AAIX,0EAA0E;AAC1E,MAAM,QAAQ,GAAG,EAAE,CAAC;AAQpB,yEAAyE;AACzE,MAAM,UAAU,GAAG,EAAE,CAAC;AAkBtB,oGAAoG;AACpG,MAAM,UAAU,aAAa,CAAC,OAAgB;IAC5C,OAAO,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AACzF,CAAC;AAED,6EAA6E;AAC7E,SAAS,eAAe,CAAC,KAA6B,EAAE,KAAgB;IACtE,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,IAAI,WAAW,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,KAAK,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,GAAG,KAAK,CAAC,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC1C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1C,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;oBAC5B,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;wBAAE,SAAS;oBACnC,IAAI,CAAC,WAAW,EAAE,CAAC;wBACjB,WAAW,GAAG,IAAI,CAAC;wBACnB,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,GAAG,QAAQ,CAAC,IAAI,SAAS,EAAE,IAAI,EAAE,UAAU,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,CAAC,CAAC;oBACtG,CAAC;oBACD,QAAQ,CAAC,IAAI,CAAC;wBACZ,GAAG,EAAE,GAAG,KAAK,CAAC,GAAG,GAAG,QAAQ,CAAC,IAAI,IAAI,MAAM,IAAI,GAAG,IAAI,OAAO,CAAC,KAAK,EAAE;wBACrE,IAAI,EAAE,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC;qBACjE,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAgB,EAAE,WAAmB;IACpE,MAAM,KAAK,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,WAAW,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACnG,MAAM,MAAM,GAAG,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IACrC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,KAAK,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;QAC1E,IAAI,KAAK,CAAC,YAAY,KAAK,EAAE,EAAE,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,gBAAgB,OAAO,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACpE,SAAS;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,GAAG,IAAI,MAAM,MAAM,EAAE,CAAC,CAAC,CAAC;IACxH,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,SAAS,gBAAgB,CACvB,QAAgC,EAChC,WAAqC,EACrC,MAAe,EACf,OAAmC;IAEnC,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,YAAY,KAAK,EAAE,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,KAAK,IAAI,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;YACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC;YAC7F,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,YAAY,KAAK,EAAE;gBAAE,OAAO,EAAE,CAAC;YAClE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;QACH,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,SAAS;QAChC,MAAM,OAAO,GAAG,MAAM;YACpB,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC;YAC/D,CAAC,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACtE,MAAM,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,gDAAgD;AAChD,MAAM,UAAU,iBAAiB,CAAC,OAAgB,EAAE,eAAuB;IACzE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC9B,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,EAAE,UAAU,EAAE,UAAU,IAAI,CAAC,UAAU,CAAC,CAAC;IACjE,MAAM,SAAS,GAAG,MAAM,EAAE,UAAU,EAAE,SAAS,CAAC;IAChD,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QAC5B,KAAK,CAAC,IAAI,CAAC,UAAU,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IAC5C,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC;IACzE,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAgB,EAChB,IAKC;IAED,MAAM,MAAM,GAAG,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,MAAM,KAAK,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtC,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,OAAO,EAAE,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,KAAK,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;IACtG,IAAI,OAAO,KAAK,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACrC,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,EAAE,CAAC;AACxE,CAAC;AAED,6FAA6F;AAC7F,MAAM,UAAU,YAAY,CAAC,OAAgB;IAC3C,OAAO,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AACtG,CAAC;AAED,qEAAqE;AACrE,SAAS,aAAa,CAAC,KAAwB;IAC7C,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACrE,CAAC;AAED,wFAAwF;AACxF,SAAS,aAAa,CAAC,KAA6B,EAAE,KAAgB;IACpE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,GAAG,KAAK,CAAC,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC9D,IAAI,QAAQ,CAAC,YAAY,KAAK,EAAE,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,KAAK,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAC/D,SAAS;QACX,CAAC;QACD,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;YACtC,IAAI,MAAM,CAAC,YAAY,KAAK,EAAE,EAAE,CAAC;gBAC/B,KAAK,CAAC,IAAI,CAAC,KAAK,QAAQ,OAAO,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;YAChG,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,WAAW,CAAC,UAA8C,EAAE,WAAoB;IAC9F,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC;IACjC,MAAM,MAAM,GAAG,MAAM,EAAE,aAAa,IAAI,oBAAoB,CAAC,WAAW,CAAC;IACzE,MAAM,GAAG,GAAG,GAAG,MAAM,EAAE,aAAa,IAAI,CAAC,OAAO,MAAM,EAAE,UAAU,IAAI,CAAC,YAAY,CAAC;IACpF,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,oBAAoB,CAAC,YAAY;YACpC,OAAO,kCAAkC,GAAG,GAAG,CAAC;QAClD,KAAK,oBAAoB,CAAC,SAAS;YACjC,OAAO,cAAc,GAAG,GAAG,CAAC;QAC9B,KAAK,oBAAoB,CAAC,WAAW;YACnC,OAAO,WAAW,CAAC,CAAC,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,OAAO,CAAC,CAAC,CAAC;YACR,8DAA8D;YAC9D,MAAM,OAAO,GAAU,MAAM,CAAC;YAC9B,OAAO,yBAAyB,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,GAAG,CAAC;QAC9D,CAAC;IACH,CAAC;AACH,CAAC;AAED,4GAA4G;AAC5G,MAAM,UAAU,YAAY,CAC1B,OAAgB,EAChB,IAAsE;IAEtE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,EAAE,aAAa,IAAI,oBAAoB,CAAC,WAAW,CAAC;IACzE,IAAI,IAAI,CAAC,WAAW,IAAI,MAAM,KAAK,oBAAoB,CAAC,SAAS,EAAE,CAAC;QAClE,OAAO,QAAQ,CAAC,WAAW,CAAC;IAC9B,CAAC;IACD,IAAI,MAAM,KAAK,oBAAoB,CAAC,WAAW,EAAE,CAAC;QAChD,4DAA4D;QAC5D,sEAAsE;QACtE,OAAO,QAAQ,CAAC,OAAO,CAAC;IAC1B,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,YAAY,CAAC,WAAW,CAAC;IACxD,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,YAAY,CAAC,MAAM;YACtB,OAAO,QAAQ,CAAC,MAAM,CAAC;QACzB,KAAK,YAAY,CAAC,OAAO;YACvB,OAAO,QAAQ,CAAC,OAAO,CAAC;QAC1B,KAAK,YAAY,CAAC,WAAW,CAAC;QAC9B,KAAK,YAAY,CAAC,OAAO,CAAC;QAC1B,KAAK,YAAY,CAAC,OAAO,CAAC;QAC1B,KAAK,YAAY,CAAC,SAAS;YACzB,MAAM;QACR,OAAO,CAAC,CAAC,CAAC;YACR,qEAAqE;YACrE,2BAA2B;YAC3B,MAAM,OAAO,GAAU,KAAK,CAAC;YAC7B,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC1E,CAAC;IACH,CAAC;IACD,MAAM,UAAU,GAAG,MAAM,EAAE,UAAU,CAAC;IACtC,MAAM,KAAK,GAAG,UAAU,EAAE,UAAU,IAAI,CAAC,CAAC;IAC1C,IACE,KAAK,KAAK,CAAC;QACX,IAAI,CAAC,YAAY,GAAG,CAAC;QACrB,CAAC,UAAU,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,KAAK;QACtC,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,EACjC,CAAC;QACD,OAAO,QAAQ,CAAC,MAAM,CAAC;IACzB,CAAC;IACD,OAAO,QAAQ,CAAC,MAAM,CAAC;AACzB,CAAC;AAED;;;;GAIG;AACH,SAAS,qBAAqB,CAAC,QAAqB;IAClD,IAAI,QAAQ,CAAC,YAAY,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC;IAC/C,OAAO,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE,CAAC;QAC7C,OAAO,CACL,MAAM,CAAC,YAAY,KAAK,EAAE;YAC1B,KAAK,CAAC,MAAM,GAAG,CAAC;YAChB,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,eAAe,CAAC,MAAM,CAAC,CACnE,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,6CAA6C;AAC7C,MAAM,UAAU,SAAS,CAAC,UAA6D;IACrF,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,IAAI,YAAY,CAAC,WAAW,CAAC;IACnE,QAAQ,KAAK,EAAE,CAAC;QACd,KAAK,YAAY,CAAC,SAAS,CAAC;QAC5B,KAAK,YAAY,CAAC,OAAO,CAAC;QAC1B,KAAK,YAAY,CAAC,MAAM;YACtB,OAAO,IAAI,CAAC;QACd,KAAK,YAAY,CAAC,WAAW,CAAC;QAC9B,KAAK,YAAY,CAAC,OAAO,CAAC;QAC1B,KAAK,YAAY,CAAC,OAAO;YACvB,OAAO,KAAK,CAAC;QACf,OAAO,CAAC,CAAC,CAAC;YACR,yEAAyE;YACzE,MAAM,OAAO,GAAU,KAAK,CAAC;YAC7B,OAAO,OAAO,OAAO,KAAK,QAAQ,CAAC;QACrC,CAAC;IACH,CAAC;AACH,CAAC;AAUD;;;;;GAKG;AACH,SAAS,QAAQ,CAAC,OAAgB;IAChC,MAAM,OAAO,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAC/D,OAAO,EAAE,GAAG,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC,QAAQ,IAAI,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1H,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,KAAK,EAAE,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,QAAQ,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;KAC7F,CAAC,CAAC,CAAC;IACJ,MAAM,KAAK,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QAC9D,OAAO,EAAE,QAAQ,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,EAAE,SAAS,CAAC,UAAU,CAAC,EAAE;QAC/E,YAAY,EAAE,SAAS,CAAC,YAAY;QACpC,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,KAAK,EAAE,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,GAAG,EAAE,IAAI,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,EAAE;KAChF,CAAC,CAAC,CAAC;IACJ,OAAO,CAAC,GAAG,OAAO,EAAE,GAAG,KAAK,CAAC,CAAC;AAChC,CAAC;AAED,8CAA8C;AAC9C,SAAS,YAAY,CAAC,OAAgB;IACpC,OAAO,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC;AAClD,CAAC;AAED,uGAAuG;AACvG,SAAS,WAAW,CAAC,OAAgB;IACnC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,MAAM,QAAQ,GAAG,IAAI,EAAE,OAAO,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,IAAI,EAAE,gBAAgB,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC;IAC5F,OAAO,QAAQ,IAAI,IAAI,EAAE,QAAQ,KAAK,eAAe,CAAC,IAAI,CAAC;AAC7D,CAAC;AAED,0EAA0E;AAC1E,SAAS,SAAS,CAAC,IAAY,EAAE,IAAY;IAC3C,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IACjD,OAAO,KAAK,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,EAAE,CAAC;AAClD,CAAC;AAED,wGAAwG;AACxG,SAAS,aAAa,CAAC,OAAgB;IACrC,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,EAAE,gBAAgB,IAAI,EAAE,CAAC;IACvD,OAAO,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,YAAY,IAAI,EAAE,CAAC,CAAC;AACvE,CAAC;AAED,+EAA+E;AAC/E,SAAS,WAAW,CAAC,MAAgD;IACnE,OAAO,OAAO,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,YAAY,CAAC,WAAqC,EAAE,KAA6B;IACxF,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,MAAM,GAAG,GAAG,CAAC,KAAa,EAAQ,EAAE;QAClC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,CAAC,CAAC;IACF,KAAK,MAAM,MAAM,IAAI,WAAW;QAAE,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3D,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO;YAAE,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,QAAqB,EAAE,MAAyB;IACjE,OAAO;QACL,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC1B,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC;QAC5B,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC;QAC/B,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QAC3B,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAC9B,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC1B,CAAC;AACJ,CAAC;AAED,SAAS,SAAS,CAAC,QAAqB,EAAE,MAAyB;IACjE,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;IAC9B,MAAM,MAAM,GAAG,GAAG,EAAE,WAAW,IAAI,CAAC,CAAC;IACrC,OAAO;QACL,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC1B,SAAS,CAAC,GAAG,CAAC;QACd,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,EAAE,WAAW,IAAI,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG;QAC/E,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QAC3B,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;QAC9B,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACnB,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC;KAC1B,CAAC;AACJ,CAAC;AAED,+GAA+G;AAC/G,SAAS,UAAU,CAAC,QAAgB,EAAE,QAAqB;IACzD,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CACpD,MAAM,CAAC,gBAAgB,KAAK,EAAE;QAC5B,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,GAAG;QAChE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CACvE,CAAC;IACF,OAAO,GAAG,QAAQ,cAAc,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;AACvD,CAAC;AAED,0DAA0D;AAC1D,SAAS,WAAW,CAAC,OAA2C;IAC9D,OAAO,cAAc,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;AAC3E,CAAC;AAED,yFAAyF;AACzF,SAAS,OAAO,CAAC,QAAqB,EAAE,MAAyB;IAC/D,MAAM,KAAK,GAAG,MAAM,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE,CAAC;IAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC,EAAE,KAAK,CAAC;IACnE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,EAAE,OAAO,CAAC;IAChF,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,eAAe,KAAK,EAAE,CAAC,EAAE,eAAe,CAAC;IAC3F,MAAM,IAAI,GACR,KAAK;QACL,CAAC,OAAO,KAAK,SAAS;YACpB,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC;YACtB,CAAC,CAAC,SAAS,KAAK,SAAS;gBACvB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;gBAC3B,CAAC,CAAC,eAAe,SAAS,EAAE,CAAC,CAAC;IACpC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,OAAO,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AAC3E,CAAC;AAED,SAAS,MAAM,CAAC,MAAyB;IACvC,MAAM,IAAI,GAAsC,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;IAC3F,IAAI,MAAM,CAAC,aAAa,KAAK,SAAS,EAAE,CAAC;QACvC,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,OAAO,CAAC,MAAyB;IACxC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,IAAI,EAAE,CAAC,EAAE,GAAG,CAAC,MAAM,CAAC,aAAa,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC;AACvF,CAAC;AAED,SAAS,MAAM,CAAC,MAAyB;IACvC,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,UAAU,CAAC,OAAmB;IACrC,OAAO,OAAO,CAAC,KAAK,KAAK,eAAe,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,KAAK,eAAe,CAAC,UAAU,CAAC;AAClG,CAAC;AAED,SAAS,YAAY,CAAC,QAAgB,EAAE,MAAc,EAAE,GAAW,EAAE,OAAmB,EAAE,EAAU;IAClG,MAAM,OAAO,GACX,OAAO,CAAC,KAAK,KAAK,eAAe,CAAC,MAAM;QACtC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1B,CAAC,CAAC,eAAe,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,IAAI,iBAAiB,EAAE,CAAC;IAC7E,MAAM,KAAK,GACT,OAAO,CAAC,KAAK,KAAK,EAAE;QAClB,CAAC,CAAC,KAAK,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG;QAChC,CAAC,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS;YAC7B,CAAC,CAAC,KAAK,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG;YAC/C,CAAC,CAAC,EAAE,CAAC;IACX,OAAO,GAAG,QAAQ,MAAM,MAAM,MAAM,GAAG,UAAU,OAAO,CAAC,KAAK,IAAI,EAAE,MAAM,OAAO,GAAG,KAAK,EAAE,CAAC;AAC9F,CAAC;AAED,SAAS,SAAS,CAAC,GAA2B;IAC5C,OAAO,GAAG,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,MAAM,CAAC,KAAa;IAC3B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC/B,OAAO,KAAK,GAAG,CAAC,IAAI,KAAK,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;AAC1F,CAAC;AAED,SAAS,GAAG,CAAC,MAAc;IACzB,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;AACjC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,OAAO,CAAC,gCAAgC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACpE,CAAC"}
@@ -0,0 +1,10 @@
1
+ import { type Stigmer } from "@stigmer/client";
2
+ import { type EvalIo } from "./follow.js";
3
+ import type { EvalRunOptions } from "./options.js";
4
+ import { type EvalExit } from "./report.js";
5
+ export { CALL_DEADLINE_MS, POLL_INTERVAL_MS, POLL_RETRY_BUDGET_MS } from "./follow.js";
6
+ /** Runs `stigmer eval run` and returns the code to exit with. */
7
+ export declare function runEvalRun(client: Stigmer, org: string, options: EvalRunOptions, io: EvalIo): Promise<EvalExit>;
8
+ /** Cancels an eval run and says what happened. */
9
+ export declare function cancelEvalRun(client: Stigmer, id: string, io: Pick<EvalIo, "stderr">): Promise<void>;
10
+ //# sourceMappingURL=run.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../src/resources/eval-run/run.ts"],"names":[],"mappings":"AAoCA,OAAO,EAKL,KAAK,OAAO,EACb,MAAM,iBAAiB,CAAC;AAIzB,OAAO,EAA2B,KAAK,MAAM,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,EAAc,MAAM,cAAc,CAAC;AAC/D,OAAO,EAA2C,KAAK,QAAQ,EAAE,MAAM,aAAa,CAAC;AAErF,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AA0CvF,iEAAiE;AACjE,wBAAsB,UAAU,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,CA2BrH;AAED,kDAAkD;AAClD,wBAAsB,aAAa,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAM1G"}
@@ -0,0 +1,158 @@
1
+ // `eval run` dispatch: resolve what the eval run is about, start it, follow
2
+ // it to the end, and print or write the result; and `eval cancel`.
3
+ //
4
+ // The start, the follow loop, Ctrl+C, the deadlines and the exit on a lost
5
+ // eval run are follow.ts's (whose header holds those rules); this module
6
+ // reads what the eval run is about and creates it.
7
+ //
8
+ // The agent is read first, by id or by reference in the organization (the
9
+ // group's one rule, resources/agent/resolve.ts), so the eval run is
10
+ // created in the agent's own organization and the output names it by
11
+ // slug. Its version is not read here: create resolves a tag, refuses a
12
+ // version that does not exist, and stamps the hash, which the tables show.
13
+ // A plugin named with `--plugin` is read the same way, in the agent's
14
+ // organization beside `--agent`, since create takes its id; a plugin named
15
+ // with `--cases-from` is sent as written (an id, or a slug in the agent's
16
+ // organization, or org/slug), and the server matches it against the
17
+ // plugins a try loads. Every read is part of the start: its refusal exits
18
+ // 1, as create's does.
19
+ //
20
+ // The eval run is sent with no name: the server names it by its id, which
21
+ // is short and unique, and a person tells eval runs apart by subject and
22
+ // start time.
23
+ //
24
+ // The load findings of the plugin `--plugin` names (a case file that does
25
+ // not parse) go to stderr before the eval run starts, each as the server
26
+ // relays it, on one line: its message already starts with the file's
27
+ // path. Under `--json` they go into Stigmer's `findings` field of the
28
+ // result document beside the format's, and a finding fails the exit code.
29
+ // `--json` writes the SDK's result document with the subject's slug, and
30
+ // `stillRunning: true` when the command could not cancel an eval run it
31
+ // stopped following.
32
+ import { ApiResourceKind } from "@stigmer/apis/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
33
+ import { toResultDocument, } from "@stigmer/client";
34
+ import { requireOrganization } from "../../client/single-org.js";
35
+ import { resolveAgent } from "../agent/resolve.js";
36
+ import { idPrefixesFor, parseReference } from "../reference.js";
37
+ import { asStartRefusal, runEval } from "./follow.js";
38
+ import { evalExitCode, oneLine, renderEvalReport } from "./report.js";
39
+ export { CALL_DEADLINE_MS, POLL_INTERVAL_MS, POLL_RETRY_BUDGET_MS } from "./follow.js";
40
+ /** The commands that set the organization a slug is read in, when none is set. */
41
+ const SET_ORG_WITH = ["stigmer config context set --org <org>", "stigmer eval run --org <org> ..."];
42
+ /** Runs `stigmer eval run` and returns the code to exit with. */
43
+ export async function runEvalRun(client, org, options, io) {
44
+ const quiet = options.json.kind !== "none";
45
+ const subject = await resolveSubject(client, org, options).catch(asStartRefusal);
46
+ const findings = subject.named?.status?.evals?.findings ?? [];
47
+ if (!quiet) {
48
+ for (const finding of findings)
49
+ io.stderr.write(`evals: ${oneLine(finding.message)}\n`);
50
+ }
51
+ return runEval(client, {
52
+ cancelElsewhere: subject.cancelElsewhere,
53
+ on: subject.on,
54
+ create: () => startEvalRun(client, subject, options),
55
+ report: (evalRun, interrupted, stillRunning) => renderEvalReport(evalRun, {
56
+ subjectSlug: subject.slug,
57
+ interrupted,
58
+ durationSeconds: toResultDocument(evalRun, { interrupted, nowMs: io.now() }).durationSeconds,
59
+ stillRunning,
60
+ }),
61
+ document: (evalRun, interrupted, stillRunning) => documentOf(evalRun, subject.slug, findings, interrupted, io.now(), stillRunning),
62
+ exitCode: (evalRun, interrupted) => evalExitCode(evalRun, { interrupted, loadFindings: findings.length }),
63
+ }, options, io);
64
+ }
65
+ /** Cancels an eval run and says what happened. */
66
+ export async function cancelEvalRun(client, id, io) {
67
+ const cancelled = await client.evalrun.cancel(id);
68
+ const status = cancelled.status;
69
+ io.stderr.write(`Cancelled ${id}: tries in flight stop; ${status?.triesFinished ?? 0} of ${status?.triesTotal ?? 0} tries had finished.\n`);
70
+ }
71
+ async function resolveSubject(client, org, options) {
72
+ const subject = options.subject;
73
+ if (subject.kind === "plugin") {
74
+ const plugin = await resolvePlugin(client, subject.plugin.ref, org);
75
+ const slug = plugin.metadata?.slug || plugin.metadata?.name || subject.plugin.ref;
76
+ return {
77
+ slug,
78
+ on: slug,
79
+ cancelElsewhere: "cancel it from the plugin's Evals tab in the console",
80
+ named: plugin,
81
+ input: { org: plugin.metadata?.org ?? "", pluginId: plugin.metadata?.id ?? "", pluginDigest: subject.plugin.digest },
82
+ };
83
+ }
84
+ const agent = await resolveAgent(client, subject.agent.ref, org, SET_ORG_WITH);
85
+ const agentOrg = agent.metadata?.org ?? "";
86
+ const slug = agent.metadata?.slug || agent.metadata?.name || subject.agent.ref;
87
+ const compared = subject.compared === "" ? undefined : await resolvePlugin(client, subject.compared, agentOrg);
88
+ const on = subject.agent.version === "" ? slug : `${slug}@${subject.agent.version}`;
89
+ return {
90
+ slug,
91
+ on: compared === undefined ? on : `${on} with and without ${compared.metadata?.slug || subject.compared}`,
92
+ cancelElsewhere: "cancel it from the agent's Evals tab in the console",
93
+ ...(compared !== undefined && { named: compared }),
94
+ input: {
95
+ org: agentOrg,
96
+ agentId: agent.metadata?.id ?? "",
97
+ agentVersion: subject.agent.version,
98
+ comparedPluginId: compared?.metadata?.id ?? "",
99
+ cases: [
100
+ ...subject.casesFrom.map((ref) => caseSourceOf(ref, agentOrg)),
101
+ ...subject.evals.map((target) => evalSourceOf(target, agentOrg)),
102
+ ],
103
+ },
104
+ };
105
+ }
106
+ async function resolvePlugin(client, ref, org) {
107
+ const parsed = parseReference(ref, org, idPrefixesFor(ApiResourceKind.plugin));
108
+ if (parsed.kind === "id") {
109
+ return client.plugin.get(parsed.id);
110
+ }
111
+ await requireOrganization(client, parsed.org, SET_ORG_WITH);
112
+ return client.plugin.getByReference({ org: parsed.org, slug: parsed.slug });
113
+ }
114
+ /** A `--cases-from` reference as the create's case source: by id, or by slug in its organization. */
115
+ function caseSourceOf(ref, agentOrg) {
116
+ const parsed = parseReference(ref, agentOrg, idPrefixesFor(ApiResourceKind.plugin));
117
+ return parsed.kind === "id"
118
+ ? { plugin: { org: "", slug: "", id: parsed.id } }
119
+ : { plugin: { org: parsed.org, slug: parsed.slug } };
120
+ }
121
+ /** An `--eval` reference as the create's case source: by id, or by slug in its organization, at its version. */
122
+ function evalSourceOf(target, agentOrg) {
123
+ const parsed = parseReference(target.ref, agentOrg, idPrefixesFor(ApiResourceKind.eval));
124
+ const ref = parsed.kind === "id" ? { org: "", slug: "", id: parsed.id } : { org: parsed.org, slug: parsed.slug };
125
+ return { eval: { ...ref, ...(target.version !== "" && { version: target.version }) } };
126
+ }
127
+ async function startEvalRun(client, subject, options) {
128
+ try {
129
+ return await client.evalrun.create({
130
+ name: "",
131
+ ...subject.input,
132
+ targets: [...options.targets],
133
+ runs: options.runs,
134
+ ablation: options.ablation,
135
+ ...(options.threshold !== undefined && { threshold: options.threshold }),
136
+ caseGlob: options.caseGlob,
137
+ caseTags: [...options.caseTags],
138
+ judgeModel: options.judgeModel,
139
+ maxCostUsd: options.maxCostUsd,
140
+ concurrency: options.concurrency,
141
+ allowTools: [...options.allowTools],
142
+ realMcpServers: options.realMcpServers,
143
+ allowRealServers: options.allowRealServers,
144
+ skipScaffold: options.skipScaffold,
145
+ });
146
+ }
147
+ catch (error) {
148
+ return asStartRefusal(error);
149
+ }
150
+ }
151
+ function documentOf(evalRun, subjectSlug, findings, interrupted, nowMs, stillRunning) {
152
+ return {
153
+ ...toResultDocument(evalRun, { interrupted, nowMs, subjectSlug }),
154
+ findings: findings.map((finding) => ({ kind: finding.kind, path: finding.path, message: finding.message })),
155
+ ...(stillRunning && { stillRunning: true }),
156
+ };
157
+ }
158
+ //# sourceMappingURL=run.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"run.js","sourceRoot":"","sources":["../../src/resources/eval-run/run.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,mEAAmE;AACnE,EAAE;AACF,2EAA2E;AAC3E,yEAAyE;AACzE,mDAAmD;AACnD,EAAE;AACF,0EAA0E;AAC1E,oEAAoE;AACpE,qEAAqE;AACrE,uEAAuE;AACvE,2EAA2E;AAC3E,sEAAsE;AACtE,2EAA2E;AAC3E,0EAA0E;AAC1E,oEAAoE;AACpE,0EAA0E;AAC1E,uBAAuB;AACvB,EAAE;AACF,0EAA0E;AAC1E,yEAAyE;AACzE,cAAc;AACd,EAAE;AACF,0EAA0E;AAC1E,yEAAyE;AACzE,qEAAqE;AACrE,sEAAsE;AACtE,0EAA0E;AAC1E,yEAAyE;AACzE,wEAAwE;AACxE,qBAAqB;AAKrB,OAAO,EAAE,eAAe,EAAE,MAAM,mFAAmF,CAAC;AACpH,OAAO,EACL,gBAAgB,GAKjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAe,MAAM,aAAa,CAAC;AAEnE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,gBAAgB,EAAiB,MAAM,aAAa,CAAC;AAErF,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEvF,kFAAkF;AAClF,MAAM,YAAY,GAAG,CAAC,wCAAwC,EAAE,kCAAkC,CAAU,CAAC;AAuC7G,iEAAiE;AACjE,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,MAAe,EAAE,GAAW,EAAE,OAAuB,EAAE,EAAU;IAChG,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC;IAC3C,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACjF,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,IAAI,EAAE,CAAC;IAC9D,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,KAAK,MAAM,OAAO,IAAI,QAAQ;YAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,OAAO,CACZ,MAAM,EACN;QACE,eAAe,EAAE,OAAO,CAAC,eAAe;QACxC,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,MAAM,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC;QACpD,MAAM,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,EAAE,CAC7C,gBAAgB,CAAC,OAAO,EAAE;YACxB,WAAW,EAAE,OAAO,CAAC,IAAI;YACzB,WAAW;YACX,eAAe,EAAE,gBAAgB,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,eAAe;YAC5F,YAAY;SACb,CAAC;QACJ,QAAQ,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,EAAE,CAC/C,UAAU,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC;QAClF,QAAQ,EAAE,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,YAAY,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;KAC1G,EACD,OAAO,EACP,EAAE,CACH,CAAC;AACJ,CAAC;AAED,kDAAkD;AAClD,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,MAAe,EAAE,EAAU,EAAE,EAA0B;IACzF,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;IAChC,EAAE,CAAC,MAAM,CAAC,KAAK,CACb,aAAa,EAAE,2BAA2B,MAAM,EAAE,aAAa,IAAI,CAAC,OAAO,MAAM,EAAE,UAAU,IAAI,CAAC,wBAAwB,CAC3H,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,MAAe,EAAE,GAAW,EAAE,OAAuB;IACjF,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,IAAI,IAAI,MAAM,CAAC,QAAQ,EAAE,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;QAClF,OAAO;YACL,IAAI;YACJ,EAAE,EAAE,IAAI;YACR,eAAe,EAAE,sDAAsD;YACvE,KAAK,EAAE,MAAM;YACb,KAAK,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,QAAQ,EAAE,GAAG,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,EAAE,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE;SACrH,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,YAAY,CAAC,CAAC;IAC/E,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,GAAG,IAAI,EAAE,CAAC;IAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,EAAE,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC;IAC/E,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,KAAK,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC/G,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;IACpF,OAAO;QACL,IAAI;QACJ,EAAE,EAAE,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,qBAAqB,QAAQ,CAAC,QAAQ,EAAE,IAAI,IAAI,OAAO,CAAC,QAAQ,EAAE;QACzG,eAAe,EAAE,qDAAqD;QACtE,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;QAClD,KAAK,EAAE;YACL,GAAG,EAAE,QAAQ;YACb,OAAO,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE;YACjC,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,OAAO;YACnC,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE;YAC9C,KAAK,EAAE;gBACL,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,YAAY,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;gBAC9D,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;aACjE;SACF;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,MAAe,EAAE,GAAW,EAAE,GAAW;IACpE,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,GAAG,EAAE,aAAa,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;IAC/E,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;QACzB,OAAO,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACtC,CAAC;IACD,MAAM,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAC5D,OAAO,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;AAC9E,CAAC;AAED,qGAAqG;AACrG,SAAS,YAAY,CAAC,GAAW,EAAE,QAAgB;IACjD,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,QAAQ,EAAE,aAAa,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC;IACpF,OAAO,MAAM,CAAC,IAAI,KAAK,IAAI;QACzB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,EAAE;QAClD,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;AACzD,CAAC;AAED,gHAAgH;AAChH,SAAS,YAAY,CAAC,MAAkB,EAAE,QAAgB;IACxD,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;IACzF,MAAM,GAAG,GACP,MAAM,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;IACvG,OAAO,EAAE,IAAI,EAAE,EAAE,GAAG,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,OAAO,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC;AACzF,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAe,EAAE,OAAwB,EAAE,OAAuB;IAC5F,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;YACjC,IAAI,EAAE,EAAE;YACR,GAAG,OAAO,CAAC,KAAK;YAChB,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC;YAC7B,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,GAAG,CAAC,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC;YACxE,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,QAAQ,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC;YAC/B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,UAAU,EAAE,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC;YACnC,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,gBAAgB,EAAE,OAAO,CAAC,gBAAgB;YAC1C,YAAY,EAAE,OAAO,CAAC,YAAY;SACnC,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,cAAc,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CACjB,OAAgB,EAChB,WAAmB,EACnB,QAAkC,EAClC,WAAoB,EACpB,KAAa,EACb,YAAqB;IAErB,OAAO;QACL,GAAG,gBAAgB,CAAC,OAAO,EAAE,EAAE,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;QACjE,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3G,GAAG,CAAC,YAAY,IAAI,EAAE,YAAY,EAAE,IAAa,EAAE,CAAC;KACrD,CAAC;AACJ,CAAC"}