@meetkai/mka1 0.54.23 → 0.54.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (443) hide show
  1. package/README.md +294 -276
  2. package/bin/mcp-server.js +3844 -1779
  3. package/bin/mcp-server.js.map +42 -17
  4. package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.d.ts +19 -0
  5. package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  6. package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.js +136 -0
  7. package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.js.map +1 -0
  8. package/dist/commonjs/funcs/llmEvalsGetRunOverview.d.ts +20 -0
  9. package/dist/commonjs/funcs/llmEvalsGetRunOverview.d.ts.map +1 -0
  10. package/dist/commonjs/funcs/llmEvalsGetRunOverview.js +138 -0
  11. package/dist/commonjs/funcs/llmEvalsGetRunOverview.js.map +1 -0
  12. package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.d.ts +19 -0
  13. package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  14. package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.js +142 -0
  15. package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.js.map +1 -0
  16. package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.d.ts +20 -0
  17. package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  18. package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.js +142 -0
  19. package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  20. package/dist/commonjs/funcs/llmEvalsImportAgentRun.d.ts +19 -0
  21. package/dist/commonjs/funcs/llmEvalsImportAgentRun.d.ts.map +1 -0
  22. package/dist/commonjs/funcs/llmEvalsImportAgentRun.js +130 -0
  23. package/dist/commonjs/funcs/llmEvalsImportAgentRun.js.map +1 -0
  24. package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.d.ts +20 -0
  25. package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  26. package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.js +146 -0
  27. package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.js.map +1 -0
  28. package/dist/commonjs/lib/config.d.ts +2 -2
  29. package/dist/commonjs/lib/config.js +2 -2
  30. package/dist/commonjs/mcp-server/mcp-server.js +1 -1
  31. package/dist/commonjs/mcp-server/server.d.ts.map +1 -1
  32. package/dist/commonjs/mcp-server/server.js +13 -1
  33. package/dist/commonjs/mcp-server/server.js.map +1 -1
  34. package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts +8 -0
  35. package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  36. package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.js +64 -0
  37. package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.js.map +1 -0
  38. package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.d.ts +8 -0
  39. package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.d.ts.map +1 -0
  40. package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.js +64 -0
  41. package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.js.map +1 -0
  42. package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts +8 -0
  43. package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  44. package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.js +64 -0
  45. package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.js.map +1 -0
  46. package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts +8 -0
  47. package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  48. package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js +64 -0
  49. package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  50. package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.d.ts +8 -0
  51. package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.d.ts.map +1 -0
  52. package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.js +64 -0
  53. package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.js.map +1 -0
  54. package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts +8 -0
  55. package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  56. package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.js +64 -0
  57. package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.js.map +1 -0
  58. package/dist/commonjs/models/components/appendagenttrialsrequest.d.ts +111 -0
  59. package/dist/commonjs/models/components/appendagenttrialsrequest.d.ts.map +1 -0
  60. package/dist/commonjs/models/components/appendagenttrialsrequest.js +136 -0
  61. package/dist/commonjs/models/components/appendagenttrialsrequest.js.map +1 -0
  62. package/dist/commonjs/models/components/evalleaderboardrow.d.ts +85 -0
  63. package/dist/commonjs/models/components/evalleaderboardrow.d.ts.map +1 -0
  64. package/dist/commonjs/models/components/evalleaderboardrow.js +129 -0
  65. package/dist/commonjs/models/components/evalleaderboardrow.js.map +1 -0
  66. package/dist/commonjs/models/components/evalrunoverview.d.ts +101 -0
  67. package/dist/commonjs/models/components/evalrunoverview.d.ts.map +1 -0
  68. package/dist/commonjs/models/components/evalrunoverview.js +150 -0
  69. package/dist/commonjs/models/components/evalrunoverview.js.map +1 -0
  70. package/dist/commonjs/models/components/evalsampleobject.d.ts +32 -0
  71. package/dist/commonjs/models/components/evalsampleobject.d.ts.map +1 -1
  72. package/dist/commonjs/models/components/evalsampleobject.js +20 -0
  73. package/dist/commonjs/models/components/evalsampleobject.js.map +1 -1
  74. package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.d.ts +32 -0
  75. package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.d.ts.map +1 -0
  76. package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.js +87 -0
  77. package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.js.map +1 -0
  78. package/dist/commonjs/models/components/evalsuiteleaderboard.d.ts +35 -0
  79. package/dist/commonjs/models/components/evalsuiteleaderboard.d.ts.map +1 -0
  80. package/dist/commonjs/models/components/evalsuiteleaderboard.js +78 -0
  81. package/dist/commonjs/models/components/evalsuiteleaderboard.js.map +1 -0
  82. package/dist/commonjs/models/components/importagentrunrequest.d.ts +164 -0
  83. package/dist/commonjs/models/components/importagentrunrequest.d.ts.map +1 -0
  84. package/dist/commonjs/models/components/importagentrunrequest.js +182 -0
  85. package/dist/commonjs/models/components/importagentrunrequest.js.map +1 -0
  86. package/dist/commonjs/models/components/index.d.ts +7 -0
  87. package/dist/commonjs/models/components/index.d.ts.map +1 -1
  88. package/dist/commonjs/models/components/index.js +7 -0
  89. package/dist/commonjs/models/components/index.js.map +1 -1
  90. package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.d.ts +34 -0
  91. package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.d.ts.map +1 -0
  92. package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.js +71 -0
  93. package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.js.map +1 -0
  94. package/dist/commonjs/models/operations/appendagentevaltrials.d.ts +264 -0
  95. package/dist/commonjs/models/operations/appendagentevaltrials.d.ts.map +1 -0
  96. package/dist/commonjs/models/operations/appendagentevaltrials.js +371 -0
  97. package/dist/commonjs/models/operations/appendagentevaltrials.js.map +1 -0
  98. package/dist/commonjs/models/operations/cancelevalrun.d.ts +20 -0
  99. package/dist/commonjs/models/operations/cancelevalrun.d.ts.map +1 -1
  100. package/dist/commonjs/models/operations/cancelevalrun.js +16 -0
  101. package/dist/commonjs/models/operations/cancelevalrun.js.map +1 -1
  102. package/dist/commonjs/models/operations/createevalrun.d.ts +20 -0
  103. package/dist/commonjs/models/operations/createevalrun.d.ts.map +1 -1
  104. package/dist/commonjs/models/operations/createevalrun.js +16 -0
  105. package/dist/commonjs/models/operations/createevalrun.js.map +1 -1
  106. package/dist/commonjs/models/operations/getevalrun.d.ts +20 -0
  107. package/dist/commonjs/models/operations/getevalrun.d.ts.map +1 -1
  108. package/dist/commonjs/models/operations/getevalrun.js +16 -0
  109. package/dist/commonjs/models/operations/getevalrun.js.map +1 -1
  110. package/dist/commonjs/models/operations/getevalrunoverview.d.ts +24 -0
  111. package/dist/commonjs/models/operations/getevalrunoverview.d.ts.map +1 -0
  112. package/dist/commonjs/models/operations/getevalrunoverview.js +73 -0
  113. package/dist/commonjs/models/operations/getevalrunoverview.js.map +1 -0
  114. package/dist/commonjs/models/operations/getevalsampletrajectory.d.ts +75 -0
  115. package/dist/commonjs/models/operations/getevalsampletrajectory.d.ts.map +1 -0
  116. package/dist/commonjs/models/operations/getevalsampletrajectory.js +133 -0
  117. package/dist/commonjs/models/operations/getevalsampletrajectory.js.map +1 -0
  118. package/dist/commonjs/models/operations/getevalsuiteleaderboard.d.ts +32 -0
  119. package/dist/commonjs/models/operations/getevalsuiteleaderboard.d.ts.map +1 -0
  120. package/dist/commonjs/models/operations/getevalsuiteleaderboard.js +83 -0
  121. package/dist/commonjs/models/operations/getevalsuiteleaderboard.js.map +1 -0
  122. package/dist/commonjs/models/operations/importagentevalrun.d.ts +262 -0
  123. package/dist/commonjs/models/operations/importagentevalrun.d.ts.map +1 -0
  124. package/dist/commonjs/models/operations/importagentevalrun.js +367 -0
  125. package/dist/commonjs/models/operations/importagentevalrun.js.map +1 -0
  126. package/dist/commonjs/models/operations/index.d.ts +6 -0
  127. package/dist/commonjs/models/operations/index.d.ts.map +1 -1
  128. package/dist/commonjs/models/operations/index.js +6 -0
  129. package/dist/commonjs/models/operations/index.js.map +1 -1
  130. package/dist/commonjs/models/operations/listevalruns.d.ts +20 -0
  131. package/dist/commonjs/models/operations/listevalruns.d.ts.map +1 -1
  132. package/dist/commonjs/models/operations/listevalruns.js +16 -0
  133. package/dist/commonjs/models/operations/listevalruns.js.map +1 -1
  134. package/dist/commonjs/models/operations/listevalscheduleruns.d.ts +20 -0
  135. package/dist/commonjs/models/operations/listevalscheduleruns.d.ts.map +1 -1
  136. package/dist/commonjs/models/operations/listevalscheduleruns.js +16 -0
  137. package/dist/commonjs/models/operations/listevalscheduleruns.js.map +1 -1
  138. package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.d.ts +20 -0
  139. package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.d.ts.map +1 -1
  140. package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.js +16 -0
  141. package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.js.map +1 -1
  142. package/dist/commonjs/models/operations/rerunfailedevalsamples.d.ts +20 -0
  143. package/dist/commonjs/models/operations/rerunfailedevalsamples.d.ts.map +1 -1
  144. package/dist/commonjs/models/operations/rerunfailedevalsamples.js +16 -0
  145. package/dist/commonjs/models/operations/rerunfailedevalsamples.js.map +1 -1
  146. package/dist/commonjs/models/operations/retryfailedevalrun.d.ts +20 -0
  147. package/dist/commonjs/models/operations/retryfailedevalrun.d.ts.map +1 -1
  148. package/dist/commonjs/models/operations/retryfailedevalrun.js +16 -0
  149. package/dist/commonjs/models/operations/retryfailedevalrun.js.map +1 -1
  150. package/dist/commonjs/models/operations/uploadevalsampletrajectory.d.ts +31 -0
  151. package/dist/commonjs/models/operations/uploadevalsampletrajectory.d.ts.map +1 -0
  152. package/dist/commonjs/models/operations/uploadevalsampletrajectory.js +86 -0
  153. package/dist/commonjs/models/operations/uploadevalsampletrajectory.js.map +1 -0
  154. package/dist/commonjs/react-query/index.d.ts +6 -0
  155. package/dist/commonjs/react-query/index.d.ts.map +1 -1
  156. package/dist/commonjs/react-query/index.js +6 -0
  157. package/dist/commonjs/react-query/index.js.map +1 -1
  158. package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.d.ts +28 -0
  159. package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  160. package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.js +47 -0
  161. package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.js.map +1 -0
  162. package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.d.ts +16 -0
  163. package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.d.ts.map +1 -0
  164. package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.js +37 -0
  165. package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.js.map +1 -0
  166. package/dist/commonjs/react-query/llmEvalsGetRunOverview.d.ts +40 -0
  167. package/dist/commonjs/react-query/llmEvalsGetRunOverview.d.ts.map +1 -0
  168. package/dist/commonjs/react-query/llmEvalsGetRunOverview.js +60 -0
  169. package/dist/commonjs/react-query/llmEvalsGetRunOverview.js.map +1 -0
  170. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.d.ts +16 -0
  171. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.d.ts.map +1 -0
  172. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.js +45 -0
  173. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.js.map +1 -0
  174. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.d.ts +44 -0
  175. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  176. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.js +65 -0
  177. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.js.map +1 -0
  178. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts +20 -0
  179. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts.map +1 -0
  180. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.js +41 -0
  181. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.js.map +1 -0
  182. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.d.ts +48 -0
  183. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  184. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.js +65 -0
  185. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  186. package/dist/commonjs/react-query/llmEvalsImportAgentRun.d.ts +28 -0
  187. package/dist/commonjs/react-query/llmEvalsImportAgentRun.d.ts.map +1 -0
  188. package/dist/commonjs/react-query/llmEvalsImportAgentRun.js +47 -0
  189. package/dist/commonjs/react-query/llmEvalsImportAgentRun.js.map +1 -0
  190. package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.d.ts +29 -0
  191. package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  192. package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.js +47 -0
  193. package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.js.map +1 -0
  194. package/dist/commonjs/sdk/evals.d.ts +42 -0
  195. package/dist/commonjs/sdk/evals.d.ts.map +1 -1
  196. package/dist/commonjs/sdk/evals.js +60 -0
  197. package/dist/commonjs/sdk/evals.js.map +1 -1
  198. package/dist/esm/funcs/llmEvalsAppendAgentTrials.d.ts +19 -0
  199. package/dist/esm/funcs/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  200. package/dist/esm/funcs/llmEvalsAppendAgentTrials.js +100 -0
  201. package/dist/esm/funcs/llmEvalsAppendAgentTrials.js.map +1 -0
  202. package/dist/esm/funcs/llmEvalsGetRunOverview.d.ts +20 -0
  203. package/dist/esm/funcs/llmEvalsGetRunOverview.d.ts.map +1 -0
  204. package/dist/esm/funcs/llmEvalsGetRunOverview.js +102 -0
  205. package/dist/esm/funcs/llmEvalsGetRunOverview.js.map +1 -0
  206. package/dist/esm/funcs/llmEvalsGetSampleTrajectory.d.ts +19 -0
  207. package/dist/esm/funcs/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  208. package/dist/esm/funcs/llmEvalsGetSampleTrajectory.js +106 -0
  209. package/dist/esm/funcs/llmEvalsGetSampleTrajectory.js.map +1 -0
  210. package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.d.ts +20 -0
  211. package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  212. package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.js +106 -0
  213. package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  214. package/dist/esm/funcs/llmEvalsImportAgentRun.d.ts +19 -0
  215. package/dist/esm/funcs/llmEvalsImportAgentRun.d.ts.map +1 -0
  216. package/dist/esm/funcs/llmEvalsImportAgentRun.js +94 -0
  217. package/dist/esm/funcs/llmEvalsImportAgentRun.js.map +1 -0
  218. package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.d.ts +20 -0
  219. package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  220. package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.js +110 -0
  221. package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.js.map +1 -0
  222. package/dist/esm/lib/config.d.ts +2 -2
  223. package/dist/esm/lib/config.js +2 -2
  224. package/dist/esm/mcp-server/mcp-server.js +1 -1
  225. package/dist/esm/mcp-server/server.d.ts.map +1 -1
  226. package/dist/esm/mcp-server/server.js +13 -1
  227. package/dist/esm/mcp-server/server.js.map +1 -1
  228. package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts +8 -0
  229. package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  230. package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.js +28 -0
  231. package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.js.map +1 -0
  232. package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.d.ts +8 -0
  233. package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.d.ts.map +1 -0
  234. package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.js +28 -0
  235. package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.js.map +1 -0
  236. package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts +8 -0
  237. package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  238. package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.js +28 -0
  239. package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.js.map +1 -0
  240. package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts +8 -0
  241. package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  242. package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js +28 -0
  243. package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  244. package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.d.ts +8 -0
  245. package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.d.ts.map +1 -0
  246. package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.js +28 -0
  247. package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.js.map +1 -0
  248. package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts +8 -0
  249. package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  250. package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.js +28 -0
  251. package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.js.map +1 -0
  252. package/dist/esm/models/components/appendagenttrialsrequest.d.ts +111 -0
  253. package/dist/esm/models/components/appendagenttrialsrequest.d.ts.map +1 -0
  254. package/dist/esm/models/components/appendagenttrialsrequest.js +96 -0
  255. package/dist/esm/models/components/appendagenttrialsrequest.js.map +1 -0
  256. package/dist/esm/models/components/evalleaderboardrow.d.ts +85 -0
  257. package/dist/esm/models/components/evalleaderboardrow.d.ts.map +1 -0
  258. package/dist/esm/models/components/evalleaderboardrow.js +89 -0
  259. package/dist/esm/models/components/evalleaderboardrow.js.map +1 -0
  260. package/dist/esm/models/components/evalrunoverview.d.ts +101 -0
  261. package/dist/esm/models/components/evalrunoverview.d.ts.map +1 -0
  262. package/dist/esm/models/components/evalrunoverview.js +108 -0
  263. package/dist/esm/models/components/evalrunoverview.js.map +1 -0
  264. package/dist/esm/models/components/evalsampleobject.d.ts +32 -0
  265. package/dist/esm/models/components/evalsampleobject.d.ts.map +1 -1
  266. package/dist/esm/models/components/evalsampleobject.js +20 -0
  267. package/dist/esm/models/components/evalsampleobject.js.map +1 -1
  268. package/dist/esm/models/components/evalsampletrajectoryuploadedobject.d.ts +32 -0
  269. package/dist/esm/models/components/evalsampletrajectoryuploadedobject.d.ts.map +1 -0
  270. package/dist/esm/models/components/evalsampletrajectoryuploadedobject.js +49 -0
  271. package/dist/esm/models/components/evalsampletrajectoryuploadedobject.js.map +1 -0
  272. package/dist/esm/models/components/evalsuiteleaderboard.d.ts +35 -0
  273. package/dist/esm/models/components/evalsuiteleaderboard.d.ts.map +1 -0
  274. package/dist/esm/models/components/evalsuiteleaderboard.js +40 -0
  275. package/dist/esm/models/components/evalsuiteleaderboard.js.map +1 -0
  276. package/dist/esm/models/components/importagentrunrequest.d.ts +164 -0
  277. package/dist/esm/models/components/importagentrunrequest.d.ts.map +1 -0
  278. package/dist/esm/models/components/importagentrunrequest.js +142 -0
  279. package/dist/esm/models/components/importagentrunrequest.js.map +1 -0
  280. package/dist/esm/models/components/index.d.ts +7 -0
  281. package/dist/esm/models/components/index.d.ts.map +1 -1
  282. package/dist/esm/models/components/index.js +7 -0
  283. package/dist/esm/models/components/index.js.map +1 -1
  284. package/dist/esm/models/components/uploadevalsampletrajectoryrequest.d.ts +34 -0
  285. package/dist/esm/models/components/uploadevalsampletrajectoryrequest.d.ts.map +1 -0
  286. package/dist/esm/models/components/uploadevalsampletrajectoryrequest.js +33 -0
  287. package/dist/esm/models/components/uploadevalsampletrajectoryrequest.js.map +1 -0
  288. package/dist/esm/models/operations/appendagentevaltrials.d.ts +264 -0
  289. package/dist/esm/models/operations/appendagentevaltrials.d.ts.map +1 -0
  290. package/dist/esm/models/operations/appendagentevaltrials.js +323 -0
  291. package/dist/esm/models/operations/appendagentevaltrials.js.map +1 -0
  292. package/dist/esm/models/operations/cancelevalrun.d.ts +20 -0
  293. package/dist/esm/models/operations/cancelevalrun.d.ts.map +1 -1
  294. package/dist/esm/models/operations/cancelevalrun.js +16 -0
  295. package/dist/esm/models/operations/cancelevalrun.js.map +1 -1
  296. package/dist/esm/models/operations/createevalrun.d.ts +20 -0
  297. package/dist/esm/models/operations/createevalrun.d.ts.map +1 -1
  298. package/dist/esm/models/operations/createevalrun.js +16 -0
  299. package/dist/esm/models/operations/createevalrun.js.map +1 -1
  300. package/dist/esm/models/operations/getevalrun.d.ts +20 -0
  301. package/dist/esm/models/operations/getevalrun.d.ts.map +1 -1
  302. package/dist/esm/models/operations/getevalrun.js +16 -0
  303. package/dist/esm/models/operations/getevalrun.js.map +1 -1
  304. package/dist/esm/models/operations/getevalrunoverview.d.ts +24 -0
  305. package/dist/esm/models/operations/getevalrunoverview.d.ts.map +1 -0
  306. package/dist/esm/models/operations/getevalrunoverview.js +35 -0
  307. package/dist/esm/models/operations/getevalrunoverview.js.map +1 -0
  308. package/dist/esm/models/operations/getevalsampletrajectory.d.ts +75 -0
  309. package/dist/esm/models/operations/getevalsampletrajectory.d.ts.map +1 -0
  310. package/dist/esm/models/operations/getevalsampletrajectory.js +93 -0
  311. package/dist/esm/models/operations/getevalsampletrajectory.js.map +1 -0
  312. package/dist/esm/models/operations/getevalsuiteleaderboard.d.ts +32 -0
  313. package/dist/esm/models/operations/getevalsuiteleaderboard.d.ts.map +1 -0
  314. package/dist/esm/models/operations/getevalsuiteleaderboard.js +45 -0
  315. package/dist/esm/models/operations/getevalsuiteleaderboard.js.map +1 -0
  316. package/dist/esm/models/operations/importagentevalrun.d.ts +262 -0
  317. package/dist/esm/models/operations/importagentevalrun.d.ts.map +1 -0
  318. package/dist/esm/models/operations/importagentevalrun.js +319 -0
  319. package/dist/esm/models/operations/importagentevalrun.js.map +1 -0
  320. package/dist/esm/models/operations/index.d.ts +6 -0
  321. package/dist/esm/models/operations/index.d.ts.map +1 -1
  322. package/dist/esm/models/operations/index.js +6 -0
  323. package/dist/esm/models/operations/index.js.map +1 -1
  324. package/dist/esm/models/operations/listevalruns.d.ts +20 -0
  325. package/dist/esm/models/operations/listevalruns.d.ts.map +1 -1
  326. package/dist/esm/models/operations/listevalruns.js +16 -0
  327. package/dist/esm/models/operations/listevalruns.js.map +1 -1
  328. package/dist/esm/models/operations/listevalscheduleruns.d.ts +20 -0
  329. package/dist/esm/models/operations/listevalscheduleruns.d.ts.map +1 -1
  330. package/dist/esm/models/operations/listevalscheduleruns.js +16 -0
  331. package/dist/esm/models/operations/listevalscheduleruns.js.map +1 -1
  332. package/dist/esm/models/operations/listevalsuiteleaderboardruns.d.ts +20 -0
  333. package/dist/esm/models/operations/listevalsuiteleaderboardruns.d.ts.map +1 -1
  334. package/dist/esm/models/operations/listevalsuiteleaderboardruns.js +16 -0
  335. package/dist/esm/models/operations/listevalsuiteleaderboardruns.js.map +1 -1
  336. package/dist/esm/models/operations/rerunfailedevalsamples.d.ts +20 -0
  337. package/dist/esm/models/operations/rerunfailedevalsamples.d.ts.map +1 -1
  338. package/dist/esm/models/operations/rerunfailedevalsamples.js +16 -0
  339. package/dist/esm/models/operations/rerunfailedevalsamples.js.map +1 -1
  340. package/dist/esm/models/operations/retryfailedevalrun.d.ts +20 -0
  341. package/dist/esm/models/operations/retryfailedevalrun.d.ts.map +1 -1
  342. package/dist/esm/models/operations/retryfailedevalrun.js +16 -0
  343. package/dist/esm/models/operations/retryfailedevalrun.js.map +1 -1
  344. package/dist/esm/models/operations/uploadevalsampletrajectory.d.ts +31 -0
  345. package/dist/esm/models/operations/uploadevalsampletrajectory.d.ts.map +1 -0
  346. package/dist/esm/models/operations/uploadevalsampletrajectory.js +48 -0
  347. package/dist/esm/models/operations/uploadevalsampletrajectory.js.map +1 -0
  348. package/dist/esm/react-query/index.d.ts +6 -0
  349. package/dist/esm/react-query/index.d.ts.map +1 -1
  350. package/dist/esm/react-query/index.js +6 -0
  351. package/dist/esm/react-query/index.js.map +1 -1
  352. package/dist/esm/react-query/llmEvalsAppendAgentTrials.d.ts +28 -0
  353. package/dist/esm/react-query/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  354. package/dist/esm/react-query/llmEvalsAppendAgentTrials.js +42 -0
  355. package/dist/esm/react-query/llmEvalsAppendAgentTrials.js.map +1 -0
  356. package/dist/esm/react-query/llmEvalsGetRunOverview.core.d.ts +16 -0
  357. package/dist/esm/react-query/llmEvalsGetRunOverview.core.d.ts.map +1 -0
  358. package/dist/esm/react-query/llmEvalsGetRunOverview.core.js +32 -0
  359. package/dist/esm/react-query/llmEvalsGetRunOverview.core.js.map +1 -0
  360. package/dist/esm/react-query/llmEvalsGetRunOverview.d.ts +40 -0
  361. package/dist/esm/react-query/llmEvalsGetRunOverview.d.ts.map +1 -0
  362. package/dist/esm/react-query/llmEvalsGetRunOverview.js +50 -0
  363. package/dist/esm/react-query/llmEvalsGetRunOverview.js.map +1 -0
  364. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.d.ts +16 -0
  365. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.d.ts.map +1 -0
  366. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.js +40 -0
  367. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.js.map +1 -0
  368. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.d.ts +44 -0
  369. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  370. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.js +55 -0
  371. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.js.map +1 -0
  372. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts +20 -0
  373. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts.map +1 -0
  374. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.js +36 -0
  375. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.js.map +1 -0
  376. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.d.ts +48 -0
  377. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  378. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.js +55 -0
  379. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  380. package/dist/esm/react-query/llmEvalsImportAgentRun.d.ts +28 -0
  381. package/dist/esm/react-query/llmEvalsImportAgentRun.d.ts.map +1 -0
  382. package/dist/esm/react-query/llmEvalsImportAgentRun.js +42 -0
  383. package/dist/esm/react-query/llmEvalsImportAgentRun.js.map +1 -0
  384. package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.d.ts +29 -0
  385. package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  386. package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.js +42 -0
  387. package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.js.map +1 -0
  388. package/dist/esm/sdk/evals.d.ts +42 -0
  389. package/dist/esm/sdk/evals.d.ts.map +1 -1
  390. package/dist/esm/sdk/evals.js +60 -0
  391. package/dist/esm/sdk/evals.js.map +1 -1
  392. package/jsr.json +1 -1
  393. package/package.json +1 -1
  394. package/src/funcs/llmEvalsAppendAgentTrials.ts +188 -0
  395. package/src/funcs/llmEvalsGetRunOverview.ts +192 -0
  396. package/src/funcs/llmEvalsGetSampleTrajectory.ts +197 -0
  397. package/src/funcs/llmEvalsGetSuiteLeaderboard.ts +197 -0
  398. package/src/funcs/llmEvalsImportAgentRun.ts +181 -0
  399. package/src/funcs/llmEvalsUploadSampleTrajectory.ts +201 -0
  400. package/src/lib/config.ts +2 -2
  401. package/src/mcp-server/mcp-server.ts +1 -1
  402. package/src/mcp-server/server.ts +13 -1
  403. package/src/mcp-server/tools/llmEvalsAppendAgentTrials.ts +37 -0
  404. package/src/mcp-server/tools/llmEvalsGetRunOverview.ts +37 -0
  405. package/src/mcp-server/tools/llmEvalsGetSampleTrajectory.ts +37 -0
  406. package/src/mcp-server/tools/llmEvalsGetSuiteLeaderboard.ts +37 -0
  407. package/src/mcp-server/tools/llmEvalsImportAgentRun.ts +37 -0
  408. package/src/mcp-server/tools/llmEvalsUploadSampleTrajectory.ts +38 -0
  409. package/src/models/components/appendagenttrialsrequest.ts +224 -0
  410. package/src/models/components/evalleaderboardrow.ts +200 -0
  411. package/src/models/components/evalrunoverview.ts +240 -0
  412. package/src/models/components/evalsampleobject.ts +48 -0
  413. package/src/models/components/evalsampletrajectoryuploadedobject.ts +98 -0
  414. package/src/models/components/evalsuiteleaderboard.ts +93 -0
  415. package/src/models/components/importagentrunrequest.ts +319 -0
  416. package/src/models/components/index.ts +7 -0
  417. package/src/models/components/uploadevalsampletrajectoryrequest.ts +79 -0
  418. package/src/models/operations/appendagentevaltrials.ts +678 -0
  419. package/src/models/operations/cancelevalrun.ts +36 -0
  420. package/src/models/operations/createevalrun.ts +36 -0
  421. package/src/models/operations/getevalrun.ts +36 -0
  422. package/src/models/operations/getevalrunoverview.ts +73 -0
  423. package/src/models/operations/getevalsampletrajectory.ts +199 -0
  424. package/src/models/operations/getevalsuiteleaderboard.ts +93 -0
  425. package/src/models/operations/importagentevalrun.ts +665 -0
  426. package/src/models/operations/index.ts +6 -0
  427. package/src/models/operations/listevalruns.ts +36 -0
  428. package/src/models/operations/listevalscheduleruns.ts +36 -0
  429. package/src/models/operations/listevalsuiteleaderboardruns.ts +36 -0
  430. package/src/models/operations/rerunfailedevalsamples.ts +36 -0
  431. package/src/models/operations/retryfailedevalrun.ts +36 -0
  432. package/src/models/operations/uploadevalsampletrajectory.ts +98 -0
  433. package/src/react-query/index.ts +6 -0
  434. package/src/react-query/llmEvalsAppendAgentTrials.ts +109 -0
  435. package/src/react-query/llmEvalsGetRunOverview.core.ts +77 -0
  436. package/src/react-query/llmEvalsGetRunOverview.ts +150 -0
  437. package/src/react-query/llmEvalsGetSampleTrajectory.core.ts +94 -0
  438. package/src/react-query/llmEvalsGetSampleTrajectory.ts +159 -0
  439. package/src/react-query/llmEvalsGetSuiteLeaderboard.core.ts +89 -0
  440. package/src/react-query/llmEvalsGetSuiteLeaderboard.ts +163 -0
  441. package/src/react-query/llmEvalsImportAgentRun.ts +109 -0
  442. package/src/react-query/llmEvalsUploadSampleTrajectory.ts +110 -0
  443. package/src/sdk/evals.ts +108 -0
@@ -33,7 +33,30 @@ export type EvalSampleObject = {
33
33
  extractedOutput: string | null;
34
34
  scores: { [k: string]: any };
35
35
  judge: { [k: string]: any } | null;
36
+ /**
37
+ * The exception that ended this sample, if any. For a harness trial this also carries an agent timeout — which is still a *scored* trial, so do not read a non-null `error` here as an unscored one; check `status` and `scores`.
38
+ */
36
39
  error: { [k: string]: any } | null;
40
+ /**
41
+ * Spend for this one trial in USD. Null outside harness runs, and when the harness reported no cost.
42
+ */
43
+ costUsd: number | null;
44
+ /**
45
+ * Prompt tokens for the whole trial session. Includes `cached_tokens`.
46
+ */
47
+ inputTokens: number | null;
48
+ /**
49
+ * Completion tokens for the whole trial session.
50
+ */
51
+ outputTokens: number | null;
52
+ /**
53
+ * The share of `input_tokens` served from the provider's prompt cache. Reported separately because it is what makes two agents at the same token count cost an order of magnitude apart.
54
+ */
55
+ cachedTokens: number | null;
56
+ /**
57
+ * Per-phase wall clock for a harness trial: `environment_setup`, `agent_setup`, `agent_execution`, `verifier`, each `{ started_at, finished_at }` as ISO 8601 timestamps exactly as the harness recorded them. A phase the harness did not time is absent rather than null. Drives the trial timing bar. Null outside harness runs.
58
+ */
59
+ phaseTimings: { [k: string]: any } | null;
37
60
  createdAt: number;
38
61
  startedAt: number | null;
39
62
  completedAt: number | null;
@@ -62,6 +85,11 @@ export const EvalSampleObject$inboundSchema: z.ZodType<
62
85
  scores: z.record(z.any()),
63
86
  judge: z.nullable(z.record(z.any())),
64
87
  error: z.nullable(z.record(z.any())),
88
+ cost_usd: z.nullable(z.number()),
89
+ input_tokens: z.nullable(z.number().int()),
90
+ output_tokens: z.nullable(z.number().int()),
91
+ cached_tokens: z.nullable(z.number().int()),
92
+ phase_timings: z.nullable(z.record(z.any())),
65
93
  created_at: z.number().int(),
66
94
  started_at: z.nullable(z.number().int()),
67
95
  completed_at: z.nullable(z.number().int()),
@@ -74,6 +102,11 @@ export const EvalSampleObject$inboundSchema: z.ZodType<
74
102
  "response_id": "responseId",
75
103
  "output_text": "outputText",
76
104
  "extracted_output": "extractedOutput",
105
+ "cost_usd": "costUsd",
106
+ "input_tokens": "inputTokens",
107
+ "output_tokens": "outputTokens",
108
+ "cached_tokens": "cachedTokens",
109
+ "phase_timings": "phaseTimings",
77
110
  "created_at": "createdAt",
78
111
  "started_at": "startedAt",
79
112
  "completed_at": "completedAt",
@@ -98,6 +131,11 @@ export type EvalSampleObject$Outbound = {
98
131
  scores: { [k: string]: any };
99
132
  judge: { [k: string]: any } | null;
100
133
  error: { [k: string]: any } | null;
134
+ cost_usd: number | null;
135
+ input_tokens: number | null;
136
+ output_tokens: number | null;
137
+ cached_tokens: number | null;
138
+ phase_timings: { [k: string]: any } | null;
101
139
  created_at: number;
102
140
  started_at: number | null;
103
141
  completed_at: number | null;
@@ -126,6 +164,11 @@ export const EvalSampleObject$outboundSchema: z.ZodType<
126
164
  scores: z.record(z.any()),
127
165
  judge: z.nullable(z.record(z.any())),
128
166
  error: z.nullable(z.record(z.any())),
167
+ costUsd: z.nullable(z.number()),
168
+ inputTokens: z.nullable(z.number().int()),
169
+ outputTokens: z.nullable(z.number().int()),
170
+ cachedTokens: z.nullable(z.number().int()),
171
+ phaseTimings: z.nullable(z.record(z.any())),
129
172
  createdAt: z.number().int(),
130
173
  startedAt: z.nullable(z.number().int()),
131
174
  completedAt: z.nullable(z.number().int()),
@@ -138,6 +181,11 @@ export const EvalSampleObject$outboundSchema: z.ZodType<
138
181
  responseId: "response_id",
139
182
  outputText: "output_text",
140
183
  extractedOutput: "extracted_output",
184
+ costUsd: "cost_usd",
185
+ inputTokens: "input_tokens",
186
+ outputTokens: "output_tokens",
187
+ cachedTokens: "cached_tokens",
188
+ phaseTimings: "phase_timings",
141
189
  createdAt: "created_at",
142
190
  startedAt: "started_at",
143
191
  completedAt: "completed_at",
@@ -0,0 +1,98 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ */
4
+
5
+ import * as z from "zod/v3";
6
+ import { remap as remap$ } from "../../lib/primitives.js";
7
+ import { safeParse } from "../../lib/schemas.js";
8
+ import { Result as SafeParseResult } from "../../types/fp.js";
9
+ import { SDKValidationError } from "../errors/sdkvalidationerror.js";
10
+
11
+ export type EvalSampleTrajectoryUploadedObject = {
12
+ object: "eval.sample.trajectory.uploaded";
13
+ runId: string;
14
+ sampleIndex: number;
15
+ taskId: string;
16
+ model: string;
17
+ stepCount: number;
18
+ /**
19
+ * Measured from the stored payload, so it describes what was actually written.
20
+ */
21
+ sizeBytes: number;
22
+ };
23
+
24
+ /** @internal */
25
+ export const EvalSampleTrajectoryUploadedObject$inboundSchema: z.ZodType<
26
+ EvalSampleTrajectoryUploadedObject,
27
+ z.ZodTypeDef,
28
+ unknown
29
+ > = z.object({
30
+ object: z.literal("eval.sample.trajectory.uploaded"),
31
+ run_id: z.string(),
32
+ sample_index: z.number().int(),
33
+ task_id: z.string(),
34
+ model: z.string(),
35
+ step_count: z.number().int(),
36
+ size_bytes: z.number().int(),
37
+ }).transform((v) => {
38
+ return remap$(v, {
39
+ "run_id": "runId",
40
+ "sample_index": "sampleIndex",
41
+ "task_id": "taskId",
42
+ "step_count": "stepCount",
43
+ "size_bytes": "sizeBytes",
44
+ });
45
+ });
46
+ /** @internal */
47
+ export type EvalSampleTrajectoryUploadedObject$Outbound = {
48
+ object: "eval.sample.trajectory.uploaded";
49
+ run_id: string;
50
+ sample_index: number;
51
+ task_id: string;
52
+ model: string;
53
+ step_count: number;
54
+ size_bytes: number;
55
+ };
56
+
57
+ /** @internal */
58
+ export const EvalSampleTrajectoryUploadedObject$outboundSchema: z.ZodType<
59
+ EvalSampleTrajectoryUploadedObject$Outbound,
60
+ z.ZodTypeDef,
61
+ EvalSampleTrajectoryUploadedObject
62
+ > = z.object({
63
+ object: z.literal("eval.sample.trajectory.uploaded"),
64
+ runId: z.string(),
65
+ sampleIndex: z.number().int(),
66
+ taskId: z.string(),
67
+ model: z.string(),
68
+ stepCount: z.number().int(),
69
+ sizeBytes: z.number().int(),
70
+ }).transform((v) => {
71
+ return remap$(v, {
72
+ runId: "run_id",
73
+ sampleIndex: "sample_index",
74
+ taskId: "task_id",
75
+ stepCount: "step_count",
76
+ sizeBytes: "size_bytes",
77
+ });
78
+ });
79
+
80
+ export function evalSampleTrajectoryUploadedObjectToJSON(
81
+ evalSampleTrajectoryUploadedObject: EvalSampleTrajectoryUploadedObject,
82
+ ): string {
83
+ return JSON.stringify(
84
+ EvalSampleTrajectoryUploadedObject$outboundSchema.parse(
85
+ evalSampleTrajectoryUploadedObject,
86
+ ),
87
+ );
88
+ }
89
+ export function evalSampleTrajectoryUploadedObjectFromJSON(
90
+ jsonString: string,
91
+ ): SafeParseResult<EvalSampleTrajectoryUploadedObject, SDKValidationError> {
92
+ return safeParse(
93
+ jsonString,
94
+ (x) =>
95
+ EvalSampleTrajectoryUploadedObject$inboundSchema.parse(JSON.parse(x)),
96
+ `Failed to parse 'EvalSampleTrajectoryUploadedObject' from JSON`,
97
+ );
98
+ }
@@ -0,0 +1,93 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ */
4
+
5
+ import * as z from "zod/v3";
6
+ import { remap as remap$ } from "../../lib/primitives.js";
7
+ import { safeParse } from "../../lib/schemas.js";
8
+ import { Result as SafeParseResult } from "../../types/fp.js";
9
+ import { SDKValidationError } from "../errors/sdkvalidationerror.js";
10
+ import {
11
+ EvalLeaderboardRow,
12
+ EvalLeaderboardRow$inboundSchema,
13
+ EvalLeaderboardRow$Outbound,
14
+ EvalLeaderboardRow$outboundSchema,
15
+ } from "./evalleaderboardrow.js";
16
+
17
+ export type EvalSuiteLeaderboard = {
18
+ object: "eval.leaderboard";
19
+ suiteId: string;
20
+ /**
21
+ * The version this board was restricted to, or null when it spans every version.
22
+ */
23
+ suiteVersion: number | null;
24
+ /**
25
+ * The `scores` key `accuracy` averages. Echoed back so a board rendered from a non-default metric is self-describing.
26
+ */
27
+ metric: string;
28
+ /**
29
+ * Rows ordered by `accuracy` descending, nulls last, then by agent, model, and effort.
30
+ */
31
+ data: Array<EvalLeaderboardRow>;
32
+ };
33
+
34
+ /** @internal */
35
+ export const EvalSuiteLeaderboard$inboundSchema: z.ZodType<
36
+ EvalSuiteLeaderboard,
37
+ z.ZodTypeDef,
38
+ unknown
39
+ > = z.object({
40
+ object: z.literal("eval.leaderboard"),
41
+ suite_id: z.string(),
42
+ suite_version: z.nullable(z.number().int()),
43
+ metric: z.string(),
44
+ data: z.array(EvalLeaderboardRow$inboundSchema),
45
+ }).transform((v) => {
46
+ return remap$(v, {
47
+ "suite_id": "suiteId",
48
+ "suite_version": "suiteVersion",
49
+ });
50
+ });
51
+ /** @internal */
52
+ export type EvalSuiteLeaderboard$Outbound = {
53
+ object: "eval.leaderboard";
54
+ suite_id: string;
55
+ suite_version: number | null;
56
+ metric: string;
57
+ data: Array<EvalLeaderboardRow$Outbound>;
58
+ };
59
+
60
+ /** @internal */
61
+ export const EvalSuiteLeaderboard$outboundSchema: z.ZodType<
62
+ EvalSuiteLeaderboard$Outbound,
63
+ z.ZodTypeDef,
64
+ EvalSuiteLeaderboard
65
+ > = z.object({
66
+ object: z.literal("eval.leaderboard"),
67
+ suiteId: z.string(),
68
+ suiteVersion: z.nullable(z.number().int()),
69
+ metric: z.string(),
70
+ data: z.array(EvalLeaderboardRow$outboundSchema),
71
+ }).transform((v) => {
72
+ return remap$(v, {
73
+ suiteId: "suite_id",
74
+ suiteVersion: "suite_version",
75
+ });
76
+ });
77
+
78
+ export function evalSuiteLeaderboardToJSON(
79
+ evalSuiteLeaderboard: EvalSuiteLeaderboard,
80
+ ): string {
81
+ return JSON.stringify(
82
+ EvalSuiteLeaderboard$outboundSchema.parse(evalSuiteLeaderboard),
83
+ );
84
+ }
85
+ export function evalSuiteLeaderboardFromJSON(
86
+ jsonString: string,
87
+ ): SafeParseResult<EvalSuiteLeaderboard, SDKValidationError> {
88
+ return safeParse(
89
+ jsonString,
90
+ (x) => EvalSuiteLeaderboard$inboundSchema.parse(JSON.parse(x)),
91
+ `Failed to parse 'EvalSuiteLeaderboard' from JSON`,
92
+ );
93
+ }
@@ -0,0 +1,319 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ */
4
+
5
+ import * as z from "zod/v3";
6
+ import { remap as remap$ } from "../../lib/primitives.js";
7
+ import { safeParse } from "../../lib/schemas.js";
8
+ import { ClosedEnum } from "../../types/enums.js";
9
+ import { Result as SafeParseResult } from "../../types/fp.js";
10
+ import { SDKValidationError } from "../errors/sdkvalidationerror.js";
11
+
12
+ /**
13
+ * `failed` means no score was produced at all. An agent timeout is NOT that — the harness still runs the verifier and scores the trial, so it stays `completed` and carries its exception in `error`.
14
+ */
15
+ export const ImportAgentRunRequestStatus = {
16
+ Completed: "completed",
17
+ Failed: "failed",
18
+ } as const;
19
+ /**
20
+ * `failed` means no score was produced at all. An agent timeout is NOT that — the harness still runs the verifier and scores the trial, so it stays `completed` and carries its exception in `error`.
21
+ */
22
+ export type ImportAgentRunRequestStatus = ClosedEnum<
23
+ typeof ImportAgentRunRequestStatus
24
+ >;
25
+
26
+ export type ImportAgentRunRequestTrials = {
27
+ /**
28
+ * Must already exist in the target suite version's manifest; unknown ids are rejected rather than invented.
29
+ */
30
+ taskId: string;
31
+ /**
32
+ * Defaults to the trial's position in `trials`. Unique per (task, model), not per run.
33
+ */
34
+ sampleIndex?: number | undefined;
35
+ /**
36
+ * `failed` means no score was produced at all. An agent timeout is NOT that — the harness still runs the verifier and scores the trial, so it stays `completed` and carries its exception in `error`.
37
+ */
38
+ status?: ImportAgentRunRequestStatus | undefined;
39
+ /**
40
+ * The task instruction.
41
+ */
42
+ prompt?: string | undefined;
43
+ target?: string | null | undefined;
44
+ /**
45
+ * What decided the score — the verifier's output. The agent's session goes to the trajectory route.
46
+ */
47
+ outputText?: string | null | undefined;
48
+ /**
49
+ * Metric values for this trial, e.g. `{ "reward": 1, "resolved": 1 }`. The board averages `reward` by default.
50
+ */
51
+ scores?: { [k: string]: number } | undefined;
52
+ /**
53
+ * The exception this trial recorded, if any. Does not by itself make the trial a failure.
54
+ */
55
+ error?: { [k: string]: any } | null | undefined;
56
+ costUsd?: number | null | undefined;
57
+ inputTokens?: number | null | undefined;
58
+ outputTokens?: number | null | undefined;
59
+ /**
60
+ * The share of `input_tokens` served from the provider's prompt cache.
61
+ */
62
+ cachedTokens?: number | null | undefined;
63
+ phaseTimings?: { [k: string]: any } | null | undefined;
64
+ };
65
+
66
+ export type ImportAgentRunRequest = {
67
+ /**
68
+ * Existing suite to import into. Its manifest defines the task set, so create the suite first.
69
+ */
70
+ suiteId: string;
71
+ /**
72
+ * Defaults to the suite's active version.
73
+ */
74
+ suiteVersion?: number | undefined;
75
+ /**
76
+ * The harness agent, e.g. `omp`.
77
+ */
78
+ agentName: string;
79
+ agentVersion?: string | null | undefined;
80
+ /**
81
+ * Reasoning-effort setting, e.g. `medium`. Part of the leaderboard's row identity, so the same agent and model at two efforts stay two rows.
82
+ */
83
+ agentEffort?: string | null | undefined;
84
+ model: string;
85
+ /**
86
+ * Total spend for the job. Falls back to the sum of the trials' own costs when omitted.
87
+ */
88
+ costUsd?: number | null | undefined;
89
+ /**
90
+ * When the job ran, unix seconds. Defaults to now — set it when backfilling, since the board orders by this.
91
+ */
92
+ createdAt?: number | undefined;
93
+ metadata?: { [k: string]: string } | undefined;
94
+ /**
95
+ * Reusing a key returns the run created the first time instead of importing a second copy. Without one, a retried upload silently doubles the trials behind a board row.
96
+ */
97
+ idempotencyKey?: string | undefined;
98
+ /**
99
+ * May be empty when opening a run to stream trials into. A one-shot import sends them all here.
100
+ */
101
+ trials?: Array<ImportAgentRunRequestTrials> | undefined;
102
+ /**
103
+ * How many trials the job will produce. Sets the run's total up front so the console shows real progress (12/89) while it streams, instead of 12/12 that grows.
104
+ */
105
+ expectedTrials?: number | undefined;
106
+ /**
107
+ * False opens the run `in_progress` for `POST /evals/runs/{run_id}/trials` to fill. Leaderboards only count completed runs, so a streaming run stays off the board until it is finalized.
108
+ */
109
+ finalize?: boolean | undefined;
110
+ };
111
+
112
+ /** @internal */
113
+ export const ImportAgentRunRequestStatus$inboundSchema: z.ZodNativeEnum<
114
+ typeof ImportAgentRunRequestStatus
115
+ > = z.nativeEnum(ImportAgentRunRequestStatus);
116
+ /** @internal */
117
+ export const ImportAgentRunRequestStatus$outboundSchema: z.ZodNativeEnum<
118
+ typeof ImportAgentRunRequestStatus
119
+ > = ImportAgentRunRequestStatus$inboundSchema;
120
+
121
+ /** @internal */
122
+ export const ImportAgentRunRequestTrials$inboundSchema: z.ZodType<
123
+ ImportAgentRunRequestTrials,
124
+ z.ZodTypeDef,
125
+ unknown
126
+ > = z.object({
127
+ task_id: z.string(),
128
+ sample_index: z.number().int().optional(),
129
+ status: ImportAgentRunRequestStatus$inboundSchema.default("completed"),
130
+ prompt: z.string().default(""),
131
+ target: z.nullable(z.string()).optional(),
132
+ output_text: z.nullable(z.string()).optional(),
133
+ scores: z.record(z.number()).optional(),
134
+ error: z.nullable(z.record(z.any())).optional(),
135
+ cost_usd: z.nullable(z.number()).optional(),
136
+ input_tokens: z.nullable(z.number().int()).optional(),
137
+ output_tokens: z.nullable(z.number().int()).optional(),
138
+ cached_tokens: z.nullable(z.number().int()).optional(),
139
+ phase_timings: z.nullable(z.record(z.any())).optional(),
140
+ }).transform((v) => {
141
+ return remap$(v, {
142
+ "task_id": "taskId",
143
+ "sample_index": "sampleIndex",
144
+ "output_text": "outputText",
145
+ "cost_usd": "costUsd",
146
+ "input_tokens": "inputTokens",
147
+ "output_tokens": "outputTokens",
148
+ "cached_tokens": "cachedTokens",
149
+ "phase_timings": "phaseTimings",
150
+ });
151
+ });
152
+ /** @internal */
153
+ export type ImportAgentRunRequestTrials$Outbound = {
154
+ task_id: string;
155
+ sample_index?: number | undefined;
156
+ status: string;
157
+ prompt: string;
158
+ target?: string | null | undefined;
159
+ output_text?: string | null | undefined;
160
+ scores?: { [k: string]: number } | undefined;
161
+ error?: { [k: string]: any } | null | undefined;
162
+ cost_usd?: number | null | undefined;
163
+ input_tokens?: number | null | undefined;
164
+ output_tokens?: number | null | undefined;
165
+ cached_tokens?: number | null | undefined;
166
+ phase_timings?: { [k: string]: any } | null | undefined;
167
+ };
168
+
169
+ /** @internal */
170
+ export const ImportAgentRunRequestTrials$outboundSchema: z.ZodType<
171
+ ImportAgentRunRequestTrials$Outbound,
172
+ z.ZodTypeDef,
173
+ ImportAgentRunRequestTrials
174
+ > = z.object({
175
+ taskId: z.string(),
176
+ sampleIndex: z.number().int().optional(),
177
+ status: ImportAgentRunRequestStatus$outboundSchema.default("completed"),
178
+ prompt: z.string().default(""),
179
+ target: z.nullable(z.string()).optional(),
180
+ outputText: z.nullable(z.string()).optional(),
181
+ scores: z.record(z.number()).optional(),
182
+ error: z.nullable(z.record(z.any())).optional(),
183
+ costUsd: z.nullable(z.number()).optional(),
184
+ inputTokens: z.nullable(z.number().int()).optional(),
185
+ outputTokens: z.nullable(z.number().int()).optional(),
186
+ cachedTokens: z.nullable(z.number().int()).optional(),
187
+ phaseTimings: z.nullable(z.record(z.any())).optional(),
188
+ }).transform((v) => {
189
+ return remap$(v, {
190
+ taskId: "task_id",
191
+ sampleIndex: "sample_index",
192
+ outputText: "output_text",
193
+ costUsd: "cost_usd",
194
+ inputTokens: "input_tokens",
195
+ outputTokens: "output_tokens",
196
+ cachedTokens: "cached_tokens",
197
+ phaseTimings: "phase_timings",
198
+ });
199
+ });
200
+
201
+ export function importAgentRunRequestTrialsToJSON(
202
+ importAgentRunRequestTrials: ImportAgentRunRequestTrials,
203
+ ): string {
204
+ return JSON.stringify(
205
+ ImportAgentRunRequestTrials$outboundSchema.parse(
206
+ importAgentRunRequestTrials,
207
+ ),
208
+ );
209
+ }
210
+ export function importAgentRunRequestTrialsFromJSON(
211
+ jsonString: string,
212
+ ): SafeParseResult<ImportAgentRunRequestTrials, SDKValidationError> {
213
+ return safeParse(
214
+ jsonString,
215
+ (x) => ImportAgentRunRequestTrials$inboundSchema.parse(JSON.parse(x)),
216
+ `Failed to parse 'ImportAgentRunRequestTrials' from JSON`,
217
+ );
218
+ }
219
+
220
+ /** @internal */
221
+ export const ImportAgentRunRequest$inboundSchema: z.ZodType<
222
+ ImportAgentRunRequest,
223
+ z.ZodTypeDef,
224
+ unknown
225
+ > = z.object({
226
+ suite_id: z.string(),
227
+ suite_version: z.number().int().optional(),
228
+ agent_name: z.string(),
229
+ agent_version: z.nullable(z.string()).optional(),
230
+ agent_effort: z.nullable(z.string()).optional(),
231
+ model: z.string(),
232
+ cost_usd: z.nullable(z.number()).optional(),
233
+ created_at: z.number().int().optional(),
234
+ metadata: z.record(z.string()).optional(),
235
+ idempotency_key: z.string().optional(),
236
+ trials: z.array(z.lazy(() => ImportAgentRunRequestTrials$inboundSchema))
237
+ .optional(),
238
+ expected_trials: z.number().int().optional(),
239
+ finalize: z.boolean().default(true),
240
+ }).transform((v) => {
241
+ return remap$(v, {
242
+ "suite_id": "suiteId",
243
+ "suite_version": "suiteVersion",
244
+ "agent_name": "agentName",
245
+ "agent_version": "agentVersion",
246
+ "agent_effort": "agentEffort",
247
+ "cost_usd": "costUsd",
248
+ "created_at": "createdAt",
249
+ "idempotency_key": "idempotencyKey",
250
+ "expected_trials": "expectedTrials",
251
+ });
252
+ });
253
+ /** @internal */
254
+ export type ImportAgentRunRequest$Outbound = {
255
+ suite_id: string;
256
+ suite_version?: number | undefined;
257
+ agent_name: string;
258
+ agent_version?: string | null | undefined;
259
+ agent_effort?: string | null | undefined;
260
+ model: string;
261
+ cost_usd?: number | null | undefined;
262
+ created_at?: number | undefined;
263
+ metadata?: { [k: string]: string } | undefined;
264
+ idempotency_key?: string | undefined;
265
+ trials?: Array<ImportAgentRunRequestTrials$Outbound> | undefined;
266
+ expected_trials?: number | undefined;
267
+ finalize: boolean;
268
+ };
269
+
270
+ /** @internal */
271
+ export const ImportAgentRunRequest$outboundSchema: z.ZodType<
272
+ ImportAgentRunRequest$Outbound,
273
+ z.ZodTypeDef,
274
+ ImportAgentRunRequest
275
+ > = z.object({
276
+ suiteId: z.string(),
277
+ suiteVersion: z.number().int().optional(),
278
+ agentName: z.string(),
279
+ agentVersion: z.nullable(z.string()).optional(),
280
+ agentEffort: z.nullable(z.string()).optional(),
281
+ model: z.string(),
282
+ costUsd: z.nullable(z.number()).optional(),
283
+ createdAt: z.number().int().optional(),
284
+ metadata: z.record(z.string()).optional(),
285
+ idempotencyKey: z.string().optional(),
286
+ trials: z.array(z.lazy(() => ImportAgentRunRequestTrials$outboundSchema))
287
+ .optional(),
288
+ expectedTrials: z.number().int().optional(),
289
+ finalize: z.boolean().default(true),
290
+ }).transform((v) => {
291
+ return remap$(v, {
292
+ suiteId: "suite_id",
293
+ suiteVersion: "suite_version",
294
+ agentName: "agent_name",
295
+ agentVersion: "agent_version",
296
+ agentEffort: "agent_effort",
297
+ costUsd: "cost_usd",
298
+ createdAt: "created_at",
299
+ idempotencyKey: "idempotency_key",
300
+ expectedTrials: "expected_trials",
301
+ });
302
+ });
303
+
304
+ export function importAgentRunRequestToJSON(
305
+ importAgentRunRequest: ImportAgentRunRequest,
306
+ ): string {
307
+ return JSON.stringify(
308
+ ImportAgentRunRequest$outboundSchema.parse(importAgentRunRequest),
309
+ );
310
+ }
311
+ export function importAgentRunRequestFromJSON(
312
+ jsonString: string,
313
+ ): SafeParseResult<ImportAgentRunRequest, SDKValidationError> {
314
+ return safeParse(
315
+ jsonString,
316
+ (x) => ImportAgentRunRequest$inboundSchema.parse(JSON.parse(x)),
317
+ `Failed to parse 'ImportAgentRunRequest' from JSON`,
318
+ );
319
+ }
@@ -27,6 +27,7 @@ export * from "./agenttextconfiguration.js";
27
27
  export * from "./agentversion.js";
28
28
  export * from "./agentversionlist.js";
29
29
  export * from "./allowedtools.js";
30
+ export * from "./appendagenttrialsrequest.js";
30
31
  export * from "./authorizeintent.js";
31
32
  export * from "./autochunkingstrategy.js";
32
33
  export * from "./autoendpoint.js";
@@ -233,6 +234,7 @@ export * from "./evaldatasetformat.js";
233
234
  export * from "./evalfewshotconfig.js";
234
235
  export * from "./evalgenerationconfig.js";
235
236
  export * from "./evalhistoricalimportobject.js";
237
+ export * from "./evalleaderboardrow.js";
236
238
  export * from "./evalleaderboardrungroup.js";
237
239
  export * from "./evalmetric.js";
238
240
  export * from "./evaloutputextraction.js";
@@ -242,15 +244,18 @@ export * from "./evalpythongraderexecution.js";
242
244
  export * from "./evalpythongradermodelaccess.js";
243
245
  export * from "./evalpythonpreprocessor.js";
244
246
  export * from "./evalrundeletedobject.js";
247
+ export * from "./evalrunoverview.js";
245
248
  export * from "./evalrunstatus.js";
246
249
  export * from "./evalsampleaudioobject.js";
247
250
  export * from "./evalsamplelist.js";
248
251
  export * from "./evalsampleobject.js";
249
252
  export * from "./evalsamplestatus.js";
253
+ export * from "./evalsampletrajectoryuploadedobject.js";
250
254
  export * from "./evalscheduledeletedobject.js";
251
255
  export * from "./evalscheduleoverlappolicy.js";
252
256
  export * from "./evalschedulerunconfig.js";
253
257
  export * from "./evalsuitedeletedobject.js";
258
+ export * from "./evalsuiteleaderboard.js";
254
259
  export * from "./evalsuitelist.js";
255
260
  export * from "./evalsuitemanifest.js";
256
261
  export * from "./evalsuiteobject.js";
@@ -309,6 +314,7 @@ export * from "./imagegenerationcallwithid.js";
309
314
  export * from "./imagegenerationrequest.js";
310
315
  export * from "./imagegenerationresponse.js";
311
316
  export * from "./imagegenerationtooldefinition.js";
317
+ export * from "./importagentrunrequest.js";
312
318
  export * from "./importhistoricalevalresultsrequest.js";
313
319
  export * from "./incompletedetails.js";
314
320
  export * from "./inputaudio.js";
@@ -607,6 +613,7 @@ export * from "./updatememorystorerequest.js";
607
613
  export * from "./updatepromptresponse.js";
608
614
  export * from "./updateresponserequest.js";
609
615
  export * from "./updatevectorstorefileattributesrequest.js";
616
+ export * from "./uploadevalsampletrajectoryrequest.js";
610
617
  export * from "./urlcitation.js";
611
618
  export * from "./usage.js";
612
619
  export * from "./usagecostsresponse.js";