@meetkai/mka1 0.54.23 → 0.54.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (443) hide show
  1. package/README.md +294 -276
  2. package/bin/mcp-server.js +3844 -1779
  3. package/bin/mcp-server.js.map +42 -17
  4. package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.d.ts +19 -0
  5. package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  6. package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.js +136 -0
  7. package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.js.map +1 -0
  8. package/dist/commonjs/funcs/llmEvalsGetRunOverview.d.ts +20 -0
  9. package/dist/commonjs/funcs/llmEvalsGetRunOverview.d.ts.map +1 -0
  10. package/dist/commonjs/funcs/llmEvalsGetRunOverview.js +138 -0
  11. package/dist/commonjs/funcs/llmEvalsGetRunOverview.js.map +1 -0
  12. package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.d.ts +19 -0
  13. package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  14. package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.js +142 -0
  15. package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.js.map +1 -0
  16. package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.d.ts +20 -0
  17. package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  18. package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.js +142 -0
  19. package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  20. package/dist/commonjs/funcs/llmEvalsImportAgentRun.d.ts +19 -0
  21. package/dist/commonjs/funcs/llmEvalsImportAgentRun.d.ts.map +1 -0
  22. package/dist/commonjs/funcs/llmEvalsImportAgentRun.js +130 -0
  23. package/dist/commonjs/funcs/llmEvalsImportAgentRun.js.map +1 -0
  24. package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.d.ts +20 -0
  25. package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  26. package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.js +146 -0
  27. package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.js.map +1 -0
  28. package/dist/commonjs/lib/config.d.ts +2 -2
  29. package/dist/commonjs/lib/config.js +2 -2
  30. package/dist/commonjs/mcp-server/mcp-server.js +1 -1
  31. package/dist/commonjs/mcp-server/server.d.ts.map +1 -1
  32. package/dist/commonjs/mcp-server/server.js +13 -1
  33. package/dist/commonjs/mcp-server/server.js.map +1 -1
  34. package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts +8 -0
  35. package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  36. package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.js +64 -0
  37. package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.js.map +1 -0
  38. package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.d.ts +8 -0
  39. package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.d.ts.map +1 -0
  40. package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.js +64 -0
  41. package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.js.map +1 -0
  42. package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts +8 -0
  43. package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  44. package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.js +64 -0
  45. package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.js.map +1 -0
  46. package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts +8 -0
  47. package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  48. package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js +64 -0
  49. package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  50. package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.d.ts +8 -0
  51. package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.d.ts.map +1 -0
  52. package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.js +64 -0
  53. package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.js.map +1 -0
  54. package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts +8 -0
  55. package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  56. package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.js +64 -0
  57. package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.js.map +1 -0
  58. package/dist/commonjs/models/components/appendagenttrialsrequest.d.ts +111 -0
  59. package/dist/commonjs/models/components/appendagenttrialsrequest.d.ts.map +1 -0
  60. package/dist/commonjs/models/components/appendagenttrialsrequest.js +136 -0
  61. package/dist/commonjs/models/components/appendagenttrialsrequest.js.map +1 -0
  62. package/dist/commonjs/models/components/evalleaderboardrow.d.ts +85 -0
  63. package/dist/commonjs/models/components/evalleaderboardrow.d.ts.map +1 -0
  64. package/dist/commonjs/models/components/evalleaderboardrow.js +129 -0
  65. package/dist/commonjs/models/components/evalleaderboardrow.js.map +1 -0
  66. package/dist/commonjs/models/components/evalrunoverview.d.ts +101 -0
  67. package/dist/commonjs/models/components/evalrunoverview.d.ts.map +1 -0
  68. package/dist/commonjs/models/components/evalrunoverview.js +150 -0
  69. package/dist/commonjs/models/components/evalrunoverview.js.map +1 -0
  70. package/dist/commonjs/models/components/evalsampleobject.d.ts +32 -0
  71. package/dist/commonjs/models/components/evalsampleobject.d.ts.map +1 -1
  72. package/dist/commonjs/models/components/evalsampleobject.js +20 -0
  73. package/dist/commonjs/models/components/evalsampleobject.js.map +1 -1
  74. package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.d.ts +32 -0
  75. package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.d.ts.map +1 -0
  76. package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.js +87 -0
  77. package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.js.map +1 -0
  78. package/dist/commonjs/models/components/evalsuiteleaderboard.d.ts +35 -0
  79. package/dist/commonjs/models/components/evalsuiteleaderboard.d.ts.map +1 -0
  80. package/dist/commonjs/models/components/evalsuiteleaderboard.js +78 -0
  81. package/dist/commonjs/models/components/evalsuiteleaderboard.js.map +1 -0
  82. package/dist/commonjs/models/components/importagentrunrequest.d.ts +164 -0
  83. package/dist/commonjs/models/components/importagentrunrequest.d.ts.map +1 -0
  84. package/dist/commonjs/models/components/importagentrunrequest.js +182 -0
  85. package/dist/commonjs/models/components/importagentrunrequest.js.map +1 -0
  86. package/dist/commonjs/models/components/index.d.ts +7 -0
  87. package/dist/commonjs/models/components/index.d.ts.map +1 -1
  88. package/dist/commonjs/models/components/index.js +7 -0
  89. package/dist/commonjs/models/components/index.js.map +1 -1
  90. package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.d.ts +34 -0
  91. package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.d.ts.map +1 -0
  92. package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.js +71 -0
  93. package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.js.map +1 -0
  94. package/dist/commonjs/models/operations/appendagentevaltrials.d.ts +264 -0
  95. package/dist/commonjs/models/operations/appendagentevaltrials.d.ts.map +1 -0
  96. package/dist/commonjs/models/operations/appendagentevaltrials.js +371 -0
  97. package/dist/commonjs/models/operations/appendagentevaltrials.js.map +1 -0
  98. package/dist/commonjs/models/operations/cancelevalrun.d.ts +20 -0
  99. package/dist/commonjs/models/operations/cancelevalrun.d.ts.map +1 -1
  100. package/dist/commonjs/models/operations/cancelevalrun.js +16 -0
  101. package/dist/commonjs/models/operations/cancelevalrun.js.map +1 -1
  102. package/dist/commonjs/models/operations/createevalrun.d.ts +20 -0
  103. package/dist/commonjs/models/operations/createevalrun.d.ts.map +1 -1
  104. package/dist/commonjs/models/operations/createevalrun.js +16 -0
  105. package/dist/commonjs/models/operations/createevalrun.js.map +1 -1
  106. package/dist/commonjs/models/operations/getevalrun.d.ts +20 -0
  107. package/dist/commonjs/models/operations/getevalrun.d.ts.map +1 -1
  108. package/dist/commonjs/models/operations/getevalrun.js +16 -0
  109. package/dist/commonjs/models/operations/getevalrun.js.map +1 -1
  110. package/dist/commonjs/models/operations/getevalrunoverview.d.ts +24 -0
  111. package/dist/commonjs/models/operations/getevalrunoverview.d.ts.map +1 -0
  112. package/dist/commonjs/models/operations/getevalrunoverview.js +73 -0
  113. package/dist/commonjs/models/operations/getevalrunoverview.js.map +1 -0
  114. package/dist/commonjs/models/operations/getevalsampletrajectory.d.ts +75 -0
  115. package/dist/commonjs/models/operations/getevalsampletrajectory.d.ts.map +1 -0
  116. package/dist/commonjs/models/operations/getevalsampletrajectory.js +133 -0
  117. package/dist/commonjs/models/operations/getevalsampletrajectory.js.map +1 -0
  118. package/dist/commonjs/models/operations/getevalsuiteleaderboard.d.ts +32 -0
  119. package/dist/commonjs/models/operations/getevalsuiteleaderboard.d.ts.map +1 -0
  120. package/dist/commonjs/models/operations/getevalsuiteleaderboard.js +83 -0
  121. package/dist/commonjs/models/operations/getevalsuiteleaderboard.js.map +1 -0
  122. package/dist/commonjs/models/operations/importagentevalrun.d.ts +262 -0
  123. package/dist/commonjs/models/operations/importagentevalrun.d.ts.map +1 -0
  124. package/dist/commonjs/models/operations/importagentevalrun.js +367 -0
  125. package/dist/commonjs/models/operations/importagentevalrun.js.map +1 -0
  126. package/dist/commonjs/models/operations/index.d.ts +6 -0
  127. package/dist/commonjs/models/operations/index.d.ts.map +1 -1
  128. package/dist/commonjs/models/operations/index.js +6 -0
  129. package/dist/commonjs/models/operations/index.js.map +1 -1
  130. package/dist/commonjs/models/operations/listevalruns.d.ts +20 -0
  131. package/dist/commonjs/models/operations/listevalruns.d.ts.map +1 -1
  132. package/dist/commonjs/models/operations/listevalruns.js +16 -0
  133. package/dist/commonjs/models/operations/listevalruns.js.map +1 -1
  134. package/dist/commonjs/models/operations/listevalscheduleruns.d.ts +20 -0
  135. package/dist/commonjs/models/operations/listevalscheduleruns.d.ts.map +1 -1
  136. package/dist/commonjs/models/operations/listevalscheduleruns.js +16 -0
  137. package/dist/commonjs/models/operations/listevalscheduleruns.js.map +1 -1
  138. package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.d.ts +20 -0
  139. package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.d.ts.map +1 -1
  140. package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.js +16 -0
  141. package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.js.map +1 -1
  142. package/dist/commonjs/models/operations/rerunfailedevalsamples.d.ts +20 -0
  143. package/dist/commonjs/models/operations/rerunfailedevalsamples.d.ts.map +1 -1
  144. package/dist/commonjs/models/operations/rerunfailedevalsamples.js +16 -0
  145. package/dist/commonjs/models/operations/rerunfailedevalsamples.js.map +1 -1
  146. package/dist/commonjs/models/operations/retryfailedevalrun.d.ts +20 -0
  147. package/dist/commonjs/models/operations/retryfailedevalrun.d.ts.map +1 -1
  148. package/dist/commonjs/models/operations/retryfailedevalrun.js +16 -0
  149. package/dist/commonjs/models/operations/retryfailedevalrun.js.map +1 -1
  150. package/dist/commonjs/models/operations/uploadevalsampletrajectory.d.ts +31 -0
  151. package/dist/commonjs/models/operations/uploadevalsampletrajectory.d.ts.map +1 -0
  152. package/dist/commonjs/models/operations/uploadevalsampletrajectory.js +86 -0
  153. package/dist/commonjs/models/operations/uploadevalsampletrajectory.js.map +1 -0
  154. package/dist/commonjs/react-query/index.d.ts +6 -0
  155. package/dist/commonjs/react-query/index.d.ts.map +1 -1
  156. package/dist/commonjs/react-query/index.js +6 -0
  157. package/dist/commonjs/react-query/index.js.map +1 -1
  158. package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.d.ts +28 -0
  159. package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  160. package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.js +47 -0
  161. package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.js.map +1 -0
  162. package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.d.ts +16 -0
  163. package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.d.ts.map +1 -0
  164. package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.js +37 -0
  165. package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.js.map +1 -0
  166. package/dist/commonjs/react-query/llmEvalsGetRunOverview.d.ts +40 -0
  167. package/dist/commonjs/react-query/llmEvalsGetRunOverview.d.ts.map +1 -0
  168. package/dist/commonjs/react-query/llmEvalsGetRunOverview.js +60 -0
  169. package/dist/commonjs/react-query/llmEvalsGetRunOverview.js.map +1 -0
  170. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.d.ts +16 -0
  171. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.d.ts.map +1 -0
  172. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.js +45 -0
  173. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.js.map +1 -0
  174. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.d.ts +44 -0
  175. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  176. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.js +65 -0
  177. package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.js.map +1 -0
  178. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts +20 -0
  179. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts.map +1 -0
  180. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.js +41 -0
  181. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.js.map +1 -0
  182. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.d.ts +48 -0
  183. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  184. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.js +65 -0
  185. package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  186. package/dist/commonjs/react-query/llmEvalsImportAgentRun.d.ts +28 -0
  187. package/dist/commonjs/react-query/llmEvalsImportAgentRun.d.ts.map +1 -0
  188. package/dist/commonjs/react-query/llmEvalsImportAgentRun.js +47 -0
  189. package/dist/commonjs/react-query/llmEvalsImportAgentRun.js.map +1 -0
  190. package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.d.ts +29 -0
  191. package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  192. package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.js +47 -0
  193. package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.js.map +1 -0
  194. package/dist/commonjs/sdk/evals.d.ts +42 -0
  195. package/dist/commonjs/sdk/evals.d.ts.map +1 -1
  196. package/dist/commonjs/sdk/evals.js +60 -0
  197. package/dist/commonjs/sdk/evals.js.map +1 -1
  198. package/dist/esm/funcs/llmEvalsAppendAgentTrials.d.ts +19 -0
  199. package/dist/esm/funcs/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  200. package/dist/esm/funcs/llmEvalsAppendAgentTrials.js +100 -0
  201. package/dist/esm/funcs/llmEvalsAppendAgentTrials.js.map +1 -0
  202. package/dist/esm/funcs/llmEvalsGetRunOverview.d.ts +20 -0
  203. package/dist/esm/funcs/llmEvalsGetRunOverview.d.ts.map +1 -0
  204. package/dist/esm/funcs/llmEvalsGetRunOverview.js +102 -0
  205. package/dist/esm/funcs/llmEvalsGetRunOverview.js.map +1 -0
  206. package/dist/esm/funcs/llmEvalsGetSampleTrajectory.d.ts +19 -0
  207. package/dist/esm/funcs/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  208. package/dist/esm/funcs/llmEvalsGetSampleTrajectory.js +106 -0
  209. package/dist/esm/funcs/llmEvalsGetSampleTrajectory.js.map +1 -0
  210. package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.d.ts +20 -0
  211. package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  212. package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.js +106 -0
  213. package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  214. package/dist/esm/funcs/llmEvalsImportAgentRun.d.ts +19 -0
  215. package/dist/esm/funcs/llmEvalsImportAgentRun.d.ts.map +1 -0
  216. package/dist/esm/funcs/llmEvalsImportAgentRun.js +94 -0
  217. package/dist/esm/funcs/llmEvalsImportAgentRun.js.map +1 -0
  218. package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.d.ts +20 -0
  219. package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  220. package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.js +110 -0
  221. package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.js.map +1 -0
  222. package/dist/esm/lib/config.d.ts +2 -2
  223. package/dist/esm/lib/config.js +2 -2
  224. package/dist/esm/mcp-server/mcp-server.js +1 -1
  225. package/dist/esm/mcp-server/server.d.ts.map +1 -1
  226. package/dist/esm/mcp-server/server.js +13 -1
  227. package/dist/esm/mcp-server/server.js.map +1 -1
  228. package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts +8 -0
  229. package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  230. package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.js +28 -0
  231. package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.js.map +1 -0
  232. package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.d.ts +8 -0
  233. package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.d.ts.map +1 -0
  234. package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.js +28 -0
  235. package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.js.map +1 -0
  236. package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts +8 -0
  237. package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  238. package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.js +28 -0
  239. package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.js.map +1 -0
  240. package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts +8 -0
  241. package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  242. package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js +28 -0
  243. package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  244. package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.d.ts +8 -0
  245. package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.d.ts.map +1 -0
  246. package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.js +28 -0
  247. package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.js.map +1 -0
  248. package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts +8 -0
  249. package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  250. package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.js +28 -0
  251. package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.js.map +1 -0
  252. package/dist/esm/models/components/appendagenttrialsrequest.d.ts +111 -0
  253. package/dist/esm/models/components/appendagenttrialsrequest.d.ts.map +1 -0
  254. package/dist/esm/models/components/appendagenttrialsrequest.js +96 -0
  255. package/dist/esm/models/components/appendagenttrialsrequest.js.map +1 -0
  256. package/dist/esm/models/components/evalleaderboardrow.d.ts +85 -0
  257. package/dist/esm/models/components/evalleaderboardrow.d.ts.map +1 -0
  258. package/dist/esm/models/components/evalleaderboardrow.js +89 -0
  259. package/dist/esm/models/components/evalleaderboardrow.js.map +1 -0
  260. package/dist/esm/models/components/evalrunoverview.d.ts +101 -0
  261. package/dist/esm/models/components/evalrunoverview.d.ts.map +1 -0
  262. package/dist/esm/models/components/evalrunoverview.js +108 -0
  263. package/dist/esm/models/components/evalrunoverview.js.map +1 -0
  264. package/dist/esm/models/components/evalsampleobject.d.ts +32 -0
  265. package/dist/esm/models/components/evalsampleobject.d.ts.map +1 -1
  266. package/dist/esm/models/components/evalsampleobject.js +20 -0
  267. package/dist/esm/models/components/evalsampleobject.js.map +1 -1
  268. package/dist/esm/models/components/evalsampletrajectoryuploadedobject.d.ts +32 -0
  269. package/dist/esm/models/components/evalsampletrajectoryuploadedobject.d.ts.map +1 -0
  270. package/dist/esm/models/components/evalsampletrajectoryuploadedobject.js +49 -0
  271. package/dist/esm/models/components/evalsampletrajectoryuploadedobject.js.map +1 -0
  272. package/dist/esm/models/components/evalsuiteleaderboard.d.ts +35 -0
  273. package/dist/esm/models/components/evalsuiteleaderboard.d.ts.map +1 -0
  274. package/dist/esm/models/components/evalsuiteleaderboard.js +40 -0
  275. package/dist/esm/models/components/evalsuiteleaderboard.js.map +1 -0
  276. package/dist/esm/models/components/importagentrunrequest.d.ts +164 -0
  277. package/dist/esm/models/components/importagentrunrequest.d.ts.map +1 -0
  278. package/dist/esm/models/components/importagentrunrequest.js +142 -0
  279. package/dist/esm/models/components/importagentrunrequest.js.map +1 -0
  280. package/dist/esm/models/components/index.d.ts +7 -0
  281. package/dist/esm/models/components/index.d.ts.map +1 -1
  282. package/dist/esm/models/components/index.js +7 -0
  283. package/dist/esm/models/components/index.js.map +1 -1
  284. package/dist/esm/models/components/uploadevalsampletrajectoryrequest.d.ts +34 -0
  285. package/dist/esm/models/components/uploadevalsampletrajectoryrequest.d.ts.map +1 -0
  286. package/dist/esm/models/components/uploadevalsampletrajectoryrequest.js +33 -0
  287. package/dist/esm/models/components/uploadevalsampletrajectoryrequest.js.map +1 -0
  288. package/dist/esm/models/operations/appendagentevaltrials.d.ts +264 -0
  289. package/dist/esm/models/operations/appendagentevaltrials.d.ts.map +1 -0
  290. package/dist/esm/models/operations/appendagentevaltrials.js +323 -0
  291. package/dist/esm/models/operations/appendagentevaltrials.js.map +1 -0
  292. package/dist/esm/models/operations/cancelevalrun.d.ts +20 -0
  293. package/dist/esm/models/operations/cancelevalrun.d.ts.map +1 -1
  294. package/dist/esm/models/operations/cancelevalrun.js +16 -0
  295. package/dist/esm/models/operations/cancelevalrun.js.map +1 -1
  296. package/dist/esm/models/operations/createevalrun.d.ts +20 -0
  297. package/dist/esm/models/operations/createevalrun.d.ts.map +1 -1
  298. package/dist/esm/models/operations/createevalrun.js +16 -0
  299. package/dist/esm/models/operations/createevalrun.js.map +1 -1
  300. package/dist/esm/models/operations/getevalrun.d.ts +20 -0
  301. package/dist/esm/models/operations/getevalrun.d.ts.map +1 -1
  302. package/dist/esm/models/operations/getevalrun.js +16 -0
  303. package/dist/esm/models/operations/getevalrun.js.map +1 -1
  304. package/dist/esm/models/operations/getevalrunoverview.d.ts +24 -0
  305. package/dist/esm/models/operations/getevalrunoverview.d.ts.map +1 -0
  306. package/dist/esm/models/operations/getevalrunoverview.js +35 -0
  307. package/dist/esm/models/operations/getevalrunoverview.js.map +1 -0
  308. package/dist/esm/models/operations/getevalsampletrajectory.d.ts +75 -0
  309. package/dist/esm/models/operations/getevalsampletrajectory.d.ts.map +1 -0
  310. package/dist/esm/models/operations/getevalsampletrajectory.js +93 -0
  311. package/dist/esm/models/operations/getevalsampletrajectory.js.map +1 -0
  312. package/dist/esm/models/operations/getevalsuiteleaderboard.d.ts +32 -0
  313. package/dist/esm/models/operations/getevalsuiteleaderboard.d.ts.map +1 -0
  314. package/dist/esm/models/operations/getevalsuiteleaderboard.js +45 -0
  315. package/dist/esm/models/operations/getevalsuiteleaderboard.js.map +1 -0
  316. package/dist/esm/models/operations/importagentevalrun.d.ts +262 -0
  317. package/dist/esm/models/operations/importagentevalrun.d.ts.map +1 -0
  318. package/dist/esm/models/operations/importagentevalrun.js +319 -0
  319. package/dist/esm/models/operations/importagentevalrun.js.map +1 -0
  320. package/dist/esm/models/operations/index.d.ts +6 -0
  321. package/dist/esm/models/operations/index.d.ts.map +1 -1
  322. package/dist/esm/models/operations/index.js +6 -0
  323. package/dist/esm/models/operations/index.js.map +1 -1
  324. package/dist/esm/models/operations/listevalruns.d.ts +20 -0
  325. package/dist/esm/models/operations/listevalruns.d.ts.map +1 -1
  326. package/dist/esm/models/operations/listevalruns.js +16 -0
  327. package/dist/esm/models/operations/listevalruns.js.map +1 -1
  328. package/dist/esm/models/operations/listevalscheduleruns.d.ts +20 -0
  329. package/dist/esm/models/operations/listevalscheduleruns.d.ts.map +1 -1
  330. package/dist/esm/models/operations/listevalscheduleruns.js +16 -0
  331. package/dist/esm/models/operations/listevalscheduleruns.js.map +1 -1
  332. package/dist/esm/models/operations/listevalsuiteleaderboardruns.d.ts +20 -0
  333. package/dist/esm/models/operations/listevalsuiteleaderboardruns.d.ts.map +1 -1
  334. package/dist/esm/models/operations/listevalsuiteleaderboardruns.js +16 -0
  335. package/dist/esm/models/operations/listevalsuiteleaderboardruns.js.map +1 -1
  336. package/dist/esm/models/operations/rerunfailedevalsamples.d.ts +20 -0
  337. package/dist/esm/models/operations/rerunfailedevalsamples.d.ts.map +1 -1
  338. package/dist/esm/models/operations/rerunfailedevalsamples.js +16 -0
  339. package/dist/esm/models/operations/rerunfailedevalsamples.js.map +1 -1
  340. package/dist/esm/models/operations/retryfailedevalrun.d.ts +20 -0
  341. package/dist/esm/models/operations/retryfailedevalrun.d.ts.map +1 -1
  342. package/dist/esm/models/operations/retryfailedevalrun.js +16 -0
  343. package/dist/esm/models/operations/retryfailedevalrun.js.map +1 -1
  344. package/dist/esm/models/operations/uploadevalsampletrajectory.d.ts +31 -0
  345. package/dist/esm/models/operations/uploadevalsampletrajectory.d.ts.map +1 -0
  346. package/dist/esm/models/operations/uploadevalsampletrajectory.js +48 -0
  347. package/dist/esm/models/operations/uploadevalsampletrajectory.js.map +1 -0
  348. package/dist/esm/react-query/index.d.ts +6 -0
  349. package/dist/esm/react-query/index.d.ts.map +1 -1
  350. package/dist/esm/react-query/index.js +6 -0
  351. package/dist/esm/react-query/index.js.map +1 -1
  352. package/dist/esm/react-query/llmEvalsAppendAgentTrials.d.ts +28 -0
  353. package/dist/esm/react-query/llmEvalsAppendAgentTrials.d.ts.map +1 -0
  354. package/dist/esm/react-query/llmEvalsAppendAgentTrials.js +42 -0
  355. package/dist/esm/react-query/llmEvalsAppendAgentTrials.js.map +1 -0
  356. package/dist/esm/react-query/llmEvalsGetRunOverview.core.d.ts +16 -0
  357. package/dist/esm/react-query/llmEvalsGetRunOverview.core.d.ts.map +1 -0
  358. package/dist/esm/react-query/llmEvalsGetRunOverview.core.js +32 -0
  359. package/dist/esm/react-query/llmEvalsGetRunOverview.core.js.map +1 -0
  360. package/dist/esm/react-query/llmEvalsGetRunOverview.d.ts +40 -0
  361. package/dist/esm/react-query/llmEvalsGetRunOverview.d.ts.map +1 -0
  362. package/dist/esm/react-query/llmEvalsGetRunOverview.js +50 -0
  363. package/dist/esm/react-query/llmEvalsGetRunOverview.js.map +1 -0
  364. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.d.ts +16 -0
  365. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.d.ts.map +1 -0
  366. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.js +40 -0
  367. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.js.map +1 -0
  368. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.d.ts +44 -0
  369. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
  370. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.js +55 -0
  371. package/dist/esm/react-query/llmEvalsGetSampleTrajectory.js.map +1 -0
  372. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts +20 -0
  373. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts.map +1 -0
  374. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.js +36 -0
  375. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.js.map +1 -0
  376. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.d.ts +48 -0
  377. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
  378. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.js +55 -0
  379. package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.js.map +1 -0
  380. package/dist/esm/react-query/llmEvalsImportAgentRun.d.ts +28 -0
  381. package/dist/esm/react-query/llmEvalsImportAgentRun.d.ts.map +1 -0
  382. package/dist/esm/react-query/llmEvalsImportAgentRun.js +42 -0
  383. package/dist/esm/react-query/llmEvalsImportAgentRun.js.map +1 -0
  384. package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.d.ts +29 -0
  385. package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
  386. package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.js +42 -0
  387. package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.js.map +1 -0
  388. package/dist/esm/sdk/evals.d.ts +42 -0
  389. package/dist/esm/sdk/evals.d.ts.map +1 -1
  390. package/dist/esm/sdk/evals.js +60 -0
  391. package/dist/esm/sdk/evals.js.map +1 -1
  392. package/jsr.json +1 -1
  393. package/package.json +1 -1
  394. package/src/funcs/llmEvalsAppendAgentTrials.ts +188 -0
  395. package/src/funcs/llmEvalsGetRunOverview.ts +192 -0
  396. package/src/funcs/llmEvalsGetSampleTrajectory.ts +197 -0
  397. package/src/funcs/llmEvalsGetSuiteLeaderboard.ts +197 -0
  398. package/src/funcs/llmEvalsImportAgentRun.ts +181 -0
  399. package/src/funcs/llmEvalsUploadSampleTrajectory.ts +201 -0
  400. package/src/lib/config.ts +2 -2
  401. package/src/mcp-server/mcp-server.ts +1 -1
  402. package/src/mcp-server/server.ts +13 -1
  403. package/src/mcp-server/tools/llmEvalsAppendAgentTrials.ts +37 -0
  404. package/src/mcp-server/tools/llmEvalsGetRunOverview.ts +37 -0
  405. package/src/mcp-server/tools/llmEvalsGetSampleTrajectory.ts +37 -0
  406. package/src/mcp-server/tools/llmEvalsGetSuiteLeaderboard.ts +37 -0
  407. package/src/mcp-server/tools/llmEvalsImportAgentRun.ts +37 -0
  408. package/src/mcp-server/tools/llmEvalsUploadSampleTrajectory.ts +38 -0
  409. package/src/models/components/appendagenttrialsrequest.ts +224 -0
  410. package/src/models/components/evalleaderboardrow.ts +200 -0
  411. package/src/models/components/evalrunoverview.ts +240 -0
  412. package/src/models/components/evalsampleobject.ts +48 -0
  413. package/src/models/components/evalsampletrajectoryuploadedobject.ts +98 -0
  414. package/src/models/components/evalsuiteleaderboard.ts +93 -0
  415. package/src/models/components/importagentrunrequest.ts +319 -0
  416. package/src/models/components/index.ts +7 -0
  417. package/src/models/components/uploadevalsampletrajectoryrequest.ts +79 -0
  418. package/src/models/operations/appendagentevaltrials.ts +678 -0
  419. package/src/models/operations/cancelevalrun.ts +36 -0
  420. package/src/models/operations/createevalrun.ts +36 -0
  421. package/src/models/operations/getevalrun.ts +36 -0
  422. package/src/models/operations/getevalrunoverview.ts +73 -0
  423. package/src/models/operations/getevalsampletrajectory.ts +199 -0
  424. package/src/models/operations/getevalsuiteleaderboard.ts +93 -0
  425. package/src/models/operations/importagentevalrun.ts +665 -0
  426. package/src/models/operations/index.ts +6 -0
  427. package/src/models/operations/listevalruns.ts +36 -0
  428. package/src/models/operations/listevalscheduleruns.ts +36 -0
  429. package/src/models/operations/listevalsuiteleaderboardruns.ts +36 -0
  430. package/src/models/operations/rerunfailedevalsamples.ts +36 -0
  431. package/src/models/operations/retryfailedevalrun.ts +36 -0
  432. package/src/models/operations/uploadevalsampletrajectory.ts +98 -0
  433. package/src/react-query/index.ts +6 -0
  434. package/src/react-query/llmEvalsAppendAgentTrials.ts +109 -0
  435. package/src/react-query/llmEvalsGetRunOverview.core.ts +77 -0
  436. package/src/react-query/llmEvalsGetRunOverview.ts +150 -0
  437. package/src/react-query/llmEvalsGetSampleTrajectory.core.ts +94 -0
  438. package/src/react-query/llmEvalsGetSampleTrajectory.ts +159 -0
  439. package/src/react-query/llmEvalsGetSuiteLeaderboard.core.ts +89 -0
  440. package/src/react-query/llmEvalsGetSuiteLeaderboard.ts +163 -0
  441. package/src/react-query/llmEvalsImportAgentRun.ts +109 -0
  442. package/src/react-query/llmEvalsUploadSampleTrajectory.ts +110 -0
  443. package/src/sdk/evals.ts +108 -0
@@ -0,0 +1,665 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ */
4
+
5
+ import * as z from "zod/v3";
6
+ import { remap as remap$ } from "../../lib/primitives.js";
7
+ import {
8
+ collectExtraKeys as collectExtraKeys$,
9
+ safeParse,
10
+ } from "../../lib/schemas.js";
11
+ import { ClosedEnum } from "../../types/enums.js";
12
+ import { Result as SafeParseResult } from "../../types/fp.js";
13
+ import * as components from "../components/index.js";
14
+ import { SDKValidationError } from "../errors/sdkvalidationerror.js";
15
+
16
+ export type ImportAgentEvalRunRequest = {
17
+ /**
18
+ * Optional external end-user identifier forwarded by the API gateway.
19
+ */
20
+ xOnBehalfOf?: string | undefined;
21
+ importAgentRunRequest: components.ImportAgentRunRequest;
22
+ };
23
+
24
+ export type ImportAgentEvalRunStop = string | Array<string>;
25
+
26
+ export const ImportAgentEvalRunTruncation = {
27
+ Auto: "auto",
28
+ Disabled: "disabled",
29
+ } as const;
30
+ export type ImportAgentEvalRunTruncation = ClosedEnum<
31
+ typeof ImportAgentEvalRunTruncation
32
+ >;
33
+
34
+ export const ImportAgentEvalRunServiceTier = {
35
+ Auto: "auto",
36
+ Default: "default",
37
+ Flex: "flex",
38
+ Priority: "priority",
39
+ } as const;
40
+ export type ImportAgentEvalRunServiceTier = ClosedEnum<
41
+ typeof ImportAgentEvalRunServiceTier
42
+ >;
43
+
44
+ export type ImportAgentEvalRunPrefillThink = boolean | string;
45
+
46
+ export type ImportAgentEvalRunGeneration = {
47
+ instructions?: string | undefined;
48
+ temperature?: number | undefined;
49
+ topP?: number | undefined;
50
+ maxOutputTokens?: number | undefined;
51
+ /**
52
+ * lm-eval alias for max_output_tokens.
53
+ */
54
+ maxGenToks?: number | undefined;
55
+ stop?: string | Array<string> | undefined;
56
+ /**
57
+ * lm-eval generate_until stop sequences.
58
+ */
59
+ until?: Array<string> | undefined;
60
+ maxToolCalls?: number | undefined;
61
+ reasoning?: any | undefined;
62
+ text?: any | undefined;
63
+ tools?: Array<any | null> | undefined;
64
+ toolChoice?: any | undefined;
65
+ parallelToolCalls?: boolean | undefined;
66
+ truncation?: ImportAgentEvalRunTruncation | undefined;
67
+ serviceTier?: ImportAgentEvalRunServiceTier | undefined;
68
+ presencePenalty?: number | undefined;
69
+ frequencyPenalty?: number | undefined;
70
+ topK?: number | undefined;
71
+ minP?: number | undefined;
72
+ repetitionPenalty?: number | undefined;
73
+ doSample?: boolean | undefined;
74
+ extraBody?: { [k: string]: any } | undefined;
75
+ chatTemplateKwargs?: { [k: string]: any } | undefined;
76
+ prefillThink?: boolean | string | undefined;
77
+ useCache?: boolean | undefined;
78
+ timeoutSeconds?: number | undefined;
79
+ maxRetries?: number | undefined;
80
+ maxEmptyRetries?: number | undefined;
81
+ additionalProperties?: { [k: string]: any } | undefined;
82
+ };
83
+
84
+ export type ImportAgentEvalRunRequestCounts = {
85
+ total: number;
86
+ completed: number;
87
+ failed: number;
88
+ };
89
+
90
+ /**
91
+ * OK
92
+ */
93
+ export type ImportAgentEvalRunResponseBody = {
94
+ id: string;
95
+ object: "eval.run";
96
+ suiteId: string;
97
+ suiteVersion: number;
98
+ suiteVersionId: string;
99
+ /**
100
+ * The org that owns this run.
101
+ */
102
+ orgId: string;
103
+ /**
104
+ * The team that owns this run.
105
+ */
106
+ teamId: string;
107
+ status: components.EvalRunStatus;
108
+ models: Array<string>;
109
+ taskIds: Array<string> | null;
110
+ judgeModel: string | null;
111
+ embeddingModel: string | null;
112
+ generation: ImportAgentEvalRunGeneration;
113
+ requestCounts: ImportAgentEvalRunRequestCounts;
114
+ metrics: { [k: string]: any } | null;
115
+ error: { [k: string]: any } | null;
116
+ artifactFileIds: Array<string>;
117
+ metadata: { [k: string]: string };
118
+ /**
119
+ * Coding-agent harness that produced this run (e.g. `omp`, `claude-code`). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.
120
+ */
121
+ agentName: string | null;
122
+ /**
123
+ * Version of `agent_name`, as the harness reported it. Null when `agent_name` is.
124
+ */
125
+ agentVersion: string | null;
126
+ /**
127
+ * Reasoning-effort setting the agent ran at (e.g. `medium`, `xhigh`). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when `agent_name` is.
128
+ */
129
+ agentEffort: string | null;
130
+ /**
131
+ * Total spend for the run in USD. Null when the harness did not report cost.
132
+ */
133
+ costUsd: number | null;
134
+ createdAt: number;
135
+ startedAt: number | null;
136
+ completedAt: number | null;
137
+ cancelledAt: number | null;
138
+ failedAt: number | null;
139
+ };
140
+
141
+ /** @internal */
142
+ export const ImportAgentEvalRunRequest$inboundSchema: z.ZodType<
143
+ ImportAgentEvalRunRequest,
144
+ z.ZodTypeDef,
145
+ unknown
146
+ > = z.object({
147
+ "X-On-Behalf-Of": z.string().optional(),
148
+ ImportAgentRunRequest: components.ImportAgentRunRequest$inboundSchema,
149
+ }).transform((v) => {
150
+ return remap$(v, {
151
+ "X-On-Behalf-Of": "xOnBehalfOf",
152
+ "ImportAgentRunRequest": "importAgentRunRequest",
153
+ });
154
+ });
155
+ /** @internal */
156
+ export type ImportAgentEvalRunRequest$Outbound = {
157
+ "X-On-Behalf-Of"?: string | undefined;
158
+ ImportAgentRunRequest: components.ImportAgentRunRequest$Outbound;
159
+ };
160
+
161
+ /** @internal */
162
+ export const ImportAgentEvalRunRequest$outboundSchema: z.ZodType<
163
+ ImportAgentEvalRunRequest$Outbound,
164
+ z.ZodTypeDef,
165
+ ImportAgentEvalRunRequest
166
+ > = z.object({
167
+ xOnBehalfOf: z.string().optional(),
168
+ importAgentRunRequest: components.ImportAgentRunRequest$outboundSchema,
169
+ }).transform((v) => {
170
+ return remap$(v, {
171
+ xOnBehalfOf: "X-On-Behalf-Of",
172
+ importAgentRunRequest: "ImportAgentRunRequest",
173
+ });
174
+ });
175
+
176
+ export function importAgentEvalRunRequestToJSON(
177
+ importAgentEvalRunRequest: ImportAgentEvalRunRequest,
178
+ ): string {
179
+ return JSON.stringify(
180
+ ImportAgentEvalRunRequest$outboundSchema.parse(importAgentEvalRunRequest),
181
+ );
182
+ }
183
+ export function importAgentEvalRunRequestFromJSON(
184
+ jsonString: string,
185
+ ): SafeParseResult<ImportAgentEvalRunRequest, SDKValidationError> {
186
+ return safeParse(
187
+ jsonString,
188
+ (x) => ImportAgentEvalRunRequest$inboundSchema.parse(JSON.parse(x)),
189
+ `Failed to parse 'ImportAgentEvalRunRequest' from JSON`,
190
+ );
191
+ }
192
+
193
+ /** @internal */
194
+ export const ImportAgentEvalRunStop$inboundSchema: z.ZodType<
195
+ ImportAgentEvalRunStop,
196
+ z.ZodTypeDef,
197
+ unknown
198
+ > = z.union([z.string(), z.array(z.string())]);
199
+ /** @internal */
200
+ export type ImportAgentEvalRunStop$Outbound = string | Array<string>;
201
+
202
+ /** @internal */
203
+ export const ImportAgentEvalRunStop$outboundSchema: z.ZodType<
204
+ ImportAgentEvalRunStop$Outbound,
205
+ z.ZodTypeDef,
206
+ ImportAgentEvalRunStop
207
+ > = z.union([z.string(), z.array(z.string())]);
208
+
209
+ export function importAgentEvalRunStopToJSON(
210
+ importAgentEvalRunStop: ImportAgentEvalRunStop,
211
+ ): string {
212
+ return JSON.stringify(
213
+ ImportAgentEvalRunStop$outboundSchema.parse(importAgentEvalRunStop),
214
+ );
215
+ }
216
+ export function importAgentEvalRunStopFromJSON(
217
+ jsonString: string,
218
+ ): SafeParseResult<ImportAgentEvalRunStop, SDKValidationError> {
219
+ return safeParse(
220
+ jsonString,
221
+ (x) => ImportAgentEvalRunStop$inboundSchema.parse(JSON.parse(x)),
222
+ `Failed to parse 'ImportAgentEvalRunStop' from JSON`,
223
+ );
224
+ }
225
+
226
+ /** @internal */
227
+ export const ImportAgentEvalRunTruncation$inboundSchema: z.ZodNativeEnum<
228
+ typeof ImportAgentEvalRunTruncation
229
+ > = z.nativeEnum(ImportAgentEvalRunTruncation);
230
+ /** @internal */
231
+ export const ImportAgentEvalRunTruncation$outboundSchema: z.ZodNativeEnum<
232
+ typeof ImportAgentEvalRunTruncation
233
+ > = ImportAgentEvalRunTruncation$inboundSchema;
234
+
235
+ /** @internal */
236
+ export const ImportAgentEvalRunServiceTier$inboundSchema: z.ZodNativeEnum<
237
+ typeof ImportAgentEvalRunServiceTier
238
+ > = z.nativeEnum(ImportAgentEvalRunServiceTier);
239
+ /** @internal */
240
+ export const ImportAgentEvalRunServiceTier$outboundSchema: z.ZodNativeEnum<
241
+ typeof ImportAgentEvalRunServiceTier
242
+ > = ImportAgentEvalRunServiceTier$inboundSchema;
243
+
244
+ /** @internal */
245
+ export const ImportAgentEvalRunPrefillThink$inboundSchema: z.ZodType<
246
+ ImportAgentEvalRunPrefillThink,
247
+ z.ZodTypeDef,
248
+ unknown
249
+ > = z.union([z.boolean(), z.string()]);
250
+ /** @internal */
251
+ export type ImportAgentEvalRunPrefillThink$Outbound = boolean | string;
252
+
253
+ /** @internal */
254
+ export const ImportAgentEvalRunPrefillThink$outboundSchema: z.ZodType<
255
+ ImportAgentEvalRunPrefillThink$Outbound,
256
+ z.ZodTypeDef,
257
+ ImportAgentEvalRunPrefillThink
258
+ > = z.union([z.boolean(), z.string()]);
259
+
260
+ export function importAgentEvalRunPrefillThinkToJSON(
261
+ importAgentEvalRunPrefillThink: ImportAgentEvalRunPrefillThink,
262
+ ): string {
263
+ return JSON.stringify(
264
+ ImportAgentEvalRunPrefillThink$outboundSchema.parse(
265
+ importAgentEvalRunPrefillThink,
266
+ ),
267
+ );
268
+ }
269
+ export function importAgentEvalRunPrefillThinkFromJSON(
270
+ jsonString: string,
271
+ ): SafeParseResult<ImportAgentEvalRunPrefillThink, SDKValidationError> {
272
+ return safeParse(
273
+ jsonString,
274
+ (x) => ImportAgentEvalRunPrefillThink$inboundSchema.parse(JSON.parse(x)),
275
+ `Failed to parse 'ImportAgentEvalRunPrefillThink' from JSON`,
276
+ );
277
+ }
278
+
279
+ /** @internal */
280
+ export const ImportAgentEvalRunGeneration$inboundSchema: z.ZodType<
281
+ ImportAgentEvalRunGeneration,
282
+ z.ZodTypeDef,
283
+ unknown
284
+ > = collectExtraKeys$(
285
+ z.object({
286
+ instructions: z.string().optional(),
287
+ temperature: z.number().optional(),
288
+ top_p: z.number().optional(),
289
+ max_output_tokens: z.number().int().optional(),
290
+ max_gen_toks: z.number().int().optional(),
291
+ stop: z.union([z.string(), z.array(z.string())]).optional(),
292
+ until: z.array(z.string()).optional(),
293
+ max_tool_calls: z.number().int().optional(),
294
+ reasoning: z.any().optional(),
295
+ text: z.any().optional(),
296
+ tools: z.array(z.nullable(z.any())).optional(),
297
+ tool_choice: z.any().optional(),
298
+ parallel_tool_calls: z.boolean().optional(),
299
+ truncation: ImportAgentEvalRunTruncation$inboundSchema.optional(),
300
+ service_tier: ImportAgentEvalRunServiceTier$inboundSchema.optional(),
301
+ presence_penalty: z.number().optional(),
302
+ frequency_penalty: z.number().optional(),
303
+ top_k: z.number().int().optional(),
304
+ min_p: z.number().optional(),
305
+ repetition_penalty: z.number().optional(),
306
+ do_sample: z.boolean().optional(),
307
+ extra_body: z.record(z.any()).optional(),
308
+ chat_template_kwargs: z.record(z.any()).optional(),
309
+ prefill_think: z.union([z.boolean(), z.string()]).optional(),
310
+ use_cache: z.boolean().optional(),
311
+ timeout_seconds: z.number().int().optional(),
312
+ max_retries: z.number().int().optional(),
313
+ max_empty_retries: z.number().int().optional(),
314
+ }).catchall(z.any()),
315
+ "additionalProperties",
316
+ true,
317
+ ).transform((v) => {
318
+ return remap$(v, {
319
+ "top_p": "topP",
320
+ "max_output_tokens": "maxOutputTokens",
321
+ "max_gen_toks": "maxGenToks",
322
+ "max_tool_calls": "maxToolCalls",
323
+ "tool_choice": "toolChoice",
324
+ "parallel_tool_calls": "parallelToolCalls",
325
+ "service_tier": "serviceTier",
326
+ "presence_penalty": "presencePenalty",
327
+ "frequency_penalty": "frequencyPenalty",
328
+ "top_k": "topK",
329
+ "min_p": "minP",
330
+ "repetition_penalty": "repetitionPenalty",
331
+ "do_sample": "doSample",
332
+ "extra_body": "extraBody",
333
+ "chat_template_kwargs": "chatTemplateKwargs",
334
+ "prefill_think": "prefillThink",
335
+ "use_cache": "useCache",
336
+ "timeout_seconds": "timeoutSeconds",
337
+ "max_retries": "maxRetries",
338
+ "max_empty_retries": "maxEmptyRetries",
339
+ });
340
+ });
341
+ /** @internal */
342
+ export type ImportAgentEvalRunGeneration$Outbound = {
343
+ instructions?: string | undefined;
344
+ temperature?: number | undefined;
345
+ top_p?: number | undefined;
346
+ max_output_tokens?: number | undefined;
347
+ max_gen_toks?: number | undefined;
348
+ stop?: string | Array<string> | undefined;
349
+ until?: Array<string> | undefined;
350
+ max_tool_calls?: number | undefined;
351
+ reasoning?: any | undefined;
352
+ text?: any | undefined;
353
+ tools?: Array<any | null> | undefined;
354
+ tool_choice?: any | undefined;
355
+ parallel_tool_calls?: boolean | undefined;
356
+ truncation?: string | undefined;
357
+ service_tier?: string | undefined;
358
+ presence_penalty?: number | undefined;
359
+ frequency_penalty?: number | undefined;
360
+ top_k?: number | undefined;
361
+ min_p?: number | undefined;
362
+ repetition_penalty?: number | undefined;
363
+ do_sample?: boolean | undefined;
364
+ extra_body?: { [k: string]: any } | undefined;
365
+ chat_template_kwargs?: { [k: string]: any } | undefined;
366
+ prefill_think?: boolean | string | undefined;
367
+ use_cache?: boolean | undefined;
368
+ timeout_seconds?: number | undefined;
369
+ max_retries?: number | undefined;
370
+ max_empty_retries?: number | undefined;
371
+ [additionalProperties: string]: unknown;
372
+ };
373
+
374
+ /** @internal */
375
+ export const ImportAgentEvalRunGeneration$outboundSchema: z.ZodType<
376
+ ImportAgentEvalRunGeneration$Outbound,
377
+ z.ZodTypeDef,
378
+ ImportAgentEvalRunGeneration
379
+ > = z.object({
380
+ instructions: z.string().optional(),
381
+ temperature: z.number().optional(),
382
+ topP: z.number().optional(),
383
+ maxOutputTokens: z.number().int().optional(),
384
+ maxGenToks: z.number().int().optional(),
385
+ stop: z.union([z.string(), z.array(z.string())]).optional(),
386
+ until: z.array(z.string()).optional(),
387
+ maxToolCalls: z.number().int().optional(),
388
+ reasoning: z.any().optional(),
389
+ text: z.any().optional(),
390
+ tools: z.array(z.nullable(z.any())).optional(),
391
+ toolChoice: z.any().optional(),
392
+ parallelToolCalls: z.boolean().optional(),
393
+ truncation: ImportAgentEvalRunTruncation$outboundSchema.optional(),
394
+ serviceTier: ImportAgentEvalRunServiceTier$outboundSchema.optional(),
395
+ presencePenalty: z.number().optional(),
396
+ frequencyPenalty: z.number().optional(),
397
+ topK: z.number().int().optional(),
398
+ minP: z.number().optional(),
399
+ repetitionPenalty: z.number().optional(),
400
+ doSample: z.boolean().optional(),
401
+ extraBody: z.record(z.any()).optional(),
402
+ chatTemplateKwargs: z.record(z.any()).optional(),
403
+ prefillThink: z.union([z.boolean(), z.string()]).optional(),
404
+ useCache: z.boolean().optional(),
405
+ timeoutSeconds: z.number().int().optional(),
406
+ maxRetries: z.number().int().optional(),
407
+ maxEmptyRetries: z.number().int().optional(),
408
+ additionalProperties: z.record(z.any()).optional(),
409
+ }).transform((v) => {
410
+ return {
411
+ ...v.additionalProperties,
412
+ ...remap$(v, {
413
+ topP: "top_p",
414
+ maxOutputTokens: "max_output_tokens",
415
+ maxGenToks: "max_gen_toks",
416
+ maxToolCalls: "max_tool_calls",
417
+ toolChoice: "tool_choice",
418
+ parallelToolCalls: "parallel_tool_calls",
419
+ serviceTier: "service_tier",
420
+ presencePenalty: "presence_penalty",
421
+ frequencyPenalty: "frequency_penalty",
422
+ topK: "top_k",
423
+ minP: "min_p",
424
+ repetitionPenalty: "repetition_penalty",
425
+ doSample: "do_sample",
426
+ extraBody: "extra_body",
427
+ chatTemplateKwargs: "chat_template_kwargs",
428
+ prefillThink: "prefill_think",
429
+ useCache: "use_cache",
430
+ timeoutSeconds: "timeout_seconds",
431
+ maxRetries: "max_retries",
432
+ maxEmptyRetries: "max_empty_retries",
433
+ additionalProperties: null,
434
+ }),
435
+ };
436
+ });
437
+
438
+ export function importAgentEvalRunGenerationToJSON(
439
+ importAgentEvalRunGeneration: ImportAgentEvalRunGeneration,
440
+ ): string {
441
+ return JSON.stringify(
442
+ ImportAgentEvalRunGeneration$outboundSchema.parse(
443
+ importAgentEvalRunGeneration,
444
+ ),
445
+ );
446
+ }
447
+ export function importAgentEvalRunGenerationFromJSON(
448
+ jsonString: string,
449
+ ): SafeParseResult<ImportAgentEvalRunGeneration, SDKValidationError> {
450
+ return safeParse(
451
+ jsonString,
452
+ (x) => ImportAgentEvalRunGeneration$inboundSchema.parse(JSON.parse(x)),
453
+ `Failed to parse 'ImportAgentEvalRunGeneration' from JSON`,
454
+ );
455
+ }
456
+
457
+ /** @internal */
458
+ export const ImportAgentEvalRunRequestCounts$inboundSchema: z.ZodType<
459
+ ImportAgentEvalRunRequestCounts,
460
+ z.ZodTypeDef,
461
+ unknown
462
+ > = z.object({
463
+ total: z.number().int(),
464
+ completed: z.number().int(),
465
+ failed: z.number().int(),
466
+ });
467
+ /** @internal */
468
+ export type ImportAgentEvalRunRequestCounts$Outbound = {
469
+ total: number;
470
+ completed: number;
471
+ failed: number;
472
+ };
473
+
474
+ /** @internal */
475
+ export const ImportAgentEvalRunRequestCounts$outboundSchema: z.ZodType<
476
+ ImportAgentEvalRunRequestCounts$Outbound,
477
+ z.ZodTypeDef,
478
+ ImportAgentEvalRunRequestCounts
479
+ > = z.object({
480
+ total: z.number().int(),
481
+ completed: z.number().int(),
482
+ failed: z.number().int(),
483
+ });
484
+
485
+ export function importAgentEvalRunRequestCountsToJSON(
486
+ importAgentEvalRunRequestCounts: ImportAgentEvalRunRequestCounts,
487
+ ): string {
488
+ return JSON.stringify(
489
+ ImportAgentEvalRunRequestCounts$outboundSchema.parse(
490
+ importAgentEvalRunRequestCounts,
491
+ ),
492
+ );
493
+ }
494
+ export function importAgentEvalRunRequestCountsFromJSON(
495
+ jsonString: string,
496
+ ): SafeParseResult<ImportAgentEvalRunRequestCounts, SDKValidationError> {
497
+ return safeParse(
498
+ jsonString,
499
+ (x) => ImportAgentEvalRunRequestCounts$inboundSchema.parse(JSON.parse(x)),
500
+ `Failed to parse 'ImportAgentEvalRunRequestCounts' from JSON`,
501
+ );
502
+ }
503
+
504
+ /** @internal */
505
+ export const ImportAgentEvalRunResponseBody$inboundSchema: z.ZodType<
506
+ ImportAgentEvalRunResponseBody,
507
+ z.ZodTypeDef,
508
+ unknown
509
+ > = z.object({
510
+ id: z.string(),
511
+ object: z.literal("eval.run"),
512
+ suite_id: z.string(),
513
+ suite_version: z.number().int(),
514
+ suite_version_id: z.string(),
515
+ org_id: z.string(),
516
+ team_id: z.string(),
517
+ status: components.EvalRunStatus$inboundSchema,
518
+ models: z.array(z.string()),
519
+ task_ids: z.nullable(z.array(z.string())),
520
+ judge_model: z.nullable(z.string()),
521
+ embedding_model: z.nullable(z.string()),
522
+ generation: z.lazy(() => ImportAgentEvalRunGeneration$inboundSchema),
523
+ request_counts: z.lazy(() => ImportAgentEvalRunRequestCounts$inboundSchema),
524
+ metrics: z.nullable(z.record(z.any())),
525
+ error: z.nullable(z.record(z.any())),
526
+ artifact_file_ids: z.array(z.string()),
527
+ metadata: z.record(z.string()),
528
+ agent_name: z.nullable(z.string()),
529
+ agent_version: z.nullable(z.string()),
530
+ agent_effort: z.nullable(z.string()),
531
+ cost_usd: z.nullable(z.number()),
532
+ created_at: z.number().int(),
533
+ started_at: z.nullable(z.number().int()),
534
+ completed_at: z.nullable(z.number().int()),
535
+ cancelled_at: z.nullable(z.number().int()),
536
+ failed_at: z.nullable(z.number().int()),
537
+ }).transform((v) => {
538
+ return remap$(v, {
539
+ "suite_id": "suiteId",
540
+ "suite_version": "suiteVersion",
541
+ "suite_version_id": "suiteVersionId",
542
+ "org_id": "orgId",
543
+ "team_id": "teamId",
544
+ "task_ids": "taskIds",
545
+ "judge_model": "judgeModel",
546
+ "embedding_model": "embeddingModel",
547
+ "request_counts": "requestCounts",
548
+ "artifact_file_ids": "artifactFileIds",
549
+ "agent_name": "agentName",
550
+ "agent_version": "agentVersion",
551
+ "agent_effort": "agentEffort",
552
+ "cost_usd": "costUsd",
553
+ "created_at": "createdAt",
554
+ "started_at": "startedAt",
555
+ "completed_at": "completedAt",
556
+ "cancelled_at": "cancelledAt",
557
+ "failed_at": "failedAt",
558
+ });
559
+ });
560
+ /** @internal */
561
+ export type ImportAgentEvalRunResponseBody$Outbound = {
562
+ id: string;
563
+ object: "eval.run";
564
+ suite_id: string;
565
+ suite_version: number;
566
+ suite_version_id: string;
567
+ org_id: string;
568
+ team_id: string;
569
+ status: string;
570
+ models: Array<string>;
571
+ task_ids: Array<string> | null;
572
+ judge_model: string | null;
573
+ embedding_model: string | null;
574
+ generation: ImportAgentEvalRunGeneration$Outbound;
575
+ request_counts: ImportAgentEvalRunRequestCounts$Outbound;
576
+ metrics: { [k: string]: any } | null;
577
+ error: { [k: string]: any } | null;
578
+ artifact_file_ids: Array<string>;
579
+ metadata: { [k: string]: string };
580
+ agent_name: string | null;
581
+ agent_version: string | null;
582
+ agent_effort: string | null;
583
+ cost_usd: number | null;
584
+ created_at: number;
585
+ started_at: number | null;
586
+ completed_at: number | null;
587
+ cancelled_at: number | null;
588
+ failed_at: number | null;
589
+ };
590
+
591
+ /** @internal */
592
+ export const ImportAgentEvalRunResponseBody$outboundSchema: z.ZodType<
593
+ ImportAgentEvalRunResponseBody$Outbound,
594
+ z.ZodTypeDef,
595
+ ImportAgentEvalRunResponseBody
596
+ > = z.object({
597
+ id: z.string(),
598
+ object: z.literal("eval.run"),
599
+ suiteId: z.string(),
600
+ suiteVersion: z.number().int(),
601
+ suiteVersionId: z.string(),
602
+ orgId: z.string(),
603
+ teamId: z.string(),
604
+ status: components.EvalRunStatus$outboundSchema,
605
+ models: z.array(z.string()),
606
+ taskIds: z.nullable(z.array(z.string())),
607
+ judgeModel: z.nullable(z.string()),
608
+ embeddingModel: z.nullable(z.string()),
609
+ generation: z.lazy(() => ImportAgentEvalRunGeneration$outboundSchema),
610
+ requestCounts: z.lazy(() => ImportAgentEvalRunRequestCounts$outboundSchema),
611
+ metrics: z.nullable(z.record(z.any())),
612
+ error: z.nullable(z.record(z.any())),
613
+ artifactFileIds: z.array(z.string()),
614
+ metadata: z.record(z.string()),
615
+ agentName: z.nullable(z.string()),
616
+ agentVersion: z.nullable(z.string()),
617
+ agentEffort: z.nullable(z.string()),
618
+ costUsd: z.nullable(z.number()),
619
+ createdAt: z.number().int(),
620
+ startedAt: z.nullable(z.number().int()),
621
+ completedAt: z.nullable(z.number().int()),
622
+ cancelledAt: z.nullable(z.number().int()),
623
+ failedAt: z.nullable(z.number().int()),
624
+ }).transform((v) => {
625
+ return remap$(v, {
626
+ suiteId: "suite_id",
627
+ suiteVersion: "suite_version",
628
+ suiteVersionId: "suite_version_id",
629
+ orgId: "org_id",
630
+ teamId: "team_id",
631
+ taskIds: "task_ids",
632
+ judgeModel: "judge_model",
633
+ embeddingModel: "embedding_model",
634
+ requestCounts: "request_counts",
635
+ artifactFileIds: "artifact_file_ids",
636
+ agentName: "agent_name",
637
+ agentVersion: "agent_version",
638
+ agentEffort: "agent_effort",
639
+ costUsd: "cost_usd",
640
+ createdAt: "created_at",
641
+ startedAt: "started_at",
642
+ completedAt: "completed_at",
643
+ cancelledAt: "cancelled_at",
644
+ failedAt: "failed_at",
645
+ });
646
+ });
647
+
648
+ export function importAgentEvalRunResponseBodyToJSON(
649
+ importAgentEvalRunResponseBody: ImportAgentEvalRunResponseBody,
650
+ ): string {
651
+ return JSON.stringify(
652
+ ImportAgentEvalRunResponseBody$outboundSchema.parse(
653
+ importAgentEvalRunResponseBody,
654
+ ),
655
+ );
656
+ }
657
+ export function importAgentEvalRunResponseBodyFromJSON(
658
+ jsonString: string,
659
+ ): SafeParseResult<ImportAgentEvalRunResponseBody, SDKValidationError> {
660
+ return safeParse(
661
+ jsonString,
662
+ (x) => ImportAgentEvalRunResponseBody$inboundSchema.parse(JSON.parse(x)),
663
+ `Failed to parse 'ImportAgentEvalRunResponseBody' from JSON`,
664
+ );
665
+ }
@@ -6,6 +6,7 @@ export * from "./acceptclusterinvite.js";
6
6
  export * from "./activateagentconnector.js";
7
7
  export * from "./addmodelcatalogorgaccess.js";
8
8
  export * from "./addteammember.js";
9
+ export * from "./appendagentevaltrials.js";
9
10
  export * from "./batchgetcompletionfeedback.js";
10
11
  export * from "./batchgetresponsefeedback.js";
11
12
  export * from "./cancelbatch.js";
@@ -148,9 +149,12 @@ export * from "./getembeddingsusage.js";
148
149
  export * from "./getenvironmentapiv1sandboxenvironmentsenvironmentidget.js";
149
150
  export * from "./getevalartifacts.js";
150
151
  export * from "./getevalrun.js";
152
+ export * from "./getevalrunoverview.js";
151
153
  export * from "./getevalsampleaudio.js";
154
+ export * from "./getevalsampletrajectory.js";
152
155
  export * from "./getevalschedule.js";
153
156
  export * from "./getevalsuite.js";
157
+ export * from "./getevalsuiteleaderboard.js";
154
158
  export * from "./getevalsuiteversion.js";
155
159
  export * from "./getexternaluserbudgets.js";
156
160
  export * from "./getexternaluserdefaultbudgets.js";
@@ -201,6 +205,7 @@ export * from "./getvectorstorefilebatch.js";
201
205
  export * from "./getvectorstorefilecontent.js";
202
206
  export * from "./getvectorstoresusage.js";
203
207
  export * from "./getworkspace.js";
208
+ export * from "./importagentevalrun.js";
204
209
  export * from "./importhistoricalevalresults.js";
205
210
  export * from "./listaccelerators.js";
206
211
  export * from "./listagentconnectors.js";
@@ -359,6 +364,7 @@ export * from "./updateserviceaccountapikey.js";
359
364
  export * from "./updateskill.js";
360
365
  export * from "./updateteam.js";
361
366
  export * from "./updatevectorstorefileattributes.js";
367
+ export * from "./uploadevalsampletrajectory.js";
362
368
  export * from "./uploadfile.js";
363
369
  export * from "./uploadworkspacearchive.js";
364
370
  export * from "./uploadworkspacefile.js";