@lightsage/mcp 0.3.1 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (402) hide show
  1. package/README.md +2 -2
  2. package/api/well-known-openai-apps-challenge.js +25 -0
  3. package/bin/mcp-server.js +1878 -1023
  4. package/bin/mcp-server.js.map +77 -58
  5. package/esm/funcs/evalTemplatesV2EvalTemplatesCreate.d.ts +16 -0
  6. package/esm/funcs/evalTemplatesV2EvalTemplatesCreate.d.ts.map +1 -0
  7. package/esm/funcs/evalTemplatesV2EvalTemplatesCreate.js +83 -0
  8. package/esm/funcs/evalTemplatesV2EvalTemplatesCreate.js.map +1 -0
  9. package/esm/funcs/evalTemplatesV2EvalTemplatesDelete.d.ts +16 -0
  10. package/esm/funcs/evalTemplatesV2EvalTemplatesDelete.d.ts.map +1 -0
  11. package/esm/funcs/{evalResultsV2EvalResultsList.js → evalTemplatesV2EvalTemplatesDelete.js} +17 -16
  12. package/esm/funcs/evalTemplatesV2EvalTemplatesDelete.js.map +1 -0
  13. package/esm/funcs/evalTemplatesV2EvalTemplatesList.d.ts +15 -0
  14. package/esm/funcs/evalTemplatesV2EvalTemplatesList.d.ts.map +1 -0
  15. package/esm/funcs/evalTemplatesV2EvalTemplatesList.js +72 -0
  16. package/esm/funcs/evalTemplatesV2EvalTemplatesList.js.map +1 -0
  17. package/esm/funcs/evalTemplatesV2EvalTemplatesRetrieve.d.ts +16 -0
  18. package/esm/funcs/evalTemplatesV2EvalTemplatesRetrieve.d.ts.map +1 -0
  19. package/esm/funcs/{evalResultsV2EvalResultsRetrieve.js → evalTemplatesV2EvalTemplatesRetrieve.js} +10 -10
  20. package/esm/funcs/evalTemplatesV2EvalTemplatesRetrieve.js.map +1 -0
  21. package/esm/funcs/evalTemplatesV2EvalTemplatesUpdate.d.ts +16 -0
  22. package/esm/funcs/evalTemplatesV2EvalTemplatesUpdate.d.ts.map +1 -0
  23. package/esm/funcs/evalTemplatesV2EvalTemplatesUpdate.js +89 -0
  24. package/esm/funcs/evalTemplatesV2EvalTemplatesUpdate.js.map +1 -0
  25. package/esm/funcs/evalsCancel.d.ts +16 -0
  26. package/esm/funcs/evalsCancel.d.ts.map +1 -0
  27. package/esm/funcs/{evalsAnalysis.js → evalsCancel.js} +10 -10
  28. package/esm/funcs/evalsCancel.js.map +1 -0
  29. package/esm/funcs/evalsGet.d.ts +16 -0
  30. package/esm/funcs/evalsGet.d.ts.map +1 -0
  31. package/esm/funcs/{evalRunsV2EvalRunsRetrieve.js → evalsGet.js} +5 -5
  32. package/esm/funcs/evalsGet.js.map +1 -0
  33. package/esm/funcs/evalsRun.d.ts +6 -4
  34. package/esm/funcs/evalsRun.d.ts.map +1 -1
  35. package/esm/funcs/evalsRun.js +6 -4
  36. package/esm/funcs/evalsRun.js.map +1 -1
  37. package/esm/funcs/evalsV2EvalsBatchCreate.d.ts +16 -0
  38. package/esm/funcs/evalsV2EvalsBatchCreate.d.ts.map +1 -0
  39. package/esm/funcs/evalsV2EvalsBatchCreate.js +83 -0
  40. package/esm/funcs/evalsV2EvalsBatchCreate.js.map +1 -0
  41. package/esm/funcs/evalsV2EvalsCreate.d.ts +1 -1
  42. package/esm/funcs/evalsV2EvalsCreate.js +1 -1
  43. package/esm/funcs/evalsV2EvalsUpdate.d.ts +1 -1
  44. package/esm/funcs/evalsV2EvalsUpdate.js +1 -1
  45. package/esm/funcs/promptsV2PromptRunsList.d.ts +1 -1
  46. package/esm/funcs/promptsV2PromptRunsList.js +1 -1
  47. package/esm/funcs/promptsV2PromptRunsRetrieve.d.ts +1 -1
  48. package/esm/funcs/promptsV2PromptRunsRetrieve.js +1 -1
  49. package/esm/landing-page.js +1 -1
  50. package/esm/lib/config.d.ts +3 -3
  51. package/esm/lib/config.js +3 -3
  52. package/esm/lib/config.js.map +1 -1
  53. package/esm/mcp-server/mcp-server.js +1 -1
  54. package/esm/mcp-server/server.d.ts.map +1 -1
  55. package/esm/mcp-server/server.js +17 -9
  56. package/esm/mcp-server/server.js.map +1 -1
  57. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesCreate.d.ts +7 -0
  58. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesCreate.d.ts.map +1 -0
  59. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesCreate.js +35 -0
  60. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesCreate.js.map +1 -0
  61. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesDelete.d.ts +7 -0
  62. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesDelete.d.ts.map +1 -0
  63. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesDelete.js +35 -0
  64. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesDelete.js.map +1 -0
  65. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesList.d.ts +3 -0
  66. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesList.d.ts.map +1 -0
  67. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesList.js +30 -0
  68. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesList.js.map +1 -0
  69. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesRetrieve.d.ts +7 -0
  70. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesRetrieve.d.ts.map +1 -0
  71. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesRetrieve.js +35 -0
  72. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesRetrieve.js.map +1 -0
  73. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.d.ts +7 -0
  74. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.d.ts.map +1 -0
  75. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.js +35 -0
  76. package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.js.map +1 -0
  77. package/esm/mcp-server/tools/evalsCancel.d.ts +7 -0
  78. package/esm/mcp-server/tools/evalsCancel.d.ts.map +1 -0
  79. package/esm/mcp-server/tools/evalsCancel.js +35 -0
  80. package/esm/mcp-server/tools/evalsCancel.js.map +1 -0
  81. package/esm/mcp-server/tools/{evalRunsV2EvalRunsRetrieve.d.ts → evalsGet.d.ts} +2 -2
  82. package/esm/mcp-server/tools/evalsGet.d.ts.map +1 -0
  83. package/esm/mcp-server/tools/{evalRunsV2EvalRunsRetrieve.js → evalsGet.js} +8 -8
  84. package/esm/mcp-server/tools/evalsGet.js.map +1 -0
  85. package/esm/mcp-server/tools/evalsRun.d.ts.map +1 -1
  86. package/esm/mcp-server/tools/evalsRun.js +6 -4
  87. package/esm/mcp-server/tools/evalsRun.js.map +1 -1
  88. package/esm/mcp-server/tools/evalsV2EvalsBatchCreate.d.ts +7 -0
  89. package/esm/mcp-server/tools/evalsV2EvalsBatchCreate.d.ts.map +1 -0
  90. package/esm/mcp-server/tools/evalsV2EvalsBatchCreate.js +35 -0
  91. package/esm/mcp-server/tools/evalsV2EvalsBatchCreate.js.map +1 -0
  92. package/esm/mcp-server/tools/evalsV2EvalsCreate.js +1 -1
  93. package/esm/mcp-server/tools/evalsV2EvalsCreate.js.map +1 -1
  94. package/esm/mcp-server/tools/evalsV2EvalsUpdate.js +1 -1
  95. package/esm/mcp-server/tools/evalsV2EvalsUpdate.js.map +1 -1
  96. package/esm/mcp-server/tools/promptsV2PromptRunsList.js +1 -1
  97. package/esm/mcp-server/tools/promptsV2PromptRunsList.js.map +1 -1
  98. package/esm/mcp-server/tools/promptsV2PromptRunsRetrieve.js +1 -1
  99. package/esm/mcp-server/tools/promptsV2PromptRunsRetrieve.js.map +1 -1
  100. package/esm/models/createevaltemplatebatchrequest.d.ts +12 -0
  101. package/esm/models/createevaltemplatebatchrequest.d.ts.map +1 -0
  102. package/esm/models/createevaltemplatebatchrequest.js +12 -0
  103. package/esm/models/createevaltemplatebatchrequest.js.map +1 -0
  104. package/esm/models/createevaltemplaterequest.d.ts +79 -0
  105. package/esm/models/createevaltemplaterequest.d.ts.map +1 -0
  106. package/esm/models/createevaltemplaterequest.js +74 -0
  107. package/esm/models/createevaltemplaterequest.js.map +1 -0
  108. package/esm/models/createv2evalconfigurationrequest.d.ts +2 -0
  109. package/esm/models/createv2evalconfigurationrequest.d.ts.map +1 -1
  110. package/esm/models/createv2evalconfigurationrequest.js +2 -0
  111. package/esm/models/createv2evalconfigurationrequest.js.map +1 -1
  112. package/esm/models/createv2evalrequest.d.ts +5 -5
  113. package/esm/models/createv2evalrequest.d.ts.map +1 -1
  114. package/esm/models/createv2evalrequest.js +5 -10
  115. package/esm/models/createv2evalrequest.js.map +1 -1
  116. package/esm/models/createv2evalrunrequest.d.ts +5 -5
  117. package/esm/models/createv2evalrunrequest.d.ts.map +1 -1
  118. package/esm/models/createv2evalrunrequest.js +6 -10
  119. package/esm/models/createv2evalrunrequest.js.map +1 -1
  120. package/esm/models/evaltemplate.d.ts +83 -0
  121. package/esm/models/evaltemplate.d.ts.map +1 -0
  122. package/esm/models/evaltemplate.js +81 -0
  123. package/esm/models/evaltemplate.js.map +1 -0
  124. package/esm/models/evaltemplatebatchcreatedeval.d.ts +11 -0
  125. package/esm/models/evaltemplatebatchcreatedeval.d.ts.map +1 -0
  126. package/esm/models/evaltemplatebatchcreatedeval.js +14 -0
  127. package/esm/models/evaltemplatebatchcreatedeval.js.map +1 -0
  128. package/esm/models/evaltemplatebatchitem.d.ts +32 -0
  129. package/esm/models/evaltemplatebatchitem.d.ts.map +1 -0
  130. package/esm/models/evaltemplatebatchitem.js +27 -0
  131. package/esm/models/evaltemplatebatchitem.js.map +1 -0
  132. package/esm/models/evaltemplatebatchresponse.d.ts +12 -0
  133. package/esm/models/evaltemplatebatchresponse.d.ts.map +1 -0
  134. package/esm/models/evaltemplatebatchresponse.js +15 -0
  135. package/esm/models/evaltemplatebatchresponse.js.map +1 -0
  136. package/esm/models/evaltemplatebatchresult.d.ts +10 -0
  137. package/esm/models/evaltemplatebatchresult.d.ts.map +1 -0
  138. package/esm/models/evaltemplatebatchresult.js +13 -0
  139. package/esm/models/evaltemplatebatchresult.js.map +1 -0
  140. package/esm/models/evaltemplateruntime.d.ts +20 -0
  141. package/esm/models/evaltemplateruntime.d.ts.map +1 -0
  142. package/esm/models/evaltemplateruntime.js +21 -0
  143. package/esm/models/evaltemplateruntime.js.map +1 -0
  144. package/esm/models/evaltemplateschedule.d.ts +28 -0
  145. package/esm/models/evaltemplateschedule.d.ts.map +1 -0
  146. package/esm/models/evaltemplateschedule.js +24 -0
  147. package/esm/models/evaltemplateschedule.js.map +1 -0
  148. package/esm/models/evaltemplatesresponse.d.ts +8 -0
  149. package/esm/models/evaltemplatesresponse.d.ts.map +1 -0
  150. package/esm/models/evaltemplatesresponse.js +11 -0
  151. package/esm/models/evaltemplatesresponse.js.map +1 -0
  152. package/esm/models/graderdefinition.d.ts +14 -0
  153. package/esm/models/graderdefinition.d.ts.map +1 -0
  154. package/esm/models/graderdefinition.js +17 -0
  155. package/esm/models/graderdefinition.js.map +1 -0
  156. package/esm/models/publiccompetitorrow.js +2 -2
  157. package/esm/models/publiccompetitorrow.js.map +1 -1
  158. package/esm/models/publiccriterionverdict.d.ts +26 -0
  159. package/esm/models/publiccriterionverdict.d.ts.map +1 -0
  160. package/esm/models/publiccriterionverdict.js +24 -0
  161. package/esm/models/publiccriterionverdict.js.map +1 -0
  162. package/esm/models/publiccurrentvisibilityresponse.js +2 -2
  163. package/esm/models/publiccurrentvisibilityresponse.js.map +1 -1
  164. package/esm/models/publicevalrunenvironment.d.ts +15 -0
  165. package/esm/models/publicevalrunenvironment.d.ts.map +1 -0
  166. package/esm/models/publicevalrunenvironment.js +15 -0
  167. package/esm/models/publicevalrunenvironment.js.map +1 -0
  168. package/esm/models/publicevalrunprogress.d.ts +1 -0
  169. package/esm/models/publicevalrunprogress.d.ts.map +1 -1
  170. package/esm/models/publicevalrunprogress.js +3 -2
  171. package/esm/models/publicevalrunprogress.js.map +1 -1
  172. package/esm/models/publicgraderresult.d.ts +13 -0
  173. package/esm/models/publicgraderresult.d.ts.map +1 -0
  174. package/esm/models/publicgraderresult.js +18 -0
  175. package/esm/models/publicgraderresult.js.map +1 -0
  176. package/esm/models/publicpromptvisibilitymetrics.js +2 -2
  177. package/esm/models/publicpromptvisibilitymetrics.js.map +1 -1
  178. package/esm/models/publicvisibilitysnapshot.js +2 -2
  179. package/esm/models/publicvisibilitysnapshot.js.map +1 -1
  180. package/esm/models/searchgroundingmode.d.ts +20 -0
  181. package/esm/models/searchgroundingmode.d.ts.map +1 -0
  182. package/esm/models/searchgroundingmode.js +19 -0
  183. package/esm/models/searchgroundingmode.js.map +1 -0
  184. package/esm/models/updateevaltemplaterequest.d.ts +71 -0
  185. package/esm/models/updateevaltemplaterequest.d.ts.map +1 -0
  186. package/esm/models/updateevaltemplaterequest.js +71 -0
  187. package/esm/models/updateevaltemplaterequest.js.map +1 -0
  188. package/esm/models/updatev2evalconfigurationrequest.d.ts +2 -0
  189. package/esm/models/updatev2evalconfigurationrequest.d.ts.map +1 -1
  190. package/esm/models/updatev2evalconfigurationrequest.js +3 -0
  191. package/esm/models/updatev2evalconfigurationrequest.js.map +1 -1
  192. package/esm/models/updatev2evalrequest.d.ts +5 -5
  193. package/esm/models/updatev2evalrequest.d.ts.map +1 -1
  194. package/esm/models/updatev2evalrequest.js +6 -10
  195. package/esm/models/updatev2evalrequest.js.map +1 -1
  196. package/esm/models/v2eval.d.ts +6 -2
  197. package/esm/models/v2eval.d.ts.map +1 -1
  198. package/esm/models/v2eval.js +5 -1
  199. package/esm/models/v2eval.js.map +1 -1
  200. package/esm/models/v2evalconfiguration.d.ts +2 -0
  201. package/esm/models/v2evalconfiguration.d.ts.map +1 -1
  202. package/esm/models/v2evalconfiguration.js +2 -0
  203. package/esm/models/v2evalconfiguration.js.map +1 -1
  204. package/esm/models/v2evalresultsummary.d.ts +10 -0
  205. package/esm/models/v2evalresultsummary.d.ts.map +1 -1
  206. package/esm/models/v2evalresultsummary.js +12 -1
  207. package/esm/models/v2evalresultsummary.js.map +1 -1
  208. package/esm/models/v2evalrun.d.ts.map +1 -1
  209. package/esm/models/v2evalrun.js +1 -1
  210. package/esm/models/v2evalrun.js.map +1 -1
  211. package/esm/models/v2evalrundetail.d.ts +30 -0
  212. package/esm/models/v2evalrundetail.d.ts.map +1 -0
  213. package/esm/models/v2evalrundetail.js +33 -0
  214. package/esm/models/v2evalrundetail.js.map +1 -0
  215. package/esm/models/v2evalrunscancelop.d.ts +31 -0
  216. package/esm/models/v2evalrunscancelop.d.ts.map +1 -0
  217. package/esm/models/v2evalrunscancelop.js +27 -0
  218. package/esm/models/v2evalrunscancelop.js.map +1 -0
  219. package/esm/models/v2evalrunscreateop.d.ts +23 -3
  220. package/esm/models/v2evalrunscreateop.d.ts.map +1 -1
  221. package/esm/models/v2evalrunscreateop.js +15 -2
  222. package/esm/models/v2evalrunscreateop.js.map +1 -1
  223. package/esm/models/v2evalrunsretrieveop.d.ts +9 -2
  224. package/esm/models/v2evalrunsretrieveop.d.ts.map +1 -1
  225. package/esm/models/v2evalrunsretrieveop.js +6 -2
  226. package/esm/models/v2evalrunsretrieveop.js.map +1 -1
  227. package/esm/models/v2evalruntraceevent.d.ts.map +1 -1
  228. package/esm/models/v2evalruntraceevent.js +1 -1
  229. package/esm/models/v2evalruntraceevent.js.map +1 -1
  230. package/esm/models/v2evalsbatchcreateop.d.ts +27 -0
  231. package/esm/models/v2evalsbatchcreateop.d.ts.map +1 -0
  232. package/esm/models/v2evalsbatchcreateop.js +24 -0
  233. package/esm/models/v2evalsbatchcreateop.js.map +1 -0
  234. package/esm/models/v2evaltemplatescreateop.d.ts +13 -0
  235. package/esm/models/v2evaltemplatescreateop.d.ts.map +1 -0
  236. package/esm/models/v2evaltemplatescreateop.js +16 -0
  237. package/esm/models/v2evaltemplatescreateop.js.map +1 -0
  238. package/esm/models/v2evaltemplatesdeleteop.d.ts +24 -0
  239. package/esm/models/v2evaltemplatesdeleteop.d.ts.map +1 -0
  240. package/esm/models/v2evaltemplatesdeleteop.js +23 -0
  241. package/esm/models/v2evaltemplatesdeleteop.js.map +1 -0
  242. package/esm/models/v2evaltemplatesretrieveop.d.ts +17 -0
  243. package/esm/models/v2evaltemplatesretrieveop.d.ts.map +1 -0
  244. package/esm/models/v2evaltemplatesretrieveop.js +19 -0
  245. package/esm/models/v2evaltemplatesretrieveop.js.map +1 -0
  246. package/esm/models/v2evaltemplatesupdateop.d.ts +33 -0
  247. package/esm/models/v2evaltemplatesupdateop.d.ts.map +1 -0
  248. package/esm/models/v2evaltemplatesupdateop.js +29 -0
  249. package/esm/models/v2evaltemplatesupdateop.js.map +1 -0
  250. package/esm/models/v2inlinemcpdefinition.d.ts +4 -4
  251. package/esm/models/v2inlinemcpdefinition.d.ts.map +1 -1
  252. package/esm/models/v2inlinemcpdefinition.js +3 -3
  253. package/esm/models/v2inlinemcpdefinition.js.map +1 -1
  254. package/esm/models/v2promptrundetail.d.ts +6 -0
  255. package/esm/models/v2promptrundetail.d.ts.map +1 -1
  256. package/esm/models/v2promptrundetail.js +4 -1
  257. package/esm/models/v2promptrundetail.js.map +1 -1
  258. package/esm/models/v2promptrunsummary.d.ts.map +1 -1
  259. package/esm/models/v2promptrunsummary.js +1 -1
  260. package/esm/models/v2promptrunsummary.js.map +1 -1
  261. package/esm/models/v2publictask.d.ts +2 -2
  262. package/esm/models/v2publictask.d.ts.map +1 -1
  263. package/esm/models/v2publictask.js +4 -4
  264. package/esm/models/v2publictask.js.map +1 -1
  265. package/esm/tool-names.d.ts.map +1 -1
  266. package/esm/tool-names.js +31 -15
  267. package/esm/tool-names.js.map +1 -1
  268. package/package.json +1 -1
  269. package/src/funcs/evalTemplatesV2EvalTemplatesCreate.ts +145 -0
  270. package/src/funcs/{evalResultsV2EvalResultsList.ts → evalTemplatesV2EvalTemplatesDelete.ts} +23 -19
  271. package/src/funcs/evalTemplatesV2EvalTemplatesList.ts +124 -0
  272. package/src/funcs/{evalResultsV2EvalResultsRetrieve.ts → evalTemplatesV2EvalTemplatesRetrieve.ts} +13 -13
  273. package/src/funcs/evalTemplatesV2EvalTemplatesUpdate.ts +154 -0
  274. package/src/funcs/{evalsAnalysis.ts → evalsCancel.ts} +13 -13
  275. package/src/funcs/{evalRunsV2EvalRunsRetrieve.ts → evalsGet.ts} +4 -4
  276. package/src/funcs/evalsRun.ts +6 -4
  277. package/src/funcs/evalsV2EvalsBatchCreate.ts +145 -0
  278. package/src/funcs/evalsV2EvalsCreate.ts +1 -1
  279. package/src/funcs/evalsV2EvalsUpdate.ts +1 -1
  280. package/src/funcs/promptsV2PromptRunsList.ts +1 -1
  281. package/src/funcs/promptsV2PromptRunsRetrieve.ts +1 -1
  282. package/src/landing-page.ts +1 -1
  283. package/src/lib/config.ts +3 -3
  284. package/src/mcp-server/mcp-server.ts +1 -1
  285. package/src/mcp-server/server.ts +17 -9
  286. package/src/mcp-server/tools/evalTemplatesV2EvalTemplatesCreate.ts +45 -0
  287. package/src/mcp-server/tools/evalTemplatesV2EvalTemplatesDelete.ts +45 -0
  288. package/src/mcp-server/tools/evalTemplatesV2EvalTemplatesList.ts +36 -0
  289. package/src/mcp-server/tools/{evalResultsV2EvalResultsRetrieve.ts → evalTemplatesV2EvalTemplatesRetrieve.ts} +9 -9
  290. package/src/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.ts +45 -0
  291. package/src/mcp-server/tools/evalsCancel.ts +43 -0
  292. package/src/mcp-server/tools/{evalRunsV2EvalRunsRetrieve.ts → evalsGet.ts} +7 -7
  293. package/src/mcp-server/tools/evalsRun.ts +6 -4
  294. package/src/mcp-server/tools/evalsV2EvalsBatchCreate.ts +43 -0
  295. package/src/mcp-server/tools/evalsV2EvalsCreate.ts +1 -1
  296. package/src/mcp-server/tools/evalsV2EvalsUpdate.ts +1 -1
  297. package/src/mcp-server/tools/promptsV2PromptRunsList.ts +1 -1
  298. package/src/mcp-server/tools/promptsV2PromptRunsRetrieve.ts +1 -1
  299. package/src/models/createevaltemplatebatchrequest.ts +35 -0
  300. package/src/models/createevaltemplaterequest.ts +151 -0
  301. package/src/models/createv2evalconfigurationrequest.ts +8 -0
  302. package/src/models/createv2evalrequest.ts +18 -22
  303. package/src/models/createv2evalrunrequest.ts +19 -24
  304. package/src/models/evaltemplate.ts +154 -0
  305. package/src/models/evaltemplatebatchcreatedeval.ts +26 -0
  306. package/src/models/evaltemplatebatchitem.ts +54 -0
  307. package/src/models/evaltemplatebatchresponse.ts +30 -0
  308. package/src/models/evaltemplatebatchresult.ts +26 -0
  309. package/src/models/evaltemplateruntime.ts +64 -0
  310. package/src/models/evaltemplateschedule.ts +44 -0
  311. package/src/models/evaltemplatesresponse.ts +18 -0
  312. package/src/models/graderdefinition.ts +34 -0
  313. package/src/models/publiccompetitorrow.ts +2 -2
  314. package/src/models/publiccriterionverdict.ts +48 -0
  315. package/src/models/publiccurrentvisibilityresponse.ts +2 -2
  316. package/src/models/publicevalrunenvironment.ts +47 -0
  317. package/src/models/publicevalrunprogress.ts +8 -2
  318. package/src/models/publicgraderresult.ts +44 -0
  319. package/src/models/publicpromptvisibilitymetrics.ts +2 -2
  320. package/src/models/publicvisibilitysnapshot.ts +2 -2
  321. package/src/models/searchgroundingmode.ts +26 -0
  322. package/src/models/updateevaltemplaterequest.ts +116 -0
  323. package/src/models/updatev2evalconfigurationrequest.ts +9 -0
  324. package/src/models/updatev2evalrequest.ts +20 -21
  325. package/src/models/v2eval.ts +14 -2
  326. package/src/models/v2evalconfiguration.ts +8 -0
  327. package/src/models/v2evalresultsummary.ts +36 -1
  328. package/src/models/v2evalrun.ts +3 -1
  329. package/src/models/v2evalrundetail.ts +90 -0
  330. package/src/models/v2evalrunscancelop.ts +70 -0
  331. package/src/models/v2evalrunscreateop.ts +43 -5
  332. package/src/models/v2evalrunsretrieveop.ts +23 -3
  333. package/src/models/v2evalruntraceevent.ts +3 -1
  334. package/src/models/v2evalsbatchcreateop.ts +66 -0
  335. package/src/models/v2evaltemplatescreateop.ts +37 -0
  336. package/src/models/v2evaltemplatesdeleteop.ts +56 -0
  337. package/src/models/v2evaltemplatesretrieveop.ts +43 -0
  338. package/src/models/v2evaltemplatesupdateop.ts +79 -0
  339. package/src/models/v2inlinemcpdefinition.ts +7 -5
  340. package/src/models/v2promptrundetail.ts +12 -1
  341. package/src/models/v2promptrunsummary.ts +3 -1
  342. package/src/models/v2publictask.ts +9 -7
  343. package/src/tool-names.ts +31 -15
  344. package/test/simulations.test.js +30 -3
  345. package/esm/funcs/evalResultsV2EvalResultsList.d.ts +0 -16
  346. package/esm/funcs/evalResultsV2EvalResultsList.d.ts.map +0 -1
  347. package/esm/funcs/evalResultsV2EvalResultsList.js.map +0 -1
  348. package/esm/funcs/evalResultsV2EvalResultsRetrieve.d.ts +0 -16
  349. package/esm/funcs/evalResultsV2EvalResultsRetrieve.d.ts.map +0 -1
  350. package/esm/funcs/evalResultsV2EvalResultsRetrieve.js.map +0 -1
  351. package/esm/funcs/evalRunsV2EvalRunsRetrieve.d.ts +0 -16
  352. package/esm/funcs/evalRunsV2EvalRunsRetrieve.d.ts.map +0 -1
  353. package/esm/funcs/evalRunsV2EvalRunsRetrieve.js.map +0 -1
  354. package/esm/funcs/evalsAnalysis.d.ts +0 -16
  355. package/esm/funcs/evalsAnalysis.d.ts.map +0 -1
  356. package/esm/funcs/evalsAnalysis.js.map +0 -1
  357. package/esm/mcp-server/tools/evalResultsV2EvalResultsList.d.ts +0 -7
  358. package/esm/mcp-server/tools/evalResultsV2EvalResultsList.d.ts.map +0 -1
  359. package/esm/mcp-server/tools/evalResultsV2EvalResultsList.js +0 -35
  360. package/esm/mcp-server/tools/evalResultsV2EvalResultsList.js.map +0 -1
  361. package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.d.ts +0 -7
  362. package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.d.ts.map +0 -1
  363. package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.js +0 -35
  364. package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.js.map +0 -1
  365. package/esm/mcp-server/tools/evalRunsV2EvalRunsRetrieve.d.ts.map +0 -1
  366. package/esm/mcp-server/tools/evalRunsV2EvalRunsRetrieve.js.map +0 -1
  367. package/esm/mcp-server/tools/evalsAnalysis.d.ts +0 -7
  368. package/esm/mcp-server/tools/evalsAnalysis.d.ts.map +0 -1
  369. package/esm/mcp-server/tools/evalsAnalysis.js +0 -35
  370. package/esm/mcp-server/tools/evalsAnalysis.js.map +0 -1
  371. package/esm/models/v2evalresult.d.ts +0 -23
  372. package/esm/models/v2evalresult.d.ts.map +0 -1
  373. package/esm/models/v2evalresult.js +0 -24
  374. package/esm/models/v2evalresult.js.map +0 -1
  375. package/esm/models/v2evalresultslistop.d.ts +0 -11
  376. package/esm/models/v2evalresultslistop.d.ts.map +0 -1
  377. package/esm/models/v2evalresultslistop.js +0 -16
  378. package/esm/models/v2evalresultslistop.js.map +0 -1
  379. package/esm/models/v2evalresultsresponse.d.ts +0 -9
  380. package/esm/models/v2evalresultsresponse.d.ts.map +0 -1
  381. package/esm/models/v2evalresultsresponse.js +0 -12
  382. package/esm/models/v2evalresultsresponse.js.map +0 -1
  383. package/esm/models/v2evalresultsretrieveop.d.ts +0 -10
  384. package/esm/models/v2evalresultsretrieveop.d.ts.map +0 -1
  385. package/esm/models/v2evalresultsretrieveop.js +0 -15
  386. package/esm/models/v2evalresultsretrieveop.js.map +0 -1
  387. package/esm/models/v2evalrunanalysisresponse.d.ts +0 -12
  388. package/esm/models/v2evalrunanalysisresponse.d.ts.map +0 -1
  389. package/esm/models/v2evalrunanalysisresponse.js +0 -15
  390. package/esm/models/v2evalrunanalysisresponse.js.map +0 -1
  391. package/esm/models/v2evalrunsanalysisop.d.ts +0 -17
  392. package/esm/models/v2evalrunsanalysisop.d.ts.map +0 -1
  393. package/esm/models/v2evalrunsanalysisop.js +0 -19
  394. package/esm/models/v2evalrunsanalysisop.js.map +0 -1
  395. package/src/mcp-server/tools/evalResultsV2EvalResultsList.ts +0 -43
  396. package/src/mcp-server/tools/evalsAnalysis.ts +0 -43
  397. package/src/models/v2evalresult.ts +0 -46
  398. package/src/models/v2evalresultslistop.ts +0 -39
  399. package/src/models/v2evalresultsresponse.ts +0 -23
  400. package/src/models/v2evalresultsretrieveop.ts +0 -28
  401. package/src/models/v2evalrunanalysisresponse.ts +0 -32
  402. package/src/models/v2evalrunsanalysisop.ts +0 -48
@@ -5,31 +5,18 @@
5
5
 
6
6
  import * as z from "zod";
7
7
  import {
8
- V2InlineCliReference,
9
- V2InlineCliReference$zodSchema,
10
- } from "./v2inlineclireference.js";
8
+ SearchGroundingMode,
9
+ SearchGroundingMode$zodSchema,
10
+ } from "./searchgroundingmode.js";
11
11
  import {
12
12
  V2InlineMcpDefinition,
13
13
  V2InlineMcpDefinition$zodSchema,
14
14
  } from "./v2inlinemcpdefinition.js";
15
- import {
16
- V2SavedCliReference,
17
- V2SavedCliReference$zodSchema,
18
- } from "./v2savedclireference.js";
19
15
  import {
20
16
  V2SavedMcpReference,
21
17
  V2SavedMcpReference$zodSchema,
22
18
  } from "./v2savedmcpreference.js";
23
19
 
24
- export type CreateV2EvalRequestCli = V2SavedCliReference | V2InlineCliReference;
25
-
26
- export const CreateV2EvalRequestCli$zodSchema: z.ZodType<
27
- CreateV2EvalRequestCli
28
- > = z.union([
29
- V2SavedCliReference$zodSchema,
30
- V2InlineCliReference$zodSchema,
31
- ]);
32
-
33
20
  export type CreateV2EvalRequestMcp =
34
21
  | V2InlineMcpDefinition
35
22
  | V2SavedMcpReference;
@@ -47,8 +34,11 @@ export type CreateV2EvalRequest = {
47
34
  judges: Array<string>;
48
35
  agent?: Array<string> | undefined;
49
36
  persona?: string | undefined;
50
- clis?: Array<V2SavedCliReference | V2InlineCliReference> | undefined;
37
+ clis?: Array<string> | undefined;
51
38
  mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | undefined;
39
+ setup_commands?: Array<string> | undefined;
40
+ cleanup_commands?: Array<string> | undefined;
41
+ search_grounding?: SearchGroundingMode | undefined;
52
42
  runs?: number | undefined;
53
43
  tags?: Array<string> | undefined;
54
44
  };
@@ -58,11 +48,11 @@ export const CreateV2EvalRequest$zodSchema: z.ZodType<CreateV2EvalRequest> = z
58
48
  agent: z.array(z.string()).optional().describe(
59
49
  "Runnable agents from `GET /v2/agents`. An empty array saves the eval without default agents. CLI clients expose repeatable `--agent`.",
60
50
  ),
61
- clis: z.array(z.union([
62
- V2SavedCliReference$zodSchema,
63
- V2InlineCliReference$zodSchema,
64
- ])).optional().describe(
65
- "Saved CLI references or inline installation commands.",
51
+ cleanup_commands: z.array(z.string()).optional().describe(
52
+ "Best-effort shell commands run sequentially after agent execution and result capture, before sandbox deletion. CLI clients expose repeatable `--cleanup-command`.",
53
+ ),
54
+ clis: z.array(z.string()).optional().describe(
55
+ "Saved CLI IDs or shell installation commands. UUID values resolve saved CLIs from `GET /v2/clis`; every other value is run as an installation command. CLI clients expose repeatable `--cli`.",
66
56
  ),
67
57
  judges: z.array(z.string()).describe(
68
58
  "Plain-text success criteria. CLI clients expose this as repeatable `--judge`.",
@@ -83,6 +73,12 @@ export const CreateV2EvalRequest$zodSchema: z.ZodType<CreateV2EvalRequest> = z
83
73
  runs: z.int().default(1).describe(
84
74
  "Independent attempts for every prompt-agent pair.",
85
75
  ),
76
+ search_grounding: SearchGroundingMode$zodSchema.optional().describe(
77
+ "Requested web-retrieval behavior for an eval run.",
78
+ ),
79
+ setup_commands: z.array(z.string()).optional().describe(
80
+ "Shell commands run sequentially in the prepared repository before the agent starts. CLI clients expose repeatable `--setup-command`.",
81
+ ),
86
82
  tags: z.array(z.string()).optional().describe(
87
83
  "Optional labels for organizing the eval. CLI clients may expose this string array as a repeatable `--tag` flag.",
88
84
  ),
@@ -5,33 +5,18 @@
5
5
 
6
6
  import * as z from "zod";
7
7
  import {
8
- V2InlineCliReference,
9
- V2InlineCliReference$zodSchema,
10
- } from "./v2inlineclireference.js";
8
+ SearchGroundingMode,
9
+ SearchGroundingMode$zodSchema,
10
+ } from "./searchgroundingmode.js";
11
11
  import {
12
12
  V2InlineMcpDefinition,
13
13
  V2InlineMcpDefinition$zodSchema,
14
14
  } from "./v2inlinemcpdefinition.js";
15
- import {
16
- V2SavedCliReference,
17
- V2SavedCliReference$zodSchema,
18
- } from "./v2savedclireference.js";
19
15
  import {
20
16
  V2SavedMcpReference,
21
17
  V2SavedMcpReference$zodSchema,
22
18
  } from "./v2savedmcpreference.js";
23
19
 
24
- export type CreateV2EvalRunRequestCli =
25
- | V2SavedCliReference
26
- | V2InlineCliReference;
27
-
28
- export const CreateV2EvalRunRequestCli$zodSchema: z.ZodType<
29
- CreateV2EvalRunRequestCli
30
- > = z.union([
31
- V2SavedCliReference$zodSchema,
32
- V2InlineCliReference$zodSchema,
33
- ]);
34
-
35
20
  export type CreateV2EvalRunRequestMcp =
36
21
  | V2InlineMcpDefinition
37
22
  | V2SavedMcpReference;
@@ -54,14 +39,17 @@ export type CreateV2EvalRunRequest = {
54
39
  judges?: Array<string> | null | undefined;
55
40
  agent?: Array<string> | null | undefined;
56
41
  persona?: string | undefined;
57
- clis?: Array<V2SavedCliReference | V2InlineCliReference> | null | undefined;
42
+ clis?: Array<string> | null | undefined;
58
43
  mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | null | undefined;
44
+ setup_commands?: Array<string> | null | undefined;
45
+ cleanup_commands?: Array<string> | null | undefined;
59
46
  runs?: number | undefined;
60
47
  tags?: Array<string> | undefined;
61
48
  repository?: string | undefined;
62
49
  env?: { [k: string]: string } | undefined;
63
50
  skills?: Array<string> | undefined;
64
51
  skill_ids?: Array<string> | undefined;
52
+ search_grounding?: SearchGroundingMode | null | undefined;
65
53
  };
66
54
 
67
55
  export const CreateV2EvalRunRequest$zodSchema: z.ZodType<
@@ -70,11 +58,11 @@ export const CreateV2EvalRunRequest$zodSchema: z.ZodType<
70
58
  agent: z.array(z.string()).nullable().optional().describe(
71
59
  "Runnable agents from `GET /v2/agents`. Required unless the saved eval already has agents. CLI clients expose repeatable `--agent`.",
72
60
  ),
73
- clis: z.array(z.union([
74
- V2SavedCliReference$zodSchema,
75
- V2InlineCliReference$zodSchema,
76
- ])).nullable().optional().describe(
77
- "Saved CLI references or inline installation commands. In saved mode, an explicit empty array disables saved CLIs for this launch.",
61
+ cleanup_commands: z.array(z.string()).nullable().optional().describe(
62
+ "Best-effort shell commands run sequentially after agent execution and result capture, before sandbox deletion. In saved mode, an explicit empty array disables saved cleanup commands for this launch. CLI clients expose repeatable `--cleanup-command`.",
63
+ ),
64
+ clis: z.array(z.string()).nullable().optional().describe(
65
+ "Saved CLI IDs or shell installation commands. UUID values resolve saved CLIs from `GET /v2/clis`; every other value is run as an installation command. In saved mode, an explicit empty array disables saved CLIs for this launch. CLI clients expose repeatable `--cli`.",
78
66
  ),
79
67
  configuration_id: z.string().optional().describe(
80
68
  "Saved-run form only. Optional saved repository, environment, skills, and legacy tool configuration.",
@@ -109,6 +97,13 @@ export const CreateV2EvalRunRequest$zodSchema: z.ZodType<
109
97
  runs: z.int().optional().describe(
110
98
  "Independent attempts for every prompt-agent pair. Defaults to the saved eval value or 1 for a direct run.",
111
99
  ),
100
+ search_grounding: SearchGroundingMode$zodSchema.nullable().optional()
101
+ .describe(
102
+ "Web-retrieval policy for this launch. Overrides the selected configuration or saved eval when provided; direct runs default to `default`.",
103
+ ),
104
+ setup_commands: z.array(z.string()).nullable().optional().describe(
105
+ "Shell commands run sequentially in the prepared repository before the agent starts. In saved mode, an explicit empty array disables saved setup commands for this launch. CLI clients expose repeatable `--setup-command`.",
106
+ ),
112
107
  skill_ids: z.array(z.string()).optional().describe(
113
108
  "Direct create-and-run form only. Reusable skill IDs from `GET /v2/skills` to install before the agent starts.",
114
109
  ),
@@ -0,0 +1,154 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ * @generated-id: ea68162f8775
4
+ */
5
+
6
+ import * as z from "zod";
7
+ import { ClosedEnum } from "../types/enums.js";
8
+ import {
9
+ EvalTemplateRuntime,
10
+ EvalTemplateRuntime$zodSchema,
11
+ } from "./evaltemplateruntime.js";
12
+ import {
13
+ EvalTemplateSchedule,
14
+ EvalTemplateSchedule$zodSchema,
15
+ } from "./evaltemplateschedule.js";
16
+ import {
17
+ GraderDefinition,
18
+ GraderDefinition$zodSchema,
19
+ } from "./graderdefinition.js";
20
+
21
+ /**
22
+ * Kind of eval the template creates. A template only applies to evals of the same type.
23
+ */
24
+ export const EvalTemplateEvalType = {
25
+ CustomEval: "custom_eval",
26
+ AgentPreference: "agent_preference",
27
+ } as const;
28
+ /**
29
+ * Kind of eval the template creates. A template only applies to evals of the same type.
30
+ */
31
+ export type EvalTemplateEvalType = ClosedEnum<typeof EvalTemplateEvalType>;
32
+
33
+ export const EvalTemplateEvalType$zodSchema = z.enum([
34
+ "custom_eval",
35
+ "agent_preference",
36
+ ]).describe(
37
+ "Kind of eval the template creates. A template only applies to evals of the same type.",
38
+ );
39
+
40
+ export const EvalTemplateIncludedComponent = {
41
+ Judges: "judges",
42
+ Targets: "targets",
43
+ Persona: "persona",
44
+ Competitors: "competitors",
45
+ Tag: "tag",
46
+ Category: "category",
47
+ Runs: "runs",
48
+ Schedule: "schedule",
49
+ Repository: "repository",
50
+ Environment: "environment",
51
+ SearchGrounding: "search_grounding",
52
+ Skills: "skills",
53
+ McpServers: "mcp_servers",
54
+ Clis: "clis",
55
+ BrandDocs: "brand_docs",
56
+ GithubPr: "github_pr",
57
+ RuntimeCommands: "runtime_commands",
58
+ } as const;
59
+ export type EvalTemplateIncludedComponent = ClosedEnum<
60
+ typeof EvalTemplateIncludedComponent
61
+ >;
62
+
63
+ export const EvalTemplateIncludedComponent$zodSchema = z.enum([
64
+ "judges",
65
+ "targets",
66
+ "persona",
67
+ "competitors",
68
+ "tag",
69
+ "category",
70
+ "runs",
71
+ "schedule",
72
+ "repository",
73
+ "environment",
74
+ "search_grounding",
75
+ "skills",
76
+ "mcp_servers",
77
+ "clis",
78
+ "brand_docs",
79
+ "github_pr",
80
+ "runtime_commands",
81
+ ]);
82
+
83
+ export type EvalTemplate = {
84
+ name: string;
85
+ description?: string | null | undefined;
86
+ eval_type?: EvalTemplateEvalType | undefined;
87
+ included_components: Array<EvalTemplateIncludedComponent>;
88
+ persona?: string | null | undefined;
89
+ category_name?: string | null | undefined;
90
+ tag?: string | null | undefined;
91
+ competitor_ids?: Array<string> | undefined;
92
+ judges?: Array<GraderDefinition> | undefined;
93
+ target_keys?: Array<string> | undefined;
94
+ schedule?: EvalTemplateSchedule | undefined;
95
+ runs_per_execution?: number | undefined;
96
+ runtime?: EvalTemplateRuntime | undefined;
97
+ id: string;
98
+ org_id: string;
99
+ version: number;
100
+ created_by?: string | null | undefined;
101
+ created_at: string;
102
+ updated_at: string;
103
+ deleted_at?: string | null | undefined;
104
+ };
105
+
106
+ export const EvalTemplate$zodSchema: z.ZodType<EvalTemplate> = z.object({
107
+ category_name: z.string().nullable().optional().describe(
108
+ "Optional category copied when `category` is included.",
109
+ ),
110
+ competitor_ids: z.array(z.string()).optional().describe(
111
+ "Competitor IDs that create additional variants when `competitors` is included.",
112
+ ),
113
+ created_at: z.iso.datetime({ offset: true }),
114
+ created_by: z.string().nullable().optional(),
115
+ deleted_at: z.iso.datetime({ offset: true }).nullable().optional(),
116
+ description: z.string().nullable().optional().describe(
117
+ "Optional guidance describing when to use the template.",
118
+ ),
119
+ eval_type: EvalTemplateEvalType$zodSchema.default("custom_eval").describe(
120
+ "Kind of eval the template creates. A template only applies to evals of the same type.",
121
+ ),
122
+ id: z.string(),
123
+ included_components: z.array(EvalTemplateIncludedComponent$zodSchema)
124
+ .describe(
125
+ "Settings copied when the template is applied. Omitted components do not override the new eval; included components copy their saved value, including an empty value.",
126
+ ),
127
+ judges: z.array(GraderDefinition$zodSchema).optional().describe(
128
+ "Judge definitions copied when `judges` is included. May be empty to explicitly create the eval without template judges.",
129
+ ),
130
+ name: z.string().describe(
131
+ "Organization-visible name for this reusable template.",
132
+ ),
133
+ org_id: z.string(),
134
+ persona: z.string().nullable().optional().describe(
135
+ "Optional persona copied when `persona` is included.",
136
+ ),
137
+ runs_per_execution: z.int().default(1).describe(
138
+ "Attempt count copied when `runs` is included.",
139
+ ),
140
+ runtime: EvalTemplateRuntime$zodSchema.optional().describe(
141
+ "Reusable runtime resources copied into each created eval.",
142
+ ),
143
+ schedule: EvalTemplateSchedule$zodSchema.optional().describe(
144
+ "Schedule copied into every eval created from the template.",
145
+ ),
146
+ tag: z.string().nullable().optional().describe(
147
+ "Optional tag copied when `tag` is included.",
148
+ ),
149
+ target_keys: z.array(z.string()).optional().describe(
150
+ "Runnable target keys copied when `targets` is included, such as `agent:codex:gpt-5.6-terra`.",
151
+ ),
152
+ updated_at: z.iso.datetime({ offset: true }),
153
+ version: z.int(),
154
+ });
@@ -0,0 +1,26 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ * @generated-id: 98054871a9ae
4
+ */
5
+
6
+ import * as z from "zod";
7
+
8
+ export type EvalTemplateBatchCreatedEval = {
9
+ id: string;
10
+ eval_id?: string | null | undefined;
11
+ name: string;
12
+ prompt: string;
13
+ competitor_id?: string | null | undefined;
14
+ competitor_name?: string | null | undefined;
15
+ };
16
+
17
+ export const EvalTemplateBatchCreatedEval$zodSchema: z.ZodType<
18
+ EvalTemplateBatchCreatedEval
19
+ > = z.object({
20
+ competitor_id: z.string().nullable().optional(),
21
+ competitor_name: z.string().nullable().optional(),
22
+ eval_id: z.string().nullable().optional(),
23
+ id: z.string(),
24
+ name: z.string(),
25
+ prompt: z.string(),
26
+ });
@@ -0,0 +1,54 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ * @generated-id: 23c0fa6bda51
4
+ */
5
+
6
+ import * as z from "zod";
7
+ import { ClosedEnum } from "../types/enums.js";
8
+
9
+ /**
10
+ * Eval subject type used for organization and reporting.
11
+ */
12
+ export const EvalTemplateBatchItemType = {
13
+ Workflow: "workflow",
14
+ Mcp: "mcp",
15
+ Cli: "cli",
16
+ Skill: "skill",
17
+ } as const;
18
+ /**
19
+ * Eval subject type used for organization and reporting.
20
+ */
21
+ export type EvalTemplateBatchItemType = ClosedEnum<
22
+ typeof EvalTemplateBatchItemType
23
+ >;
24
+
25
+ export const EvalTemplateBatchItemType$zodSchema = z.enum([
26
+ "workflow",
27
+ "mcp",
28
+ "cli",
29
+ "skill",
30
+ ]).describe("Eval subject type used for organization and reporting.");
31
+
32
+ /**
33
+ * Prompt-specific fields combined with the referenced template.
34
+ */
35
+ export type EvalTemplateBatchItem = {
36
+ name: string;
37
+ prompts: Array<string>;
38
+ type?: EvalTemplateBatchItemType | undefined;
39
+ subject_id?: string | null | undefined;
40
+ };
41
+
42
+ export const EvalTemplateBatchItem$zodSchema: z.ZodType<EvalTemplateBatchItem> =
43
+ z.object({
44
+ name: z.string().describe("Name of the prompt-specific eval to create."),
45
+ prompts: z.array(z.string()).describe(
46
+ "Task prompts owned by this eval. Judges, targets, scheduling, and runtime resources come from the template.",
47
+ ),
48
+ subject_id: z.string().nullable().optional().describe(
49
+ "Optional subject identifier associated with this eval.",
50
+ ),
51
+ type: EvalTemplateBatchItemType$zodSchema.default("workflow").describe(
52
+ "Eval subject type used for organization and reporting.",
53
+ ),
54
+ }).describe("Prompt-specific fields combined with the referenced template.");
@@ -0,0 +1,30 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ * @generated-id: a63a246560d8
4
+ */
5
+
6
+ import * as z from "zod";
7
+ import {
8
+ EvalTemplateBatchResult,
9
+ EvalTemplateBatchResult$zodSchema,
10
+ } from "./evaltemplatebatchresult.js";
11
+
12
+ export type EvalTemplateBatchResponse = {
13
+ template_id: string;
14
+ template_version: number;
15
+ created_count: number;
16
+ base_count: number;
17
+ variant_count: number;
18
+ results?: Array<EvalTemplateBatchResult> | undefined;
19
+ };
20
+
21
+ export const EvalTemplateBatchResponse$zodSchema: z.ZodType<
22
+ EvalTemplateBatchResponse
23
+ > = z.object({
24
+ base_count: z.int(),
25
+ created_count: z.int(),
26
+ results: z.array(EvalTemplateBatchResult$zodSchema).optional(),
27
+ template_id: z.string(),
28
+ template_version: z.int(),
29
+ variant_count: z.int(),
30
+ });
@@ -0,0 +1,26 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ * @generated-id: f2e21f14a77e
4
+ */
5
+
6
+ import * as z from "zod";
7
+ import {
8
+ EvalTemplateBatchCreatedEval,
9
+ EvalTemplateBatchCreatedEval$zodSchema,
10
+ } from "./evaltemplatebatchcreatedeval.js";
11
+
12
+ export type EvalTemplateBatchResult = {
13
+ input_index: number;
14
+ name: string;
15
+ base: EvalTemplateBatchCreatedEval;
16
+ variants?: Array<EvalTemplateBatchCreatedEval> | undefined;
17
+ };
18
+
19
+ export const EvalTemplateBatchResult$zodSchema: z.ZodType<
20
+ EvalTemplateBatchResult
21
+ > = z.object({
22
+ base: EvalTemplateBatchCreatedEval$zodSchema,
23
+ input_index: z.int(),
24
+ name: z.string(),
25
+ variants: z.array(EvalTemplateBatchCreatedEval$zodSchema).optional(),
26
+ });
@@ -0,0 +1,64 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ * @generated-id: c5af571c2713
4
+ */
5
+
6
+ import * as z from "zod";
7
+ import {
8
+ SearchGroundingMode,
9
+ SearchGroundingMode$zodSchema,
10
+ } from "./searchgroundingmode.js";
11
+
12
+ /**
13
+ * Reusable runtime resources copied into each created eval.
14
+ */
15
+ export type EvalTemplateRuntime = {
16
+ starting_project_id?: string | null | undefined;
17
+ environment_profile_ids?: Array<string> | undefined;
18
+ include_env_vars?: boolean | undefined;
19
+ search_grounding?: SearchGroundingMode | undefined;
20
+ skill_ids?: Array<string> | undefined;
21
+ mcp_server_ids?: Array<string> | undefined;
22
+ cli_ids?: Array<string> | undefined;
23
+ include_brand_docs?: boolean | undefined;
24
+ run_on_github_pr?: boolean | undefined;
25
+ setup_commands?: Array<string> | undefined;
26
+ cleanup_commands?: Array<string> | undefined;
27
+ };
28
+
29
+ export const EvalTemplateRuntime$zodSchema: z.ZodType<EvalTemplateRuntime> = z
30
+ .object({
31
+ cleanup_commands: z.array(z.string()).optional().describe(
32
+ "Shell commands run after each eval attempt, in order.",
33
+ ),
34
+ cli_ids: z.array(z.string()).optional().describe(
35
+ "Reusable CLI IDs copied into each created eval.",
36
+ ),
37
+ environment_profile_ids: z.array(z.string()).optional().describe(
38
+ "Environment profile IDs copied into each created eval.",
39
+ ),
40
+ include_brand_docs: z.boolean().default(false).describe(
41
+ "Whether organization documentation is included at runtime.",
42
+ ),
43
+ include_env_vars: z.boolean().default(true).describe(
44
+ "Whether selected environment profiles are enabled.",
45
+ ),
46
+ mcp_server_ids: z.array(z.string()).optional().describe(
47
+ "Reusable MCP server IDs copied into each created eval.",
48
+ ),
49
+ run_on_github_pr: z.boolean().default(false).describe(
50
+ "Whether each created eval is configured for GitHub pull requests.",
51
+ ),
52
+ search_grounding: SearchGroundingMode$zodSchema.optional().describe(
53
+ "Requested web-retrieval behavior for an eval run.",
54
+ ),
55
+ setup_commands: z.array(z.string()).optional().describe(
56
+ "Shell commands run before each eval attempt, in order.",
57
+ ),
58
+ skill_ids: z.array(z.string()).optional().describe(
59
+ "Reusable skill IDs copied into each created eval.",
60
+ ),
61
+ starting_project_id: z.string().nullable().optional().describe(
62
+ "Optional reusable repository workspace ID.",
63
+ ),
64
+ }).describe("Reusable runtime resources copied into each created eval.");
@@ -0,0 +1,44 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ * @generated-id: e2a92236d519
4
+ */
5
+
6
+ import * as z from "zod";
7
+ import { ClosedEnum } from "../types/enums.js";
8
+
9
+ /**
10
+ * Schedule cadence used when `enabled` is true.
11
+ */
12
+ export const Frequency = {
13
+ Daily: "daily",
14
+ EveryOtherDay: "every_other_day",
15
+ Weekly: "weekly",
16
+ } as const;
17
+ /**
18
+ * Schedule cadence used when `enabled` is true.
19
+ */
20
+ export type Frequency = ClosedEnum<typeof Frequency>;
21
+
22
+ export const Frequency$zodSchema = z.enum([
23
+ "daily",
24
+ "every_other_day",
25
+ "weekly",
26
+ ]).describe("Schedule cadence used when `enabled` is true.");
27
+
28
+ /**
29
+ * Schedule copied into every eval created from the template.
30
+ */
31
+ export type EvalTemplateSchedule = {
32
+ enabled?: boolean | undefined;
33
+ frequency?: Frequency | undefined;
34
+ };
35
+
36
+ export const EvalTemplateSchedule$zodSchema: z.ZodType<EvalTemplateSchedule> = z
37
+ .object({
38
+ enabled: z.boolean().default(false).describe(
39
+ "Whether each eval created from the template receives a schedule.",
40
+ ),
41
+ frequency: Frequency$zodSchema.default("weekly").describe(
42
+ "Schedule cadence used when `enabled` is true.",
43
+ ),
44
+ }).describe("Schedule copied into every eval created from the template.");
@@ -0,0 +1,18 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ * @generated-id: 72a47845de15
4
+ */
5
+
6
+ import * as z from "zod";
7
+ import { EvalTemplate, EvalTemplate$zodSchema } from "./evaltemplate.js";
8
+
9
+ export type EvalTemplatesResponse = {
10
+ data?: Array<EvalTemplate> | undefined;
11
+ count: number;
12
+ };
13
+
14
+ export const EvalTemplatesResponse$zodSchema: z.ZodType<EvalTemplatesResponse> =
15
+ z.object({
16
+ count: z.int(),
17
+ data: z.array(EvalTemplate$zodSchema).optional(),
18
+ });
@@ -0,0 +1,34 @@
1
+ /*
2
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
3
+ * @generated-id: e683eba11663
4
+ */
5
+
6
+ import * as z from "zod";
7
+
8
+ export type GraderDefinition = {
9
+ name: string;
10
+ type: string;
11
+ needle?: string | null | undefined;
12
+ needles?: Array<string> | null | undefined;
13
+ pattern?: string | null | undefined;
14
+ case_sensitive?: boolean | undefined;
15
+ criteria?: string | null | undefined;
16
+ weight?: number | undefined;
17
+ blocking?: boolean | undefined;
18
+ };
19
+
20
+ export const GraderDefinition$zodSchema: z.ZodType<GraderDefinition> = z.object(
21
+ {
22
+ blocking: z.boolean().default(false).describe(
23
+ "Whether a failed or missing result from this grader fails the entire eval regardless of the aggregate score.",
24
+ ),
25
+ case_sensitive: z.boolean().default(false),
26
+ criteria: z.string().nullable().optional(),
27
+ name: z.string(),
28
+ needle: z.string().nullable().optional(),
29
+ needles: z.array(z.string()).nullable().optional(),
30
+ pattern: z.string().nullable().optional(),
31
+ type: z.string(),
32
+ weight: z.number().default(1),
33
+ },
34
+ );
@@ -26,11 +26,11 @@ export const PublicCompetitorRow$zodSchema: z.ZodType<PublicCompetitorRow> = z
26
26
  ),
27
27
  company: PublicCompany$zodSchema,
28
28
  share_of_voice: z.number().nullable().optional().describe(
29
- "Share of all brand mentions across tracked companies, 0 to 1; rows sum to about 1.",
29
+ "Fraction of successful prompt runs that mention the company, from 0 to 1; for example, 0.25 means one in four runs. Failed provider runs are excluded. A run can mention several companies, so rows do not sum to 1.",
30
30
  ),
31
31
  share_of_voice_rank: z.int().nullable().optional(),
32
32
  visibility_score: z.number().nullable().optional().describe(
33
- "The company's visibility score, from 0 to 1. Rows do not sum to 1.",
33
+ "Mean recommendation strength across successful prompt runs, from 0 to 1. Each run contributes the company's recommendation strength (0 to 1) when it is mentioned and 0 when it is not. Failed provider runs are excluded. Rows do not sum to 1.",
34
34
  ),
35
35
  visibility_score_rank: z.int().nullable().optional(),
36
36
  });