@meetkai/mka1 0.54.23 → 0.54.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +294 -276
- package/bin/mcp-server.js +3844 -1779
- package/bin/mcp-server.js.map +42 -17
- package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.d.ts +19 -0
- package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.d.ts.map +1 -0
- package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.js +136 -0
- package/dist/commonjs/funcs/llmEvalsAppendAgentTrials.js.map +1 -0
- package/dist/commonjs/funcs/llmEvalsGetRunOverview.d.ts +20 -0
- package/dist/commonjs/funcs/llmEvalsGetRunOverview.d.ts.map +1 -0
- package/dist/commonjs/funcs/llmEvalsGetRunOverview.js +138 -0
- package/dist/commonjs/funcs/llmEvalsGetRunOverview.js.map +1 -0
- package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.d.ts +19 -0
- package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
- package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.js +142 -0
- package/dist/commonjs/funcs/llmEvalsGetSampleTrajectory.js.map +1 -0
- package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.d.ts +20 -0
- package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
- package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.js +142 -0
- package/dist/commonjs/funcs/llmEvalsGetSuiteLeaderboard.js.map +1 -0
- package/dist/commonjs/funcs/llmEvalsImportAgentRun.d.ts +19 -0
- package/dist/commonjs/funcs/llmEvalsImportAgentRun.d.ts.map +1 -0
- package/dist/commonjs/funcs/llmEvalsImportAgentRun.js +130 -0
- package/dist/commonjs/funcs/llmEvalsImportAgentRun.js.map +1 -0
- package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.d.ts +20 -0
- package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
- package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.js +146 -0
- package/dist/commonjs/funcs/llmEvalsUploadSampleTrajectory.js.map +1 -0
- package/dist/commonjs/lib/config.d.ts +2 -2
- package/dist/commonjs/lib/config.js +2 -2
- package/dist/commonjs/mcp-server/mcp-server.js +1 -1
- package/dist/commonjs/mcp-server/server.d.ts.map +1 -1
- package/dist/commonjs/mcp-server/server.js +13 -1
- package/dist/commonjs/mcp-server/server.js.map +1 -1
- package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts +8 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.js +64 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsAppendAgentTrials.js.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.d.ts +8 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.d.ts.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.js +64 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetRunOverview.js.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts +8 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.js +64 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetSampleTrajectory.js.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts +8 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js +64 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.d.ts +8 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.d.ts.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.js +64 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsImportAgentRun.js.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts +8 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.js +64 -0
- package/dist/commonjs/mcp-server/tools/llmEvalsUploadSampleTrajectory.js.map +1 -0
- package/dist/commonjs/models/components/appendagenttrialsrequest.d.ts +111 -0
- package/dist/commonjs/models/components/appendagenttrialsrequest.d.ts.map +1 -0
- package/dist/commonjs/models/components/appendagenttrialsrequest.js +136 -0
- package/dist/commonjs/models/components/appendagenttrialsrequest.js.map +1 -0
- package/dist/commonjs/models/components/evalleaderboardrow.d.ts +85 -0
- package/dist/commonjs/models/components/evalleaderboardrow.d.ts.map +1 -0
- package/dist/commonjs/models/components/evalleaderboardrow.js +129 -0
- package/dist/commonjs/models/components/evalleaderboardrow.js.map +1 -0
- package/dist/commonjs/models/components/evalrunoverview.d.ts +101 -0
- package/dist/commonjs/models/components/evalrunoverview.d.ts.map +1 -0
- package/dist/commonjs/models/components/evalrunoverview.js +150 -0
- package/dist/commonjs/models/components/evalrunoverview.js.map +1 -0
- package/dist/commonjs/models/components/evalsampleobject.d.ts +32 -0
- package/dist/commonjs/models/components/evalsampleobject.d.ts.map +1 -1
- package/dist/commonjs/models/components/evalsampleobject.js +20 -0
- package/dist/commonjs/models/components/evalsampleobject.js.map +1 -1
- package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.d.ts +32 -0
- package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.d.ts.map +1 -0
- package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.js +87 -0
- package/dist/commonjs/models/components/evalsampletrajectoryuploadedobject.js.map +1 -0
- package/dist/commonjs/models/components/evalsuiteleaderboard.d.ts +35 -0
- package/dist/commonjs/models/components/evalsuiteleaderboard.d.ts.map +1 -0
- package/dist/commonjs/models/components/evalsuiteleaderboard.js +78 -0
- package/dist/commonjs/models/components/evalsuiteleaderboard.js.map +1 -0
- package/dist/commonjs/models/components/importagentrunrequest.d.ts +164 -0
- package/dist/commonjs/models/components/importagentrunrequest.d.ts.map +1 -0
- package/dist/commonjs/models/components/importagentrunrequest.js +182 -0
- package/dist/commonjs/models/components/importagentrunrequest.js.map +1 -0
- package/dist/commonjs/models/components/index.d.ts +7 -0
- package/dist/commonjs/models/components/index.d.ts.map +1 -1
- package/dist/commonjs/models/components/index.js +7 -0
- package/dist/commonjs/models/components/index.js.map +1 -1
- package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.d.ts +34 -0
- package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.d.ts.map +1 -0
- package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.js +71 -0
- package/dist/commonjs/models/components/uploadevalsampletrajectoryrequest.js.map +1 -0
- package/dist/commonjs/models/operations/appendagentevaltrials.d.ts +264 -0
- package/dist/commonjs/models/operations/appendagentevaltrials.d.ts.map +1 -0
- package/dist/commonjs/models/operations/appendagentevaltrials.js +371 -0
- package/dist/commonjs/models/operations/appendagentevaltrials.js.map +1 -0
- package/dist/commonjs/models/operations/cancelevalrun.d.ts +20 -0
- package/dist/commonjs/models/operations/cancelevalrun.d.ts.map +1 -1
- package/dist/commonjs/models/operations/cancelevalrun.js +16 -0
- package/dist/commonjs/models/operations/cancelevalrun.js.map +1 -1
- package/dist/commonjs/models/operations/createevalrun.d.ts +20 -0
- package/dist/commonjs/models/operations/createevalrun.d.ts.map +1 -1
- package/dist/commonjs/models/operations/createevalrun.js +16 -0
- package/dist/commonjs/models/operations/createevalrun.js.map +1 -1
- package/dist/commonjs/models/operations/getevalrun.d.ts +20 -0
- package/dist/commonjs/models/operations/getevalrun.d.ts.map +1 -1
- package/dist/commonjs/models/operations/getevalrun.js +16 -0
- package/dist/commonjs/models/operations/getevalrun.js.map +1 -1
- package/dist/commonjs/models/operations/getevalrunoverview.d.ts +24 -0
- package/dist/commonjs/models/operations/getevalrunoverview.d.ts.map +1 -0
- package/dist/commonjs/models/operations/getevalrunoverview.js +73 -0
- package/dist/commonjs/models/operations/getevalrunoverview.js.map +1 -0
- package/dist/commonjs/models/operations/getevalsampletrajectory.d.ts +75 -0
- package/dist/commonjs/models/operations/getevalsampletrajectory.d.ts.map +1 -0
- package/dist/commonjs/models/operations/getevalsampletrajectory.js +133 -0
- package/dist/commonjs/models/operations/getevalsampletrajectory.js.map +1 -0
- package/dist/commonjs/models/operations/getevalsuiteleaderboard.d.ts +32 -0
- package/dist/commonjs/models/operations/getevalsuiteleaderboard.d.ts.map +1 -0
- package/dist/commonjs/models/operations/getevalsuiteleaderboard.js +83 -0
- package/dist/commonjs/models/operations/getevalsuiteleaderboard.js.map +1 -0
- package/dist/commonjs/models/operations/importagentevalrun.d.ts +262 -0
- package/dist/commonjs/models/operations/importagentevalrun.d.ts.map +1 -0
- package/dist/commonjs/models/operations/importagentevalrun.js +367 -0
- package/dist/commonjs/models/operations/importagentevalrun.js.map +1 -0
- package/dist/commonjs/models/operations/index.d.ts +6 -0
- package/dist/commonjs/models/operations/index.d.ts.map +1 -1
- package/dist/commonjs/models/operations/index.js +6 -0
- package/dist/commonjs/models/operations/index.js.map +1 -1
- package/dist/commonjs/models/operations/listevalruns.d.ts +20 -0
- package/dist/commonjs/models/operations/listevalruns.d.ts.map +1 -1
- package/dist/commonjs/models/operations/listevalruns.js +16 -0
- package/dist/commonjs/models/operations/listevalruns.js.map +1 -1
- package/dist/commonjs/models/operations/listevalscheduleruns.d.ts +20 -0
- package/dist/commonjs/models/operations/listevalscheduleruns.d.ts.map +1 -1
- package/dist/commonjs/models/operations/listevalscheduleruns.js +16 -0
- package/dist/commonjs/models/operations/listevalscheduleruns.js.map +1 -1
- package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.d.ts +20 -0
- package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.d.ts.map +1 -1
- package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.js +16 -0
- package/dist/commonjs/models/operations/listevalsuiteleaderboardruns.js.map +1 -1
- package/dist/commonjs/models/operations/rerunfailedevalsamples.d.ts +20 -0
- package/dist/commonjs/models/operations/rerunfailedevalsamples.d.ts.map +1 -1
- package/dist/commonjs/models/operations/rerunfailedevalsamples.js +16 -0
- package/dist/commonjs/models/operations/rerunfailedevalsamples.js.map +1 -1
- package/dist/commonjs/models/operations/retryfailedevalrun.d.ts +20 -0
- package/dist/commonjs/models/operations/retryfailedevalrun.d.ts.map +1 -1
- package/dist/commonjs/models/operations/retryfailedevalrun.js +16 -0
- package/dist/commonjs/models/operations/retryfailedevalrun.js.map +1 -1
- package/dist/commonjs/models/operations/uploadevalsampletrajectory.d.ts +31 -0
- package/dist/commonjs/models/operations/uploadevalsampletrajectory.d.ts.map +1 -0
- package/dist/commonjs/models/operations/uploadevalsampletrajectory.js +86 -0
- package/dist/commonjs/models/operations/uploadevalsampletrajectory.js.map +1 -0
- package/dist/commonjs/react-query/index.d.ts +6 -0
- package/dist/commonjs/react-query/index.d.ts.map +1 -1
- package/dist/commonjs/react-query/index.js +6 -0
- package/dist/commonjs/react-query/index.js.map +1 -1
- package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.d.ts +28 -0
- package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.d.ts.map +1 -0
- package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.js +47 -0
- package/dist/commonjs/react-query/llmEvalsAppendAgentTrials.js.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.d.ts +16 -0
- package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.d.ts.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.js +37 -0
- package/dist/commonjs/react-query/llmEvalsGetRunOverview.core.js.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetRunOverview.d.ts +40 -0
- package/dist/commonjs/react-query/llmEvalsGetRunOverview.d.ts.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetRunOverview.js +60 -0
- package/dist/commonjs/react-query/llmEvalsGetRunOverview.js.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.d.ts +16 -0
- package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.d.ts.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.js +45 -0
- package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.core.js.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.d.ts +44 -0
- package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.js +65 -0
- package/dist/commonjs/react-query/llmEvalsGetSampleTrajectory.js.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts +20 -0
- package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.js +41 -0
- package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.core.js.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.d.ts +48 -0
- package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
- package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.js +65 -0
- package/dist/commonjs/react-query/llmEvalsGetSuiteLeaderboard.js.map +1 -0
- package/dist/commonjs/react-query/llmEvalsImportAgentRun.d.ts +28 -0
- package/dist/commonjs/react-query/llmEvalsImportAgentRun.d.ts.map +1 -0
- package/dist/commonjs/react-query/llmEvalsImportAgentRun.js +47 -0
- package/dist/commonjs/react-query/llmEvalsImportAgentRun.js.map +1 -0
- package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.d.ts +29 -0
- package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
- package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.js +47 -0
- package/dist/commonjs/react-query/llmEvalsUploadSampleTrajectory.js.map +1 -0
- package/dist/commonjs/sdk/evals.d.ts +42 -0
- package/dist/commonjs/sdk/evals.d.ts.map +1 -1
- package/dist/commonjs/sdk/evals.js +60 -0
- package/dist/commonjs/sdk/evals.js.map +1 -1
- package/dist/esm/funcs/llmEvalsAppendAgentTrials.d.ts +19 -0
- package/dist/esm/funcs/llmEvalsAppendAgentTrials.d.ts.map +1 -0
- package/dist/esm/funcs/llmEvalsAppendAgentTrials.js +100 -0
- package/dist/esm/funcs/llmEvalsAppendAgentTrials.js.map +1 -0
- package/dist/esm/funcs/llmEvalsGetRunOverview.d.ts +20 -0
- package/dist/esm/funcs/llmEvalsGetRunOverview.d.ts.map +1 -0
- package/dist/esm/funcs/llmEvalsGetRunOverview.js +102 -0
- package/dist/esm/funcs/llmEvalsGetRunOverview.js.map +1 -0
- package/dist/esm/funcs/llmEvalsGetSampleTrajectory.d.ts +19 -0
- package/dist/esm/funcs/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
- package/dist/esm/funcs/llmEvalsGetSampleTrajectory.js +106 -0
- package/dist/esm/funcs/llmEvalsGetSampleTrajectory.js.map +1 -0
- package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.d.ts +20 -0
- package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
- package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.js +106 -0
- package/dist/esm/funcs/llmEvalsGetSuiteLeaderboard.js.map +1 -0
- package/dist/esm/funcs/llmEvalsImportAgentRun.d.ts +19 -0
- package/dist/esm/funcs/llmEvalsImportAgentRun.d.ts.map +1 -0
- package/dist/esm/funcs/llmEvalsImportAgentRun.js +94 -0
- package/dist/esm/funcs/llmEvalsImportAgentRun.js.map +1 -0
- package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.d.ts +20 -0
- package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
- package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.js +110 -0
- package/dist/esm/funcs/llmEvalsUploadSampleTrajectory.js.map +1 -0
- package/dist/esm/lib/config.d.ts +2 -2
- package/dist/esm/lib/config.js +2 -2
- package/dist/esm/mcp-server/mcp-server.js +1 -1
- package/dist/esm/mcp-server/server.d.ts.map +1 -1
- package/dist/esm/mcp-server/server.js +13 -1
- package/dist/esm/mcp-server/server.js.map +1 -1
- package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts +8 -0
- package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.d.ts.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.js +28 -0
- package/dist/esm/mcp-server/tools/llmEvalsAppendAgentTrials.js.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.d.ts +8 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.d.ts.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.js +28 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetRunOverview.js.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts +8 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.js +28 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetSampleTrajectory.js.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts +8 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js +28 -0
- package/dist/esm/mcp-server/tools/llmEvalsGetSuiteLeaderboard.js.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.d.ts +8 -0
- package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.d.ts.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.js +28 -0
- package/dist/esm/mcp-server/tools/llmEvalsImportAgentRun.js.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts +8 -0
- package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
- package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.js +28 -0
- package/dist/esm/mcp-server/tools/llmEvalsUploadSampleTrajectory.js.map +1 -0
- package/dist/esm/models/components/appendagenttrialsrequest.d.ts +111 -0
- package/dist/esm/models/components/appendagenttrialsrequest.d.ts.map +1 -0
- package/dist/esm/models/components/appendagenttrialsrequest.js +96 -0
- package/dist/esm/models/components/appendagenttrialsrequest.js.map +1 -0
- package/dist/esm/models/components/evalleaderboardrow.d.ts +85 -0
- package/dist/esm/models/components/evalleaderboardrow.d.ts.map +1 -0
- package/dist/esm/models/components/evalleaderboardrow.js +89 -0
- package/dist/esm/models/components/evalleaderboardrow.js.map +1 -0
- package/dist/esm/models/components/evalrunoverview.d.ts +101 -0
- package/dist/esm/models/components/evalrunoverview.d.ts.map +1 -0
- package/dist/esm/models/components/evalrunoverview.js +108 -0
- package/dist/esm/models/components/evalrunoverview.js.map +1 -0
- package/dist/esm/models/components/evalsampleobject.d.ts +32 -0
- package/dist/esm/models/components/evalsampleobject.d.ts.map +1 -1
- package/dist/esm/models/components/evalsampleobject.js +20 -0
- package/dist/esm/models/components/evalsampleobject.js.map +1 -1
- package/dist/esm/models/components/evalsampletrajectoryuploadedobject.d.ts +32 -0
- package/dist/esm/models/components/evalsampletrajectoryuploadedobject.d.ts.map +1 -0
- package/dist/esm/models/components/evalsampletrajectoryuploadedobject.js +49 -0
- package/dist/esm/models/components/evalsampletrajectoryuploadedobject.js.map +1 -0
- package/dist/esm/models/components/evalsuiteleaderboard.d.ts +35 -0
- package/dist/esm/models/components/evalsuiteleaderboard.d.ts.map +1 -0
- package/dist/esm/models/components/evalsuiteleaderboard.js +40 -0
- package/dist/esm/models/components/evalsuiteleaderboard.js.map +1 -0
- package/dist/esm/models/components/importagentrunrequest.d.ts +164 -0
- package/dist/esm/models/components/importagentrunrequest.d.ts.map +1 -0
- package/dist/esm/models/components/importagentrunrequest.js +142 -0
- package/dist/esm/models/components/importagentrunrequest.js.map +1 -0
- package/dist/esm/models/components/index.d.ts +7 -0
- package/dist/esm/models/components/index.d.ts.map +1 -1
- package/dist/esm/models/components/index.js +7 -0
- package/dist/esm/models/components/index.js.map +1 -1
- package/dist/esm/models/components/uploadevalsampletrajectoryrequest.d.ts +34 -0
- package/dist/esm/models/components/uploadevalsampletrajectoryrequest.d.ts.map +1 -0
- package/dist/esm/models/components/uploadevalsampletrajectoryrequest.js +33 -0
- package/dist/esm/models/components/uploadevalsampletrajectoryrequest.js.map +1 -0
- package/dist/esm/models/operations/appendagentevaltrials.d.ts +264 -0
- package/dist/esm/models/operations/appendagentevaltrials.d.ts.map +1 -0
- package/dist/esm/models/operations/appendagentevaltrials.js +323 -0
- package/dist/esm/models/operations/appendagentevaltrials.js.map +1 -0
- package/dist/esm/models/operations/cancelevalrun.d.ts +20 -0
- package/dist/esm/models/operations/cancelevalrun.d.ts.map +1 -1
- package/dist/esm/models/operations/cancelevalrun.js +16 -0
- package/dist/esm/models/operations/cancelevalrun.js.map +1 -1
- package/dist/esm/models/operations/createevalrun.d.ts +20 -0
- package/dist/esm/models/operations/createevalrun.d.ts.map +1 -1
- package/dist/esm/models/operations/createevalrun.js +16 -0
- package/dist/esm/models/operations/createevalrun.js.map +1 -1
- package/dist/esm/models/operations/getevalrun.d.ts +20 -0
- package/dist/esm/models/operations/getevalrun.d.ts.map +1 -1
- package/dist/esm/models/operations/getevalrun.js +16 -0
- package/dist/esm/models/operations/getevalrun.js.map +1 -1
- package/dist/esm/models/operations/getevalrunoverview.d.ts +24 -0
- package/dist/esm/models/operations/getevalrunoverview.d.ts.map +1 -0
- package/dist/esm/models/operations/getevalrunoverview.js +35 -0
- package/dist/esm/models/operations/getevalrunoverview.js.map +1 -0
- package/dist/esm/models/operations/getevalsampletrajectory.d.ts +75 -0
- package/dist/esm/models/operations/getevalsampletrajectory.d.ts.map +1 -0
- package/dist/esm/models/operations/getevalsampletrajectory.js +93 -0
- package/dist/esm/models/operations/getevalsampletrajectory.js.map +1 -0
- package/dist/esm/models/operations/getevalsuiteleaderboard.d.ts +32 -0
- package/dist/esm/models/operations/getevalsuiteleaderboard.d.ts.map +1 -0
- package/dist/esm/models/operations/getevalsuiteleaderboard.js +45 -0
- package/dist/esm/models/operations/getevalsuiteleaderboard.js.map +1 -0
- package/dist/esm/models/operations/importagentevalrun.d.ts +262 -0
- package/dist/esm/models/operations/importagentevalrun.d.ts.map +1 -0
- package/dist/esm/models/operations/importagentevalrun.js +319 -0
- package/dist/esm/models/operations/importagentevalrun.js.map +1 -0
- package/dist/esm/models/operations/index.d.ts +6 -0
- package/dist/esm/models/operations/index.d.ts.map +1 -1
- package/dist/esm/models/operations/index.js +6 -0
- package/dist/esm/models/operations/index.js.map +1 -1
- package/dist/esm/models/operations/listevalruns.d.ts +20 -0
- package/dist/esm/models/operations/listevalruns.d.ts.map +1 -1
- package/dist/esm/models/operations/listevalruns.js +16 -0
- package/dist/esm/models/operations/listevalruns.js.map +1 -1
- package/dist/esm/models/operations/listevalscheduleruns.d.ts +20 -0
- package/dist/esm/models/operations/listevalscheduleruns.d.ts.map +1 -1
- package/dist/esm/models/operations/listevalscheduleruns.js +16 -0
- package/dist/esm/models/operations/listevalscheduleruns.js.map +1 -1
- package/dist/esm/models/operations/listevalsuiteleaderboardruns.d.ts +20 -0
- package/dist/esm/models/operations/listevalsuiteleaderboardruns.d.ts.map +1 -1
- package/dist/esm/models/operations/listevalsuiteleaderboardruns.js +16 -0
- package/dist/esm/models/operations/listevalsuiteleaderboardruns.js.map +1 -1
- package/dist/esm/models/operations/rerunfailedevalsamples.d.ts +20 -0
- package/dist/esm/models/operations/rerunfailedevalsamples.d.ts.map +1 -1
- package/dist/esm/models/operations/rerunfailedevalsamples.js +16 -0
- package/dist/esm/models/operations/rerunfailedevalsamples.js.map +1 -1
- package/dist/esm/models/operations/retryfailedevalrun.d.ts +20 -0
- package/dist/esm/models/operations/retryfailedevalrun.d.ts.map +1 -1
- package/dist/esm/models/operations/retryfailedevalrun.js +16 -0
- package/dist/esm/models/operations/retryfailedevalrun.js.map +1 -1
- package/dist/esm/models/operations/uploadevalsampletrajectory.d.ts +31 -0
- package/dist/esm/models/operations/uploadevalsampletrajectory.d.ts.map +1 -0
- package/dist/esm/models/operations/uploadevalsampletrajectory.js +48 -0
- package/dist/esm/models/operations/uploadevalsampletrajectory.js.map +1 -0
- package/dist/esm/react-query/index.d.ts +6 -0
- package/dist/esm/react-query/index.d.ts.map +1 -1
- package/dist/esm/react-query/index.js +6 -0
- package/dist/esm/react-query/index.js.map +1 -1
- package/dist/esm/react-query/llmEvalsAppendAgentTrials.d.ts +28 -0
- package/dist/esm/react-query/llmEvalsAppendAgentTrials.d.ts.map +1 -0
- package/dist/esm/react-query/llmEvalsAppendAgentTrials.js +42 -0
- package/dist/esm/react-query/llmEvalsAppendAgentTrials.js.map +1 -0
- package/dist/esm/react-query/llmEvalsGetRunOverview.core.d.ts +16 -0
- package/dist/esm/react-query/llmEvalsGetRunOverview.core.d.ts.map +1 -0
- package/dist/esm/react-query/llmEvalsGetRunOverview.core.js +32 -0
- package/dist/esm/react-query/llmEvalsGetRunOverview.core.js.map +1 -0
- package/dist/esm/react-query/llmEvalsGetRunOverview.d.ts +40 -0
- package/dist/esm/react-query/llmEvalsGetRunOverview.d.ts.map +1 -0
- package/dist/esm/react-query/llmEvalsGetRunOverview.js +50 -0
- package/dist/esm/react-query/llmEvalsGetRunOverview.js.map +1 -0
- package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.d.ts +16 -0
- package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.d.ts.map +1 -0
- package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.js +40 -0
- package/dist/esm/react-query/llmEvalsGetSampleTrajectory.core.js.map +1 -0
- package/dist/esm/react-query/llmEvalsGetSampleTrajectory.d.ts +44 -0
- package/dist/esm/react-query/llmEvalsGetSampleTrajectory.d.ts.map +1 -0
- package/dist/esm/react-query/llmEvalsGetSampleTrajectory.js +55 -0
- package/dist/esm/react-query/llmEvalsGetSampleTrajectory.js.map +1 -0
- package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts +20 -0
- package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.d.ts.map +1 -0
- package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.js +36 -0
- package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.core.js.map +1 -0
- package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.d.ts +48 -0
- package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.d.ts.map +1 -0
- package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.js +55 -0
- package/dist/esm/react-query/llmEvalsGetSuiteLeaderboard.js.map +1 -0
- package/dist/esm/react-query/llmEvalsImportAgentRun.d.ts +28 -0
- package/dist/esm/react-query/llmEvalsImportAgentRun.d.ts.map +1 -0
- package/dist/esm/react-query/llmEvalsImportAgentRun.js +42 -0
- package/dist/esm/react-query/llmEvalsImportAgentRun.js.map +1 -0
- package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.d.ts +29 -0
- package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.d.ts.map +1 -0
- package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.js +42 -0
- package/dist/esm/react-query/llmEvalsUploadSampleTrajectory.js.map +1 -0
- package/dist/esm/sdk/evals.d.ts +42 -0
- package/dist/esm/sdk/evals.d.ts.map +1 -1
- package/dist/esm/sdk/evals.js +60 -0
- package/dist/esm/sdk/evals.js.map +1 -1
- package/jsr.json +1 -1
- package/package.json +1 -1
- package/src/funcs/llmEvalsAppendAgentTrials.ts +188 -0
- package/src/funcs/llmEvalsGetRunOverview.ts +192 -0
- package/src/funcs/llmEvalsGetSampleTrajectory.ts +197 -0
- package/src/funcs/llmEvalsGetSuiteLeaderboard.ts +197 -0
- package/src/funcs/llmEvalsImportAgentRun.ts +181 -0
- package/src/funcs/llmEvalsUploadSampleTrajectory.ts +201 -0
- package/src/lib/config.ts +2 -2
- package/src/mcp-server/mcp-server.ts +1 -1
- package/src/mcp-server/server.ts +13 -1
- package/src/mcp-server/tools/llmEvalsAppendAgentTrials.ts +37 -0
- package/src/mcp-server/tools/llmEvalsGetRunOverview.ts +37 -0
- package/src/mcp-server/tools/llmEvalsGetSampleTrajectory.ts +37 -0
- package/src/mcp-server/tools/llmEvalsGetSuiteLeaderboard.ts +37 -0
- package/src/mcp-server/tools/llmEvalsImportAgentRun.ts +37 -0
- package/src/mcp-server/tools/llmEvalsUploadSampleTrajectory.ts +38 -0
- package/src/models/components/appendagenttrialsrequest.ts +224 -0
- package/src/models/components/evalleaderboardrow.ts +200 -0
- package/src/models/components/evalrunoverview.ts +240 -0
- package/src/models/components/evalsampleobject.ts +48 -0
- package/src/models/components/evalsampletrajectoryuploadedobject.ts +98 -0
- package/src/models/components/evalsuiteleaderboard.ts +93 -0
- package/src/models/components/importagentrunrequest.ts +319 -0
- package/src/models/components/index.ts +7 -0
- package/src/models/components/uploadevalsampletrajectoryrequest.ts +79 -0
- package/src/models/operations/appendagentevaltrials.ts +678 -0
- package/src/models/operations/cancelevalrun.ts +36 -0
- package/src/models/operations/createevalrun.ts +36 -0
- package/src/models/operations/getevalrun.ts +36 -0
- package/src/models/operations/getevalrunoverview.ts +73 -0
- package/src/models/operations/getevalsampletrajectory.ts +199 -0
- package/src/models/operations/getevalsuiteleaderboard.ts +93 -0
- package/src/models/operations/importagentevalrun.ts +665 -0
- package/src/models/operations/index.ts +6 -0
- package/src/models/operations/listevalruns.ts +36 -0
- package/src/models/operations/listevalscheduleruns.ts +36 -0
- package/src/models/operations/listevalsuiteleaderboardruns.ts +36 -0
- package/src/models/operations/rerunfailedevalsamples.ts +36 -0
- package/src/models/operations/retryfailedevalrun.ts +36 -0
- package/src/models/operations/uploadevalsampletrajectory.ts +98 -0
- package/src/react-query/index.ts +6 -0
- package/src/react-query/llmEvalsAppendAgentTrials.ts +109 -0
- package/src/react-query/llmEvalsGetRunOverview.core.ts +77 -0
- package/src/react-query/llmEvalsGetRunOverview.ts +150 -0
- package/src/react-query/llmEvalsGetSampleTrajectory.core.ts +94 -0
- package/src/react-query/llmEvalsGetSampleTrajectory.ts +159 -0
- package/src/react-query/llmEvalsGetSuiteLeaderboard.core.ts +89 -0
- package/src/react-query/llmEvalsGetSuiteLeaderboard.ts +163 -0
- package/src/react-query/llmEvalsImportAgentRun.ts +109 -0
- package/src/react-query/llmEvalsUploadSampleTrajectory.ts +110 -0
- package/src/sdk/evals.ts +108 -0
|
@@ -125,6 +125,22 @@ export type ListEvalRunsData = {
|
|
|
125
125
|
error: { [k: string]: any } | null;
|
|
126
126
|
artifactFileIds: Array<string>;
|
|
127
127
|
metadata: { [k: string]: string };
|
|
128
|
+
/**
|
|
129
|
+
* Coding-agent harness that produced this run (e.g. `omp`, `claude-code`). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.
|
|
130
|
+
*/
|
|
131
|
+
agentName: string | null;
|
|
132
|
+
/**
|
|
133
|
+
* Version of `agent_name`, as the harness reported it. Null when `agent_name` is.
|
|
134
|
+
*/
|
|
135
|
+
agentVersion: string | null;
|
|
136
|
+
/**
|
|
137
|
+
* Reasoning-effort setting the agent ran at (e.g. `medium`, `xhigh`). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when `agent_name` is.
|
|
138
|
+
*/
|
|
139
|
+
agentEffort: string | null;
|
|
140
|
+
/**
|
|
141
|
+
* Total spend for the run in USD. Null when the harness did not report cost.
|
|
142
|
+
*/
|
|
143
|
+
costUsd: number | null;
|
|
128
144
|
createdAt: number;
|
|
129
145
|
startedAt: number | null;
|
|
130
146
|
completedAt: number | null;
|
|
@@ -545,6 +561,10 @@ export const ListEvalRunsData$inboundSchema: z.ZodType<
|
|
|
545
561
|
error: z.nullable(z.record(z.any())),
|
|
546
562
|
artifact_file_ids: z.array(z.string()),
|
|
547
563
|
metadata: z.record(z.string()),
|
|
564
|
+
agent_name: z.nullable(z.string()),
|
|
565
|
+
agent_version: z.nullable(z.string()),
|
|
566
|
+
agent_effort: z.nullable(z.string()),
|
|
567
|
+
cost_usd: z.nullable(z.number()),
|
|
548
568
|
created_at: z.number().int(),
|
|
549
569
|
started_at: z.nullable(z.number().int()),
|
|
550
570
|
completed_at: z.nullable(z.number().int()),
|
|
@@ -562,6 +582,10 @@ export const ListEvalRunsData$inboundSchema: z.ZodType<
|
|
|
562
582
|
"embedding_model": "embeddingModel",
|
|
563
583
|
"request_counts": "requestCounts",
|
|
564
584
|
"artifact_file_ids": "artifactFileIds",
|
|
585
|
+
"agent_name": "agentName",
|
|
586
|
+
"agent_version": "agentVersion",
|
|
587
|
+
"agent_effort": "agentEffort",
|
|
588
|
+
"cost_usd": "costUsd",
|
|
565
589
|
"created_at": "createdAt",
|
|
566
590
|
"started_at": "startedAt",
|
|
567
591
|
"completed_at": "completedAt",
|
|
@@ -589,6 +613,10 @@ export type ListEvalRunsData$Outbound = {
|
|
|
589
613
|
error: { [k: string]: any } | null;
|
|
590
614
|
artifact_file_ids: Array<string>;
|
|
591
615
|
metadata: { [k: string]: string };
|
|
616
|
+
agent_name: string | null;
|
|
617
|
+
agent_version: string | null;
|
|
618
|
+
agent_effort: string | null;
|
|
619
|
+
cost_usd: number | null;
|
|
592
620
|
created_at: number;
|
|
593
621
|
started_at: number | null;
|
|
594
622
|
completed_at: number | null;
|
|
@@ -620,6 +648,10 @@ export const ListEvalRunsData$outboundSchema: z.ZodType<
|
|
|
620
648
|
error: z.nullable(z.record(z.any())),
|
|
621
649
|
artifactFileIds: z.array(z.string()),
|
|
622
650
|
metadata: z.record(z.string()),
|
|
651
|
+
agentName: z.nullable(z.string()),
|
|
652
|
+
agentVersion: z.nullable(z.string()),
|
|
653
|
+
agentEffort: z.nullable(z.string()),
|
|
654
|
+
costUsd: z.nullable(z.number()),
|
|
623
655
|
createdAt: z.number().int(),
|
|
624
656
|
startedAt: z.nullable(z.number().int()),
|
|
625
657
|
completedAt: z.nullable(z.number().int()),
|
|
@@ -637,6 +669,10 @@ export const ListEvalRunsData$outboundSchema: z.ZodType<
|
|
|
637
669
|
embeddingModel: "embedding_model",
|
|
638
670
|
requestCounts: "request_counts",
|
|
639
671
|
artifactFileIds: "artifact_file_ids",
|
|
672
|
+
agentName: "agent_name",
|
|
673
|
+
agentVersion: "agent_version",
|
|
674
|
+
agentEffort: "agent_effort",
|
|
675
|
+
costUsd: "cost_usd",
|
|
640
676
|
createdAt: "created_at",
|
|
641
677
|
startedAt: "started_at",
|
|
642
678
|
completedAt: "completed_at",
|
|
@@ -130,6 +130,22 @@ export type ListEvalScheduleRunsData = {
|
|
|
130
130
|
error: { [k: string]: any } | null;
|
|
131
131
|
artifactFileIds: Array<string>;
|
|
132
132
|
metadata: { [k: string]: string };
|
|
133
|
+
/**
|
|
134
|
+
* Coding-agent harness that produced this run (e.g. `omp`, `claude-code`). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.
|
|
135
|
+
*/
|
|
136
|
+
agentName: string | null;
|
|
137
|
+
/**
|
|
138
|
+
* Version of `agent_name`, as the harness reported it. Null when `agent_name` is.
|
|
139
|
+
*/
|
|
140
|
+
agentVersion: string | null;
|
|
141
|
+
/**
|
|
142
|
+
* Reasoning-effort setting the agent ran at (e.g. `medium`, `xhigh`). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when `agent_name` is.
|
|
143
|
+
*/
|
|
144
|
+
agentEffort: string | null;
|
|
145
|
+
/**
|
|
146
|
+
* Total spend for the run in USD. Null when the harness did not report cost.
|
|
147
|
+
*/
|
|
148
|
+
costUsd: number | null;
|
|
133
149
|
createdAt: number;
|
|
134
150
|
startedAt: number | null;
|
|
135
151
|
completedAt: number | null;
|
|
@@ -566,6 +582,10 @@ export const ListEvalScheduleRunsData$inboundSchema: z.ZodType<
|
|
|
566
582
|
error: z.nullable(z.record(z.any())),
|
|
567
583
|
artifact_file_ids: z.array(z.string()),
|
|
568
584
|
metadata: z.record(z.string()),
|
|
585
|
+
agent_name: z.nullable(z.string()),
|
|
586
|
+
agent_version: z.nullable(z.string()),
|
|
587
|
+
agent_effort: z.nullable(z.string()),
|
|
588
|
+
cost_usd: z.nullable(z.number()),
|
|
569
589
|
created_at: z.number().int(),
|
|
570
590
|
started_at: z.nullable(z.number().int()),
|
|
571
591
|
completed_at: z.nullable(z.number().int()),
|
|
@@ -583,6 +603,10 @@ export const ListEvalScheduleRunsData$inboundSchema: z.ZodType<
|
|
|
583
603
|
"embedding_model": "embeddingModel",
|
|
584
604
|
"request_counts": "requestCounts",
|
|
585
605
|
"artifact_file_ids": "artifactFileIds",
|
|
606
|
+
"agent_name": "agentName",
|
|
607
|
+
"agent_version": "agentVersion",
|
|
608
|
+
"agent_effort": "agentEffort",
|
|
609
|
+
"cost_usd": "costUsd",
|
|
586
610
|
"created_at": "createdAt",
|
|
587
611
|
"started_at": "startedAt",
|
|
588
612
|
"completed_at": "completedAt",
|
|
@@ -610,6 +634,10 @@ export type ListEvalScheduleRunsData$Outbound = {
|
|
|
610
634
|
error: { [k: string]: any } | null;
|
|
611
635
|
artifact_file_ids: Array<string>;
|
|
612
636
|
metadata: { [k: string]: string };
|
|
637
|
+
agent_name: string | null;
|
|
638
|
+
agent_version: string | null;
|
|
639
|
+
agent_effort: string | null;
|
|
640
|
+
cost_usd: number | null;
|
|
613
641
|
created_at: number;
|
|
614
642
|
started_at: number | null;
|
|
615
643
|
completed_at: number | null;
|
|
@@ -641,6 +669,10 @@ export const ListEvalScheduleRunsData$outboundSchema: z.ZodType<
|
|
|
641
669
|
error: z.nullable(z.record(z.any())),
|
|
642
670
|
artifactFileIds: z.array(z.string()),
|
|
643
671
|
metadata: z.record(z.string()),
|
|
672
|
+
agentName: z.nullable(z.string()),
|
|
673
|
+
agentVersion: z.nullable(z.string()),
|
|
674
|
+
agentEffort: z.nullable(z.string()),
|
|
675
|
+
costUsd: z.nullable(z.number()),
|
|
644
676
|
createdAt: z.number().int(),
|
|
645
677
|
startedAt: z.nullable(z.number().int()),
|
|
646
678
|
completedAt: z.nullable(z.number().int()),
|
|
@@ -658,6 +690,10 @@ export const ListEvalScheduleRunsData$outboundSchema: z.ZodType<
|
|
|
658
690
|
embeddingModel: "embedding_model",
|
|
659
691
|
requestCounts: "request_counts",
|
|
660
692
|
artifactFileIds: "artifact_file_ids",
|
|
693
|
+
agentName: "agent_name",
|
|
694
|
+
agentVersion: "agent_version",
|
|
695
|
+
agentEffort: "agent_effort",
|
|
696
|
+
costUsd: "cost_usd",
|
|
661
697
|
createdAt: "created_at",
|
|
662
698
|
startedAt: "started_at",
|
|
663
699
|
completedAt: "completed_at",
|
|
@@ -116,6 +116,22 @@ export type ListEvalSuiteLeaderboardRunsData = {
|
|
|
116
116
|
error: { [k: string]: any } | null;
|
|
117
117
|
artifactFileIds: Array<string>;
|
|
118
118
|
metadata: { [k: string]: string };
|
|
119
|
+
/**
|
|
120
|
+
* Coding-agent harness that produced this run (e.g. `omp`, `claude-code`). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.
|
|
121
|
+
*/
|
|
122
|
+
agentName: string | null;
|
|
123
|
+
/**
|
|
124
|
+
* Version of `agent_name`, as the harness reported it. Null when `agent_name` is.
|
|
125
|
+
*/
|
|
126
|
+
agentVersion: string | null;
|
|
127
|
+
/**
|
|
128
|
+
* Reasoning-effort setting the agent ran at (e.g. `medium`, `xhigh`). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when `agent_name` is.
|
|
129
|
+
*/
|
|
130
|
+
agentEffort: string | null;
|
|
131
|
+
/**
|
|
132
|
+
* Total spend for the run in USD. Null when the harness did not report cost.
|
|
133
|
+
*/
|
|
134
|
+
costUsd: number | null;
|
|
119
135
|
createdAt: number;
|
|
120
136
|
startedAt: number | null;
|
|
121
137
|
completedAt: number | null;
|
|
@@ -580,6 +596,10 @@ export const ListEvalSuiteLeaderboardRunsData$inboundSchema: z.ZodType<
|
|
|
580
596
|
error: z.nullable(z.record(z.any())),
|
|
581
597
|
artifact_file_ids: z.array(z.string()),
|
|
582
598
|
metadata: z.record(z.string()),
|
|
599
|
+
agent_name: z.nullable(z.string()),
|
|
600
|
+
agent_version: z.nullable(z.string()),
|
|
601
|
+
agent_effort: z.nullable(z.string()),
|
|
602
|
+
cost_usd: z.nullable(z.number()),
|
|
583
603
|
created_at: z.number().int(),
|
|
584
604
|
started_at: z.nullable(z.number().int()),
|
|
585
605
|
completed_at: z.nullable(z.number().int()),
|
|
@@ -597,6 +617,10 @@ export const ListEvalSuiteLeaderboardRunsData$inboundSchema: z.ZodType<
|
|
|
597
617
|
"embedding_model": "embeddingModel",
|
|
598
618
|
"request_counts": "requestCounts",
|
|
599
619
|
"artifact_file_ids": "artifactFileIds",
|
|
620
|
+
"agent_name": "agentName",
|
|
621
|
+
"agent_version": "agentVersion",
|
|
622
|
+
"agent_effort": "agentEffort",
|
|
623
|
+
"cost_usd": "costUsd",
|
|
600
624
|
"created_at": "createdAt",
|
|
601
625
|
"started_at": "startedAt",
|
|
602
626
|
"completed_at": "completedAt",
|
|
@@ -624,6 +648,10 @@ export type ListEvalSuiteLeaderboardRunsData$Outbound = {
|
|
|
624
648
|
error: { [k: string]: any } | null;
|
|
625
649
|
artifact_file_ids: Array<string>;
|
|
626
650
|
metadata: { [k: string]: string };
|
|
651
|
+
agent_name: string | null;
|
|
652
|
+
agent_version: string | null;
|
|
653
|
+
agent_effort: string | null;
|
|
654
|
+
cost_usd: number | null;
|
|
627
655
|
created_at: number;
|
|
628
656
|
started_at: number | null;
|
|
629
657
|
completed_at: number | null;
|
|
@@ -659,6 +687,10 @@ export const ListEvalSuiteLeaderboardRunsData$outboundSchema: z.ZodType<
|
|
|
659
687
|
error: z.nullable(z.record(z.any())),
|
|
660
688
|
artifactFileIds: z.array(z.string()),
|
|
661
689
|
metadata: z.record(z.string()),
|
|
690
|
+
agentName: z.nullable(z.string()),
|
|
691
|
+
agentVersion: z.nullable(z.string()),
|
|
692
|
+
agentEffort: z.nullable(z.string()),
|
|
693
|
+
costUsd: z.nullable(z.number()),
|
|
662
694
|
createdAt: z.number().int(),
|
|
663
695
|
startedAt: z.nullable(z.number().int()),
|
|
664
696
|
completedAt: z.nullable(z.number().int()),
|
|
@@ -676,6 +708,10 @@ export const ListEvalSuiteLeaderboardRunsData$outboundSchema: z.ZodType<
|
|
|
676
708
|
embeddingModel: "embedding_model",
|
|
677
709
|
requestCounts: "request_counts",
|
|
678
710
|
artifactFileIds: "artifact_file_ids",
|
|
711
|
+
agentName: "agent_name",
|
|
712
|
+
agentVersion: "agent_version",
|
|
713
|
+
agentEffort: "agent_effort",
|
|
714
|
+
costUsd: "cost_usd",
|
|
679
715
|
createdAt: "created_at",
|
|
680
716
|
startedAt: "started_at",
|
|
681
717
|
completedAt: "completed_at",
|
|
@@ -115,6 +115,22 @@ export type RerunFailedEvalSamplesResponseBody = {
|
|
|
115
115
|
error: { [k: string]: any } | null;
|
|
116
116
|
artifactFileIds: Array<string>;
|
|
117
117
|
metadata: { [k: string]: string };
|
|
118
|
+
/**
|
|
119
|
+
* Coding-agent harness that produced this run (e.g. `omp`, `claude-code`). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.
|
|
120
|
+
*/
|
|
121
|
+
agentName: string | null;
|
|
122
|
+
/**
|
|
123
|
+
* Version of `agent_name`, as the harness reported it. Null when `agent_name` is.
|
|
124
|
+
*/
|
|
125
|
+
agentVersion: string | null;
|
|
126
|
+
/**
|
|
127
|
+
* Reasoning-effort setting the agent ran at (e.g. `medium`, `xhigh`). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when `agent_name` is.
|
|
128
|
+
*/
|
|
129
|
+
agentEffort: string | null;
|
|
130
|
+
/**
|
|
131
|
+
* Total spend for the run in USD. Null when the harness did not report cost.
|
|
132
|
+
*/
|
|
133
|
+
costUsd: number | null;
|
|
118
134
|
createdAt: number;
|
|
119
135
|
startedAt: number | null;
|
|
120
136
|
completedAt: number | null;
|
|
@@ -515,6 +531,10 @@ export const RerunFailedEvalSamplesResponseBody$inboundSchema: z.ZodType<
|
|
|
515
531
|
error: z.nullable(z.record(z.any())),
|
|
516
532
|
artifact_file_ids: z.array(z.string()),
|
|
517
533
|
metadata: z.record(z.string()),
|
|
534
|
+
agent_name: z.nullable(z.string()),
|
|
535
|
+
agent_version: z.nullable(z.string()),
|
|
536
|
+
agent_effort: z.nullable(z.string()),
|
|
537
|
+
cost_usd: z.nullable(z.number()),
|
|
518
538
|
created_at: z.number().int(),
|
|
519
539
|
started_at: z.nullable(z.number().int()),
|
|
520
540
|
completed_at: z.nullable(z.number().int()),
|
|
@@ -532,6 +552,10 @@ export const RerunFailedEvalSamplesResponseBody$inboundSchema: z.ZodType<
|
|
|
532
552
|
"embedding_model": "embeddingModel",
|
|
533
553
|
"request_counts": "requestCounts",
|
|
534
554
|
"artifact_file_ids": "artifactFileIds",
|
|
555
|
+
"agent_name": "agentName",
|
|
556
|
+
"agent_version": "agentVersion",
|
|
557
|
+
"agent_effort": "agentEffort",
|
|
558
|
+
"cost_usd": "costUsd",
|
|
535
559
|
"created_at": "createdAt",
|
|
536
560
|
"started_at": "startedAt",
|
|
537
561
|
"completed_at": "completedAt",
|
|
@@ -559,6 +583,10 @@ export type RerunFailedEvalSamplesResponseBody$Outbound = {
|
|
|
559
583
|
error: { [k: string]: any } | null;
|
|
560
584
|
artifact_file_ids: Array<string>;
|
|
561
585
|
metadata: { [k: string]: string };
|
|
586
|
+
agent_name: string | null;
|
|
587
|
+
agent_version: string | null;
|
|
588
|
+
agent_effort: string | null;
|
|
589
|
+
cost_usd: number | null;
|
|
562
590
|
created_at: number;
|
|
563
591
|
started_at: number | null;
|
|
564
592
|
completed_at: number | null;
|
|
@@ -592,6 +620,10 @@ export const RerunFailedEvalSamplesResponseBody$outboundSchema: z.ZodType<
|
|
|
592
620
|
error: z.nullable(z.record(z.any())),
|
|
593
621
|
artifactFileIds: z.array(z.string()),
|
|
594
622
|
metadata: z.record(z.string()),
|
|
623
|
+
agentName: z.nullable(z.string()),
|
|
624
|
+
agentVersion: z.nullable(z.string()),
|
|
625
|
+
agentEffort: z.nullable(z.string()),
|
|
626
|
+
costUsd: z.nullable(z.number()),
|
|
595
627
|
createdAt: z.number().int(),
|
|
596
628
|
startedAt: z.nullable(z.number().int()),
|
|
597
629
|
completedAt: z.nullable(z.number().int()),
|
|
@@ -609,6 +641,10 @@ export const RerunFailedEvalSamplesResponseBody$outboundSchema: z.ZodType<
|
|
|
609
641
|
embeddingModel: "embedding_model",
|
|
610
642
|
requestCounts: "request_counts",
|
|
611
643
|
artifactFileIds: "artifact_file_ids",
|
|
644
|
+
agentName: "agent_name",
|
|
645
|
+
agentVersion: "agent_version",
|
|
646
|
+
agentEffort: "agent_effort",
|
|
647
|
+
costUsd: "cost_usd",
|
|
612
648
|
createdAt: "created_at",
|
|
613
649
|
startedAt: "started_at",
|
|
614
650
|
completedAt: "completed_at",
|
|
@@ -115,6 +115,22 @@ export type RetryFailedEvalRunResponseBody = {
|
|
|
115
115
|
error: { [k: string]: any } | null;
|
|
116
116
|
artifactFileIds: Array<string>;
|
|
117
117
|
metadata: { [k: string]: string };
|
|
118
|
+
/**
|
|
119
|
+
* Coding-agent harness that produced this run (e.g. `omp`, `claude-code`). Null for every non-harness eval kind — a null here is not a missing value, it means the run is not an agent run.
|
|
120
|
+
*/
|
|
121
|
+
agentName: string | null;
|
|
122
|
+
/**
|
|
123
|
+
* Version of `agent_name`, as the harness reported it. Null when `agent_name` is.
|
|
124
|
+
*/
|
|
125
|
+
agentVersion: string | null;
|
|
126
|
+
/**
|
|
127
|
+
* Reasoning-effort setting the agent ran at (e.g. `medium`, `xhigh`). Part of the leaderboard's row identity: the same agent and model at two efforts are two rows, not one. Null when `agent_name` is.
|
|
128
|
+
*/
|
|
129
|
+
agentEffort: string | null;
|
|
130
|
+
/**
|
|
131
|
+
* Total spend for the run in USD. Null when the harness did not report cost.
|
|
132
|
+
*/
|
|
133
|
+
costUsd: number | null;
|
|
118
134
|
createdAt: number;
|
|
119
135
|
startedAt: number | null;
|
|
120
136
|
completedAt: number | null;
|
|
@@ -509,6 +525,10 @@ export const RetryFailedEvalRunResponseBody$inboundSchema: z.ZodType<
|
|
|
509
525
|
error: z.nullable(z.record(z.any())),
|
|
510
526
|
artifact_file_ids: z.array(z.string()),
|
|
511
527
|
metadata: z.record(z.string()),
|
|
528
|
+
agent_name: z.nullable(z.string()),
|
|
529
|
+
agent_version: z.nullable(z.string()),
|
|
530
|
+
agent_effort: z.nullable(z.string()),
|
|
531
|
+
cost_usd: z.nullable(z.number()),
|
|
512
532
|
created_at: z.number().int(),
|
|
513
533
|
started_at: z.nullable(z.number().int()),
|
|
514
534
|
completed_at: z.nullable(z.number().int()),
|
|
@@ -526,6 +546,10 @@ export const RetryFailedEvalRunResponseBody$inboundSchema: z.ZodType<
|
|
|
526
546
|
"embedding_model": "embeddingModel",
|
|
527
547
|
"request_counts": "requestCounts",
|
|
528
548
|
"artifact_file_ids": "artifactFileIds",
|
|
549
|
+
"agent_name": "agentName",
|
|
550
|
+
"agent_version": "agentVersion",
|
|
551
|
+
"agent_effort": "agentEffort",
|
|
552
|
+
"cost_usd": "costUsd",
|
|
529
553
|
"created_at": "createdAt",
|
|
530
554
|
"started_at": "startedAt",
|
|
531
555
|
"completed_at": "completedAt",
|
|
@@ -553,6 +577,10 @@ export type RetryFailedEvalRunResponseBody$Outbound = {
|
|
|
553
577
|
error: { [k: string]: any } | null;
|
|
554
578
|
artifact_file_ids: Array<string>;
|
|
555
579
|
metadata: { [k: string]: string };
|
|
580
|
+
agent_name: string | null;
|
|
581
|
+
agent_version: string | null;
|
|
582
|
+
agent_effort: string | null;
|
|
583
|
+
cost_usd: number | null;
|
|
556
584
|
created_at: number;
|
|
557
585
|
started_at: number | null;
|
|
558
586
|
completed_at: number | null;
|
|
@@ -584,6 +612,10 @@ export const RetryFailedEvalRunResponseBody$outboundSchema: z.ZodType<
|
|
|
584
612
|
error: z.nullable(z.record(z.any())),
|
|
585
613
|
artifactFileIds: z.array(z.string()),
|
|
586
614
|
metadata: z.record(z.string()),
|
|
615
|
+
agentName: z.nullable(z.string()),
|
|
616
|
+
agentVersion: z.nullable(z.string()),
|
|
617
|
+
agentEffort: z.nullable(z.string()),
|
|
618
|
+
costUsd: z.nullable(z.number()),
|
|
587
619
|
createdAt: z.number().int(),
|
|
588
620
|
startedAt: z.nullable(z.number().int()),
|
|
589
621
|
completedAt: z.nullable(z.number().int()),
|
|
@@ -601,6 +633,10 @@ export const RetryFailedEvalRunResponseBody$outboundSchema: z.ZodType<
|
|
|
601
633
|
embeddingModel: "embedding_model",
|
|
602
634
|
requestCounts: "request_counts",
|
|
603
635
|
artifactFileIds: "artifact_file_ids",
|
|
636
|
+
agentName: "agent_name",
|
|
637
|
+
agentVersion: "agent_version",
|
|
638
|
+
agentEffort: "agent_effort",
|
|
639
|
+
costUsd: "cost_usd",
|
|
604
640
|
createdAt: "created_at",
|
|
605
641
|
startedAt: "started_at",
|
|
606
642
|
completedAt: "completed_at",
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import * as z from "zod/v3";
|
|
6
|
+
import { remap as remap$ } from "../../lib/primitives.js";
|
|
7
|
+
import { safeParse } from "../../lib/schemas.js";
|
|
8
|
+
import { Result as SafeParseResult } from "../../types/fp.js";
|
|
9
|
+
import * as components from "../components/index.js";
|
|
10
|
+
import { SDKValidationError } from "../errors/sdkvalidationerror.js";
|
|
11
|
+
|
|
12
|
+
export type UploadEvalSampleTrajectoryRequest = {
|
|
13
|
+
runId: string;
|
|
14
|
+
sampleIndex: number;
|
|
15
|
+
taskId?: string | undefined;
|
|
16
|
+
model?: string | undefined;
|
|
17
|
+
/**
|
|
18
|
+
* Optional external end-user identifier forwarded by the API gateway.
|
|
19
|
+
*/
|
|
20
|
+
xOnBehalfOf?: string | undefined;
|
|
21
|
+
uploadEvalSampleTrajectoryRequest:
|
|
22
|
+
components.UploadEvalSampleTrajectoryRequest;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/** @internal */
|
|
26
|
+
export const UploadEvalSampleTrajectoryRequest$inboundSchema: z.ZodType<
|
|
27
|
+
UploadEvalSampleTrajectoryRequest,
|
|
28
|
+
z.ZodTypeDef,
|
|
29
|
+
unknown
|
|
30
|
+
> = z.object({
|
|
31
|
+
run_id: z.string(),
|
|
32
|
+
sample_index: z.number().int(),
|
|
33
|
+
task_id: z.string().optional(),
|
|
34
|
+
model: z.string().optional(),
|
|
35
|
+
"X-On-Behalf-Of": z.string().optional(),
|
|
36
|
+
UploadEvalSampleTrajectoryRequest:
|
|
37
|
+
components.UploadEvalSampleTrajectoryRequest$inboundSchema,
|
|
38
|
+
}).transform((v) => {
|
|
39
|
+
return remap$(v, {
|
|
40
|
+
"run_id": "runId",
|
|
41
|
+
"sample_index": "sampleIndex",
|
|
42
|
+
"task_id": "taskId",
|
|
43
|
+
"X-On-Behalf-Of": "xOnBehalfOf",
|
|
44
|
+
"UploadEvalSampleTrajectoryRequest": "uploadEvalSampleTrajectoryRequest",
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
/** @internal */
|
|
48
|
+
export type UploadEvalSampleTrajectoryRequest$Outbound = {
|
|
49
|
+
run_id: string;
|
|
50
|
+
sample_index: number;
|
|
51
|
+
task_id?: string | undefined;
|
|
52
|
+
model?: string | undefined;
|
|
53
|
+
"X-On-Behalf-Of"?: string | undefined;
|
|
54
|
+
UploadEvalSampleTrajectoryRequest:
|
|
55
|
+
components.UploadEvalSampleTrajectoryRequest$Outbound;
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
/** @internal */
|
|
59
|
+
export const UploadEvalSampleTrajectoryRequest$outboundSchema: z.ZodType<
|
|
60
|
+
UploadEvalSampleTrajectoryRequest$Outbound,
|
|
61
|
+
z.ZodTypeDef,
|
|
62
|
+
UploadEvalSampleTrajectoryRequest
|
|
63
|
+
> = z.object({
|
|
64
|
+
runId: z.string(),
|
|
65
|
+
sampleIndex: z.number().int(),
|
|
66
|
+
taskId: z.string().optional(),
|
|
67
|
+
model: z.string().optional(),
|
|
68
|
+
xOnBehalfOf: z.string().optional(),
|
|
69
|
+
uploadEvalSampleTrajectoryRequest:
|
|
70
|
+
components.UploadEvalSampleTrajectoryRequest$outboundSchema,
|
|
71
|
+
}).transform((v) => {
|
|
72
|
+
return remap$(v, {
|
|
73
|
+
runId: "run_id",
|
|
74
|
+
sampleIndex: "sample_index",
|
|
75
|
+
taskId: "task_id",
|
|
76
|
+
xOnBehalfOf: "X-On-Behalf-Of",
|
|
77
|
+
uploadEvalSampleTrajectoryRequest: "UploadEvalSampleTrajectoryRequest",
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
export function uploadEvalSampleTrajectoryRequestToJSON(
|
|
82
|
+
uploadEvalSampleTrajectoryRequest: UploadEvalSampleTrajectoryRequest,
|
|
83
|
+
): string {
|
|
84
|
+
return JSON.stringify(
|
|
85
|
+
UploadEvalSampleTrajectoryRequest$outboundSchema.parse(
|
|
86
|
+
uploadEvalSampleTrajectoryRequest,
|
|
87
|
+
),
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
export function uploadEvalSampleTrajectoryRequestFromJSON(
|
|
91
|
+
jsonString: string,
|
|
92
|
+
): SafeParseResult<UploadEvalSampleTrajectoryRequest, SDKValidationError> {
|
|
93
|
+
return safeParse(
|
|
94
|
+
jsonString,
|
|
95
|
+
(x) => UploadEvalSampleTrajectoryRequest$inboundSchema.parse(JSON.parse(x)),
|
|
96
|
+
`Failed to parse 'UploadEvalSampleTrajectoryRequest' from JSON`,
|
|
97
|
+
);
|
|
98
|
+
}
|
package/src/react-query/index.ts
CHANGED
|
@@ -163,6 +163,7 @@ export * from "./llmConversationsListItems.js";
|
|
|
163
163
|
export * from "./llmConversationsUpdate.js";
|
|
164
164
|
export * from "./llmEmbeddingsEmbed.js";
|
|
165
165
|
export * from "./llmEmbeddingsListModels.js";
|
|
166
|
+
export * from "./llmEvalsAppendAgentTrials.js";
|
|
166
167
|
export * from "./llmEvalsCancelRun.js";
|
|
167
168
|
export * from "./llmEvalsCreateRun.js";
|
|
168
169
|
export * from "./llmEvalsCreateSchedule.js";
|
|
@@ -173,10 +174,14 @@ export * from "./llmEvalsDeleteSchedule.js";
|
|
|
173
174
|
export * from "./llmEvalsDeleteSuite.js";
|
|
174
175
|
export * from "./llmEvalsGetArtifacts.js";
|
|
175
176
|
export * from "./llmEvalsGetRun.js";
|
|
177
|
+
export * from "./llmEvalsGetRunOverview.js";
|
|
176
178
|
export * from "./llmEvalsGetSampleAudio.js";
|
|
179
|
+
export * from "./llmEvalsGetSampleTrajectory.js";
|
|
177
180
|
export * from "./llmEvalsGetSchedule.js";
|
|
178
181
|
export * from "./llmEvalsGetSuite.js";
|
|
182
|
+
export * from "./llmEvalsGetSuiteLeaderboard.js";
|
|
179
183
|
export * from "./llmEvalsGetSuiteVersion.js";
|
|
184
|
+
export * from "./llmEvalsImportAgentRun.js";
|
|
180
185
|
export * from "./llmEvalsImportHistoricalResults.js";
|
|
181
186
|
export * from "./llmEvalsListRuns.js";
|
|
182
187
|
export * from "./llmEvalsListSamples.js";
|
|
@@ -189,6 +194,7 @@ export * from "./llmEvalsRerunFailedSamples.js";
|
|
|
189
194
|
export * from "./llmEvalsRetryFailedRun.js";
|
|
190
195
|
export * from "./llmEvalsTriggerSchedule.js";
|
|
191
196
|
export * from "./llmEvalsUpdateSchedule.js";
|
|
197
|
+
export * from "./llmEvalsUploadSampleTrajectory.js";
|
|
192
198
|
export * from "./llmExtractCreateSchema.js";
|
|
193
199
|
export * from "./llmExtractDeleteSchema.js";
|
|
194
200
|
export * from "./llmExtractExtract.js";
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
import {
|
|
6
|
+
MutationKey,
|
|
7
|
+
useMutation,
|
|
8
|
+
UseMutationResult,
|
|
9
|
+
} from "@tanstack/react-query";
|
|
10
|
+
import { SDKCore } from "../core.js";
|
|
11
|
+
import { llmEvalsAppendAgentTrials } from "../funcs/llmEvalsAppendAgentTrials.js";
|
|
12
|
+
import { combineSignals } from "../lib/primitives.js";
|
|
13
|
+
import { RequestOptions } from "../lib/sdks.js";
|
|
14
|
+
import {
|
|
15
|
+
ConnectionError,
|
|
16
|
+
InvalidRequestError,
|
|
17
|
+
RequestAbortedError,
|
|
18
|
+
RequestTimeoutError,
|
|
19
|
+
UnexpectedClientError,
|
|
20
|
+
} from "../models/errors/httpclienterrors.js";
|
|
21
|
+
import { ResponseValidationError } from "../models/errors/responsevalidationerror.js";
|
|
22
|
+
import { SDKError } from "../models/errors/sdkerror.js";
|
|
23
|
+
import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
|
|
24
|
+
import * as operations from "../models/operations/index.js";
|
|
25
|
+
import { unwrapAsync } from "../types/fp.js";
|
|
26
|
+
import { useSDKContext } from "./_context.js";
|
|
27
|
+
import { MutationHookOptions } from "./_types.js";
|
|
28
|
+
|
|
29
|
+
export type LlmEvalsAppendAgentTrialsMutationVariables = {
|
|
30
|
+
request: operations.AppendAgentEvalTrialsRequest;
|
|
31
|
+
options?: RequestOptions;
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
export type LlmEvalsAppendAgentTrialsMutationData =
|
|
35
|
+
operations.AppendAgentEvalTrialsResponseBody;
|
|
36
|
+
|
|
37
|
+
export type LlmEvalsAppendAgentTrialsMutationError =
|
|
38
|
+
| SDKError
|
|
39
|
+
| ResponseValidationError
|
|
40
|
+
| ConnectionError
|
|
41
|
+
| RequestAbortedError
|
|
42
|
+
| RequestTimeoutError
|
|
43
|
+
| InvalidRequestError
|
|
44
|
+
| UnexpectedClientError
|
|
45
|
+
| SDKValidationError;
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* Append trials to an open coding-agent run
|
|
49
|
+
*
|
|
50
|
+
* @remarks
|
|
51
|
+
* Adds finished trials to a run opened with `finalize: false`, so a job started from MKA1 can be watched while it runs instead of appearing complete out of nowhere at the end. Each trial pins its own `sample_index` and upserts on (task, model, sample_index): a harness cannot tell a timeout from a lost response, so re-sending a batch is safe and replaces rather than duplicates. The whole batch and its aggregate recompute land in one transaction, serialized per run. Counts, metrics and cost are recomputed from the stored rows on every call, never accumulated from the request, so a replayed batch cannot double them. Send `finalize: true` (usually with an empty `trials`) to close the run — until then it stays `in_progress` and off the leaderboards, which only count completed runs. Appending to an already-completed or cancelled run is rejected: that record is finished, and rewriting it would silently move a number someone may have already read.
|
|
52
|
+
*/
|
|
53
|
+
export function useLlmEvalsAppendAgentTrialsMutation(
|
|
54
|
+
options?: MutationHookOptions<
|
|
55
|
+
LlmEvalsAppendAgentTrialsMutationData,
|
|
56
|
+
LlmEvalsAppendAgentTrialsMutationError,
|
|
57
|
+
LlmEvalsAppendAgentTrialsMutationVariables
|
|
58
|
+
>,
|
|
59
|
+
): UseMutationResult<
|
|
60
|
+
LlmEvalsAppendAgentTrialsMutationData,
|
|
61
|
+
LlmEvalsAppendAgentTrialsMutationError,
|
|
62
|
+
LlmEvalsAppendAgentTrialsMutationVariables
|
|
63
|
+
> {
|
|
64
|
+
const client = useSDKContext();
|
|
65
|
+
return useMutation({
|
|
66
|
+
...buildLlmEvalsAppendAgentTrialsMutation(client, options),
|
|
67
|
+
...options,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function mutationKeyLlmEvalsAppendAgentTrials(): MutationKey {
|
|
72
|
+
return ["@meetkai/mka1", "evals", "appendAgentTrials"];
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function buildLlmEvalsAppendAgentTrialsMutation(
|
|
76
|
+
client$: SDKCore,
|
|
77
|
+
hookOptions?: RequestOptions,
|
|
78
|
+
): {
|
|
79
|
+
mutationKey: MutationKey;
|
|
80
|
+
mutationFn: (
|
|
81
|
+
variables: LlmEvalsAppendAgentTrialsMutationVariables,
|
|
82
|
+
) => Promise<LlmEvalsAppendAgentTrialsMutationData>;
|
|
83
|
+
} {
|
|
84
|
+
return {
|
|
85
|
+
mutationKey: mutationKeyLlmEvalsAppendAgentTrials(),
|
|
86
|
+
mutationFn: function llmEvalsAppendAgentTrialsMutationFn({
|
|
87
|
+
request,
|
|
88
|
+
options,
|
|
89
|
+
}): Promise<LlmEvalsAppendAgentTrialsMutationData> {
|
|
90
|
+
const mergedOptions = {
|
|
91
|
+
...hookOptions,
|
|
92
|
+
...options,
|
|
93
|
+
fetchOptions: {
|
|
94
|
+
...hookOptions?.fetchOptions,
|
|
95
|
+
...options?.fetchOptions,
|
|
96
|
+
signal: combineSignals(
|
|
97
|
+
hookOptions?.fetchOptions?.signal,
|
|
98
|
+
options?.fetchOptions?.signal,
|
|
99
|
+
),
|
|
100
|
+
},
|
|
101
|
+
};
|
|
102
|
+
return unwrapAsync(llmEvalsAppendAgentTrials(
|
|
103
|
+
client$,
|
|
104
|
+
request,
|
|
105
|
+
mergedOptions,
|
|
106
|
+
));
|
|
107
|
+
},
|
|
108
|
+
};
|
|
109
|
+
}
|