@lightsage/mcp 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/api/well-known-openai-apps-challenge.js +25 -0
- package/bin/mcp-server.js +1878 -1023
- package/bin/mcp-server.js.map +77 -58
- package/esm/funcs/evalTemplatesV2EvalTemplatesCreate.d.ts +16 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesCreate.d.ts.map +1 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesCreate.js +83 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesCreate.js.map +1 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesDelete.d.ts +16 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesDelete.d.ts.map +1 -0
- package/esm/funcs/{evalResultsV2EvalResultsList.js → evalTemplatesV2EvalTemplatesDelete.js} +17 -16
- package/esm/funcs/evalTemplatesV2EvalTemplatesDelete.js.map +1 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesList.d.ts +15 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesList.d.ts.map +1 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesList.js +72 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesList.js.map +1 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesRetrieve.d.ts +16 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesRetrieve.d.ts.map +1 -0
- package/esm/funcs/{evalResultsV2EvalResultsRetrieve.js → evalTemplatesV2EvalTemplatesRetrieve.js} +10 -10
- package/esm/funcs/evalTemplatesV2EvalTemplatesRetrieve.js.map +1 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesUpdate.d.ts +16 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesUpdate.d.ts.map +1 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesUpdate.js +89 -0
- package/esm/funcs/evalTemplatesV2EvalTemplatesUpdate.js.map +1 -0
- package/esm/funcs/evalsCancel.d.ts +16 -0
- package/esm/funcs/evalsCancel.d.ts.map +1 -0
- package/esm/funcs/{evalsAnalysis.js → evalsCancel.js} +10 -10
- package/esm/funcs/evalsCancel.js.map +1 -0
- package/esm/funcs/evalsGet.d.ts +16 -0
- package/esm/funcs/evalsGet.d.ts.map +1 -0
- package/esm/funcs/{evalRunsV2EvalRunsRetrieve.js → evalsGet.js} +5 -5
- package/esm/funcs/evalsGet.js.map +1 -0
- package/esm/funcs/evalsRun.d.ts +6 -4
- package/esm/funcs/evalsRun.d.ts.map +1 -1
- package/esm/funcs/evalsRun.js +6 -4
- package/esm/funcs/evalsRun.js.map +1 -1
- package/esm/funcs/evalsV2EvalsBatchCreate.d.ts +16 -0
- package/esm/funcs/evalsV2EvalsBatchCreate.d.ts.map +1 -0
- package/esm/funcs/evalsV2EvalsBatchCreate.js +83 -0
- package/esm/funcs/evalsV2EvalsBatchCreate.js.map +1 -0
- package/esm/funcs/evalsV2EvalsCreate.d.ts +1 -1
- package/esm/funcs/evalsV2EvalsCreate.js +1 -1
- package/esm/funcs/evalsV2EvalsUpdate.d.ts +1 -1
- package/esm/funcs/evalsV2EvalsUpdate.js +1 -1
- package/esm/funcs/promptsV2PromptRunsList.d.ts +1 -1
- package/esm/funcs/promptsV2PromptRunsList.js +1 -1
- package/esm/funcs/promptsV2PromptRunsRetrieve.d.ts +1 -1
- package/esm/funcs/promptsV2PromptRunsRetrieve.js +1 -1
- package/esm/landing-page.js +1 -1
- package/esm/lib/config.d.ts +3 -3
- package/esm/lib/config.js +3 -3
- package/esm/lib/config.js.map +1 -1
- package/esm/mcp-server/mcp-server.js +1 -1
- package/esm/mcp-server/server.d.ts.map +1 -1
- package/esm/mcp-server/server.js +17 -9
- package/esm/mcp-server/server.js.map +1 -1
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesCreate.d.ts +7 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesCreate.d.ts.map +1 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesCreate.js +35 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesCreate.js.map +1 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesDelete.d.ts +7 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesDelete.d.ts.map +1 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesDelete.js +35 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesDelete.js.map +1 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesList.d.ts +3 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesList.d.ts.map +1 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesList.js +30 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesList.js.map +1 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesRetrieve.d.ts +7 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesRetrieve.d.ts.map +1 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesRetrieve.js +35 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesRetrieve.js.map +1 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.d.ts +7 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.d.ts.map +1 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.js +35 -0
- package/esm/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.js.map +1 -0
- package/esm/mcp-server/tools/evalsCancel.d.ts +7 -0
- package/esm/mcp-server/tools/evalsCancel.d.ts.map +1 -0
- package/esm/mcp-server/tools/evalsCancel.js +35 -0
- package/esm/mcp-server/tools/evalsCancel.js.map +1 -0
- package/esm/mcp-server/tools/{evalRunsV2EvalRunsRetrieve.d.ts → evalsGet.d.ts} +2 -2
- package/esm/mcp-server/tools/evalsGet.d.ts.map +1 -0
- package/esm/mcp-server/tools/{evalRunsV2EvalRunsRetrieve.js → evalsGet.js} +8 -8
- package/esm/mcp-server/tools/evalsGet.js.map +1 -0
- package/esm/mcp-server/tools/evalsRun.d.ts.map +1 -1
- package/esm/mcp-server/tools/evalsRun.js +6 -4
- package/esm/mcp-server/tools/evalsRun.js.map +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsBatchCreate.d.ts +7 -0
- package/esm/mcp-server/tools/evalsV2EvalsBatchCreate.d.ts.map +1 -0
- package/esm/mcp-server/tools/evalsV2EvalsBatchCreate.js +35 -0
- package/esm/mcp-server/tools/evalsV2EvalsBatchCreate.js.map +1 -0
- package/esm/mcp-server/tools/evalsV2EvalsCreate.js +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsCreate.js.map +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsUpdate.js +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsUpdate.js.map +1 -1
- package/esm/mcp-server/tools/promptsV2PromptRunsList.js +1 -1
- package/esm/mcp-server/tools/promptsV2PromptRunsList.js.map +1 -1
- package/esm/mcp-server/tools/promptsV2PromptRunsRetrieve.js +1 -1
- package/esm/mcp-server/tools/promptsV2PromptRunsRetrieve.js.map +1 -1
- package/esm/models/createevaltemplatebatchrequest.d.ts +12 -0
- package/esm/models/createevaltemplatebatchrequest.d.ts.map +1 -0
- package/esm/models/createevaltemplatebatchrequest.js +12 -0
- package/esm/models/createevaltemplatebatchrequest.js.map +1 -0
- package/esm/models/createevaltemplaterequest.d.ts +79 -0
- package/esm/models/createevaltemplaterequest.d.ts.map +1 -0
- package/esm/models/createevaltemplaterequest.js +74 -0
- package/esm/models/createevaltemplaterequest.js.map +1 -0
- package/esm/models/createv2evalconfigurationrequest.d.ts +2 -0
- package/esm/models/createv2evalconfigurationrequest.d.ts.map +1 -1
- package/esm/models/createv2evalconfigurationrequest.js +2 -0
- package/esm/models/createv2evalconfigurationrequest.js.map +1 -1
- package/esm/models/createv2evalrequest.d.ts +5 -5
- package/esm/models/createv2evalrequest.d.ts.map +1 -1
- package/esm/models/createv2evalrequest.js +5 -10
- package/esm/models/createv2evalrequest.js.map +1 -1
- package/esm/models/createv2evalrunrequest.d.ts +5 -5
- package/esm/models/createv2evalrunrequest.d.ts.map +1 -1
- package/esm/models/createv2evalrunrequest.js +6 -10
- package/esm/models/createv2evalrunrequest.js.map +1 -1
- package/esm/models/evaltemplate.d.ts +83 -0
- package/esm/models/evaltemplate.d.ts.map +1 -0
- package/esm/models/evaltemplate.js +81 -0
- package/esm/models/evaltemplate.js.map +1 -0
- package/esm/models/evaltemplatebatchcreatedeval.d.ts +11 -0
- package/esm/models/evaltemplatebatchcreatedeval.d.ts.map +1 -0
- package/esm/models/evaltemplatebatchcreatedeval.js +14 -0
- package/esm/models/evaltemplatebatchcreatedeval.js.map +1 -0
- package/esm/models/evaltemplatebatchitem.d.ts +32 -0
- package/esm/models/evaltemplatebatchitem.d.ts.map +1 -0
- package/esm/models/evaltemplatebatchitem.js +27 -0
- package/esm/models/evaltemplatebatchitem.js.map +1 -0
- package/esm/models/evaltemplatebatchresponse.d.ts +12 -0
- package/esm/models/evaltemplatebatchresponse.d.ts.map +1 -0
- package/esm/models/evaltemplatebatchresponse.js +15 -0
- package/esm/models/evaltemplatebatchresponse.js.map +1 -0
- package/esm/models/evaltemplatebatchresult.d.ts +10 -0
- package/esm/models/evaltemplatebatchresult.d.ts.map +1 -0
- package/esm/models/evaltemplatebatchresult.js +13 -0
- package/esm/models/evaltemplatebatchresult.js.map +1 -0
- package/esm/models/evaltemplateruntime.d.ts +20 -0
- package/esm/models/evaltemplateruntime.d.ts.map +1 -0
- package/esm/models/evaltemplateruntime.js +21 -0
- package/esm/models/evaltemplateruntime.js.map +1 -0
- package/esm/models/evaltemplateschedule.d.ts +28 -0
- package/esm/models/evaltemplateschedule.d.ts.map +1 -0
- package/esm/models/evaltemplateschedule.js +24 -0
- package/esm/models/evaltemplateschedule.js.map +1 -0
- package/esm/models/evaltemplatesresponse.d.ts +8 -0
- package/esm/models/evaltemplatesresponse.d.ts.map +1 -0
- package/esm/models/evaltemplatesresponse.js +11 -0
- package/esm/models/evaltemplatesresponse.js.map +1 -0
- package/esm/models/graderdefinition.d.ts +14 -0
- package/esm/models/graderdefinition.d.ts.map +1 -0
- package/esm/models/graderdefinition.js +17 -0
- package/esm/models/graderdefinition.js.map +1 -0
- package/esm/models/publiccompetitorrow.js +2 -2
- package/esm/models/publiccompetitorrow.js.map +1 -1
- package/esm/models/publiccriterionverdict.d.ts +26 -0
- package/esm/models/publiccriterionverdict.d.ts.map +1 -0
- package/esm/models/publiccriterionverdict.js +24 -0
- package/esm/models/publiccriterionverdict.js.map +1 -0
- package/esm/models/publiccurrentvisibilityresponse.js +2 -2
- package/esm/models/publiccurrentvisibilityresponse.js.map +1 -1
- package/esm/models/publicevalrunenvironment.d.ts +15 -0
- package/esm/models/publicevalrunenvironment.d.ts.map +1 -0
- package/esm/models/publicevalrunenvironment.js +15 -0
- package/esm/models/publicevalrunenvironment.js.map +1 -0
- package/esm/models/publicevalrunprogress.d.ts +1 -0
- package/esm/models/publicevalrunprogress.d.ts.map +1 -1
- package/esm/models/publicevalrunprogress.js +3 -2
- package/esm/models/publicevalrunprogress.js.map +1 -1
- package/esm/models/publicgraderresult.d.ts +13 -0
- package/esm/models/publicgraderresult.d.ts.map +1 -0
- package/esm/models/publicgraderresult.js +18 -0
- package/esm/models/publicgraderresult.js.map +1 -0
- package/esm/models/publicpromptvisibilitymetrics.js +2 -2
- package/esm/models/publicpromptvisibilitymetrics.js.map +1 -1
- package/esm/models/publicvisibilitysnapshot.js +2 -2
- package/esm/models/publicvisibilitysnapshot.js.map +1 -1
- package/esm/models/searchgroundingmode.d.ts +20 -0
- package/esm/models/searchgroundingmode.d.ts.map +1 -0
- package/esm/models/searchgroundingmode.js +19 -0
- package/esm/models/searchgroundingmode.js.map +1 -0
- package/esm/models/updateevaltemplaterequest.d.ts +71 -0
- package/esm/models/updateevaltemplaterequest.d.ts.map +1 -0
- package/esm/models/updateevaltemplaterequest.js +71 -0
- package/esm/models/updateevaltemplaterequest.js.map +1 -0
- package/esm/models/updatev2evalconfigurationrequest.d.ts +2 -0
- package/esm/models/updatev2evalconfigurationrequest.d.ts.map +1 -1
- package/esm/models/updatev2evalconfigurationrequest.js +3 -0
- package/esm/models/updatev2evalconfigurationrequest.js.map +1 -1
- package/esm/models/updatev2evalrequest.d.ts +5 -5
- package/esm/models/updatev2evalrequest.d.ts.map +1 -1
- package/esm/models/updatev2evalrequest.js +6 -10
- package/esm/models/updatev2evalrequest.js.map +1 -1
- package/esm/models/v2eval.d.ts +6 -2
- package/esm/models/v2eval.d.ts.map +1 -1
- package/esm/models/v2eval.js +5 -1
- package/esm/models/v2eval.js.map +1 -1
- package/esm/models/v2evalconfiguration.d.ts +2 -0
- package/esm/models/v2evalconfiguration.d.ts.map +1 -1
- package/esm/models/v2evalconfiguration.js +2 -0
- package/esm/models/v2evalconfiguration.js.map +1 -1
- package/esm/models/v2evalresultsummary.d.ts +10 -0
- package/esm/models/v2evalresultsummary.d.ts.map +1 -1
- package/esm/models/v2evalresultsummary.js +12 -1
- package/esm/models/v2evalresultsummary.js.map +1 -1
- package/esm/models/v2evalrun.d.ts.map +1 -1
- package/esm/models/v2evalrun.js +1 -1
- package/esm/models/v2evalrun.js.map +1 -1
- package/esm/models/v2evalrundetail.d.ts +30 -0
- package/esm/models/v2evalrundetail.d.ts.map +1 -0
- package/esm/models/v2evalrundetail.js +33 -0
- package/esm/models/v2evalrundetail.js.map +1 -0
- package/esm/models/v2evalrunscancelop.d.ts +31 -0
- package/esm/models/v2evalrunscancelop.d.ts.map +1 -0
- package/esm/models/v2evalrunscancelop.js +27 -0
- package/esm/models/v2evalrunscancelop.js.map +1 -0
- package/esm/models/v2evalrunscreateop.d.ts +23 -3
- package/esm/models/v2evalrunscreateop.d.ts.map +1 -1
- package/esm/models/v2evalrunscreateop.js +15 -2
- package/esm/models/v2evalrunscreateop.js.map +1 -1
- package/esm/models/v2evalrunsretrieveop.d.ts +9 -2
- package/esm/models/v2evalrunsretrieveop.d.ts.map +1 -1
- package/esm/models/v2evalrunsretrieveop.js +6 -2
- package/esm/models/v2evalrunsretrieveop.js.map +1 -1
- package/esm/models/v2evalruntraceevent.d.ts.map +1 -1
- package/esm/models/v2evalruntraceevent.js +1 -1
- package/esm/models/v2evalruntraceevent.js.map +1 -1
- package/esm/models/v2evalsbatchcreateop.d.ts +27 -0
- package/esm/models/v2evalsbatchcreateop.d.ts.map +1 -0
- package/esm/models/v2evalsbatchcreateop.js +24 -0
- package/esm/models/v2evalsbatchcreateop.js.map +1 -0
- package/esm/models/v2evaltemplatescreateop.d.ts +13 -0
- package/esm/models/v2evaltemplatescreateop.d.ts.map +1 -0
- package/esm/models/v2evaltemplatescreateop.js +16 -0
- package/esm/models/v2evaltemplatescreateop.js.map +1 -0
- package/esm/models/v2evaltemplatesdeleteop.d.ts +24 -0
- package/esm/models/v2evaltemplatesdeleteop.d.ts.map +1 -0
- package/esm/models/v2evaltemplatesdeleteop.js +23 -0
- package/esm/models/v2evaltemplatesdeleteop.js.map +1 -0
- package/esm/models/v2evaltemplatesretrieveop.d.ts +17 -0
- package/esm/models/v2evaltemplatesretrieveop.d.ts.map +1 -0
- package/esm/models/v2evaltemplatesretrieveop.js +19 -0
- package/esm/models/v2evaltemplatesretrieveop.js.map +1 -0
- package/esm/models/v2evaltemplatesupdateop.d.ts +33 -0
- package/esm/models/v2evaltemplatesupdateop.d.ts.map +1 -0
- package/esm/models/v2evaltemplatesupdateop.js +29 -0
- package/esm/models/v2evaltemplatesupdateop.js.map +1 -0
- package/esm/models/v2inlinemcpdefinition.d.ts +4 -4
- package/esm/models/v2inlinemcpdefinition.d.ts.map +1 -1
- package/esm/models/v2inlinemcpdefinition.js +3 -3
- package/esm/models/v2inlinemcpdefinition.js.map +1 -1
- package/esm/models/v2promptrundetail.d.ts +6 -0
- package/esm/models/v2promptrundetail.d.ts.map +1 -1
- package/esm/models/v2promptrundetail.js +4 -1
- package/esm/models/v2promptrundetail.js.map +1 -1
- package/esm/models/v2promptrunsummary.d.ts.map +1 -1
- package/esm/models/v2promptrunsummary.js +1 -1
- package/esm/models/v2promptrunsummary.js.map +1 -1
- package/esm/models/v2publictask.d.ts +2 -2
- package/esm/models/v2publictask.d.ts.map +1 -1
- package/esm/models/v2publictask.js +4 -4
- package/esm/models/v2publictask.js.map +1 -1
- package/esm/tool-names.d.ts.map +1 -1
- package/esm/tool-names.js +31 -15
- package/esm/tool-names.js.map +1 -1
- package/package.json +1 -1
- package/src/funcs/evalTemplatesV2EvalTemplatesCreate.ts +145 -0
- package/src/funcs/{evalResultsV2EvalResultsList.ts → evalTemplatesV2EvalTemplatesDelete.ts} +23 -19
- package/src/funcs/evalTemplatesV2EvalTemplatesList.ts +124 -0
- package/src/funcs/{evalResultsV2EvalResultsRetrieve.ts → evalTemplatesV2EvalTemplatesRetrieve.ts} +13 -13
- package/src/funcs/evalTemplatesV2EvalTemplatesUpdate.ts +154 -0
- package/src/funcs/{evalsAnalysis.ts → evalsCancel.ts} +13 -13
- package/src/funcs/{evalRunsV2EvalRunsRetrieve.ts → evalsGet.ts} +4 -4
- package/src/funcs/evalsRun.ts +6 -4
- package/src/funcs/evalsV2EvalsBatchCreate.ts +145 -0
- package/src/funcs/evalsV2EvalsCreate.ts +1 -1
- package/src/funcs/evalsV2EvalsUpdate.ts +1 -1
- package/src/funcs/promptsV2PromptRunsList.ts +1 -1
- package/src/funcs/promptsV2PromptRunsRetrieve.ts +1 -1
- package/src/landing-page.ts +1 -1
- package/src/lib/config.ts +3 -3
- package/src/mcp-server/mcp-server.ts +1 -1
- package/src/mcp-server/server.ts +17 -9
- package/src/mcp-server/tools/evalTemplatesV2EvalTemplatesCreate.ts +45 -0
- package/src/mcp-server/tools/evalTemplatesV2EvalTemplatesDelete.ts +45 -0
- package/src/mcp-server/tools/evalTemplatesV2EvalTemplatesList.ts +36 -0
- package/src/mcp-server/tools/{evalResultsV2EvalResultsRetrieve.ts → evalTemplatesV2EvalTemplatesRetrieve.ts} +9 -9
- package/src/mcp-server/tools/evalTemplatesV2EvalTemplatesUpdate.ts +45 -0
- package/src/mcp-server/tools/evalsCancel.ts +43 -0
- package/src/mcp-server/tools/{evalRunsV2EvalRunsRetrieve.ts → evalsGet.ts} +7 -7
- package/src/mcp-server/tools/evalsRun.ts +6 -4
- package/src/mcp-server/tools/evalsV2EvalsBatchCreate.ts +43 -0
- package/src/mcp-server/tools/evalsV2EvalsCreate.ts +1 -1
- package/src/mcp-server/tools/evalsV2EvalsUpdate.ts +1 -1
- package/src/mcp-server/tools/promptsV2PromptRunsList.ts +1 -1
- package/src/mcp-server/tools/promptsV2PromptRunsRetrieve.ts +1 -1
- package/src/models/createevaltemplatebatchrequest.ts +35 -0
- package/src/models/createevaltemplaterequest.ts +151 -0
- package/src/models/createv2evalconfigurationrequest.ts +8 -0
- package/src/models/createv2evalrequest.ts +18 -22
- package/src/models/createv2evalrunrequest.ts +19 -24
- package/src/models/evaltemplate.ts +154 -0
- package/src/models/evaltemplatebatchcreatedeval.ts +26 -0
- package/src/models/evaltemplatebatchitem.ts +54 -0
- package/src/models/evaltemplatebatchresponse.ts +30 -0
- package/src/models/evaltemplatebatchresult.ts +26 -0
- package/src/models/evaltemplateruntime.ts +64 -0
- package/src/models/evaltemplateschedule.ts +44 -0
- package/src/models/evaltemplatesresponse.ts +18 -0
- package/src/models/graderdefinition.ts +34 -0
- package/src/models/publiccompetitorrow.ts +2 -2
- package/src/models/publiccriterionverdict.ts +48 -0
- package/src/models/publiccurrentvisibilityresponse.ts +2 -2
- package/src/models/publicevalrunenvironment.ts +47 -0
- package/src/models/publicevalrunprogress.ts +8 -2
- package/src/models/publicgraderresult.ts +44 -0
- package/src/models/publicpromptvisibilitymetrics.ts +2 -2
- package/src/models/publicvisibilitysnapshot.ts +2 -2
- package/src/models/searchgroundingmode.ts +26 -0
- package/src/models/updateevaltemplaterequest.ts +116 -0
- package/src/models/updatev2evalconfigurationrequest.ts +9 -0
- package/src/models/updatev2evalrequest.ts +20 -21
- package/src/models/v2eval.ts +14 -2
- package/src/models/v2evalconfiguration.ts +8 -0
- package/src/models/v2evalresultsummary.ts +36 -1
- package/src/models/v2evalrun.ts +3 -1
- package/src/models/v2evalrundetail.ts +90 -0
- package/src/models/v2evalrunscancelop.ts +70 -0
- package/src/models/v2evalrunscreateop.ts +43 -5
- package/src/models/v2evalrunsretrieveop.ts +23 -3
- package/src/models/v2evalruntraceevent.ts +3 -1
- package/src/models/v2evalsbatchcreateop.ts +66 -0
- package/src/models/v2evaltemplatescreateop.ts +37 -0
- package/src/models/v2evaltemplatesdeleteop.ts +56 -0
- package/src/models/v2evaltemplatesretrieveop.ts +43 -0
- package/src/models/v2evaltemplatesupdateop.ts +79 -0
- package/src/models/v2inlinemcpdefinition.ts +7 -5
- package/src/models/v2promptrundetail.ts +12 -1
- package/src/models/v2promptrunsummary.ts +3 -1
- package/src/models/v2publictask.ts +9 -7
- package/src/tool-names.ts +31 -15
- package/test/simulations.test.js +30 -3
- package/esm/funcs/evalResultsV2EvalResultsList.d.ts +0 -16
- package/esm/funcs/evalResultsV2EvalResultsList.d.ts.map +0 -1
- package/esm/funcs/evalResultsV2EvalResultsList.js.map +0 -1
- package/esm/funcs/evalResultsV2EvalResultsRetrieve.d.ts +0 -16
- package/esm/funcs/evalResultsV2EvalResultsRetrieve.d.ts.map +0 -1
- package/esm/funcs/evalResultsV2EvalResultsRetrieve.js.map +0 -1
- package/esm/funcs/evalRunsV2EvalRunsRetrieve.d.ts +0 -16
- package/esm/funcs/evalRunsV2EvalRunsRetrieve.d.ts.map +0 -1
- package/esm/funcs/evalRunsV2EvalRunsRetrieve.js.map +0 -1
- package/esm/funcs/evalsAnalysis.d.ts +0 -16
- package/esm/funcs/evalsAnalysis.d.ts.map +0 -1
- package/esm/funcs/evalsAnalysis.js.map +0 -1
- package/esm/mcp-server/tools/evalResultsV2EvalResultsList.d.ts +0 -7
- package/esm/mcp-server/tools/evalResultsV2EvalResultsList.d.ts.map +0 -1
- package/esm/mcp-server/tools/evalResultsV2EvalResultsList.js +0 -35
- package/esm/mcp-server/tools/evalResultsV2EvalResultsList.js.map +0 -1
- package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.d.ts +0 -7
- package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.d.ts.map +0 -1
- package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.js +0 -35
- package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.js.map +0 -1
- package/esm/mcp-server/tools/evalRunsV2EvalRunsRetrieve.d.ts.map +0 -1
- package/esm/mcp-server/tools/evalRunsV2EvalRunsRetrieve.js.map +0 -1
- package/esm/mcp-server/tools/evalsAnalysis.d.ts +0 -7
- package/esm/mcp-server/tools/evalsAnalysis.d.ts.map +0 -1
- package/esm/mcp-server/tools/evalsAnalysis.js +0 -35
- package/esm/mcp-server/tools/evalsAnalysis.js.map +0 -1
- package/esm/models/v2evalresult.d.ts +0 -23
- package/esm/models/v2evalresult.d.ts.map +0 -1
- package/esm/models/v2evalresult.js +0 -24
- package/esm/models/v2evalresult.js.map +0 -1
- package/esm/models/v2evalresultslistop.d.ts +0 -11
- package/esm/models/v2evalresultslistop.d.ts.map +0 -1
- package/esm/models/v2evalresultslistop.js +0 -16
- package/esm/models/v2evalresultslistop.js.map +0 -1
- package/esm/models/v2evalresultsresponse.d.ts +0 -9
- package/esm/models/v2evalresultsresponse.d.ts.map +0 -1
- package/esm/models/v2evalresultsresponse.js +0 -12
- package/esm/models/v2evalresultsresponse.js.map +0 -1
- package/esm/models/v2evalresultsretrieveop.d.ts +0 -10
- package/esm/models/v2evalresultsretrieveop.d.ts.map +0 -1
- package/esm/models/v2evalresultsretrieveop.js +0 -15
- package/esm/models/v2evalresultsretrieveop.js.map +0 -1
- package/esm/models/v2evalrunanalysisresponse.d.ts +0 -12
- package/esm/models/v2evalrunanalysisresponse.d.ts.map +0 -1
- package/esm/models/v2evalrunanalysisresponse.js +0 -15
- package/esm/models/v2evalrunanalysisresponse.js.map +0 -1
- package/esm/models/v2evalrunsanalysisop.d.ts +0 -17
- package/esm/models/v2evalrunsanalysisop.d.ts.map +0 -1
- package/esm/models/v2evalrunsanalysisop.js +0 -19
- package/esm/models/v2evalrunsanalysisop.js.map +0 -1
- package/src/mcp-server/tools/evalResultsV2EvalResultsList.ts +0 -43
- package/src/mcp-server/tools/evalsAnalysis.ts +0 -43
- package/src/models/v2evalresult.ts +0 -46
- package/src/models/v2evalresultslistop.ts +0 -39
- package/src/models/v2evalresultsresponse.ts +0 -23
- package/src/models/v2evalresultsretrieveop.ts +0 -28
- package/src/models/v2evalrunanalysisresponse.ts +0 -32
- package/src/models/v2evalrunsanalysisop.ts +0 -48
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: a4eb73f4b2a9
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
PublicEvalRunProgress,
|
|
9
|
+
PublicEvalRunProgress$zodSchema,
|
|
10
|
+
} from "./publicevalrunprogress.js";
|
|
11
|
+
import {
|
|
12
|
+
V2EvalResultSummary,
|
|
13
|
+
V2EvalResultSummary$zodSchema,
|
|
14
|
+
} from "./v2evalresultsummary.js";
|
|
15
|
+
import {
|
|
16
|
+
V2EvalRunAnalysis,
|
|
17
|
+
V2EvalRunAnalysis$zodSchema,
|
|
18
|
+
} from "./v2evalrunanalysis.js";
|
|
19
|
+
|
|
20
|
+
export type V2EvalRunDetail = {
|
|
21
|
+
id: string;
|
|
22
|
+
job_id: string;
|
|
23
|
+
eval_id?: string | null | undefined;
|
|
24
|
+
eval_name?: string | null | undefined;
|
|
25
|
+
configuration_id?: string | null | undefined;
|
|
26
|
+
persona?: string | null | undefined;
|
|
27
|
+
agent?: Array<string> | undefined;
|
|
28
|
+
prompt_count?: number | undefined;
|
|
29
|
+
runs?: number | undefined;
|
|
30
|
+
attempt_count?: number | undefined;
|
|
31
|
+
status: string;
|
|
32
|
+
stage?: string | null | undefined;
|
|
33
|
+
dispatch_id?: string | null | undefined;
|
|
34
|
+
progress?: PublicEvalRunProgress | undefined;
|
|
35
|
+
estimated_cost?: number | null | undefined;
|
|
36
|
+
error?: string | null | undefined;
|
|
37
|
+
created_at?: string | null | undefined;
|
|
38
|
+
started_at?: string | null | undefined;
|
|
39
|
+
completed_at?: string | null | undefined;
|
|
40
|
+
results?: Array<V2EvalResultSummary> | undefined;
|
|
41
|
+
analysis?: Array<V2EvalRunAnalysis> | undefined;
|
|
42
|
+
results_truncated?: boolean | undefined;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export const V2EvalRunDetail$zodSchema: z.ZodType<V2EvalRunDetail> = z.object({
|
|
46
|
+
agent: z.array(z.string()).optional().describe(
|
|
47
|
+
"Runnable agents selected for this simulation.",
|
|
48
|
+
),
|
|
49
|
+
analysis: z.array(V2EvalRunAnalysis$zodSchema).optional().describe(
|
|
50
|
+
"Per-attempt analysis for this run. Empty until an attempt has been analyzed.",
|
|
51
|
+
),
|
|
52
|
+
attempt_count: z.int().default(1).describe(
|
|
53
|
+
"Total concrete attempts after all fan-out dimensions.",
|
|
54
|
+
),
|
|
55
|
+
completed_at: z.string().nullable().optional(),
|
|
56
|
+
configuration_id: z.string().nullable().optional(),
|
|
57
|
+
created_at: z.string().nullable().optional(),
|
|
58
|
+
dispatch_id: z.string().nullable().optional(),
|
|
59
|
+
error: z.string().nullable().optional(),
|
|
60
|
+
estimated_cost: z.number().nullable().optional(),
|
|
61
|
+
eval_id: z.string().nullable().optional().describe(
|
|
62
|
+
"Saved eval definition ID, including one created by a direct run.",
|
|
63
|
+
),
|
|
64
|
+
eval_name: z.string().nullable().optional().describe(
|
|
65
|
+
"Display name captured for the eval when the run was created.",
|
|
66
|
+
),
|
|
67
|
+
id: z.string().describe("Run ID to use when polling eval-run status."),
|
|
68
|
+
job_id: z.string().describe("Alias of `id` for the durable parent job."),
|
|
69
|
+
persona: z.string().nullable().optional().describe(
|
|
70
|
+
"Perspective applied to every attempt in this simulation.",
|
|
71
|
+
),
|
|
72
|
+
progress: PublicEvalRunProgress$zodSchema.optional(),
|
|
73
|
+
prompt_count: z.int().default(1).describe(
|
|
74
|
+
"Number of prompts included in the simulation.",
|
|
75
|
+
),
|
|
76
|
+
results: z.array(V2EvalResultSummary$zodSchema).optional().describe(
|
|
77
|
+
"Per-attempt outcomes for this run. Empty while the run is active; populated as attempts finish.",
|
|
78
|
+
),
|
|
79
|
+
results_truncated: z.boolean().default(false).describe(
|
|
80
|
+
"True when the run produced more results than the inline cap. The remaining results are not included in this response.",
|
|
81
|
+
),
|
|
82
|
+
runs: z.int().default(1).describe(
|
|
83
|
+
"Independent attempts requested for every prompt-agent pair.",
|
|
84
|
+
),
|
|
85
|
+
stage: z.string().nullable().optional(),
|
|
86
|
+
started_at: z.string().nullable().optional(),
|
|
87
|
+
status: z.string().describe(
|
|
88
|
+
"Terminal values: `completed`, `failed`, and `cancelled`. Recoverable interruptions are reported as `running` with stage `retrying`.",
|
|
89
|
+
),
|
|
90
|
+
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 15553511dcfe
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
HTTPValidationError,
|
|
9
|
+
HTTPValidationError$zodSchema,
|
|
10
|
+
} from "./httpvalidationerror.js";
|
|
11
|
+
import { V2EvalRun, V2EvalRun$zodSchema } from "./v2evalrun.js";
|
|
12
|
+
|
|
13
|
+
export type V2EvalRunsCancelRequest = { run_id: string };
|
|
14
|
+
|
|
15
|
+
export const V2EvalRunsCancelRequest$zodSchema: z.ZodType<
|
|
16
|
+
V2EvalRunsCancelRequest
|
|
17
|
+
> = z.object({
|
|
18
|
+
run_id: z.string(),
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Eval run cancellation is temporarily unavailable.
|
|
23
|
+
*/
|
|
24
|
+
export type V2EvalRunsCancelServiceUnavailableResponseBody = { detail: string };
|
|
25
|
+
|
|
26
|
+
export const V2EvalRunsCancelServiceUnavailableResponseBody$zodSchema:
|
|
27
|
+
z.ZodType<V2EvalRunsCancelServiceUnavailableResponseBody> = z.object({
|
|
28
|
+
detail: z.string(),
|
|
29
|
+
}).describe("Eval run cancellation is temporarily unavailable.");
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The eval run has already completed or failed.
|
|
33
|
+
*/
|
|
34
|
+
export type V2EvalRunsCancelConflictResponseBody = { detail: string };
|
|
35
|
+
|
|
36
|
+
export const V2EvalRunsCancelConflictResponseBody$zodSchema: z.ZodType<
|
|
37
|
+
V2EvalRunsCancelConflictResponseBody
|
|
38
|
+
> = z.object({
|
|
39
|
+
detail: z.string(),
|
|
40
|
+
}).describe("The eval run has already completed or failed.");
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Eval run not found. Use the run ID returned when you started the eval.
|
|
44
|
+
*/
|
|
45
|
+
export type V2EvalRunsCancelNotFoundResponseBody = { detail: string };
|
|
46
|
+
|
|
47
|
+
export const V2EvalRunsCancelNotFoundResponseBody$zodSchema: z.ZodType<
|
|
48
|
+
V2EvalRunsCancelNotFoundResponseBody
|
|
49
|
+
> = z.object({
|
|
50
|
+
detail: z.string(),
|
|
51
|
+
}).describe(
|
|
52
|
+
"Eval run not found. Use the run ID returned when you started the eval.",
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
export type V2EvalRunsCancelResponse =
|
|
56
|
+
| V2EvalRun
|
|
57
|
+
| V2EvalRunsCancelNotFoundResponseBody
|
|
58
|
+
| V2EvalRunsCancelConflictResponseBody
|
|
59
|
+
| V2EvalRunsCancelServiceUnavailableResponseBody
|
|
60
|
+
| HTTPValidationError;
|
|
61
|
+
|
|
62
|
+
export const V2EvalRunsCancelResponse$zodSchema: z.ZodType<
|
|
63
|
+
V2EvalRunsCancelResponse
|
|
64
|
+
> = z.union([
|
|
65
|
+
V2EvalRun$zodSchema,
|
|
66
|
+
z.lazy(() => V2EvalRunsCancelNotFoundResponseBody$zodSchema),
|
|
67
|
+
z.lazy(() => V2EvalRunsCancelConflictResponseBody$zodSchema),
|
|
68
|
+
z.lazy(() => V2EvalRunsCancelServiceUnavailableResponseBody$zodSchema),
|
|
69
|
+
HTTPValidationError$zodSchema,
|
|
70
|
+
]);
|
|
@@ -13,25 +13,63 @@ import { V2EvalRun, V2EvalRun$zodSchema } from "./v2evalrun.js";
|
|
|
13
13
|
/**
|
|
14
14
|
* The requested coding agent is unavailable on the organization's current plan.
|
|
15
15
|
*/
|
|
16
|
-
export type
|
|
16
|
+
export type V2EvalRunsCreateForbiddenResponseBody = { detail: string };
|
|
17
17
|
|
|
18
|
-
export const
|
|
19
|
-
|
|
18
|
+
export const V2EvalRunsCreateForbiddenResponseBody$zodSchema: z.ZodType<
|
|
19
|
+
V2EvalRunsCreateForbiddenResponseBody
|
|
20
20
|
> = z.object({
|
|
21
21
|
detail: z.string(),
|
|
22
22
|
}).describe(
|
|
23
23
|
"The requested coding agent is unavailable on the organization's current plan.",
|
|
24
24
|
);
|
|
25
25
|
|
|
26
|
+
export type Detail = { error: string; required: number; balance: number };
|
|
27
|
+
|
|
28
|
+
export const Detail$zodSchema: z.ZodType<Detail> = z.object({
|
|
29
|
+
balance: z.number().describe("The organization's current credit balance."),
|
|
30
|
+
error: z.string(),
|
|
31
|
+
required: z.number().describe("Usage-based credits the requested run needs."),
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* The organization does not currently have enough credits for the requested run.
|
|
36
|
+
*/
|
|
37
|
+
export type V2EvalRunsCreatePaymentRequiredResponseBody = { detail: Detail };
|
|
38
|
+
|
|
39
|
+
export const V2EvalRunsCreatePaymentRequiredResponseBody$zodSchema: z.ZodType<
|
|
40
|
+
V2EvalRunsCreatePaymentRequiredResponseBody
|
|
41
|
+
> = z.object({
|
|
42
|
+
detail: z.lazy(() => Detail$zodSchema),
|
|
43
|
+
}).describe(
|
|
44
|
+
"The organization does not currently have enough credits for the requested run.",
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The saved/direct mode fields, referenced resource, or environment credentials are invalid.
|
|
49
|
+
*/
|
|
50
|
+
export type V2EvalRunsCreateBadRequestResponseBody = { detail: string };
|
|
51
|
+
|
|
52
|
+
export const V2EvalRunsCreateBadRequestResponseBody$zodSchema: z.ZodType<
|
|
53
|
+
V2EvalRunsCreateBadRequestResponseBody
|
|
54
|
+
> = z.object({
|
|
55
|
+
detail: z.string(),
|
|
56
|
+
}).describe(
|
|
57
|
+
"The saved/direct mode fields, referenced resource, or environment credentials are invalid.",
|
|
58
|
+
);
|
|
59
|
+
|
|
26
60
|
export type V2EvalRunsCreateResponse =
|
|
27
61
|
| V2EvalRun
|
|
28
|
-
|
|
|
62
|
+
| V2EvalRunsCreateBadRequestResponseBody
|
|
63
|
+
| V2EvalRunsCreatePaymentRequiredResponseBody
|
|
64
|
+
| V2EvalRunsCreateForbiddenResponseBody
|
|
29
65
|
| HTTPValidationError;
|
|
30
66
|
|
|
31
67
|
export const V2EvalRunsCreateResponse$zodSchema: z.ZodType<
|
|
32
68
|
V2EvalRunsCreateResponse
|
|
33
69
|
> = z.union([
|
|
34
70
|
V2EvalRun$zodSchema,
|
|
35
|
-
z.lazy(() =>
|
|
71
|
+
z.lazy(() => V2EvalRunsCreateBadRequestResponseBody$zodSchema),
|
|
72
|
+
z.lazy(() => V2EvalRunsCreatePaymentRequiredResponseBody$zodSchema),
|
|
73
|
+
z.lazy(() => V2EvalRunsCreateForbiddenResponseBody$zodSchema),
|
|
36
74
|
HTTPValidationError$zodSchema,
|
|
37
75
|
]);
|
|
@@ -8,7 +8,10 @@ import {
|
|
|
8
8
|
HTTPValidationError,
|
|
9
9
|
HTTPValidationError$zodSchema,
|
|
10
10
|
} from "./httpvalidationerror.js";
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
V2EvalRunDetail,
|
|
13
|
+
V2EvalRunDetail$zodSchema,
|
|
14
|
+
} from "./v2evalrundetail.js";
|
|
12
15
|
|
|
13
16
|
export type V2EvalRunsRetrieveRequest = { run_id: string };
|
|
14
17
|
|
|
@@ -18,11 +21,28 @@ export const V2EvalRunsRetrieveRequest$zodSchema: z.ZodType<
|
|
|
18
21
|
run_id: z.string(),
|
|
19
22
|
});
|
|
20
23
|
|
|
21
|
-
|
|
24
|
+
/**
|
|
25
|
+
* Eval run not found. Use the run ID returned when you started the eval.
|
|
26
|
+
*/
|
|
27
|
+
export type V2EvalRunsRetrieveResponseBody = { detail: string };
|
|
28
|
+
|
|
29
|
+
export const V2EvalRunsRetrieveResponseBody$zodSchema: z.ZodType<
|
|
30
|
+
V2EvalRunsRetrieveResponseBody
|
|
31
|
+
> = z.object({
|
|
32
|
+
detail: z.string(),
|
|
33
|
+
}).describe(
|
|
34
|
+
"Eval run not found. Use the run ID returned when you started the eval.",
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
export type V2EvalRunsRetrieveResponse =
|
|
38
|
+
| V2EvalRunDetail
|
|
39
|
+
| V2EvalRunsRetrieveResponseBody
|
|
40
|
+
| HTTPValidationError;
|
|
22
41
|
|
|
23
42
|
export const V2EvalRunsRetrieveResponse$zodSchema: z.ZodType<
|
|
24
43
|
V2EvalRunsRetrieveResponse
|
|
25
44
|
> = z.union([
|
|
26
|
-
|
|
45
|
+
V2EvalRunDetail$zodSchema,
|
|
46
|
+
z.lazy(() => V2EvalRunsRetrieveResponseBody$zodSchema),
|
|
27
47
|
HTTPValidationError$zodSchema,
|
|
28
48
|
]);
|
|
@@ -41,5 +41,7 @@ export const V2EvalRunTraceEvent$zodSchema: z.ZodType<V2EvalRunTraceEvent> = z
|
|
|
41
41
|
repeat_index: z.int().default(0),
|
|
42
42
|
run_id: z.string(),
|
|
43
43
|
timestamp: z.string().nullable().optional(),
|
|
44
|
-
type: z.string()
|
|
44
|
+
type: z.string().describe(
|
|
45
|
+
"Event type. Events are not tool invocations: one invocation usually emits a `tool_call` event and a `tool_result` event, and can emit extra `tool_call` updates as its arguments stream. Count invocations with the result's `tool_calls`, not with trace events.",
|
|
46
|
+
),
|
|
45
47
|
});
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 49d1b9f52fa6
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
EvalTemplateBatchResponse,
|
|
9
|
+
EvalTemplateBatchResponse$zodSchema,
|
|
10
|
+
} from "./evaltemplatebatchresponse.js";
|
|
11
|
+
import {
|
|
12
|
+
HTTPValidationError,
|
|
13
|
+
HTTPValidationError$zodSchema,
|
|
14
|
+
} from "./httpvalidationerror.js";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The supplied template version is stale.
|
|
18
|
+
*/
|
|
19
|
+
export type V2EvalsBatchCreateConflictResponseBody = { detail: string };
|
|
20
|
+
|
|
21
|
+
export const V2EvalsBatchCreateConflictResponseBody$zodSchema: z.ZodType<
|
|
22
|
+
V2EvalsBatchCreateConflictResponseBody
|
|
23
|
+
> = z.object({
|
|
24
|
+
detail: z.string(),
|
|
25
|
+
}).describe("The supplied template version is stale.");
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* The Eval template does not exist in the organization.
|
|
29
|
+
*/
|
|
30
|
+
export type V2EvalsBatchCreateNotFoundResponseBody = { detail: string };
|
|
31
|
+
|
|
32
|
+
export const V2EvalsBatchCreateNotFoundResponseBody$zodSchema: z.ZodType<
|
|
33
|
+
V2EvalsBatchCreateNotFoundResponseBody
|
|
34
|
+
> = z.object({
|
|
35
|
+
detail: z.string(),
|
|
36
|
+
}).describe("The Eval template does not exist in the organization.");
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* The template contains invalid judges, targets, scheduling, metadata, or organization resources.
|
|
40
|
+
*/
|
|
41
|
+
export type V2EvalsBatchCreateBadRequestResponseBody = { detail: string };
|
|
42
|
+
|
|
43
|
+
export const V2EvalsBatchCreateBadRequestResponseBody$zodSchema: z.ZodType<
|
|
44
|
+
V2EvalsBatchCreateBadRequestResponseBody
|
|
45
|
+
> = z.object({
|
|
46
|
+
detail: z.string(),
|
|
47
|
+
}).describe(
|
|
48
|
+
"The template contains invalid judges, targets, scheduling, metadata, or organization resources.",
|
|
49
|
+
);
|
|
50
|
+
|
|
51
|
+
export type V2EvalsBatchCreateResponse =
|
|
52
|
+
| EvalTemplateBatchResponse
|
|
53
|
+
| V2EvalsBatchCreateBadRequestResponseBody
|
|
54
|
+
| V2EvalsBatchCreateNotFoundResponseBody
|
|
55
|
+
| V2EvalsBatchCreateConflictResponseBody
|
|
56
|
+
| HTTPValidationError;
|
|
57
|
+
|
|
58
|
+
export const V2EvalsBatchCreateResponse$zodSchema: z.ZodType<
|
|
59
|
+
V2EvalsBatchCreateResponse
|
|
60
|
+
> = z.union([
|
|
61
|
+
EvalTemplateBatchResponse$zodSchema,
|
|
62
|
+
z.lazy(() => V2EvalsBatchCreateBadRequestResponseBody$zodSchema),
|
|
63
|
+
z.lazy(() => V2EvalsBatchCreateNotFoundResponseBody$zodSchema),
|
|
64
|
+
z.lazy(() => V2EvalsBatchCreateConflictResponseBody$zodSchema),
|
|
65
|
+
HTTPValidationError$zodSchema,
|
|
66
|
+
]);
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: c84d659a1689
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import { EvalTemplate, EvalTemplate$zodSchema } from "./evaltemplate.js";
|
|
8
|
+
import {
|
|
9
|
+
HTTPValidationError,
|
|
10
|
+
HTTPValidationError$zodSchema,
|
|
11
|
+
} from "./httpvalidationerror.js";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* The template contains invalid judges, targets, scheduling, metadata, or organization resources.
|
|
15
|
+
*/
|
|
16
|
+
export type V2EvalTemplatesCreateResponseBody = { detail: string };
|
|
17
|
+
|
|
18
|
+
export const V2EvalTemplatesCreateResponseBody$zodSchema: z.ZodType<
|
|
19
|
+
V2EvalTemplatesCreateResponseBody
|
|
20
|
+
> = z.object({
|
|
21
|
+
detail: z.string(),
|
|
22
|
+
}).describe(
|
|
23
|
+
"The template contains invalid judges, targets, scheduling, metadata, or organization resources.",
|
|
24
|
+
);
|
|
25
|
+
|
|
26
|
+
export type V2EvalTemplatesCreateResponse =
|
|
27
|
+
| EvalTemplate
|
|
28
|
+
| V2EvalTemplatesCreateResponseBody
|
|
29
|
+
| HTTPValidationError;
|
|
30
|
+
|
|
31
|
+
export const V2EvalTemplatesCreateResponse$zodSchema: z.ZodType<
|
|
32
|
+
V2EvalTemplatesCreateResponse
|
|
33
|
+
> = z.union([
|
|
34
|
+
EvalTemplate$zodSchema,
|
|
35
|
+
z.lazy(() => V2EvalTemplatesCreateResponseBody$zodSchema),
|
|
36
|
+
HTTPValidationError$zodSchema,
|
|
37
|
+
]);
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 7108af503e2f
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import { EvalTemplate, EvalTemplate$zodSchema } from "./evaltemplate.js";
|
|
8
|
+
import {
|
|
9
|
+
HTTPValidationError,
|
|
10
|
+
HTTPValidationError$zodSchema,
|
|
11
|
+
} from "./httpvalidationerror.js";
|
|
12
|
+
|
|
13
|
+
export type V2EvalTemplatesDeleteRequest = { template_id: string };
|
|
14
|
+
|
|
15
|
+
export const V2EvalTemplatesDeleteRequest$zodSchema: z.ZodType<
|
|
16
|
+
V2EvalTemplatesDeleteRequest
|
|
17
|
+
> = z.object({
|
|
18
|
+
template_id: z.string(),
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* The supplied template version is stale.
|
|
23
|
+
*/
|
|
24
|
+
export type V2EvalTemplatesDeleteConflictResponseBody = { detail: string };
|
|
25
|
+
|
|
26
|
+
export const V2EvalTemplatesDeleteConflictResponseBody$zodSchema: z.ZodType<
|
|
27
|
+
V2EvalTemplatesDeleteConflictResponseBody
|
|
28
|
+
> = z.object({
|
|
29
|
+
detail: z.string(),
|
|
30
|
+
}).describe("The supplied template version is stale.");
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The Eval template does not exist in the organization.
|
|
34
|
+
*/
|
|
35
|
+
export type V2EvalTemplatesDeleteNotFoundResponseBody = { detail: string };
|
|
36
|
+
|
|
37
|
+
export const V2EvalTemplatesDeleteNotFoundResponseBody$zodSchema: z.ZodType<
|
|
38
|
+
V2EvalTemplatesDeleteNotFoundResponseBody
|
|
39
|
+
> = z.object({
|
|
40
|
+
detail: z.string(),
|
|
41
|
+
}).describe("The Eval template does not exist in the organization.");
|
|
42
|
+
|
|
43
|
+
export type V2EvalTemplatesDeleteResponse =
|
|
44
|
+
| EvalTemplate
|
|
45
|
+
| V2EvalTemplatesDeleteNotFoundResponseBody
|
|
46
|
+
| V2EvalTemplatesDeleteConflictResponseBody
|
|
47
|
+
| HTTPValidationError;
|
|
48
|
+
|
|
49
|
+
export const V2EvalTemplatesDeleteResponse$zodSchema: z.ZodType<
|
|
50
|
+
V2EvalTemplatesDeleteResponse
|
|
51
|
+
> = z.union([
|
|
52
|
+
EvalTemplate$zodSchema,
|
|
53
|
+
z.lazy(() => V2EvalTemplatesDeleteNotFoundResponseBody$zodSchema),
|
|
54
|
+
z.lazy(() => V2EvalTemplatesDeleteConflictResponseBody$zodSchema),
|
|
55
|
+
HTTPValidationError$zodSchema,
|
|
56
|
+
]);
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 8bfad89e06a7
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import { EvalTemplate, EvalTemplate$zodSchema } from "./evaltemplate.js";
|
|
8
|
+
import {
|
|
9
|
+
HTTPValidationError,
|
|
10
|
+
HTTPValidationError$zodSchema,
|
|
11
|
+
} from "./httpvalidationerror.js";
|
|
12
|
+
|
|
13
|
+
export type V2EvalTemplatesRetrieveRequest = { template_id: string };
|
|
14
|
+
|
|
15
|
+
export const V2EvalTemplatesRetrieveRequest$zodSchema: z.ZodType<
|
|
16
|
+
V2EvalTemplatesRetrieveRequest
|
|
17
|
+
> = z.object({
|
|
18
|
+
template_id: z.string(),
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* The Eval template does not exist in the organization.
|
|
23
|
+
*/
|
|
24
|
+
export type V2EvalTemplatesRetrieveResponseBody = { detail: string };
|
|
25
|
+
|
|
26
|
+
export const V2EvalTemplatesRetrieveResponseBody$zodSchema: z.ZodType<
|
|
27
|
+
V2EvalTemplatesRetrieveResponseBody
|
|
28
|
+
> = z.object({
|
|
29
|
+
detail: z.string(),
|
|
30
|
+
}).describe("The Eval template does not exist in the organization.");
|
|
31
|
+
|
|
32
|
+
export type V2EvalTemplatesRetrieveResponse =
|
|
33
|
+
| EvalTemplate
|
|
34
|
+
| V2EvalTemplatesRetrieveResponseBody
|
|
35
|
+
| HTTPValidationError;
|
|
36
|
+
|
|
37
|
+
export const V2EvalTemplatesRetrieveResponse$zodSchema: z.ZodType<
|
|
38
|
+
V2EvalTemplatesRetrieveResponse
|
|
39
|
+
> = z.union([
|
|
40
|
+
EvalTemplate$zodSchema,
|
|
41
|
+
z.lazy(() => V2EvalTemplatesRetrieveResponseBody$zodSchema),
|
|
42
|
+
HTTPValidationError$zodSchema,
|
|
43
|
+
]);
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: c59f083905c6
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import { EvalTemplate, EvalTemplate$zodSchema } from "./evaltemplate.js";
|
|
8
|
+
import {
|
|
9
|
+
HTTPValidationError,
|
|
10
|
+
HTTPValidationError$zodSchema,
|
|
11
|
+
} from "./httpvalidationerror.js";
|
|
12
|
+
import {
|
|
13
|
+
UpdateEvalTemplateRequest,
|
|
14
|
+
UpdateEvalTemplateRequest$zodSchema,
|
|
15
|
+
} from "./updateevaltemplaterequest.js";
|
|
16
|
+
|
|
17
|
+
export type V2EvalTemplatesUpdateRequest = {
|
|
18
|
+
template_id: string;
|
|
19
|
+
body: UpdateEvalTemplateRequest;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export const V2EvalTemplatesUpdateRequest$zodSchema: z.ZodType<
|
|
23
|
+
V2EvalTemplatesUpdateRequest
|
|
24
|
+
> = z.object({
|
|
25
|
+
body: UpdateEvalTemplateRequest$zodSchema,
|
|
26
|
+
template_id: z.string(),
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* The supplied template version is stale.
|
|
31
|
+
*/
|
|
32
|
+
export type V2EvalTemplatesUpdateConflictResponseBody = { detail: string };
|
|
33
|
+
|
|
34
|
+
export const V2EvalTemplatesUpdateConflictResponseBody$zodSchema: z.ZodType<
|
|
35
|
+
V2EvalTemplatesUpdateConflictResponseBody
|
|
36
|
+
> = z.object({
|
|
37
|
+
detail: z.string(),
|
|
38
|
+
}).describe("The supplied template version is stale.");
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* The Eval template does not exist in the organization.
|
|
42
|
+
*/
|
|
43
|
+
export type V2EvalTemplatesUpdateNotFoundResponseBody = { detail: string };
|
|
44
|
+
|
|
45
|
+
export const V2EvalTemplatesUpdateNotFoundResponseBody$zodSchema: z.ZodType<
|
|
46
|
+
V2EvalTemplatesUpdateNotFoundResponseBody
|
|
47
|
+
> = z.object({
|
|
48
|
+
detail: z.string(),
|
|
49
|
+
}).describe("The Eval template does not exist in the organization.");
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* The template contains invalid judges, targets, scheduling, metadata, or organization resources.
|
|
53
|
+
*/
|
|
54
|
+
export type V2EvalTemplatesUpdateBadRequestResponseBody = { detail: string };
|
|
55
|
+
|
|
56
|
+
export const V2EvalTemplatesUpdateBadRequestResponseBody$zodSchema: z.ZodType<
|
|
57
|
+
V2EvalTemplatesUpdateBadRequestResponseBody
|
|
58
|
+
> = z.object({
|
|
59
|
+
detail: z.string(),
|
|
60
|
+
}).describe(
|
|
61
|
+
"The template contains invalid judges, targets, scheduling, metadata, or organization resources.",
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
export type V2EvalTemplatesUpdateResponse =
|
|
65
|
+
| EvalTemplate
|
|
66
|
+
| V2EvalTemplatesUpdateBadRequestResponseBody
|
|
67
|
+
| V2EvalTemplatesUpdateNotFoundResponseBody
|
|
68
|
+
| V2EvalTemplatesUpdateConflictResponseBody
|
|
69
|
+
| HTTPValidationError;
|
|
70
|
+
|
|
71
|
+
export const V2EvalTemplatesUpdateResponse$zodSchema: z.ZodType<
|
|
72
|
+
V2EvalTemplatesUpdateResponse
|
|
73
|
+
> = z.union([
|
|
74
|
+
EvalTemplate$zodSchema,
|
|
75
|
+
z.lazy(() => V2EvalTemplatesUpdateBadRequestResponseBody$zodSchema),
|
|
76
|
+
z.lazy(() => V2EvalTemplatesUpdateNotFoundResponseBody$zodSchema),
|
|
77
|
+
z.lazy(() => V2EvalTemplatesUpdateConflictResponseBody$zodSchema),
|
|
78
|
+
HTTPValidationError$zodSchema,
|
|
79
|
+
]);
|
|
@@ -6,14 +6,16 @@
|
|
|
6
6
|
import * as z from "zod";
|
|
7
7
|
import { ClosedEnum } from "../types/enums.js";
|
|
8
8
|
|
|
9
|
-
export const
|
|
9
|
+
export const V2InlineMcpDefinitionType = {
|
|
10
10
|
Http: "http",
|
|
11
11
|
Sse: "sse",
|
|
12
12
|
Stdio: "stdio",
|
|
13
13
|
} as const;
|
|
14
|
-
export type
|
|
14
|
+
export type V2InlineMcpDefinitionType = ClosedEnum<
|
|
15
|
+
typeof V2InlineMcpDefinitionType
|
|
16
|
+
>;
|
|
15
17
|
|
|
16
|
-
export const
|
|
18
|
+
export const V2InlineMcpDefinitionType$zodSchema = z.enum([
|
|
17
19
|
"http",
|
|
18
20
|
"sse",
|
|
19
21
|
"stdio",
|
|
@@ -21,7 +23,7 @@ export const Type$zodSchema = z.enum([
|
|
|
21
23
|
|
|
22
24
|
export type V2InlineMcpDefinition = {
|
|
23
25
|
name: string;
|
|
24
|
-
type:
|
|
26
|
+
type: V2InlineMcpDefinitionType;
|
|
25
27
|
url?: string | null | undefined;
|
|
26
28
|
headers?: { [k: string]: string } | undefined;
|
|
27
29
|
command?: string | null | undefined;
|
|
@@ -44,7 +46,7 @@ export const V2InlineMcpDefinition$zodSchema: z.ZodType<V2InlineMcpDefinition> =
|
|
|
44
46
|
name: z.string().describe(
|
|
45
47
|
"Stable server name used in the selected agent's MCP config.",
|
|
46
48
|
),
|
|
47
|
-
type:
|
|
49
|
+
type: V2InlineMcpDefinitionType$zodSchema,
|
|
48
50
|
url: z.string().nullable().optional().describe(
|
|
49
51
|
"HTTPS endpoint for an `http` or `sse` MCP server.",
|
|
50
52
|
),
|
|
@@ -52,12 +52,18 @@ export type V2PromptRunDetail = {
|
|
|
52
52
|
response?: string | null | undefined;
|
|
53
53
|
mentions?: Array<V2PromptRunMention> | undefined;
|
|
54
54
|
citations?: Array<V2PromptRunCitation> | undefined;
|
|
55
|
+
tool_calls?: Array<{ [k: string]: any }> | undefined;
|
|
56
|
+
conversation_turns?: Array<{ [k: string]: any }> | undefined;
|
|
55
57
|
};
|
|
56
58
|
|
|
57
59
|
export const V2PromptRunDetail$zodSchema: z.ZodType<V2PromptRunDetail> = z
|
|
58
60
|
.object({
|
|
59
61
|
citation_count: z.int().default(0),
|
|
60
62
|
citations: z.array(V2PromptRunCitation$zodSchema).optional(),
|
|
63
|
+
conversation_turns: z.array(z.record(z.string(), z.any())).optional()
|
|
64
|
+
.describe(
|
|
65
|
+
"Sanitized interleaved text and tool-call trace captured for this run. It interleaves the agent's `text` turns with `tool_call` turns, so its length is not a tool-call count; use `tool_calls` or `tool_call_count`.",
|
|
66
|
+
),
|
|
61
67
|
created_at: z.string().nullable().optional(),
|
|
62
68
|
harness: z.string().nullable().optional().describe(
|
|
63
69
|
"Coding-agent harness for agent runs; otherwise the answer platform identifier.",
|
|
@@ -75,5 +81,10 @@ export const V2PromptRunDetail$zodSchema: z.ZodType<V2PromptRunDetail> = z
|
|
|
75
81
|
response_preview: z.string().nullable().optional(),
|
|
76
82
|
run_date: z.string(),
|
|
77
83
|
target_kind: V2PromptRunDetailTargetKind$zodSchema.nullable().optional(),
|
|
78
|
-
tool_call_count: z.int().nullable().optional()
|
|
84
|
+
tool_call_count: z.int().nullable().optional().describe(
|
|
85
|
+
"Coding-agent tool invocations, counting a call and its result once. Matches the length of `tool_calls` on the detail response; null for answer-engine runs.",
|
|
86
|
+
),
|
|
87
|
+
tool_calls: z.array(z.record(z.string(), z.any())).optional().describe(
|
|
88
|
+
"Sanitized coding-agent tool invocations captured for this run, one entry per invocation with its result merged in.",
|
|
89
|
+
),
|
|
79
90
|
});
|
|
@@ -61,5 +61,7 @@ export const V2PromptRunSummary$zodSchema: z.ZodType<V2PromptRunSummary> = z
|
|
|
61
61
|
response_preview: z.string().nullable().optional(),
|
|
62
62
|
run_date: z.string(),
|
|
63
63
|
target_kind: V2PromptRunSummaryTargetKind$zodSchema.nullable().optional(),
|
|
64
|
-
tool_call_count: z.int().nullable().optional()
|
|
64
|
+
tool_call_count: z.int().nullable().optional().describe(
|
|
65
|
+
"Coding-agent tool invocations, counting a call and its result once. Matches the length of `tool_calls` on the detail response; null for answer-engine runs.",
|
|
66
|
+
),
|
|
65
67
|
});
|