@lightsage/mcp 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/api/well-known-openai-apps-challenge.js +25 -0
- package/bin/mcp-server.js +660 -797
- package/bin/mcp-server.js.map +53 -57
- package/esm/funcs/evalsCancel.d.ts +16 -0
- package/esm/funcs/evalsCancel.d.ts.map +1 -0
- package/esm/funcs/{evalsAnalysis.js → evalsCancel.js} +10 -10
- package/esm/funcs/evalsCancel.js.map +1 -0
- package/esm/funcs/evalsGet.d.ts +16 -0
- package/esm/funcs/evalsGet.d.ts.map +1 -0
- package/esm/funcs/{evalRunsV2EvalRunsRetrieve.js → evalsGet.js} +5 -5
- package/esm/funcs/evalsGet.js.map +1 -0
- package/esm/funcs/evalsRun.d.ts +6 -4
- package/esm/funcs/evalsRun.d.ts.map +1 -1
- package/esm/funcs/evalsRun.js +6 -4
- package/esm/funcs/evalsRun.js.map +1 -1
- package/esm/funcs/evalsV2EvalsCreate.d.ts +1 -1
- package/esm/funcs/evalsV2EvalsCreate.js +1 -1
- package/esm/funcs/evalsV2EvalsUpdate.d.ts +1 -1
- package/esm/funcs/evalsV2EvalsUpdate.js +1 -1
- package/esm/funcs/promptsV2PromptRunsList.d.ts +1 -1
- package/esm/funcs/promptsV2PromptRunsList.js +1 -1
- package/esm/funcs/promptsV2PromptRunsRetrieve.d.ts +1 -1
- package/esm/funcs/promptsV2PromptRunsRetrieve.js +1 -1
- package/esm/landing-page.js +1 -1
- package/esm/lib/config.d.ts +3 -3
- package/esm/lib/config.js +3 -3
- package/esm/lib/config.js.map +1 -1
- package/esm/mcp-server/mcp-server.js +1 -1
- package/esm/mcp-server/server.d.ts.map +1 -1
- package/esm/mcp-server/server.js +5 -9
- package/esm/mcp-server/server.js.map +1 -1
- package/esm/mcp-server/tools/evalsCancel.d.ts +7 -0
- package/esm/mcp-server/tools/evalsCancel.d.ts.map +1 -0
- package/esm/mcp-server/tools/evalsCancel.js +35 -0
- package/esm/mcp-server/tools/evalsCancel.js.map +1 -0
- package/esm/mcp-server/tools/{evalRunsV2EvalRunsRetrieve.d.ts → evalsGet.d.ts} +2 -2
- package/esm/mcp-server/tools/evalsGet.d.ts.map +1 -0
- package/esm/mcp-server/tools/{evalRunsV2EvalRunsRetrieve.js → evalsGet.js} +8 -8
- package/esm/mcp-server/tools/evalsGet.js.map +1 -0
- package/esm/mcp-server/tools/evalsRun.d.ts.map +1 -1
- package/esm/mcp-server/tools/evalsRun.js +6 -4
- package/esm/mcp-server/tools/evalsRun.js.map +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsCreate.js +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsCreate.js.map +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsUpdate.js +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsUpdate.js.map +1 -1
- package/esm/mcp-server/tools/promptsV2PromptRunsList.js +1 -1
- package/esm/mcp-server/tools/promptsV2PromptRunsList.js.map +1 -1
- package/esm/mcp-server/tools/promptsV2PromptRunsRetrieve.js +1 -1
- package/esm/mcp-server/tools/promptsV2PromptRunsRetrieve.js.map +1 -1
- package/esm/models/createv2evalconfigurationrequest.d.ts +2 -0
- package/esm/models/createv2evalconfigurationrequest.d.ts.map +1 -1
- package/esm/models/createv2evalconfigurationrequest.js +2 -0
- package/esm/models/createv2evalconfigurationrequest.js.map +1 -1
- package/esm/models/createv2evalrequest.d.ts +5 -5
- package/esm/models/createv2evalrequest.d.ts.map +1 -1
- package/esm/models/createv2evalrequest.js +5 -10
- package/esm/models/createv2evalrequest.js.map +1 -1
- package/esm/models/createv2evalrunrequest.d.ts +5 -5
- package/esm/models/createv2evalrunrequest.d.ts.map +1 -1
- package/esm/models/createv2evalrunrequest.js +6 -10
- package/esm/models/createv2evalrunrequest.js.map +1 -1
- package/esm/models/publiccompetitorrow.js +2 -2
- package/esm/models/publiccompetitorrow.js.map +1 -1
- package/esm/models/publiccriterionverdict.d.ts +26 -0
- package/esm/models/publiccriterionverdict.d.ts.map +1 -0
- package/esm/models/publiccriterionverdict.js +24 -0
- package/esm/models/publiccriterionverdict.js.map +1 -0
- package/esm/models/publiccurrentvisibilityresponse.js +2 -2
- package/esm/models/publiccurrentvisibilityresponse.js.map +1 -1
- package/esm/models/publicevalrunenvironment.d.ts +15 -0
- package/esm/models/publicevalrunenvironment.d.ts.map +1 -0
- package/esm/models/publicevalrunenvironment.js +15 -0
- package/esm/models/publicevalrunenvironment.js.map +1 -0
- package/esm/models/publicevalrunprogress.d.ts +1 -0
- package/esm/models/publicevalrunprogress.d.ts.map +1 -1
- package/esm/models/publicevalrunprogress.js +3 -2
- package/esm/models/publicevalrunprogress.js.map +1 -1
- package/esm/models/publicgraderresult.d.ts +13 -0
- package/esm/models/publicgraderresult.d.ts.map +1 -0
- package/esm/models/publicgraderresult.js +18 -0
- package/esm/models/publicgraderresult.js.map +1 -0
- package/esm/models/publicpromptvisibilitymetrics.js +2 -2
- package/esm/models/publicpromptvisibilitymetrics.js.map +1 -1
- package/esm/models/publicvisibilitysnapshot.js +2 -2
- package/esm/models/publicvisibilitysnapshot.js.map +1 -1
- package/esm/models/searchgroundingmode.d.ts +20 -0
- package/esm/models/searchgroundingmode.d.ts.map +1 -0
- package/esm/models/searchgroundingmode.js +19 -0
- package/esm/models/searchgroundingmode.js.map +1 -0
- package/esm/models/updatev2evalconfigurationrequest.d.ts +2 -0
- package/esm/models/updatev2evalconfigurationrequest.d.ts.map +1 -1
- package/esm/models/updatev2evalconfigurationrequest.js +3 -0
- package/esm/models/updatev2evalconfigurationrequest.js.map +1 -1
- package/esm/models/updatev2evalrequest.d.ts +5 -5
- package/esm/models/updatev2evalrequest.d.ts.map +1 -1
- package/esm/models/updatev2evalrequest.js +6 -10
- package/esm/models/updatev2evalrequest.js.map +1 -1
- package/esm/models/v2eval.d.ts +6 -2
- package/esm/models/v2eval.d.ts.map +1 -1
- package/esm/models/v2eval.js +5 -1
- package/esm/models/v2eval.js.map +1 -1
- package/esm/models/v2evalconfiguration.d.ts +2 -0
- package/esm/models/v2evalconfiguration.d.ts.map +1 -1
- package/esm/models/v2evalconfiguration.js +2 -0
- package/esm/models/v2evalconfiguration.js.map +1 -1
- package/esm/models/v2evalresultsummary.d.ts +10 -0
- package/esm/models/v2evalresultsummary.d.ts.map +1 -1
- package/esm/models/v2evalresultsummary.js +12 -1
- package/esm/models/v2evalresultsummary.js.map +1 -1
- package/esm/models/v2evalrun.d.ts.map +1 -1
- package/esm/models/v2evalrun.js +1 -1
- package/esm/models/v2evalrun.js.map +1 -1
- package/esm/models/v2evalrundetail.d.ts +30 -0
- package/esm/models/v2evalrundetail.d.ts.map +1 -0
- package/esm/models/v2evalrundetail.js +33 -0
- package/esm/models/v2evalrundetail.js.map +1 -0
- package/esm/models/v2evalrunscancelop.d.ts +31 -0
- package/esm/models/v2evalrunscancelop.d.ts.map +1 -0
- package/esm/models/v2evalrunscancelop.js +27 -0
- package/esm/models/v2evalrunscancelop.js.map +1 -0
- package/esm/models/v2evalrunscreateop.d.ts +23 -3
- package/esm/models/v2evalrunscreateop.d.ts.map +1 -1
- package/esm/models/v2evalrunscreateop.js +15 -2
- package/esm/models/v2evalrunscreateop.js.map +1 -1
- package/esm/models/v2evalrunsretrieveop.d.ts +9 -2
- package/esm/models/v2evalrunsretrieveop.d.ts.map +1 -1
- package/esm/models/v2evalrunsretrieveop.js +6 -2
- package/esm/models/v2evalrunsretrieveop.js.map +1 -1
- package/esm/models/v2evalruntraceevent.d.ts.map +1 -1
- package/esm/models/v2evalruntraceevent.js +1 -1
- package/esm/models/v2evalruntraceevent.js.map +1 -1
- package/esm/models/v2promptrundetail.d.ts +6 -0
- package/esm/models/v2promptrundetail.d.ts.map +1 -1
- package/esm/models/v2promptrundetail.js +4 -1
- package/esm/models/v2promptrundetail.js.map +1 -1
- package/esm/models/v2promptrunsummary.d.ts.map +1 -1
- package/esm/models/v2promptrunsummary.js +1 -1
- package/esm/models/v2promptrunsummary.js.map +1 -1
- package/esm/models/v2publictask.d.ts +2 -2
- package/esm/models/v2publictask.d.ts.map +1 -1
- package/esm/models/v2publictask.js +4 -4
- package/esm/models/v2publictask.js.map +1 -1
- package/esm/tool-names.d.ts.map +1 -1
- package/esm/tool-names.js +11 -19
- package/esm/tool-names.js.map +1 -1
- package/package.json +1 -1
- package/src/funcs/{evalsAnalysis.ts → evalsCancel.ts} +13 -13
- package/src/funcs/{evalRunsV2EvalRunsRetrieve.ts → evalsGet.ts} +4 -4
- package/src/funcs/evalsRun.ts +6 -4
- package/src/funcs/evalsV2EvalsCreate.ts +1 -1
- package/src/funcs/evalsV2EvalsUpdate.ts +1 -1
- package/src/funcs/promptsV2PromptRunsList.ts +1 -1
- package/src/funcs/promptsV2PromptRunsRetrieve.ts +1 -1
- package/src/landing-page.ts +1 -1
- package/src/lib/config.ts +3 -3
- package/src/mcp-server/mcp-server.ts +1 -1
- package/src/mcp-server/server.ts +5 -9
- package/src/mcp-server/tools/evalsCancel.ts +43 -0
- package/src/mcp-server/tools/{evalRunsV2EvalRunsRetrieve.ts → evalsGet.ts} +7 -7
- package/src/mcp-server/tools/evalsRun.ts +6 -4
- package/src/mcp-server/tools/evalsV2EvalsCreate.ts +1 -1
- package/src/mcp-server/tools/evalsV2EvalsUpdate.ts +1 -1
- package/src/mcp-server/tools/promptsV2PromptRunsList.ts +1 -1
- package/src/mcp-server/tools/promptsV2PromptRunsRetrieve.ts +1 -1
- package/src/models/createv2evalconfigurationrequest.ts +8 -0
- package/src/models/createv2evalrequest.ts +18 -22
- package/src/models/createv2evalrunrequest.ts +19 -24
- package/src/models/publiccompetitorrow.ts +2 -2
- package/src/models/publiccriterionverdict.ts +48 -0
- package/src/models/publiccurrentvisibilityresponse.ts +2 -2
- package/src/models/publicevalrunenvironment.ts +47 -0
- package/src/models/publicevalrunprogress.ts +8 -2
- package/src/models/publicgraderresult.ts +44 -0
- package/src/models/publicpromptvisibilitymetrics.ts +2 -2
- package/src/models/publicvisibilitysnapshot.ts +2 -2
- package/src/models/searchgroundingmode.ts +26 -0
- package/src/models/updatev2evalconfigurationrequest.ts +9 -0
- package/src/models/updatev2evalrequest.ts +20 -21
- package/src/models/v2eval.ts +14 -2
- package/src/models/v2evalconfiguration.ts +8 -0
- package/src/models/v2evalresultsummary.ts +36 -1
- package/src/models/v2evalrun.ts +3 -1
- package/src/models/v2evalrundetail.ts +90 -0
- package/src/models/v2evalrunscancelop.ts +70 -0
- package/src/models/v2evalrunscreateop.ts +43 -5
- package/src/models/v2evalrunsretrieveop.ts +23 -3
- package/src/models/v2evalruntraceevent.ts +3 -1
- package/src/models/v2promptrundetail.ts +12 -1
- package/src/models/v2promptrunsummary.ts +3 -1
- package/src/models/v2publictask.ts +9 -7
- package/src/tool-names.ts +11 -19
- package/test/simulations.test.js +30 -3
- package/esm/funcs/evalResultsV2EvalResultsList.d.ts +0 -16
- package/esm/funcs/evalResultsV2EvalResultsList.d.ts.map +0 -1
- package/esm/funcs/evalResultsV2EvalResultsList.js +0 -87
- package/esm/funcs/evalResultsV2EvalResultsList.js.map +0 -1
- package/esm/funcs/evalResultsV2EvalResultsRetrieve.d.ts +0 -16
- package/esm/funcs/evalResultsV2EvalResultsRetrieve.d.ts.map +0 -1
- package/esm/funcs/evalResultsV2EvalResultsRetrieve.js +0 -88
- package/esm/funcs/evalResultsV2EvalResultsRetrieve.js.map +0 -1
- package/esm/funcs/evalRunsV2EvalRunsRetrieve.d.ts +0 -16
- package/esm/funcs/evalRunsV2EvalRunsRetrieve.d.ts.map +0 -1
- package/esm/funcs/evalRunsV2EvalRunsRetrieve.js.map +0 -1
- package/esm/funcs/evalsAnalysis.d.ts +0 -16
- package/esm/funcs/evalsAnalysis.d.ts.map +0 -1
- package/esm/funcs/evalsAnalysis.js.map +0 -1
- package/esm/mcp-server/tools/evalResultsV2EvalResultsList.d.ts +0 -7
- package/esm/mcp-server/tools/evalResultsV2EvalResultsList.d.ts.map +0 -1
- package/esm/mcp-server/tools/evalResultsV2EvalResultsList.js +0 -35
- package/esm/mcp-server/tools/evalResultsV2EvalResultsList.js.map +0 -1
- package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.d.ts +0 -7
- package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.d.ts.map +0 -1
- package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.js +0 -35
- package/esm/mcp-server/tools/evalResultsV2EvalResultsRetrieve.js.map +0 -1
- package/esm/mcp-server/tools/evalRunsV2EvalRunsRetrieve.d.ts.map +0 -1
- package/esm/mcp-server/tools/evalRunsV2EvalRunsRetrieve.js.map +0 -1
- package/esm/mcp-server/tools/evalsAnalysis.d.ts +0 -7
- package/esm/mcp-server/tools/evalsAnalysis.d.ts.map +0 -1
- package/esm/mcp-server/tools/evalsAnalysis.js +0 -35
- package/esm/mcp-server/tools/evalsAnalysis.js.map +0 -1
- package/esm/models/v2evalresult.d.ts +0 -23
- package/esm/models/v2evalresult.d.ts.map +0 -1
- package/esm/models/v2evalresult.js +0 -24
- package/esm/models/v2evalresult.js.map +0 -1
- package/esm/models/v2evalresultslistop.d.ts +0 -11
- package/esm/models/v2evalresultslistop.d.ts.map +0 -1
- package/esm/models/v2evalresultslistop.js +0 -16
- package/esm/models/v2evalresultslistop.js.map +0 -1
- package/esm/models/v2evalresultsresponse.d.ts +0 -9
- package/esm/models/v2evalresultsresponse.d.ts.map +0 -1
- package/esm/models/v2evalresultsresponse.js +0 -12
- package/esm/models/v2evalresultsresponse.js.map +0 -1
- package/esm/models/v2evalresultsretrieveop.d.ts +0 -10
- package/esm/models/v2evalresultsretrieveop.d.ts.map +0 -1
- package/esm/models/v2evalresultsretrieveop.js +0 -15
- package/esm/models/v2evalresultsretrieveop.js.map +0 -1
- package/esm/models/v2evalrunanalysisresponse.d.ts +0 -12
- package/esm/models/v2evalrunanalysisresponse.d.ts.map +0 -1
- package/esm/models/v2evalrunanalysisresponse.js +0 -15
- package/esm/models/v2evalrunanalysisresponse.js.map +0 -1
- package/esm/models/v2evalrunsanalysisop.d.ts +0 -17
- package/esm/models/v2evalrunsanalysisop.d.ts.map +0 -1
- package/esm/models/v2evalrunsanalysisop.js +0 -19
- package/esm/models/v2evalrunsanalysisop.js.map +0 -1
- package/src/funcs/evalResultsV2EvalResultsList.ts +0 -149
- package/src/funcs/evalResultsV2EvalResultsRetrieve.ts +0 -153
- package/src/mcp-server/tools/evalResultsV2EvalResultsList.ts +0 -43
- package/src/mcp-server/tools/evalResultsV2EvalResultsRetrieve.ts +0 -45
- package/src/mcp-server/tools/evalsAnalysis.ts +0 -43
- package/src/models/v2evalresult.ts +0 -46
- package/src/models/v2evalresultslistop.ts +0 -39
- package/src/models/v2evalresultsresponse.ts +0 -23
- package/src/models/v2evalresultsretrieveop.ts +0 -28
- package/src/models/v2evalrunanalysisresponse.ts +0 -32
- package/src/models/v2evalrunsanalysisop.ts +0 -48
|
@@ -4,6 +4,10 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
SearchGroundingMode,
|
|
9
|
+
SearchGroundingMode$zodSchema,
|
|
10
|
+
} from "./searchgroundingmode.js";
|
|
7
11
|
|
|
8
12
|
export type UpdateV2EvalConfigurationRequest = {
|
|
9
13
|
name?: string | undefined;
|
|
@@ -14,6 +18,7 @@ export type UpdateV2EvalConfigurationRequest = {
|
|
|
14
18
|
skill_ids?: Array<string> | null | undefined;
|
|
15
19
|
skills?: Array<string> | null | undefined;
|
|
16
20
|
include_docs?: boolean | undefined;
|
|
21
|
+
search_grounding?: SearchGroundingMode | null | undefined;
|
|
17
22
|
};
|
|
18
23
|
|
|
19
24
|
export const UpdateV2EvalConfigurationRequest$zodSchema: z.ZodType<
|
|
@@ -25,6 +30,10 @@ export const UpdateV2EvalConfigurationRequest$zodSchema: z.ZodType<
|
|
|
25
30
|
mcp_server_ids: z.array(z.string()).nullable().optional(),
|
|
26
31
|
name: z.string().optional(),
|
|
27
32
|
repository_id: z.string().optional(),
|
|
33
|
+
search_grounding: SearchGroundingMode$zodSchema.nullable().optional()
|
|
34
|
+
.describe(
|
|
35
|
+
"Replacement web-retrieval policy. Omit it to keep the saved value.",
|
|
36
|
+
),
|
|
28
37
|
skill_ids: z.array(z.string()).nullable().optional(),
|
|
29
38
|
skills: z.array(z.string()).nullable().optional().describe(
|
|
30
39
|
"Public skill packages saved with this configuration, using GitHub `owner/repository` references.",
|
|
@@ -5,31 +5,18 @@
|
|
|
5
5
|
|
|
6
6
|
import * as z from "zod";
|
|
7
7
|
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
} from "./
|
|
8
|
+
SearchGroundingMode,
|
|
9
|
+
SearchGroundingMode$zodSchema,
|
|
10
|
+
} from "./searchgroundingmode.js";
|
|
11
11
|
import {
|
|
12
12
|
V2InlineMcpDefinition,
|
|
13
13
|
V2InlineMcpDefinition$zodSchema,
|
|
14
14
|
} from "./v2inlinemcpdefinition.js";
|
|
15
|
-
import {
|
|
16
|
-
V2SavedCliReference,
|
|
17
|
-
V2SavedCliReference$zodSchema,
|
|
18
|
-
} from "./v2savedclireference.js";
|
|
19
15
|
import {
|
|
20
16
|
V2SavedMcpReference,
|
|
21
17
|
V2SavedMcpReference$zodSchema,
|
|
22
18
|
} from "./v2savedmcpreference.js";
|
|
23
19
|
|
|
24
|
-
export type UpdateV2EvalRequestCli = V2SavedCliReference | V2InlineCliReference;
|
|
25
|
-
|
|
26
|
-
export const UpdateV2EvalRequestCli$zodSchema: z.ZodType<
|
|
27
|
-
UpdateV2EvalRequestCli
|
|
28
|
-
> = z.union([
|
|
29
|
-
V2SavedCliReference$zodSchema,
|
|
30
|
-
V2InlineCliReference$zodSchema,
|
|
31
|
-
]);
|
|
32
|
-
|
|
33
20
|
export type UpdateV2EvalRequestMcp =
|
|
34
21
|
| V2InlineMcpDefinition
|
|
35
22
|
| V2SavedMcpReference;
|
|
@@ -47,8 +34,11 @@ export type UpdateV2EvalRequest = {
|
|
|
47
34
|
judges?: Array<string> | null | undefined;
|
|
48
35
|
agent?: Array<string> | null | undefined;
|
|
49
36
|
persona?: string | undefined;
|
|
50
|
-
clis?: Array<
|
|
37
|
+
clis?: Array<string> | null | undefined;
|
|
51
38
|
mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | null | undefined;
|
|
39
|
+
setup_commands?: Array<string> | null | undefined;
|
|
40
|
+
cleanup_commands?: Array<string> | null | undefined;
|
|
41
|
+
search_grounding?: SearchGroundingMode | null | undefined;
|
|
52
42
|
runs?: number | undefined;
|
|
53
43
|
tags?: Array<string> | null | undefined;
|
|
54
44
|
};
|
|
@@ -56,10 +46,12 @@ export type UpdateV2EvalRequest = {
|
|
|
56
46
|
export const UpdateV2EvalRequest$zodSchema: z.ZodType<UpdateV2EvalRequest> = z
|
|
57
47
|
.object({
|
|
58
48
|
agent: z.array(z.string()).nullable().optional(),
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
49
|
+
cleanup_commands: z.array(z.string()).nullable().optional().describe(
|
|
50
|
+
"Replacement cleanup commands. Pass an empty array to clear all saved cleanup commands.",
|
|
51
|
+
),
|
|
52
|
+
clis: z.array(z.string()).nullable().optional().describe(
|
|
53
|
+
"Replacement saved CLI IDs or shell installation commands. UUID values resolve saved CLIs; every other value is an installation command. Pass an empty array to clear all CLIs.",
|
|
54
|
+
),
|
|
63
55
|
judges: z.array(z.string()).nullable().optional().describe(
|
|
64
56
|
"Replacement success criteria for the eval.",
|
|
65
57
|
),
|
|
@@ -73,6 +65,13 @@ export const UpdateV2EvalRequest$zodSchema: z.ZodType<UpdateV2EvalRequest> = z
|
|
|
73
65
|
),
|
|
74
66
|
prompts: z.array(z.string()).nullable().optional(),
|
|
75
67
|
runs: z.int().optional(),
|
|
68
|
+
search_grounding: SearchGroundingMode$zodSchema.nullable().optional()
|
|
69
|
+
.describe(
|
|
70
|
+
"Replacement web-retrieval policy. Omit it to keep the saved value.",
|
|
71
|
+
),
|
|
72
|
+
setup_commands: z.array(z.string()).nullable().optional().describe(
|
|
73
|
+
"Replacement setup commands. Pass an empty array to clear all saved setup commands.",
|
|
74
|
+
),
|
|
76
75
|
tags: z.array(z.string()).nullable().optional().describe(
|
|
77
76
|
"Replacement labels for the eval. Pass an empty array to clear all tags.",
|
|
78
77
|
),
|
package/src/models/v2eval.ts
CHANGED
|
@@ -4,6 +4,10 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
SearchGroundingMode,
|
|
9
|
+
SearchGroundingMode$zodSchema,
|
|
10
|
+
} from "./searchgroundingmode.js";
|
|
7
11
|
import {
|
|
8
12
|
V2InlineCliReference,
|
|
9
13
|
V2InlineCliReference$zodSchema,
|
|
@@ -21,9 +25,9 @@ import {
|
|
|
21
25
|
V2SavedMcpReference$zodSchema,
|
|
22
26
|
} from "./v2savedmcpreference.js";
|
|
23
27
|
|
|
24
|
-
export type
|
|
28
|
+
export type Cli = V2SavedCliReference | V2InlineCliReference;
|
|
25
29
|
|
|
26
|
-
export const
|
|
30
|
+
export const Cli$zodSchema: z.ZodType<Cli> = z.union([
|
|
27
31
|
V2SavedCliReference$zodSchema,
|
|
28
32
|
V2InlineCliReference$zodSchema,
|
|
29
33
|
]);
|
|
@@ -44,6 +48,9 @@ export type V2Eval = {
|
|
|
44
48
|
persona?: string | null | undefined;
|
|
45
49
|
clis?: Array<V2SavedCliReference | V2InlineCliReference> | undefined;
|
|
46
50
|
mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | undefined;
|
|
51
|
+
setup_commands?: Array<string> | undefined;
|
|
52
|
+
cleanup_commands?: Array<string> | undefined;
|
|
53
|
+
search_grounding?: SearchGroundingMode | undefined;
|
|
47
54
|
runs?: number | undefined;
|
|
48
55
|
tags?: Array<string> | undefined;
|
|
49
56
|
created_at?: string | null | undefined;
|
|
@@ -52,6 +59,7 @@ export type V2Eval = {
|
|
|
52
59
|
|
|
53
60
|
export const V2Eval$zodSchema: z.ZodType<V2Eval> = z.object({
|
|
54
61
|
agent: z.array(z.string()).optional(),
|
|
62
|
+
cleanup_commands: z.array(z.string()).optional(),
|
|
55
63
|
clis: z.array(z.union([
|
|
56
64
|
V2SavedCliReference$zodSchema,
|
|
57
65
|
V2InlineCliReference$zodSchema,
|
|
@@ -67,6 +75,10 @@ export const V2Eval$zodSchema: z.ZodType<V2Eval> = z.object({
|
|
|
67
75
|
persona: z.string().nullable().optional(),
|
|
68
76
|
prompts: z.array(z.string()).optional(),
|
|
69
77
|
runs: z.int().default(1),
|
|
78
|
+
search_grounding: SearchGroundingMode$zodSchema.optional().describe(
|
|
79
|
+
"Requested web-retrieval behavior for an eval run.",
|
|
80
|
+
),
|
|
81
|
+
setup_commands: z.array(z.string()).optional(),
|
|
70
82
|
tags: z.array(z.string()).optional(),
|
|
71
83
|
updated_at: z.string().nullable().optional(),
|
|
72
84
|
});
|
|
@@ -4,6 +4,10 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
SearchGroundingMode,
|
|
9
|
+
SearchGroundingMode$zodSchema,
|
|
10
|
+
} from "./searchgroundingmode.js";
|
|
7
11
|
|
|
8
12
|
export type V2EvalConfiguration = {
|
|
9
13
|
id: string;
|
|
@@ -16,6 +20,7 @@ export type V2EvalConfiguration = {
|
|
|
16
20
|
skill_ids?: Array<string> | undefined;
|
|
17
21
|
skills?: Array<string> | undefined;
|
|
18
22
|
include_docs?: boolean | undefined;
|
|
23
|
+
search_grounding?: SearchGroundingMode | undefined;
|
|
19
24
|
created_at?: string | null | undefined;
|
|
20
25
|
updated_at?: string | null | undefined;
|
|
21
26
|
};
|
|
@@ -31,6 +36,9 @@ export const V2EvalConfiguration$zodSchema: z.ZodType<V2EvalConfiguration> = z
|
|
|
31
36
|
mcp_server_ids: z.array(z.string()).optional(),
|
|
32
37
|
name: z.string(),
|
|
33
38
|
repository_id: z.string().nullable().optional(),
|
|
39
|
+
search_grounding: SearchGroundingMode$zodSchema.optional().describe(
|
|
40
|
+
"Requested web-retrieval behavior for an eval run.",
|
|
41
|
+
),
|
|
34
42
|
skill_ids: z.array(z.string()).optional(),
|
|
35
43
|
skills: z.array(z.string()).optional().describe(
|
|
36
44
|
"Public GitHub `owner/repository` skill packages installed for this configuration.",
|
|
@@ -4,6 +4,18 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
PublicEvalRunEnvironment,
|
|
9
|
+
PublicEvalRunEnvironment$zodSchema,
|
|
10
|
+
} from "./publicevalrunenvironment.js";
|
|
11
|
+
import {
|
|
12
|
+
PublicGraderResult,
|
|
13
|
+
PublicGraderResult$zodSchema,
|
|
14
|
+
} from "./publicgraderresult.js";
|
|
15
|
+
import {
|
|
16
|
+
SearchGroundingMode,
|
|
17
|
+
SearchGroundingMode$zodSchema,
|
|
18
|
+
} from "./searchgroundingmode.js";
|
|
7
19
|
|
|
8
20
|
export type V2EvalResultSummary = {
|
|
9
21
|
id: string;
|
|
@@ -15,8 +27,15 @@ export type V2EvalResultSummary = {
|
|
|
15
27
|
passed?: boolean | null | undefined;
|
|
16
28
|
score?: number | null | undefined;
|
|
17
29
|
failure_reasons?: Array<string> | null | undefined;
|
|
30
|
+
grader_results?: Array<PublicGraderResult> | undefined;
|
|
18
31
|
latency_ms?: number | null | undefined;
|
|
32
|
+
search_grounding?: SearchGroundingMode | null | undefined;
|
|
33
|
+
web_retrieval_observed?: boolean | null | undefined;
|
|
34
|
+
search_observed?: boolean | null | undefined;
|
|
35
|
+
grounding_qualified?: boolean | null | undefined;
|
|
36
|
+
grounding_qualification_reason?: string | null | undefined;
|
|
19
37
|
created_at?: string | null | undefined;
|
|
38
|
+
environment?: PublicEvalRunEnvironment | null | undefined;
|
|
20
39
|
};
|
|
21
40
|
|
|
22
41
|
export const V2EvalResultSummary$zodSchema: z.ZodType<V2EvalResultSummary> = z
|
|
@@ -25,8 +44,21 @@ export const V2EvalResultSummary$zodSchema: z.ZodType<V2EvalResultSummary> = z
|
|
|
25
44
|
"Runnable agent combining the harness and model.",
|
|
26
45
|
),
|
|
27
46
|
created_at: z.string().nullable().optional(),
|
|
47
|
+
environment: PublicEvalRunEnvironment$zodSchema.nullable().optional()
|
|
48
|
+
.describe(
|
|
49
|
+
"The environment this attempt used, without secret values. Null when the attempt had no environment.",
|
|
50
|
+
),
|
|
28
51
|
eval_id: z.string().nullable().optional(),
|
|
29
|
-
failure_reasons: z.array(z.string()).nullable().optional()
|
|
52
|
+
failure_reasons: z.array(z.string()).nullable().optional().describe(
|
|
53
|
+
"Checks that failed, listed even when the result passed overall. Deductions from checks that passed are in `grader_results[].deductions`, not here.",
|
|
54
|
+
),
|
|
55
|
+
grader_results: z.array(PublicGraderResult$zodSchema).optional().describe(
|
|
56
|
+
"Per-check verdicts behind `score`, including deductions recorded by passing checks.",
|
|
57
|
+
),
|
|
58
|
+
grounding_qualification_reason: z.string().nullable().optional(),
|
|
59
|
+
grounding_qualified: z.boolean().nullable().optional().describe(
|
|
60
|
+
"Whether the attempt satisfied the requested grounding experiment. Null means the default mode was used or isolation could not be proven.",
|
|
61
|
+
),
|
|
30
62
|
id: z.string().describe("Result ID, not an eval job ID."),
|
|
31
63
|
latency_ms: z.int().nullable().optional(),
|
|
32
64
|
passed: z.boolean().nullable().optional(),
|
|
@@ -34,4 +66,7 @@ export const V2EvalResultSummary$zodSchema: z.ZodType<V2EvalResultSummary> = z
|
|
|
34
66
|
prompt_index: z.int().default(0),
|
|
35
67
|
repeat_index: z.int().default(0),
|
|
36
68
|
score: z.number().nullable().optional().describe("Score from 0 to 1."),
|
|
69
|
+
search_grounding: SearchGroundingMode$zodSchema.nullable().optional(),
|
|
70
|
+
search_observed: z.boolean().nullable().optional(),
|
|
71
|
+
web_retrieval_observed: z.boolean().nullable().optional(),
|
|
37
72
|
});
|
package/src/models/v2evalrun.ts
CHANGED
|
@@ -64,5 +64,7 @@ export const V2EvalRun$zodSchema: z.ZodType<V2EvalRun> = z.object({
|
|
|
64
64
|
),
|
|
65
65
|
stage: z.string().nullable().optional(),
|
|
66
66
|
started_at: z.string().nullable().optional(),
|
|
67
|
-
status: z.string()
|
|
67
|
+
status: z.string().describe(
|
|
68
|
+
"Terminal values: `completed`, `failed`, and `cancelled`. Recoverable interruptions are reported as `running` with stage `retrying`.",
|
|
69
|
+
),
|
|
68
70
|
});
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: a4eb73f4b2a9
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
PublicEvalRunProgress,
|
|
9
|
+
PublicEvalRunProgress$zodSchema,
|
|
10
|
+
} from "./publicevalrunprogress.js";
|
|
11
|
+
import {
|
|
12
|
+
V2EvalResultSummary,
|
|
13
|
+
V2EvalResultSummary$zodSchema,
|
|
14
|
+
} from "./v2evalresultsummary.js";
|
|
15
|
+
import {
|
|
16
|
+
V2EvalRunAnalysis,
|
|
17
|
+
V2EvalRunAnalysis$zodSchema,
|
|
18
|
+
} from "./v2evalrunanalysis.js";
|
|
19
|
+
|
|
20
|
+
export type V2EvalRunDetail = {
|
|
21
|
+
id: string;
|
|
22
|
+
job_id: string;
|
|
23
|
+
eval_id?: string | null | undefined;
|
|
24
|
+
eval_name?: string | null | undefined;
|
|
25
|
+
configuration_id?: string | null | undefined;
|
|
26
|
+
persona?: string | null | undefined;
|
|
27
|
+
agent?: Array<string> | undefined;
|
|
28
|
+
prompt_count?: number | undefined;
|
|
29
|
+
runs?: number | undefined;
|
|
30
|
+
attempt_count?: number | undefined;
|
|
31
|
+
status: string;
|
|
32
|
+
stage?: string | null | undefined;
|
|
33
|
+
dispatch_id?: string | null | undefined;
|
|
34
|
+
progress?: PublicEvalRunProgress | undefined;
|
|
35
|
+
estimated_cost?: number | null | undefined;
|
|
36
|
+
error?: string | null | undefined;
|
|
37
|
+
created_at?: string | null | undefined;
|
|
38
|
+
started_at?: string | null | undefined;
|
|
39
|
+
completed_at?: string | null | undefined;
|
|
40
|
+
results?: Array<V2EvalResultSummary> | undefined;
|
|
41
|
+
analysis?: Array<V2EvalRunAnalysis> | undefined;
|
|
42
|
+
results_truncated?: boolean | undefined;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
export const V2EvalRunDetail$zodSchema: z.ZodType<V2EvalRunDetail> = z.object({
|
|
46
|
+
agent: z.array(z.string()).optional().describe(
|
|
47
|
+
"Runnable agents selected for this simulation.",
|
|
48
|
+
),
|
|
49
|
+
analysis: z.array(V2EvalRunAnalysis$zodSchema).optional().describe(
|
|
50
|
+
"Per-attempt analysis for this run. Empty until an attempt has been analyzed.",
|
|
51
|
+
),
|
|
52
|
+
attempt_count: z.int().default(1).describe(
|
|
53
|
+
"Total concrete attempts after all fan-out dimensions.",
|
|
54
|
+
),
|
|
55
|
+
completed_at: z.string().nullable().optional(),
|
|
56
|
+
configuration_id: z.string().nullable().optional(),
|
|
57
|
+
created_at: z.string().nullable().optional(),
|
|
58
|
+
dispatch_id: z.string().nullable().optional(),
|
|
59
|
+
error: z.string().nullable().optional(),
|
|
60
|
+
estimated_cost: z.number().nullable().optional(),
|
|
61
|
+
eval_id: z.string().nullable().optional().describe(
|
|
62
|
+
"Saved eval definition ID, including one created by a direct run.",
|
|
63
|
+
),
|
|
64
|
+
eval_name: z.string().nullable().optional().describe(
|
|
65
|
+
"Display name captured for the eval when the run was created.",
|
|
66
|
+
),
|
|
67
|
+
id: z.string().describe("Run ID to use when polling eval-run status."),
|
|
68
|
+
job_id: z.string().describe("Alias of `id` for the durable parent job."),
|
|
69
|
+
persona: z.string().nullable().optional().describe(
|
|
70
|
+
"Perspective applied to every attempt in this simulation.",
|
|
71
|
+
),
|
|
72
|
+
progress: PublicEvalRunProgress$zodSchema.optional(),
|
|
73
|
+
prompt_count: z.int().default(1).describe(
|
|
74
|
+
"Number of prompts included in the simulation.",
|
|
75
|
+
),
|
|
76
|
+
results: z.array(V2EvalResultSummary$zodSchema).optional().describe(
|
|
77
|
+
"Per-attempt outcomes for this run. Empty while the run is active; populated as attempts finish.",
|
|
78
|
+
),
|
|
79
|
+
results_truncated: z.boolean().default(false).describe(
|
|
80
|
+
"True when the run produced more results than the inline cap. The remaining results are not included in this response.",
|
|
81
|
+
),
|
|
82
|
+
runs: z.int().default(1).describe(
|
|
83
|
+
"Independent attempts requested for every prompt-agent pair.",
|
|
84
|
+
),
|
|
85
|
+
stage: z.string().nullable().optional(),
|
|
86
|
+
started_at: z.string().nullable().optional(),
|
|
87
|
+
status: z.string().describe(
|
|
88
|
+
"Terminal values: `completed`, `failed`, and `cancelled`. Recoverable interruptions are reported as `running` with stage `retrying`.",
|
|
89
|
+
),
|
|
90
|
+
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 15553511dcfe
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
HTTPValidationError,
|
|
9
|
+
HTTPValidationError$zodSchema,
|
|
10
|
+
} from "./httpvalidationerror.js";
|
|
11
|
+
import { V2EvalRun, V2EvalRun$zodSchema } from "./v2evalrun.js";
|
|
12
|
+
|
|
13
|
+
export type V2EvalRunsCancelRequest = { run_id: string };
|
|
14
|
+
|
|
15
|
+
export const V2EvalRunsCancelRequest$zodSchema: z.ZodType<
|
|
16
|
+
V2EvalRunsCancelRequest
|
|
17
|
+
> = z.object({
|
|
18
|
+
run_id: z.string(),
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Eval run cancellation is temporarily unavailable.
|
|
23
|
+
*/
|
|
24
|
+
export type V2EvalRunsCancelServiceUnavailableResponseBody = { detail: string };
|
|
25
|
+
|
|
26
|
+
export const V2EvalRunsCancelServiceUnavailableResponseBody$zodSchema:
|
|
27
|
+
z.ZodType<V2EvalRunsCancelServiceUnavailableResponseBody> = z.object({
|
|
28
|
+
detail: z.string(),
|
|
29
|
+
}).describe("Eval run cancellation is temporarily unavailable.");
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* The eval run has already completed or failed.
|
|
33
|
+
*/
|
|
34
|
+
export type V2EvalRunsCancelConflictResponseBody = { detail: string };
|
|
35
|
+
|
|
36
|
+
export const V2EvalRunsCancelConflictResponseBody$zodSchema: z.ZodType<
|
|
37
|
+
V2EvalRunsCancelConflictResponseBody
|
|
38
|
+
> = z.object({
|
|
39
|
+
detail: z.string(),
|
|
40
|
+
}).describe("The eval run has already completed or failed.");
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Eval run not found. Use the run ID returned when you started the eval.
|
|
44
|
+
*/
|
|
45
|
+
export type V2EvalRunsCancelNotFoundResponseBody = { detail: string };
|
|
46
|
+
|
|
47
|
+
export const V2EvalRunsCancelNotFoundResponseBody$zodSchema: z.ZodType<
|
|
48
|
+
V2EvalRunsCancelNotFoundResponseBody
|
|
49
|
+
> = z.object({
|
|
50
|
+
detail: z.string(),
|
|
51
|
+
}).describe(
|
|
52
|
+
"Eval run not found. Use the run ID returned when you started the eval.",
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
export type V2EvalRunsCancelResponse =
|
|
56
|
+
| V2EvalRun
|
|
57
|
+
| V2EvalRunsCancelNotFoundResponseBody
|
|
58
|
+
| V2EvalRunsCancelConflictResponseBody
|
|
59
|
+
| V2EvalRunsCancelServiceUnavailableResponseBody
|
|
60
|
+
| HTTPValidationError;
|
|
61
|
+
|
|
62
|
+
export const V2EvalRunsCancelResponse$zodSchema: z.ZodType<
|
|
63
|
+
V2EvalRunsCancelResponse
|
|
64
|
+
> = z.union([
|
|
65
|
+
V2EvalRun$zodSchema,
|
|
66
|
+
z.lazy(() => V2EvalRunsCancelNotFoundResponseBody$zodSchema),
|
|
67
|
+
z.lazy(() => V2EvalRunsCancelConflictResponseBody$zodSchema),
|
|
68
|
+
z.lazy(() => V2EvalRunsCancelServiceUnavailableResponseBody$zodSchema),
|
|
69
|
+
HTTPValidationError$zodSchema,
|
|
70
|
+
]);
|
|
@@ -13,25 +13,63 @@ import { V2EvalRun, V2EvalRun$zodSchema } from "./v2evalrun.js";
|
|
|
13
13
|
/**
|
|
14
14
|
* The requested coding agent is unavailable on the organization's current plan.
|
|
15
15
|
*/
|
|
16
|
-
export type
|
|
16
|
+
export type V2EvalRunsCreateForbiddenResponseBody = { detail: string };
|
|
17
17
|
|
|
18
|
-
export const
|
|
19
|
-
|
|
18
|
+
export const V2EvalRunsCreateForbiddenResponseBody$zodSchema: z.ZodType<
|
|
19
|
+
V2EvalRunsCreateForbiddenResponseBody
|
|
20
20
|
> = z.object({
|
|
21
21
|
detail: z.string(),
|
|
22
22
|
}).describe(
|
|
23
23
|
"The requested coding agent is unavailable on the organization's current plan.",
|
|
24
24
|
);
|
|
25
25
|
|
|
26
|
+
export type Detail = { error: string; required: number; balance: number };
|
|
27
|
+
|
|
28
|
+
export const Detail$zodSchema: z.ZodType<Detail> = z.object({
|
|
29
|
+
balance: z.number().describe("The organization's current credit balance."),
|
|
30
|
+
error: z.string(),
|
|
31
|
+
required: z.number().describe("Usage-based credits the requested run needs."),
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* The organization does not currently have enough credits for the requested run.
|
|
36
|
+
*/
|
|
37
|
+
export type V2EvalRunsCreatePaymentRequiredResponseBody = { detail: Detail };
|
|
38
|
+
|
|
39
|
+
export const V2EvalRunsCreatePaymentRequiredResponseBody$zodSchema: z.ZodType<
|
|
40
|
+
V2EvalRunsCreatePaymentRequiredResponseBody
|
|
41
|
+
> = z.object({
|
|
42
|
+
detail: z.lazy(() => Detail$zodSchema),
|
|
43
|
+
}).describe(
|
|
44
|
+
"The organization does not currently have enough credits for the requested run.",
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* The saved/direct mode fields, referenced resource, or environment credentials are invalid.
|
|
49
|
+
*/
|
|
50
|
+
export type V2EvalRunsCreateBadRequestResponseBody = { detail: string };
|
|
51
|
+
|
|
52
|
+
export const V2EvalRunsCreateBadRequestResponseBody$zodSchema: z.ZodType<
|
|
53
|
+
V2EvalRunsCreateBadRequestResponseBody
|
|
54
|
+
> = z.object({
|
|
55
|
+
detail: z.string(),
|
|
56
|
+
}).describe(
|
|
57
|
+
"The saved/direct mode fields, referenced resource, or environment credentials are invalid.",
|
|
58
|
+
);
|
|
59
|
+
|
|
26
60
|
export type V2EvalRunsCreateResponse =
|
|
27
61
|
| V2EvalRun
|
|
28
|
-
|
|
|
62
|
+
| V2EvalRunsCreateBadRequestResponseBody
|
|
63
|
+
| V2EvalRunsCreatePaymentRequiredResponseBody
|
|
64
|
+
| V2EvalRunsCreateForbiddenResponseBody
|
|
29
65
|
| HTTPValidationError;
|
|
30
66
|
|
|
31
67
|
export const V2EvalRunsCreateResponse$zodSchema: z.ZodType<
|
|
32
68
|
V2EvalRunsCreateResponse
|
|
33
69
|
> = z.union([
|
|
34
70
|
V2EvalRun$zodSchema,
|
|
35
|
-
z.lazy(() =>
|
|
71
|
+
z.lazy(() => V2EvalRunsCreateBadRequestResponseBody$zodSchema),
|
|
72
|
+
z.lazy(() => V2EvalRunsCreatePaymentRequiredResponseBody$zodSchema),
|
|
73
|
+
z.lazy(() => V2EvalRunsCreateForbiddenResponseBody$zodSchema),
|
|
36
74
|
HTTPValidationError$zodSchema,
|
|
37
75
|
]);
|
|
@@ -8,7 +8,10 @@ import {
|
|
|
8
8
|
HTTPValidationError,
|
|
9
9
|
HTTPValidationError$zodSchema,
|
|
10
10
|
} from "./httpvalidationerror.js";
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
V2EvalRunDetail,
|
|
13
|
+
V2EvalRunDetail$zodSchema,
|
|
14
|
+
} from "./v2evalrundetail.js";
|
|
12
15
|
|
|
13
16
|
export type V2EvalRunsRetrieveRequest = { run_id: string };
|
|
14
17
|
|
|
@@ -18,11 +21,28 @@ export const V2EvalRunsRetrieveRequest$zodSchema: z.ZodType<
|
|
|
18
21
|
run_id: z.string(),
|
|
19
22
|
});
|
|
20
23
|
|
|
21
|
-
|
|
24
|
+
/**
|
|
25
|
+
* Eval run not found. Use the run ID returned when you started the eval.
|
|
26
|
+
*/
|
|
27
|
+
export type V2EvalRunsRetrieveResponseBody = { detail: string };
|
|
28
|
+
|
|
29
|
+
export const V2EvalRunsRetrieveResponseBody$zodSchema: z.ZodType<
|
|
30
|
+
V2EvalRunsRetrieveResponseBody
|
|
31
|
+
> = z.object({
|
|
32
|
+
detail: z.string(),
|
|
33
|
+
}).describe(
|
|
34
|
+
"Eval run not found. Use the run ID returned when you started the eval.",
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
export type V2EvalRunsRetrieveResponse =
|
|
38
|
+
| V2EvalRunDetail
|
|
39
|
+
| V2EvalRunsRetrieveResponseBody
|
|
40
|
+
| HTTPValidationError;
|
|
22
41
|
|
|
23
42
|
export const V2EvalRunsRetrieveResponse$zodSchema: z.ZodType<
|
|
24
43
|
V2EvalRunsRetrieveResponse
|
|
25
44
|
> = z.union([
|
|
26
|
-
|
|
45
|
+
V2EvalRunDetail$zodSchema,
|
|
46
|
+
z.lazy(() => V2EvalRunsRetrieveResponseBody$zodSchema),
|
|
27
47
|
HTTPValidationError$zodSchema,
|
|
28
48
|
]);
|
|
@@ -41,5 +41,7 @@ export const V2EvalRunTraceEvent$zodSchema: z.ZodType<V2EvalRunTraceEvent> = z
|
|
|
41
41
|
repeat_index: z.int().default(0),
|
|
42
42
|
run_id: z.string(),
|
|
43
43
|
timestamp: z.string().nullable().optional(),
|
|
44
|
-
type: z.string()
|
|
44
|
+
type: z.string().describe(
|
|
45
|
+
"Event type. Events are not tool invocations: one invocation usually emits a `tool_call` event and a `tool_result` event, and can emit extra `tool_call` updates as its arguments stream. Count invocations with the result's `tool_calls`, not with trace events.",
|
|
46
|
+
),
|
|
45
47
|
});
|
|
@@ -52,12 +52,18 @@ export type V2PromptRunDetail = {
|
|
|
52
52
|
response?: string | null | undefined;
|
|
53
53
|
mentions?: Array<V2PromptRunMention> | undefined;
|
|
54
54
|
citations?: Array<V2PromptRunCitation> | undefined;
|
|
55
|
+
tool_calls?: Array<{ [k: string]: any }> | undefined;
|
|
56
|
+
conversation_turns?: Array<{ [k: string]: any }> | undefined;
|
|
55
57
|
};
|
|
56
58
|
|
|
57
59
|
export const V2PromptRunDetail$zodSchema: z.ZodType<V2PromptRunDetail> = z
|
|
58
60
|
.object({
|
|
59
61
|
citation_count: z.int().default(0),
|
|
60
62
|
citations: z.array(V2PromptRunCitation$zodSchema).optional(),
|
|
63
|
+
conversation_turns: z.array(z.record(z.string(), z.any())).optional()
|
|
64
|
+
.describe(
|
|
65
|
+
"Sanitized interleaved text and tool-call trace captured for this run. It interleaves the agent's `text` turns with `tool_call` turns, so its length is not a tool-call count; use `tool_calls` or `tool_call_count`.",
|
|
66
|
+
),
|
|
61
67
|
created_at: z.string().nullable().optional(),
|
|
62
68
|
harness: z.string().nullable().optional().describe(
|
|
63
69
|
"Coding-agent harness for agent runs; otherwise the answer platform identifier.",
|
|
@@ -75,5 +81,10 @@ export const V2PromptRunDetail$zodSchema: z.ZodType<V2PromptRunDetail> = z
|
|
|
75
81
|
response_preview: z.string().nullable().optional(),
|
|
76
82
|
run_date: z.string(),
|
|
77
83
|
target_kind: V2PromptRunDetailTargetKind$zodSchema.nullable().optional(),
|
|
78
|
-
tool_call_count: z.int().nullable().optional()
|
|
84
|
+
tool_call_count: z.int().nullable().optional().describe(
|
|
85
|
+
"Coding-agent tool invocations, counting a call and its result once. Matches the length of `tool_calls` on the detail response; null for answer-engine runs.",
|
|
86
|
+
),
|
|
87
|
+
tool_calls: z.array(z.record(z.string(), z.any())).optional().describe(
|
|
88
|
+
"Sanitized coding-agent tool invocations captured for this run, one entry per invocation with its result merged in.",
|
|
89
|
+
),
|
|
79
90
|
});
|
|
@@ -61,5 +61,7 @@ export const V2PromptRunSummary$zodSchema: z.ZodType<V2PromptRunSummary> = z
|
|
|
61
61
|
response_preview: z.string().nullable().optional(),
|
|
62
62
|
run_date: z.string(),
|
|
63
63
|
target_kind: V2PromptRunSummaryTargetKind$zodSchema.nullable().optional(),
|
|
64
|
-
tool_call_count: z.int().nullable().optional()
|
|
64
|
+
tool_call_count: z.int().nullable().optional().describe(
|
|
65
|
+
"Coding-agent tool invocations, counting a call and its result once. Matches the length of `tool_calls` on the detail response; null for answer-engine runs.",
|
|
66
|
+
),
|
|
65
67
|
});
|
|
@@ -20,7 +20,7 @@ export type V2PublicTask = {
|
|
|
20
20
|
execution_prompt: string;
|
|
21
21
|
reasoning?: string | null | undefined;
|
|
22
22
|
evidence?: { [k: string]: any } | undefined;
|
|
23
|
-
score
|
|
23
|
+
score?: number | null | undefined;
|
|
24
24
|
effort: string;
|
|
25
25
|
priority?: string | null | undefined;
|
|
26
26
|
status: string;
|
|
@@ -28,7 +28,7 @@ export type V2PublicTask = {
|
|
|
28
28
|
function_area?: string | null | undefined;
|
|
29
29
|
action_type?: string | null | undefined;
|
|
30
30
|
source?: string | null | undefined;
|
|
31
|
-
source_type
|
|
31
|
+
source_type?: string | null | undefined;
|
|
32
32
|
source_refs?: { [k: string]: any } | undefined;
|
|
33
33
|
metadata?: { [k: string]: any } | undefined;
|
|
34
34
|
created_at?: string | null | undefined;
|
|
@@ -63,7 +63,9 @@ export const V2PublicTask$zodSchema: z.ZodType<V2PublicTask> =
|
|
|
63
63
|
"Sort order within the batch, ascending; values need not start at 1.",
|
|
64
64
|
),
|
|
65
65
|
reasoning: z.string().nullable().optional(),
|
|
66
|
-
score: z.number()
|
|
66
|
+
score: z.number().nullable().optional().describe(
|
|
67
|
+
"Planner impact score; manually created tasks may be unscored.",
|
|
68
|
+
),
|
|
67
69
|
source: z.string().nullable().optional(),
|
|
68
70
|
source_key: z.string().describe(
|
|
69
71
|
"A finding key namespaced by source, such as `api_perf:https://example.com/docs/cli`.",
|
|
@@ -71,8 +73,8 @@ export const V2PublicTask$zodSchema: z.ZodType<V2PublicTask> =
|
|
|
71
73
|
source_refs: z.record(z.string(), z.any()).optional().describe(
|
|
72
74
|
"Source references needed to understand or execute the task.",
|
|
73
75
|
),
|
|
74
|
-
source_type: z.string().describe(
|
|
75
|
-
"A free-form source type such as `api_performance`;
|
|
76
|
+
source_type: z.string().nullable().optional().describe(
|
|
77
|
+
"A free-form source type such as `api_performance`; manually created tasks may not have a source.",
|
|
76
78
|
),
|
|
77
79
|
status: z.string().describe(
|
|
78
80
|
"The workflow status. New tasks start as `new`; `added_to_sheet` and `snoozed` tasks are listed only when `status=all`.",
|
|
@@ -106,11 +108,11 @@ export const V2PublicTask$zodSchemaOutbound: z.ZodType<any> = z.object({
|
|
|
106
108
|
priority: z.nullable(z.string()).optional(),
|
|
107
109
|
rank: z.int(),
|
|
108
110
|
reasoning: z.nullable(z.string()).optional(),
|
|
109
|
-
score: z.number(),
|
|
111
|
+
score: z.nullable(z.number()).optional(),
|
|
110
112
|
source: z.nullable(z.string()).optional(),
|
|
111
113
|
source_key: z.string(),
|
|
112
114
|
source_refs: z.record(z.string(), z.any()).optional(),
|
|
113
|
-
source_type: z.string(),
|
|
115
|
+
source_type: z.nullable(z.string()).optional(),
|
|
114
116
|
status: z.string(),
|
|
115
117
|
task_type: z.string(),
|
|
116
118
|
title: z.string(),
|