@lightsage/mcp 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/bin/mcp-server.js +525 -262
- package/bin/mcp-server.js.map +26 -19
- package/esm/funcs/evalResultsV2EvalResultsList.d.ts +1 -1
- package/esm/funcs/evalResultsV2EvalResultsList.js +1 -1
- package/esm/funcs/evalsAnalysis.d.ts +16 -0
- package/esm/funcs/evalsAnalysis.d.ts.map +1 -0
- package/esm/funcs/evalsAnalysis.js +88 -0
- package/esm/funcs/evalsAnalysis.js.map +1 -0
- package/esm/funcs/evalsTrace.d.ts +16 -0
- package/esm/funcs/evalsTrace.d.ts.map +1 -0
- package/esm/funcs/{evalsMonitor.js → evalsTrace.js} +19 -20
- package/esm/funcs/evalsTrace.js.map +1 -0
- package/esm/funcs/evalsV2EvalsCreate.d.ts +1 -1
- package/esm/funcs/evalsV2EvalsCreate.js +1 -1
- package/esm/funcs/evalsV2EvalsUpdate.d.ts +1 -1
- package/esm/funcs/evalsV2EvalsUpdate.js +1 -1
- package/esm/landing-page.js +1 -1
- package/esm/lib/config.d.ts +2 -2
- package/esm/lib/config.js +2 -2
- package/esm/mcp-server/mcp-server.js +1 -1
- package/esm/mcp-server/server.d.ts.map +1 -1
- package/esm/mcp-server/server.js +5 -3
- package/esm/mcp-server/server.js.map +1 -1
- package/esm/mcp-server/tools/evalResultsV2EvalResultsList.js +1 -1
- package/esm/mcp-server/tools/evalResultsV2EvalResultsList.js.map +1 -1
- package/esm/mcp-server/tools/evalsAnalysis.d.ts +7 -0
- package/esm/mcp-server/tools/evalsAnalysis.d.ts.map +1 -0
- package/esm/mcp-server/tools/evalsAnalysis.js +35 -0
- package/esm/mcp-server/tools/evalsAnalysis.js.map +1 -0
- package/esm/mcp-server/tools/evalsTrace.d.ts +7 -0
- package/esm/mcp-server/tools/evalsTrace.d.ts.map +1 -0
- package/esm/mcp-server/tools/evalsTrace.js +35 -0
- package/esm/mcp-server/tools/evalsTrace.js.map +1 -0
- package/esm/mcp-server/tools/evalsV2EvalsCreate.js +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsCreate.js.map +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsUpdate.js +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsUpdate.js.map +1 -1
- package/esm/models/createv2evalrequest.d.ts +1 -2
- package/esm/models/createv2evalrequest.d.ts.map +1 -1
- package/esm/models/createv2evalrequest.js +1 -2
- package/esm/models/createv2evalrequest.js.map +1 -1
- package/esm/models/updatev2evalrequest.d.ts +1 -2
- package/esm/models/updatev2evalrequest.d.ts.map +1 -1
- package/esm/models/updatev2evalrequest.js +1 -2
- package/esm/models/updatev2evalrequest.js.map +1 -1
- package/esm/models/v2evalrunanalysis.d.ts +18 -0
- package/esm/models/v2evalrunanalysis.d.ts.map +1 -0
- package/esm/models/v2evalrunanalysis.js +20 -0
- package/esm/models/v2evalrunanalysis.js.map +1 -0
- package/esm/models/v2evalrunanalysisresponse.d.ts +12 -0
- package/esm/models/v2evalrunanalysisresponse.d.ts.map +1 -0
- package/esm/models/v2evalrunanalysisresponse.js +15 -0
- package/esm/models/v2evalrunanalysisresponse.js.map +1 -0
- package/esm/models/v2evalrunsanalysisop.d.ts +17 -0
- package/esm/models/v2evalrunsanalysisop.d.ts.map +1 -0
- package/esm/models/v2evalrunsanalysisop.js +19 -0
- package/esm/models/v2evalrunsanalysisop.js.map +1 -0
- package/esm/models/v2evalrunstraceop.d.ts +19 -0
- package/esm/models/v2evalrunstraceop.d.ts.map +1 -0
- package/esm/models/v2evalrunstraceop.js +21 -0
- package/esm/models/v2evalrunstraceop.js.map +1 -0
- package/esm/models/v2evalruntraceevent.d.ts +20 -0
- package/esm/models/v2evalruntraceevent.d.ts.map +1 -0
- package/esm/models/v2evalruntraceevent.js +22 -0
- package/esm/models/v2evalruntraceevent.js.map +1 -0
- package/esm/models/v2evalruntraceresponse.d.ts +14 -0
- package/esm/models/v2evalruntraceresponse.d.ts.map +1 -0
- package/esm/models/v2evalruntraceresponse.js +17 -0
- package/esm/models/v2evalruntraceresponse.js.map +1 -0
- package/esm/tool-names.d.ts.map +1 -1
- package/esm/tool-names.js +9 -5
- package/esm/tool-names.js.map +1 -1
- package/package.json +1 -1
- package/src/funcs/evalResultsV2EvalResultsList.ts +1 -1
- package/src/funcs/evalsAnalysis.ts +153 -0
- package/src/funcs/{evalsMonitor.ts → evalsTrace.ts} +26 -24
- package/src/funcs/evalsV2EvalsCreate.ts +1 -1
- package/src/funcs/evalsV2EvalsUpdate.ts +1 -1
- package/src/landing-page.ts +1 -1
- package/src/lib/config.ts +2 -2
- package/src/mcp-server/mcp-server.ts +1 -1
- package/src/mcp-server/server.ts +5 -3
- package/src/mcp-server/tools/evalResultsV2EvalResultsList.ts +1 -1
- package/src/mcp-server/tools/evalsAnalysis.ts +43 -0
- package/src/mcp-server/tools/{evalsMonitor.ts → evalsTrace.ts} +9 -9
- package/src/mcp-server/tools/evalsV2EvalsCreate.ts +1 -1
- package/src/mcp-server/tools/evalsV2EvalsUpdate.ts +1 -1
- package/src/models/createv2evalrequest.ts +2 -3
- package/src/models/updatev2evalrequest.ts +4 -3
- package/src/models/v2evalrunanalysis.ts +51 -0
- package/src/models/v2evalrunanalysisresponse.ts +32 -0
- package/src/models/v2evalrunsanalysisop.ts +48 -0
- package/src/models/v2evalrunstraceop.ts +58 -0
- package/src/models/v2evalruntraceevent.ts +43 -0
- package/src/models/v2evalruntraceresponse.ts +38 -0
- package/src/tool-names.ts +9 -5
- package/esm/funcs/evalsMonitor.d.ts +0 -20
- package/esm/funcs/evalsMonitor.d.ts.map +0 -1
- package/esm/funcs/evalsMonitor.js.map +0 -1
- package/esm/mcp-server/tools/evalsMonitor.d.ts +0 -7
- package/esm/mcp-server/tools/evalsMonitor.d.ts.map +0 -1
- package/esm/mcp-server/tools/evalsMonitor.js +0 -35
- package/esm/mcp-server/tools/evalsMonitor.js.map +0 -1
- package/esm/models/v2evalrunsmonitorop.d.ts +0 -27
- package/esm/models/v2evalrunsmonitorop.d.ts.map +0 -1
- package/esm/models/v2evalrunsmonitorop.js +0 -27
- package/esm/models/v2evalrunsmonitorop.js.map +0 -1
- package/src/models/v2evalrunsmonitorop.ts +0 -59
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/*
|
|
2
2
|
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
-
* @generated-id:
|
|
3
|
+
* @generated-id: 696bcb18693c
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { LightsageMcpCore } from "../core.js";
|
|
7
|
-
import { encodeFormQuery } from "../lib/encodings.js";
|
|
7
|
+
import { encodeFormQuery, encodeSimple } from "../lib/encodings.js";
|
|
8
8
|
import { compactMap } from "../lib/primitives.js";
|
|
9
9
|
import { safeParse } from "../lib/schemas.js";
|
|
10
10
|
import { RequestOptions } from "../lib/sdks.js";
|
|
@@ -20,26 +20,21 @@ import {
|
|
|
20
20
|
} from "../models/errors/httpclienterrors.js";
|
|
21
21
|
import { SDKValidationError } from "../models/errors/sdkvalidationerror.js";
|
|
22
22
|
import {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
} from "../models/
|
|
23
|
+
V2EvalRunsTraceRequest,
|
|
24
|
+
V2EvalRunsTraceRequest$zodSchema,
|
|
25
|
+
} from "../models/v2evalrunstraceop.js";
|
|
26
26
|
import { APICall, APIPromise } from "../types/async.js";
|
|
27
27
|
import { Result } from "../types/fp.js";
|
|
28
28
|
|
|
29
|
-
export enum MonitorAcceptEnum {
|
|
30
|
-
applicationJsonAccept = "application/json",
|
|
31
|
-
textEventStreamAccept = "text/event-stream",
|
|
32
|
-
}
|
|
33
|
-
|
|
34
29
|
/**
|
|
35
|
-
*
|
|
30
|
+
* Retrieve an eval run trace
|
|
36
31
|
*
|
|
37
32
|
* @remarks
|
|
38
|
-
*
|
|
33
|
+
* Returns a finite page of sanitized execution events for one eval run. Use `next_after_id` as `after_id` to retrieve the next page. Use the monitor operation instead when following a run live.
|
|
39
34
|
*/
|
|
40
|
-
export function
|
|
35
|
+
export function evalsTrace(
|
|
41
36
|
client$: LightsageMcpCore,
|
|
42
|
-
request
|
|
37
|
+
request: V2EvalRunsTraceRequest,
|
|
43
38
|
options?: RequestOptions,
|
|
44
39
|
): APIPromise<
|
|
45
40
|
Result<
|
|
@@ -62,8 +57,8 @@ export function evalsMonitor(
|
|
|
62
57
|
|
|
63
58
|
async function $do(
|
|
64
59
|
client$: LightsageMcpCore,
|
|
65
|
-
request
|
|
66
|
-
options?: RequestOptions
|
|
60
|
+
request: V2EvalRunsTraceRequest,
|
|
61
|
+
options?: RequestOptions,
|
|
67
62
|
): Promise<
|
|
68
63
|
[
|
|
69
64
|
Result<
|
|
@@ -81,7 +76,7 @@ async function $do(
|
|
|
81
76
|
> {
|
|
82
77
|
const parsed$ = safeParse(
|
|
83
78
|
request,
|
|
84
|
-
(value$) =>
|
|
79
|
+
(value$) => V2EvalRunsTraceRequest$zodSchema.parse(value$),
|
|
85
80
|
"Input validation failed",
|
|
86
81
|
);
|
|
87
82
|
if (!parsed$.ok) {
|
|
@@ -89,16 +84,23 @@ async function $do(
|
|
|
89
84
|
}
|
|
90
85
|
const payload$ = parsed$.value;
|
|
91
86
|
const body$ = null;
|
|
92
|
-
|
|
87
|
+
|
|
88
|
+
const pathParams$ = {
|
|
89
|
+
run_id: encodeSimple("run_id", payload$.run_id, {
|
|
90
|
+
explode: false,
|
|
91
|
+
charEncoding: "percent",
|
|
92
|
+
}),
|
|
93
|
+
};
|
|
94
|
+
const path$ = pathToFunc("/v2/eval-runs/{run_id}/trace")(
|
|
95
|
+
pathParams$,
|
|
96
|
+
);
|
|
93
97
|
const query$ = encodeFormQuery({
|
|
94
|
-
"after_id": payload
|
|
95
|
-
"
|
|
96
|
-
"run_id": payload$?.run_id,
|
|
98
|
+
"after_id": payload$.after_id,
|
|
99
|
+
"limit": payload$.limit,
|
|
97
100
|
});
|
|
98
101
|
|
|
99
102
|
const headers$ = new Headers(compactMap({
|
|
100
|
-
Accept:
|
|
101
|
-
|| "application/json;q=1, text/event-stream;q=0",
|
|
103
|
+
Accept: "application/json",
|
|
102
104
|
}));
|
|
103
105
|
const securityInput = await extractSecurity(client$._options.security);
|
|
104
106
|
const requestSecurity = resolveGlobalSecurity(securityInput);
|
|
@@ -106,7 +108,7 @@ async function $do(
|
|
|
106
108
|
const context = {
|
|
107
109
|
options: client$._options,
|
|
108
110
|
baseURL: options?.serverURL ?? client$._baseURL ?? "",
|
|
109
|
-
operationID: "
|
|
111
|
+
operationID: "v2_eval_runs_trace",
|
|
110
112
|
oAuth2Scopes: null,
|
|
111
113
|
resolvedSecurity: requestSecurity,
|
|
112
114
|
securitySource: client$._options.security,
|
|
@@ -30,7 +30,7 @@ import { Result } from "../types/fp.js";
|
|
|
30
30
|
* Create Eval
|
|
31
31
|
*
|
|
32
32
|
* @remarks
|
|
33
|
-
* Creates an eval with a task prompt, subject, and judge
|
|
33
|
+
* Creates an eval with a task prompt, subject, and plain-text success criterion in `judge`. The eval can then be run through eval runs.
|
|
34
34
|
*/
|
|
35
35
|
export function evalsV2EvalsCreate(
|
|
36
36
|
client$: LightsageMcpCore,
|
|
@@ -30,7 +30,7 @@ import { Result } from "../types/fp.js";
|
|
|
30
30
|
* Update Eval
|
|
31
31
|
*
|
|
32
32
|
* @remarks
|
|
33
|
-
* Updates an eval's task prompt, subject, judge
|
|
33
|
+
* Updates an eval's task prompt, subject, plain-text `judge`, or name. Omitted fields remain unchanged.
|
|
34
34
|
*/
|
|
35
35
|
export function evalsV2EvalsUpdate(
|
|
36
36
|
client$: LightsageMcpCore,
|
package/src/landing-page.ts
CHANGED
|
@@ -931,7 +931,7 @@ http_headers = { "lightsage-api-key-auth" = "YOUR_LIGHTSAGE_API_KEY_AUTH" }`;
|
|
|
931
931
|
<h1>Instructions</h1>
|
|
932
932
|
<p>One-click installation for Claude Desktop users</p>
|
|
933
933
|
<div class="instruction-item">
|
|
934
|
-
<a href="https://github.com/lightsagehq/mcp/releases/download/v0.
|
|
934
|
+
<a href="https://github.com/lightsagehq/mcp/releases/download/v0.3.0/mcp-server.mcpb" download="mcp-server.mcpb" class="action-button header-action" style="display: inline-flex; margin-bottom: 16px;">
|
|
935
935
|
📥 Download MCP Bundle
|
|
936
936
|
</a>
|
|
937
937
|
</div>
|
package/src/lib/config.ts
CHANGED
|
@@ -66,7 +66,7 @@ export function serverURLFromOptions(options: SDKOptions): URL | null {
|
|
|
66
66
|
export const SDK_METADATA = {
|
|
67
67
|
language: "typescript",
|
|
68
68
|
openapiDocVersion: "1.0.0",
|
|
69
|
-
sdkVersion: "0.
|
|
69
|
+
sdkVersion: "0.3.0",
|
|
70
70
|
genVersion: "2.935.1",
|
|
71
|
-
userAgent: "speakeasy-sdk/mcp-typescript 0.
|
|
71
|
+
userAgent: "speakeasy-sdk/mcp-typescript 0.3.0 2.935.1 1.0.0 @lightsage/mcp",
|
|
72
72
|
} as const;
|
package/src/mcp-server/server.ts
CHANGED
|
@@ -28,8 +28,9 @@ import { tool$evalResultsV2EvalResultsList } from "./tools/evalResultsV2EvalResu
|
|
|
28
28
|
import { tool$evalResultsV2EvalResultsRetrieve } from "./tools/evalResultsV2EvalResultsRetrieve.js";
|
|
29
29
|
import { tool$evalRunsV2EvalRunsList } from "./tools/evalRunsV2EvalRunsList.js";
|
|
30
30
|
import { tool$evalRunsV2EvalRunsRetrieve } from "./tools/evalRunsV2EvalRunsRetrieve.js";
|
|
31
|
-
import { tool$
|
|
31
|
+
import { tool$evalsAnalysis } from "./tools/evalsAnalysis.js";
|
|
32
32
|
import { tool$evalsRun } from "./tools/evalsRun.js";
|
|
33
|
+
import { tool$evalsTrace } from "./tools/evalsTrace.js";
|
|
33
34
|
import { tool$evalsV2EvalConfigurationsCreate } from "./tools/evalsV2EvalConfigurationsCreate.js";
|
|
34
35
|
import { tool$evalsV2EvalConfigurationsDelete } from "./tools/evalsV2EvalConfigurationsDelete.js";
|
|
35
36
|
import { tool$evalsV2EvalConfigurationsList } from "./tools/evalsV2EvalConfigurationsList.js";
|
|
@@ -91,7 +92,7 @@ export function createMCPServer(deps: {
|
|
|
91
92
|
sizes: ["any"],
|
|
92
93
|
}],
|
|
93
94
|
name: "LightsageMcp",
|
|
94
|
-
version: "0.
|
|
95
|
+
version: "0.3.0",
|
|
95
96
|
});
|
|
96
97
|
|
|
97
98
|
const getClient = deps.getSDK || (() =>
|
|
@@ -174,7 +175,8 @@ export function createMCPServer(deps: {
|
|
|
174
175
|
tool(tool$evalsV2EvalConfigurationsUpdate);
|
|
175
176
|
tool(tool$evalsV2EvalConfigurationsDelete);
|
|
176
177
|
tool(tool$evalsRun);
|
|
177
|
-
tool(tool$
|
|
178
|
+
tool(tool$evalsAnalysis);
|
|
179
|
+
tool(tool$evalsTrace);
|
|
178
180
|
tool(tool$evalRunsV2EvalRunsList);
|
|
179
181
|
tool(tool$evalRunsV2EvalRunsRetrieve);
|
|
180
182
|
tool(tool$evalResultsV2EvalResultsList);
|
|
@@ -15,7 +15,7 @@ export const tool$evalResultsV2EvalResultsList: ToolDefinition<typeof args> = {
|
|
|
15
15
|
name: "eval-results-v2-eval-results-list",
|
|
16
16
|
description: `List Results
|
|
17
17
|
|
|
18
|
-
Returns completed custom-eval results from
|
|
18
|
+
Returns completed custom-eval results from all supported run flows, ordered newest first. Result IDs are different from eval run IDs.`,
|
|
19
19
|
annotations: {
|
|
20
20
|
"title": "",
|
|
21
21
|
"destructiveHint": false,
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 323a92ae7245
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { evalsAnalysis } from "../../funcs/evalsAnalysis.js";
|
|
7
|
+
import { V2EvalRunsAnalysisRequest$zodSchema } from "../../models/v2evalrunsanalysisop.js";
|
|
8
|
+
import { formatResult, ToolDefinition } from "../tools.js";
|
|
9
|
+
|
|
10
|
+
const args = {
|
|
11
|
+
request: V2EvalRunsAnalysisRequest$zodSchema,
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
export const tool$evalsAnalysis: ToolDefinition<typeof args> = {
|
|
15
|
+
name: "evals-analysis",
|
|
16
|
+
description: `Retrieve eval run analysis
|
|
17
|
+
|
|
18
|
+
Returns the persisted analysis produced for each completed attempt in one eval run. The analysis includes the verdict, outcome explanation, findings, bottleneck, and analysis sections. Active runs return the analysis available so far together with current run status and progress.`,
|
|
19
|
+
annotations: {
|
|
20
|
+
"title": "",
|
|
21
|
+
"destructiveHint": false,
|
|
22
|
+
"idempotentHint": false,
|
|
23
|
+
"openWorldHint": false,
|
|
24
|
+
"readOnlyHint": true,
|
|
25
|
+
},
|
|
26
|
+
args,
|
|
27
|
+
tool: async (client, args, ctx) => {
|
|
28
|
+
const [result] = await evalsAnalysis(
|
|
29
|
+
client,
|
|
30
|
+
args.request,
|
|
31
|
+
{ fetchOptions: { signal: ctx.signal } },
|
|
32
|
+
).$inspect();
|
|
33
|
+
|
|
34
|
+
if (!result.ok) {
|
|
35
|
+
return {
|
|
36
|
+
content: [{ type: "text", text: result.error.message }],
|
|
37
|
+
isError: true,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
return formatResult(result.value);
|
|
42
|
+
},
|
|
43
|
+
};
|
|
@@ -1,21 +1,21 @@
|
|
|
1
1
|
/*
|
|
2
2
|
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
-
* @generated-id:
|
|
3
|
+
* @generated-id: d4ce799d0d36
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { evalsTrace } from "../../funcs/evalsTrace.js";
|
|
7
|
+
import { V2EvalRunsTraceRequest$zodSchema } from "../../models/v2evalrunstraceop.js";
|
|
8
8
|
import { formatResult, ToolDefinition } from "../tools.js";
|
|
9
9
|
|
|
10
10
|
const args = {
|
|
11
|
-
request:
|
|
11
|
+
request: V2EvalRunsTraceRequest$zodSchema,
|
|
12
12
|
};
|
|
13
13
|
|
|
14
|
-
export const tool$
|
|
15
|
-
name: "evals-
|
|
16
|
-
description: `
|
|
14
|
+
export const tool$evalsTrace: ToolDefinition<typeof args> = {
|
|
15
|
+
name: "evals-trace",
|
|
16
|
+
description: `Retrieve an eval run trace
|
|
17
17
|
|
|
18
|
-
|
|
18
|
+
Returns a finite page of sanitized execution events for one eval run. Use \`next_after_id\` as \`after_id\` to retrieve the next page. Use the monitor operation instead when following a run live.`,
|
|
19
19
|
annotations: {
|
|
20
20
|
"title": "",
|
|
21
21
|
"destructiveHint": false,
|
|
@@ -25,7 +25,7 @@ Streams an immediate snapshot followed by live status and sanitized activity. Pr
|
|
|
25
25
|
},
|
|
26
26
|
args,
|
|
27
27
|
tool: async (client, args, ctx) => {
|
|
28
|
-
const [result] = await
|
|
28
|
+
const [result] = await evalsTrace(
|
|
29
29
|
client,
|
|
30
30
|
args.request,
|
|
31
31
|
{ fetchOptions: { signal: ctx.signal } },
|
|
@@ -15,7 +15,7 @@ export const tool$evalsV2EvalsCreate: ToolDefinition<typeof args> = {
|
|
|
15
15
|
name: "evals-v2-evals-create",
|
|
16
16
|
description: `Create Eval
|
|
17
17
|
|
|
18
|
-
Creates an eval with a task prompt, subject, and judge
|
|
18
|
+
Creates an eval with a task prompt, subject, and plain-text success criterion in \`judge\`. The eval can then be run through eval runs.`,
|
|
19
19
|
annotations: {
|
|
20
20
|
"title": "",
|
|
21
21
|
"destructiveHint": false,
|
|
@@ -15,7 +15,7 @@ export const tool$evalsV2EvalsUpdate: ToolDefinition<typeof args> = {
|
|
|
15
15
|
name: "evals-v2-evals-update",
|
|
16
16
|
description: `Update Eval
|
|
17
17
|
|
|
18
|
-
Updates an eval's task prompt, subject, judge
|
|
18
|
+
Updates an eval's task prompt, subject, plain-text \`judge\`, or name. Omitted fields remain unchanged.`,
|
|
19
19
|
annotations: {
|
|
20
20
|
"title": "",
|
|
21
21
|
"destructiveHint": false,
|
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
|
|
6
6
|
import * as z from "zod";
|
|
7
7
|
import { ClosedEnum } from "../types/enums.js";
|
|
8
|
-
import { V2EvalJudge, V2EvalJudge$zodSchema } from "./v2evaljudge.js";
|
|
9
8
|
|
|
10
9
|
export const CreateV2EvalRequestType = {
|
|
11
10
|
Workflow: "workflow",
|
|
@@ -28,7 +27,7 @@ export type CreateV2EvalRequest = {
|
|
|
28
27
|
name: string;
|
|
29
28
|
type: CreateV2EvalRequestType;
|
|
30
29
|
prompt: string;
|
|
31
|
-
judge:
|
|
30
|
+
judge: string;
|
|
32
31
|
mcp_server_id?: string | undefined;
|
|
33
32
|
cli_id?: string | undefined;
|
|
34
33
|
skill_id?: string | undefined;
|
|
@@ -37,7 +36,7 @@ export type CreateV2EvalRequest = {
|
|
|
37
36
|
export const CreateV2EvalRequest$zodSchema: z.ZodType<CreateV2EvalRequest> = z
|
|
38
37
|
.object({
|
|
39
38
|
cli_id: z.string().optional(),
|
|
40
|
-
judge:
|
|
39
|
+
judge: z.string().describe("Plain-text outcome that counts as success."),
|
|
41
40
|
mcp_server_id: z.string().optional(),
|
|
42
41
|
name: z.string(),
|
|
43
42
|
prompt: z.string(),
|
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
|
|
6
6
|
import * as z from "zod";
|
|
7
7
|
import { ClosedEnum } from "../types/enums.js";
|
|
8
|
-
import { V2EvalJudge, V2EvalJudge$zodSchema } from "./v2evaljudge.js";
|
|
9
8
|
|
|
10
9
|
export const UpdateV2EvalRequestType = {
|
|
11
10
|
Workflow: "workflow",
|
|
@@ -28,7 +27,7 @@ export type UpdateV2EvalRequest = {
|
|
|
28
27
|
name?: string | undefined;
|
|
29
28
|
type?: UpdateV2EvalRequestType | undefined;
|
|
30
29
|
prompt?: string | undefined;
|
|
31
|
-
judge?:
|
|
30
|
+
judge?: string | undefined;
|
|
32
31
|
mcp_server_id?: string | undefined;
|
|
33
32
|
cli_id?: string | undefined;
|
|
34
33
|
skill_id?: string | undefined;
|
|
@@ -37,7 +36,9 @@ export type UpdateV2EvalRequest = {
|
|
|
37
36
|
export const UpdateV2EvalRequest$zodSchema: z.ZodType<UpdateV2EvalRequest> = z
|
|
38
37
|
.object({
|
|
39
38
|
cli_id: z.string().optional(),
|
|
40
|
-
judge:
|
|
39
|
+
judge: z.string().optional().describe(
|
|
40
|
+
"Plain-text outcome that counts as success.",
|
|
41
|
+
),
|
|
41
42
|
mcp_server_id: z.string().optional(),
|
|
42
43
|
name: z.string().optional(),
|
|
43
44
|
prompt: z.string().optional(),
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: ce9f97cba8e0
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
|
|
8
|
+
export type V2EvalRunAnalysis = {
|
|
9
|
+
id: string;
|
|
10
|
+
result_id: string;
|
|
11
|
+
attempt_id: string;
|
|
12
|
+
eval_id?: string | null | undefined;
|
|
13
|
+
agent_id?: string | null | undefined;
|
|
14
|
+
variant_key?: string | undefined;
|
|
15
|
+
repeat_index?: number | undefined;
|
|
16
|
+
analysis_version?: string | null | undefined;
|
|
17
|
+
analysis?: { [k: string]: any } | undefined;
|
|
18
|
+
created_at?: string | null | undefined;
|
|
19
|
+
updated_at?: string | null | undefined;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export const V2EvalRunAnalysis$zodSchema: z.ZodType<V2EvalRunAnalysis> = z
|
|
23
|
+
.object({
|
|
24
|
+
agent_id: z.string().nullable().optional().describe(
|
|
25
|
+
"Runnable agent ID used for this concrete attempt.",
|
|
26
|
+
),
|
|
27
|
+
analysis: z.record(z.string(), z.any()).optional().describe(
|
|
28
|
+
"Persisted eval_run_analysis payload, including verdict, outcome, findings, bottleneck, and analysis sections.",
|
|
29
|
+
),
|
|
30
|
+
analysis_version: z.string().nullable().optional().describe(
|
|
31
|
+
"Version of the analysis pipeline that produced this payload.",
|
|
32
|
+
),
|
|
33
|
+
attempt_id: z.string().describe(
|
|
34
|
+
"Child eval-attempt ID used to correlate the analysis with trace events.",
|
|
35
|
+
),
|
|
36
|
+
created_at: z.string().nullable().optional(),
|
|
37
|
+
eval_id: z.string().nullable().optional().describe(
|
|
38
|
+
"Public eval definition ID, not the backing executable eval ID.",
|
|
39
|
+
),
|
|
40
|
+
id: z.string().describe("Persisted eval analysis ID."),
|
|
41
|
+
repeat_index: z.int().default(0).describe(
|
|
42
|
+
"Zero-based repeat index within the submitted run.",
|
|
43
|
+
),
|
|
44
|
+
result_id: z.string().describe(
|
|
45
|
+
"Concrete eval result ID analyzed for this attempt.",
|
|
46
|
+
),
|
|
47
|
+
updated_at: z.string().nullable().optional(),
|
|
48
|
+
variant_key: z.string().default("default").describe(
|
|
49
|
+
"Runtime variant that produced this result.",
|
|
50
|
+
),
|
|
51
|
+
});
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 16648f81799b
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
PublicEvalRunProgress,
|
|
9
|
+
PublicEvalRunProgress$zodSchema,
|
|
10
|
+
} from "./publicevalrunprogress.js";
|
|
11
|
+
import {
|
|
12
|
+
V2EvalRunAnalysis,
|
|
13
|
+
V2EvalRunAnalysis$zodSchema,
|
|
14
|
+
} from "./v2evalrunanalysis.js";
|
|
15
|
+
|
|
16
|
+
export type V2EvalRunAnalysisResponse = {
|
|
17
|
+
run_id: string;
|
|
18
|
+
status: string;
|
|
19
|
+
progress?: PublicEvalRunProgress | undefined;
|
|
20
|
+
error?: string | null | undefined;
|
|
21
|
+
data?: Array<V2EvalRunAnalysis> | undefined;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export const V2EvalRunAnalysisResponse$zodSchema: z.ZodType<
|
|
25
|
+
V2EvalRunAnalysisResponse
|
|
26
|
+
> = z.object({
|
|
27
|
+
data: z.array(V2EvalRunAnalysis$zodSchema).optional(),
|
|
28
|
+
error: z.string().nullable().optional(),
|
|
29
|
+
progress: PublicEvalRunProgress$zodSchema.optional(),
|
|
30
|
+
run_id: z.string(),
|
|
31
|
+
status: z.string(),
|
|
32
|
+
});
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 4c00eef582d7
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
HTTPValidationError,
|
|
9
|
+
HTTPValidationError$zodSchema,
|
|
10
|
+
} from "./httpvalidationerror.js";
|
|
11
|
+
import {
|
|
12
|
+
V2EvalRunAnalysisResponse,
|
|
13
|
+
V2EvalRunAnalysisResponse$zodSchema,
|
|
14
|
+
} from "./v2evalrunanalysisresponse.js";
|
|
15
|
+
|
|
16
|
+
export type V2EvalRunsAnalysisRequest = { run_id: string };
|
|
17
|
+
|
|
18
|
+
export const V2EvalRunsAnalysisRequest$zodSchema: z.ZodType<
|
|
19
|
+
V2EvalRunsAnalysisRequest
|
|
20
|
+
> = z.object({
|
|
21
|
+
run_id: z.string(),
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Eval run not found. Use the run ID returned when you started the eval.
|
|
26
|
+
*/
|
|
27
|
+
export type V2EvalRunsAnalysisResponseBody = { detail: string };
|
|
28
|
+
|
|
29
|
+
export const V2EvalRunsAnalysisResponseBody$zodSchema: z.ZodType<
|
|
30
|
+
V2EvalRunsAnalysisResponseBody
|
|
31
|
+
> = z.object({
|
|
32
|
+
detail: z.string(),
|
|
33
|
+
}).describe(
|
|
34
|
+
"Eval run not found. Use the run ID returned when you started the eval.",
|
|
35
|
+
);
|
|
36
|
+
|
|
37
|
+
export type V2EvalRunsAnalysisResponse =
|
|
38
|
+
| V2EvalRunAnalysisResponse
|
|
39
|
+
| V2EvalRunsAnalysisResponseBody
|
|
40
|
+
| HTTPValidationError;
|
|
41
|
+
|
|
42
|
+
export const V2EvalRunsAnalysisResponse$zodSchema: z.ZodType<
|
|
43
|
+
V2EvalRunsAnalysisResponse
|
|
44
|
+
> = z.union([
|
|
45
|
+
V2EvalRunAnalysisResponse$zodSchema,
|
|
46
|
+
z.lazy(() => V2EvalRunsAnalysisResponseBody$zodSchema),
|
|
47
|
+
HTTPValidationError$zodSchema,
|
|
48
|
+
]);
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: f7cdcd3e7f8d
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
HTTPValidationError,
|
|
9
|
+
HTTPValidationError$zodSchema,
|
|
10
|
+
} from "./httpvalidationerror.js";
|
|
11
|
+
import {
|
|
12
|
+
V2EvalRunTraceResponse,
|
|
13
|
+
V2EvalRunTraceResponse$zodSchema,
|
|
14
|
+
} from "./v2evalruntraceresponse.js";
|
|
15
|
+
|
|
16
|
+
export type V2EvalRunsTraceRequest = {
|
|
17
|
+
run_id: string;
|
|
18
|
+
after_id?: number | undefined;
|
|
19
|
+
limit?: number | undefined;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export const V2EvalRunsTraceRequest$zodSchema: z.ZodType<
|
|
23
|
+
V2EvalRunsTraceRequest
|
|
24
|
+
> = z.object({
|
|
25
|
+
after_id: z.int().default(0).describe(
|
|
26
|
+
"Return events with IDs greater than this cursor.",
|
|
27
|
+
),
|
|
28
|
+
limit: z.int().default(500).describe(
|
|
29
|
+
"Maximum number of trace events to return.",
|
|
30
|
+
),
|
|
31
|
+
run_id: z.string(),
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Eval run not found. Use the run ID returned when you started the eval.
|
|
36
|
+
*/
|
|
37
|
+
export type V2EvalRunsTraceResponseBody = { detail: string };
|
|
38
|
+
|
|
39
|
+
export const V2EvalRunsTraceResponseBody$zodSchema: z.ZodType<
|
|
40
|
+
V2EvalRunsTraceResponseBody
|
|
41
|
+
> = z.object({
|
|
42
|
+
detail: z.string(),
|
|
43
|
+
}).describe(
|
|
44
|
+
"Eval run not found. Use the run ID returned when you started the eval.",
|
|
45
|
+
);
|
|
46
|
+
|
|
47
|
+
export type V2EvalRunsTraceResponse =
|
|
48
|
+
| V2EvalRunTraceResponse
|
|
49
|
+
| V2EvalRunsTraceResponseBody
|
|
50
|
+
| HTTPValidationError;
|
|
51
|
+
|
|
52
|
+
export const V2EvalRunsTraceResponse$zodSchema: z.ZodType<
|
|
53
|
+
V2EvalRunsTraceResponse
|
|
54
|
+
> = z.union([
|
|
55
|
+
V2EvalRunTraceResponse$zodSchema,
|
|
56
|
+
z.lazy(() => V2EvalRunsTraceResponseBody$zodSchema),
|
|
57
|
+
HTTPValidationError$zodSchema,
|
|
58
|
+
]);
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: cf33bec99838
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
|
|
8
|
+
export type V2EvalRunTraceEvent = {
|
|
9
|
+
run_id: string;
|
|
10
|
+
eval_id?: string | null | undefined;
|
|
11
|
+
eval_name?: string | null | undefined;
|
|
12
|
+
configuration_id?: string | null | undefined;
|
|
13
|
+
agent_id?: string | null | undefined;
|
|
14
|
+
agent_name?: string | null | undefined;
|
|
15
|
+
child_job_id: string;
|
|
16
|
+
event_id: number;
|
|
17
|
+
timestamp?: string | null | undefined;
|
|
18
|
+
category: string;
|
|
19
|
+
type: string;
|
|
20
|
+
message: string;
|
|
21
|
+
data?: { [k: string]: any } | undefined;
|
|
22
|
+
};
|
|
23
|
+
|
|
24
|
+
export const V2EvalRunTraceEvent$zodSchema: z.ZodType<V2EvalRunTraceEvent> = z
|
|
25
|
+
.object({
|
|
26
|
+
agent_id: z.string().nullable().optional(),
|
|
27
|
+
agent_name: z.string().nullable().optional(),
|
|
28
|
+
category: z.string(),
|
|
29
|
+
child_job_id: z.string().describe(
|
|
30
|
+
"Child eval-attempt ID associated with the event.",
|
|
31
|
+
),
|
|
32
|
+
configuration_id: z.string().nullable().optional(),
|
|
33
|
+
data: z.record(z.string(), z.any()).optional().describe(
|
|
34
|
+
"Sanitized event payload.",
|
|
35
|
+
),
|
|
36
|
+
eval_id: z.string().nullable().optional(),
|
|
37
|
+
eval_name: z.string().nullable().optional(),
|
|
38
|
+
event_id: z.int(),
|
|
39
|
+
message: z.string(),
|
|
40
|
+
run_id: z.string(),
|
|
41
|
+
timestamp: z.string().nullable().optional(),
|
|
42
|
+
type: z.string(),
|
|
43
|
+
});
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 25d2b266ce5f
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import {
|
|
8
|
+
PublicEvalRunProgress,
|
|
9
|
+
PublicEvalRunProgress$zodSchema,
|
|
10
|
+
} from "./publicevalrunprogress.js";
|
|
11
|
+
import {
|
|
12
|
+
V2EvalRunTraceEvent,
|
|
13
|
+
V2EvalRunTraceEvent$zodSchema,
|
|
14
|
+
} from "./v2evalruntraceevent.js";
|
|
15
|
+
|
|
16
|
+
export type V2EvalRunTraceResponse = {
|
|
17
|
+
run_id: string;
|
|
18
|
+
status: string;
|
|
19
|
+
progress?: PublicEvalRunProgress | undefined;
|
|
20
|
+
error?: string | null | undefined;
|
|
21
|
+
data?: Array<V2EvalRunTraceEvent> | undefined;
|
|
22
|
+
next_after_id: number;
|
|
23
|
+
has_more: boolean;
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export const V2EvalRunTraceResponse$zodSchema: z.ZodType<
|
|
27
|
+
V2EvalRunTraceResponse
|
|
28
|
+
> = z.object({
|
|
29
|
+
data: z.array(V2EvalRunTraceEvent$zodSchema).optional(),
|
|
30
|
+
error: z.string().nullable().optional(),
|
|
31
|
+
has_more: z.boolean(),
|
|
32
|
+
next_after_id: z.int().describe(
|
|
33
|
+
"Cursor to pass as `after_id` when requesting the next page.",
|
|
34
|
+
),
|
|
35
|
+
progress: PublicEvalRunProgress$zodSchema.optional(),
|
|
36
|
+
run_id: z.string(),
|
|
37
|
+
status: z.string(),
|
|
38
|
+
});
|