@lightsage/mcp 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/bin/mcp-server.js +198 -81
- package/bin/mcp-server.js.map +32 -29
- package/esm/funcs/evalsRun.d.ts +3 -3
- package/esm/funcs/evalsRun.js +3 -3
- package/esm/funcs/evalsV2EvalsCreate.d.ts +1 -1
- package/esm/funcs/evalsV2EvalsCreate.js +1 -1
- package/esm/funcs/evalsV2EvalsList.d.ts +1 -1
- package/esm/funcs/evalsV2EvalsList.js +1 -1
- package/esm/funcs/evalsV2EvalsRetrieve.d.ts +1 -1
- package/esm/funcs/evalsV2EvalsRetrieve.js +1 -1
- package/esm/funcs/evalsV2EvalsUpdate.d.ts +1 -1
- package/esm/funcs/evalsV2EvalsUpdate.js +1 -1
- package/esm/landing-page.js +1 -1
- package/esm/lib/config.d.ts +3 -3
- package/esm/lib/config.js +3 -3
- package/esm/lib/config.js.map +1 -1
- package/esm/mcp-server/mcp-server.js +1 -1
- package/esm/mcp-server/server.js +1 -1
- package/esm/mcp-server/tools/evalsRun.js +3 -3
- package/esm/mcp-server/tools/evalsRun.js.map +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsCreate.js +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsCreate.js.map +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsList.js +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsList.js.map +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsRetrieve.js +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsRetrieve.js.map +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsUpdate.js +1 -1
- package/esm/mcp-server/tools/evalsV2EvalsUpdate.js.map +1 -1
- package/esm/models/createv2evalrequest.d.ts +16 -20
- package/esm/models/createv2evalrequest.d.ts.map +1 -1
- package/esm/models/createv2evalrequest.js +25 -17
- package/esm/models/createv2evalrequest.js.map +1 -1
- package/esm/models/createv2evalrunrequest.d.ts +19 -4
- package/esm/models/createv2evalrunrequest.d.ts.map +1 -1
- package/esm/models/createv2evalrunrequest.js +31 -6
- package/esm/models/createv2evalrunrequest.js.map +1 -1
- package/esm/models/updatev2evalrequest.d.ts +16 -20
- package/esm/models/updatev2evalrequest.d.ts.map +1 -1
- package/esm/models/updatev2evalrequest.js +25 -17
- package/esm/models/updatev2evalrequest.js.map +1 -1
- package/esm/models/v2eval.d.ts +16 -21
- package/esm/models/v2eval.d.ts.map +1 -1
- package/esm/models/v2eval.js +25 -18
- package/esm/models/v2eval.js.map +1 -1
- package/esm/models/v2evalresult.d.ts +4 -2
- package/esm/models/v2evalresult.d.ts.map +1 -1
- package/esm/models/v2evalresult.js +3 -1
- package/esm/models/v2evalresult.js.map +1 -1
- package/esm/models/v2evalresultsummary.d.ts +4 -1
- package/esm/models/v2evalresultsummary.d.ts.map +1 -1
- package/esm/models/v2evalresultsummary.js +4 -1
- package/esm/models/v2evalresultsummary.js.map +1 -1
- package/esm/models/v2evalrun.d.ts +5 -2
- package/esm/models/v2evalrun.d.ts.map +1 -1
- package/esm/models/v2evalrun.js +5 -2
- package/esm/models/v2evalrun.js.map +1 -1
- package/esm/models/v2evalrunanalysis.d.ts +3 -1
- package/esm/models/v2evalrunanalysis.d.ts.map +1 -1
- package/esm/models/v2evalrunanalysis.js +3 -1
- package/esm/models/v2evalrunanalysis.js.map +1 -1
- package/esm/models/v2evalruntraceevent.d.ts +3 -2
- package/esm/models/v2evalruntraceevent.d.ts.map +1 -1
- package/esm/models/v2evalruntraceevent.js +3 -2
- package/esm/models/v2evalruntraceevent.js.map +1 -1
- package/esm/models/v2inlineclireference.d.ts +6 -0
- package/esm/models/v2inlineclireference.d.ts.map +1 -0
- package/esm/models/v2inlineclireference.js +10 -0
- package/esm/models/v2inlineclireference.js.map +1 -0
- package/esm/models/v2inlinemcpdefinition.d.ts +28 -0
- package/esm/models/v2inlinemcpdefinition.d.ts.map +1 -0
- package/esm/models/v2inlinemcpdefinition.js +25 -0
- package/esm/models/v2inlinemcpdefinition.js.map +1 -0
- package/esm/models/v2savedclireference.d.ts +6 -0
- package/esm/models/v2savedclireference.d.ts.map +1 -0
- package/esm/models/v2savedclireference.js +10 -0
- package/esm/models/v2savedclireference.js.map +1 -0
- package/esm/models/v2savedmcpreference.d.ts +6 -0
- package/esm/models/v2savedmcpreference.d.ts.map +1 -0
- package/esm/models/v2savedmcpreference.js +10 -0
- package/esm/models/v2savedmcpreference.js.map +1 -0
- package/esm/tool-names.js +5 -5
- package/esm/tool-names.js.map +1 -1
- package/package.json +1 -1
- package/src/funcs/evalsRun.ts +3 -3
- package/src/funcs/evalsV2EvalsCreate.ts +1 -1
- package/src/funcs/evalsV2EvalsList.ts +1 -1
- package/src/funcs/evalsV2EvalsRetrieve.ts +1 -1
- package/src/funcs/evalsV2EvalsUpdate.ts +1 -1
- package/src/landing-page.ts +1 -1
- package/src/lib/config.ts +3 -3
- package/src/mcp-server/mcp-server.ts +1 -1
- package/src/mcp-server/server.ts +1 -1
- package/src/mcp-server/tools/evalsRun.ts +3 -3
- package/src/mcp-server/tools/evalsV2EvalsCreate.ts +1 -1
- package/src/mcp-server/tools/evalsV2EvalsList.ts +1 -1
- package/src/mcp-server/tools/evalsV2EvalsRetrieve.ts +1 -1
- package/src/mcp-server/tools/evalsV2EvalsUpdate.ts +1 -1
- package/src/models/createv2evalrequest.ts +71 -27
- package/src/models/createv2evalrunrequest.ts +85 -13
- package/src/models/updatev2evalrequest.ts +60 -28
- package/src/models/v2eval.ts +49 -26
- package/src/models/v2evalresult.ts +8 -4
- package/src/models/v2evalresultsummary.ts +9 -3
- package/src/models/v2evalrun.ts +18 -6
- package/src/models/v2evalrunanalysis.ts +11 -3
- package/src/models/v2evalruntraceevent.ts +6 -4
- package/src/models/v2inlineclireference.ts +15 -0
- package/src/models/v2inlinemcpdefinition.ts +51 -0
- package/src/models/v2savedclireference.ts +13 -0
- package/src/models/v2savedmcpreference.ts +13 -0
- package/src/tool-names.ts +5 -5
- package/test/simulations.test.js +162 -0
- package/esm/models/v2evaljudge.d.ts +0 -6
- package/esm/models/v2evaljudge.d.ts.map +0 -1
- package/esm/models/v2evaljudge.js +0 -9
- package/esm/models/v2evaljudge.js.map +0 -1
- package/src/models/v2evaljudge.ts +0 -12
package/src/models/v2eval.ts
CHANGED
|
@@ -4,46 +4,69 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import * as z from "zod";
|
|
7
|
-
import {
|
|
8
|
-
|
|
7
|
+
import {
|
|
8
|
+
V2InlineCliReference,
|
|
9
|
+
V2InlineCliReference$zodSchema,
|
|
10
|
+
} from "./v2inlineclireference.js";
|
|
11
|
+
import {
|
|
12
|
+
V2InlineMcpDefinition,
|
|
13
|
+
V2InlineMcpDefinition$zodSchema,
|
|
14
|
+
} from "./v2inlinemcpdefinition.js";
|
|
15
|
+
import {
|
|
16
|
+
V2SavedCliReference,
|
|
17
|
+
V2SavedCliReference$zodSchema,
|
|
18
|
+
} from "./v2savedclireference.js";
|
|
19
|
+
import {
|
|
20
|
+
V2SavedMcpReference,
|
|
21
|
+
V2SavedMcpReference$zodSchema,
|
|
22
|
+
} from "./v2savedmcpreference.js";
|
|
9
23
|
|
|
10
|
-
export
|
|
11
|
-
Workflow: "workflow",
|
|
12
|
-
Mcp: "mcp",
|
|
13
|
-
Cli: "cli",
|
|
14
|
-
Skill: "skill",
|
|
15
|
-
} as const;
|
|
16
|
-
export type V2EvalType = ClosedEnum<typeof V2EvalType>;
|
|
24
|
+
export type V2EvalCli = V2SavedCliReference | V2InlineCliReference;
|
|
17
25
|
|
|
18
|
-
export const
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
26
|
+
export const V2EvalCli$zodSchema: z.ZodType<V2EvalCli> = z.union([
|
|
27
|
+
V2SavedCliReference$zodSchema,
|
|
28
|
+
V2InlineCliReference$zodSchema,
|
|
29
|
+
]);
|
|
30
|
+
|
|
31
|
+
export type V2EvalMcp = V2InlineMcpDefinition | V2SavedMcpReference;
|
|
32
|
+
|
|
33
|
+
export const V2EvalMcp$zodSchema: z.ZodType<V2EvalMcp> = z.union([
|
|
34
|
+
V2InlineMcpDefinition$zodSchema,
|
|
35
|
+
V2SavedMcpReference$zodSchema,
|
|
23
36
|
]);
|
|
24
37
|
|
|
25
38
|
export type V2Eval = {
|
|
26
39
|
id: string;
|
|
27
40
|
name: string;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
41
|
+
prompts?: Array<string> | undefined;
|
|
42
|
+
judges?: Array<string> | undefined;
|
|
43
|
+
agent?: Array<string> | undefined;
|
|
44
|
+
persona?: string | null | undefined;
|
|
45
|
+
clis?: Array<V2SavedCliReference | V2InlineCliReference> | undefined;
|
|
46
|
+
mcps?: Array<V2InlineMcpDefinition | V2SavedMcpReference> | undefined;
|
|
47
|
+
runs?: number | undefined;
|
|
48
|
+
tags?: Array<string> | undefined;
|
|
34
49
|
created_at?: string | null | undefined;
|
|
35
50
|
updated_at?: string | null | undefined;
|
|
36
51
|
};
|
|
37
52
|
|
|
38
53
|
export const V2Eval$zodSchema: z.ZodType<V2Eval> = z.object({
|
|
39
|
-
|
|
54
|
+
agent: z.array(z.string()).optional(),
|
|
55
|
+
clis: z.array(z.union([
|
|
56
|
+
V2SavedCliReference$zodSchema,
|
|
57
|
+
V2InlineCliReference$zodSchema,
|
|
58
|
+
])).optional(),
|
|
40
59
|
created_at: z.string().nullable().optional(),
|
|
41
60
|
id: z.string(),
|
|
42
|
-
|
|
43
|
-
|
|
61
|
+
judges: z.array(z.string()).optional(),
|
|
62
|
+
mcps: z.array(z.union([
|
|
63
|
+
V2InlineMcpDefinition$zodSchema,
|
|
64
|
+
V2SavedMcpReference$zodSchema,
|
|
65
|
+
])).optional(),
|
|
44
66
|
name: z.string(),
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
67
|
+
persona: z.string().nullable().optional(),
|
|
68
|
+
prompts: z.array(z.string()).optional(),
|
|
69
|
+
runs: z.int().default(1),
|
|
70
|
+
tags: z.array(z.string()).optional(),
|
|
48
71
|
updated_at: z.string().nullable().optional(),
|
|
49
72
|
});
|
|
@@ -8,13 +8,15 @@ import * as z from "zod";
|
|
|
8
8
|
export type V2EvalResult = {
|
|
9
9
|
id: string;
|
|
10
10
|
eval_id?: string | null | undefined;
|
|
11
|
-
|
|
11
|
+
agent?: string | null | undefined;
|
|
12
|
+
prompt?: string | null | undefined;
|
|
13
|
+
prompt_index?: number | undefined;
|
|
14
|
+
repeat_index?: number | undefined;
|
|
12
15
|
passed?: boolean | null | undefined;
|
|
13
16
|
score?: number | null | undefined;
|
|
14
17
|
failure_reasons?: Array<string> | null | undefined;
|
|
15
18
|
latency_ms?: number | null | undefined;
|
|
16
19
|
created_at?: string | null | undefined;
|
|
17
|
-
prompt?: string | null | undefined;
|
|
18
20
|
output?: string | null | undefined;
|
|
19
21
|
execution_stdout?: string | null | undefined;
|
|
20
22
|
execution_stderr?: string | null | undefined;
|
|
@@ -23,8 +25,8 @@ export type V2EvalResult = {
|
|
|
23
25
|
};
|
|
24
26
|
|
|
25
27
|
export const V2EvalResult$zodSchema: z.ZodType<V2EvalResult> = z.object({
|
|
26
|
-
|
|
27
|
-
"Runnable agent
|
|
28
|
+
agent: z.string().nullable().optional().describe(
|
|
29
|
+
"Runnable agent combining the harness and model.",
|
|
28
30
|
),
|
|
29
31
|
created_at: z.string().nullable().optional(),
|
|
30
32
|
eval_id: z.string().nullable().optional(),
|
|
@@ -37,6 +39,8 @@ export const V2EvalResult$zodSchema: z.ZodType<V2EvalResult> = z.object({
|
|
|
37
39
|
output: z.string().nullable().optional(),
|
|
38
40
|
passed: z.boolean().nullable().optional(),
|
|
39
41
|
prompt: z.string().nullable().optional(),
|
|
42
|
+
prompt_index: z.int().default(0),
|
|
43
|
+
repeat_index: z.int().default(0),
|
|
40
44
|
score: z.number().nullable().optional().describe("Score from 0 to 1."),
|
|
41
45
|
tool_calls: z.array(z.record(z.string(), z.any())).nullable().optional(),
|
|
42
46
|
});
|
|
@@ -8,7 +8,10 @@ import * as z from "zod";
|
|
|
8
8
|
export type V2EvalResultSummary = {
|
|
9
9
|
id: string;
|
|
10
10
|
eval_id?: string | null | undefined;
|
|
11
|
-
|
|
11
|
+
agent?: string | null | undefined;
|
|
12
|
+
prompt?: string | null | undefined;
|
|
13
|
+
prompt_index?: number | undefined;
|
|
14
|
+
repeat_index?: number | undefined;
|
|
12
15
|
passed?: boolean | null | undefined;
|
|
13
16
|
score?: number | null | undefined;
|
|
14
17
|
failure_reasons?: Array<string> | null | undefined;
|
|
@@ -18,8 +21,8 @@ export type V2EvalResultSummary = {
|
|
|
18
21
|
|
|
19
22
|
export const V2EvalResultSummary$zodSchema: z.ZodType<V2EvalResultSummary> = z
|
|
20
23
|
.object({
|
|
21
|
-
|
|
22
|
-
"Runnable agent
|
|
24
|
+
agent: z.string().nullable().optional().describe(
|
|
25
|
+
"Runnable agent combining the harness and model.",
|
|
23
26
|
),
|
|
24
27
|
created_at: z.string().nullable().optional(),
|
|
25
28
|
eval_id: z.string().nullable().optional(),
|
|
@@ -27,5 +30,8 @@ export const V2EvalResultSummary$zodSchema: z.ZodType<V2EvalResultSummary> = z
|
|
|
27
30
|
id: z.string().describe("Result ID, not an eval job ID."),
|
|
28
31
|
latency_ms: z.int().nullable().optional(),
|
|
29
32
|
passed: z.boolean().nullable().optional(),
|
|
33
|
+
prompt: z.string().nullable().optional(),
|
|
34
|
+
prompt_index: z.int().default(0),
|
|
35
|
+
repeat_index: z.int().default(0),
|
|
30
36
|
score: z.number().nullable().optional().describe("Score from 0 to 1."),
|
|
31
37
|
});
|
package/src/models/v2evalrun.ts
CHANGED
|
@@ -15,8 +15,11 @@ export type V2EvalRun = {
|
|
|
15
15
|
eval_id?: string | null | undefined;
|
|
16
16
|
eval_name?: string | null | undefined;
|
|
17
17
|
configuration_id?: string | null | undefined;
|
|
18
|
-
|
|
19
|
-
|
|
18
|
+
persona?: string | null | undefined;
|
|
19
|
+
agent?: Array<string> | undefined;
|
|
20
|
+
prompt_count?: number | undefined;
|
|
21
|
+
runs?: number | undefined;
|
|
22
|
+
attempt_count?: number | undefined;
|
|
20
23
|
status: string;
|
|
21
24
|
stage?: string | null | undefined;
|
|
22
25
|
dispatch_id?: string | null | undefined;
|
|
@@ -29,11 +32,11 @@ export type V2EvalRun = {
|
|
|
29
32
|
};
|
|
30
33
|
|
|
31
34
|
export const V2EvalRun$zodSchema: z.ZodType<V2EvalRun> = z.object({
|
|
32
|
-
|
|
33
|
-
"Runnable
|
|
35
|
+
agent: z.array(z.string()).optional().describe(
|
|
36
|
+
"Runnable agents selected for this simulation.",
|
|
34
37
|
),
|
|
35
|
-
|
|
36
|
-
"
|
|
38
|
+
attempt_count: z.int().default(1).describe(
|
|
39
|
+
"Total concrete attempts after all fan-out dimensions.",
|
|
37
40
|
),
|
|
38
41
|
completed_at: z.string().nullable().optional(),
|
|
39
42
|
configuration_id: z.string().nullable().optional(),
|
|
@@ -49,7 +52,16 @@ export const V2EvalRun$zodSchema: z.ZodType<V2EvalRun> = z.object({
|
|
|
49
52
|
),
|
|
50
53
|
id: z.string().describe("Run ID to use when polling eval-run status."),
|
|
51
54
|
job_id: z.string().describe("Alias of `id` for the durable parent job."),
|
|
55
|
+
persona: z.string().nullable().optional().describe(
|
|
56
|
+
"Perspective applied to every attempt in this simulation.",
|
|
57
|
+
),
|
|
52
58
|
progress: PublicEvalRunProgress$zodSchema.optional(),
|
|
59
|
+
prompt_count: z.int().default(1).describe(
|
|
60
|
+
"Number of prompts included in the simulation.",
|
|
61
|
+
),
|
|
62
|
+
runs: z.int().default(1).describe(
|
|
63
|
+
"Independent attempts requested for every prompt-agent pair.",
|
|
64
|
+
),
|
|
53
65
|
stage: z.string().nullable().optional(),
|
|
54
66
|
started_at: z.string().nullable().optional(),
|
|
55
67
|
status: z.string(),
|
|
@@ -10,7 +10,9 @@ export type V2EvalRunAnalysis = {
|
|
|
10
10
|
result_id: string;
|
|
11
11
|
attempt_id: string;
|
|
12
12
|
eval_id?: string | null | undefined;
|
|
13
|
-
|
|
13
|
+
agent?: string | null | undefined;
|
|
14
|
+
prompt?: string | null | undefined;
|
|
15
|
+
prompt_index?: number | undefined;
|
|
14
16
|
variant_key?: string | undefined;
|
|
15
17
|
repeat_index?: number | undefined;
|
|
16
18
|
analysis_version?: string | null | undefined;
|
|
@@ -21,8 +23,8 @@ export type V2EvalRunAnalysis = {
|
|
|
21
23
|
|
|
22
24
|
export const V2EvalRunAnalysis$zodSchema: z.ZodType<V2EvalRunAnalysis> = z
|
|
23
25
|
.object({
|
|
24
|
-
|
|
25
|
-
"Runnable agent
|
|
26
|
+
agent: z.string().nullable().optional().describe(
|
|
27
|
+
"Runnable agent used for this concrete attempt.",
|
|
26
28
|
),
|
|
27
29
|
analysis: z.record(z.string(), z.any()).optional().describe(
|
|
28
30
|
"Persisted eval_run_analysis payload, including verdict, outcome, findings, bottleneck, and analysis sections.",
|
|
@@ -38,6 +40,12 @@ export const V2EvalRunAnalysis$zodSchema: z.ZodType<V2EvalRunAnalysis> = z
|
|
|
38
40
|
"Public eval definition ID, not the backing executable eval ID.",
|
|
39
41
|
),
|
|
40
42
|
id: z.string().describe("Persisted eval analysis ID."),
|
|
43
|
+
prompt: z.string().nullable().optional().describe(
|
|
44
|
+
"Exact task prompt used for this concrete attempt.",
|
|
45
|
+
),
|
|
46
|
+
prompt_index: z.int().default(0).describe(
|
|
47
|
+
"Zero-based prompt index within the submitted simulation.",
|
|
48
|
+
),
|
|
41
49
|
repeat_index: z.int().default(0).describe(
|
|
42
50
|
"Zero-based repeat index within the submitted run.",
|
|
43
51
|
),
|
|
@@ -10,8 +10,9 @@ export type V2EvalRunTraceEvent = {
|
|
|
10
10
|
eval_id?: string | null | undefined;
|
|
11
11
|
eval_name?: string | null | undefined;
|
|
12
12
|
configuration_id?: string | null | undefined;
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
agent?: string | null | undefined;
|
|
14
|
+
prompt_index?: number | undefined;
|
|
15
|
+
repeat_index?: number | undefined;
|
|
15
16
|
child_job_id: string;
|
|
16
17
|
event_id: number;
|
|
17
18
|
timestamp?: string | null | undefined;
|
|
@@ -23,8 +24,7 @@ export type V2EvalRunTraceEvent = {
|
|
|
23
24
|
|
|
24
25
|
export const V2EvalRunTraceEvent$zodSchema: z.ZodType<V2EvalRunTraceEvent> = z
|
|
25
26
|
.object({
|
|
26
|
-
|
|
27
|
-
agent_name: z.string().nullable().optional(),
|
|
27
|
+
agent: z.string().nullable().optional(),
|
|
28
28
|
category: z.string(),
|
|
29
29
|
child_job_id: z.string().describe(
|
|
30
30
|
"Child eval-attempt ID associated with the event.",
|
|
@@ -37,6 +37,8 @@ export const V2EvalRunTraceEvent$zodSchema: z.ZodType<V2EvalRunTraceEvent> = z
|
|
|
37
37
|
eval_name: z.string().nullable().optional(),
|
|
38
38
|
event_id: z.int(),
|
|
39
39
|
message: z.string(),
|
|
40
|
+
prompt_index: z.int().default(0),
|
|
41
|
+
repeat_index: z.int().default(0),
|
|
40
42
|
run_id: z.string(),
|
|
41
43
|
timestamp: z.string().nullable().optional(),
|
|
42
44
|
type: z.string(),
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: d13cb884e6f6
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
|
|
8
|
+
export type V2InlineCliReference = { install_command: string };
|
|
9
|
+
|
|
10
|
+
export const V2InlineCliReference$zodSchema: z.ZodType<V2InlineCliReference> = z
|
|
11
|
+
.object({
|
|
12
|
+
install_command: z.string().describe(
|
|
13
|
+
"Shell command to install this CLI before the agent starts.",
|
|
14
|
+
),
|
|
15
|
+
});
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 96ca65e675c2
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
import { ClosedEnum } from "../types/enums.js";
|
|
8
|
+
|
|
9
|
+
export const Type = {
|
|
10
|
+
Http: "http",
|
|
11
|
+
Sse: "sse",
|
|
12
|
+
Stdio: "stdio",
|
|
13
|
+
} as const;
|
|
14
|
+
export type Type = ClosedEnum<typeof Type>;
|
|
15
|
+
|
|
16
|
+
export const Type$zodSchema = z.enum([
|
|
17
|
+
"http",
|
|
18
|
+
"sse",
|
|
19
|
+
"stdio",
|
|
20
|
+
]);
|
|
21
|
+
|
|
22
|
+
export type V2InlineMcpDefinition = {
|
|
23
|
+
name: string;
|
|
24
|
+
type: Type;
|
|
25
|
+
url?: string | null | undefined;
|
|
26
|
+
headers?: { [k: string]: string } | undefined;
|
|
27
|
+
command?: string | null | undefined;
|
|
28
|
+
args?: Array<string> | undefined;
|
|
29
|
+
env?: { [k: string]: string } | undefined;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
export const V2InlineMcpDefinition$zodSchema: z.ZodType<V2InlineMcpDefinition> =
|
|
33
|
+
z.object({
|
|
34
|
+
args: z.array(z.string()).optional(),
|
|
35
|
+
command: z.string().nullable().optional().describe(
|
|
36
|
+
"Executable command for a `stdio` MCP server.",
|
|
37
|
+
),
|
|
38
|
+
env: z.record(z.string(), z.string()).optional().describe(
|
|
39
|
+
"Environment mapping for a `stdio` MCP server. Values may reference run secrets with `${VARIABLE_NAME}`.",
|
|
40
|
+
),
|
|
41
|
+
headers: z.record(z.string(), z.string()).optional().describe(
|
|
42
|
+
"HTTP headers. Reference run secrets with `${VARIABLE_NAME}` instead of embedding credentials.",
|
|
43
|
+
),
|
|
44
|
+
name: z.string().describe(
|
|
45
|
+
"Stable server name used in the selected agent's MCP config.",
|
|
46
|
+
),
|
|
47
|
+
type: Type$zodSchema,
|
|
48
|
+
url: z.string().nullable().optional().describe(
|
|
49
|
+
"HTTPS endpoint for an `http` or `sse` MCP server.",
|
|
50
|
+
),
|
|
51
|
+
});
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 99afd03df958
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
|
|
8
|
+
export type V2SavedCliReference = { id: string };
|
|
9
|
+
|
|
10
|
+
export const V2SavedCliReference$zodSchema: z.ZodType<V2SavedCliReference> = z
|
|
11
|
+
.object({
|
|
12
|
+
id: z.string().describe("Saved CLI ID from `GET /v2/clis`."),
|
|
13
|
+
});
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
+
* @generated-id: 9cc0c2e77b20
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as z from "zod";
|
|
7
|
+
|
|
8
|
+
export type V2SavedMcpReference = { id: string };
|
|
9
|
+
|
|
10
|
+
export const V2SavedMcpReference$zodSchema: z.ZodType<V2SavedMcpReference> = z
|
|
11
|
+
.object({
|
|
12
|
+
id: z.string().describe("Saved MCP server ID from `GET /v2/mcp-servers`."),
|
|
13
|
+
});
|
package/src/tool-names.ts
CHANGED
|
@@ -110,19 +110,19 @@ export const toolNames: Array<{ name: string; description: string }>= [
|
|
|
110
110
|
},
|
|
111
111
|
{
|
|
112
112
|
"name": "evals-v2-evals-list",
|
|
113
|
-
"description": "List Evals\n\nReturns all active
|
|
113
|
+
"description": "List Evals\n\nReturns all active eval definitions owned by the organization."
|
|
114
114
|
},
|
|
115
115
|
{
|
|
116
116
|
"name": "evals-v2-evals-create",
|
|
117
|
-
"description": "Create Eval\n\nCreates
|
|
117
|
+
"description": "Create Eval\n\nCreates a reusable simulation definition with one or more task prompts and judges. Optional agents, persona, CLI installations, MCP servers, repeat count, and tags are saved with the eval. Creating the eval does not start a run and does not accept secret environment values."
|
|
118
118
|
},
|
|
119
119
|
{
|
|
120
120
|
"name": "evals-v2-evals-retrieve",
|
|
121
|
-
"description": "Retrieve Eval\n\nReturns one eval
|
|
121
|
+
"description": "Retrieve Eval\n\nReturns one reusable eval simulation definition."
|
|
122
122
|
},
|
|
123
123
|
{
|
|
124
124
|
"name": "evals-v2-evals-update",
|
|
125
|
-
"description": "Update Eval\n\nUpdates
|
|
125
|
+
"description": "Update Eval\n\nUpdates a reusable eval simulation definition. Omitted fields remain unchanged. Empty `agent`, `clis`, `mcps`, or `tags` arrays clear those collections, and `persona: null` clears the saved persona."
|
|
126
126
|
},
|
|
127
127
|
{
|
|
128
128
|
"name": "evals-v2-evals-delete",
|
|
@@ -150,7 +150,7 @@ export const toolNames: Array<{ name: string; description: string }>= [
|
|
|
150
150
|
},
|
|
151
151
|
{
|
|
152
152
|
"name": "evals-run",
|
|
153
|
-
"description": "Start an eval run\n\nQueues a durable eval job and returns immediately with a polling ID. Poll `GET /v2/eval-runs/{run_id}` until its status reaches a terminal value.\n\nSend one of these flat request forms:\n\n- **Saved eval:** provide `eval_id
|
|
153
|
+
"description": "Start an eval run\n\nQueues a durable eval job and returns immediately with a polling ID. Poll `GET /v2/eval-runs/{run_id}` until its status reaches a terminal value.\n\nSend one of these flat request forms:\n\n- **Saved eval:** provide `eval_id`; optionally replace its prompts, persona, agents, CLIs, MCPs, or runs for this launch. A saved configuration can still provide repository, environment, skills, and legacy tool selections.\n- **Direct create-and-run:** provide `prompts`, `judges`, and `agent`. You may also provide a persona, tags, repository, write-only environment variables, saved or inline CLIs and MCP servers, and skills.\n\nThe run fans out across prompts × agents × runs. Persona is shared metadata and does not multiply attempts. No separate `mode` field is required."
|
|
154
154
|
},
|
|
155
155
|
{
|
|
156
156
|
"name": "evals-analysis",
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import http from "node:http";
|
|
3
|
+
import test from "node:test";
|
|
4
|
+
|
|
5
|
+
const { LightsageMcpCore } = await import("../esm/core.js");
|
|
6
|
+
const { CreateV2EvalRequest$zodSchema } = await import(
|
|
7
|
+
"../esm/models/createv2evalrequest.js"
|
|
8
|
+
);
|
|
9
|
+
const { CreateV2EvalRunRequest$zodSchema } = await import(
|
|
10
|
+
"../esm/models/createv2evalrunrequest.js"
|
|
11
|
+
);
|
|
12
|
+
const { tool$evalsV2EvalsCreate } = await import(
|
|
13
|
+
"../esm/mcp-server/tools/evalsV2EvalsCreate.js"
|
|
14
|
+
);
|
|
15
|
+
const { tool$evalsRun } = await import(
|
|
16
|
+
"../esm/mcp-server/tools/evalsRun.js"
|
|
17
|
+
);
|
|
18
|
+
|
|
19
|
+
const savedCli = { id: "230f0c71-ed91-4a4a-9932-9cef267411bc" };
|
|
20
|
+
const inlineCli = { install_command: "brew install lightsagehq/tools/lightsage" };
|
|
21
|
+
const savedMcp = { id: "550e8400-e29b-41d4-a716-446655440000" };
|
|
22
|
+
const inlineMcp = {
|
|
23
|
+
name: "lightsage",
|
|
24
|
+
type: "http",
|
|
25
|
+
url: "https://mcp.lightsage.com/mcp",
|
|
26
|
+
headers: {
|
|
27
|
+
"X-Lightsage-Api-Key": "${LIGHTSAGE_API_KEY}",
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
function createSimulationServer(requests) {
|
|
32
|
+
return http.createServer(async (req, res) => {
|
|
33
|
+
const chunks = [];
|
|
34
|
+
for await (const chunk of req) chunks.push(chunk);
|
|
35
|
+
requests.push({
|
|
36
|
+
method: req.method,
|
|
37
|
+
url: req.url,
|
|
38
|
+
apiKey: req.headers["x-lightsage-api-key"],
|
|
39
|
+
body: JSON.parse(Buffer.concat(chunks).toString()),
|
|
40
|
+
});
|
|
41
|
+
res.setHeader("content-type", "application/json");
|
|
42
|
+
res.end(JSON.stringify({ accepted: true }));
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
test("create eval accepts the canonical simulation matrix", () => {
|
|
47
|
+
const parsed = CreateV2EvalRequest$zodSchema.parse({
|
|
48
|
+
name: "Lightsage interface simulation",
|
|
49
|
+
prompts: ["Use the CLI.", "Use the MCP server."],
|
|
50
|
+
judges: ["The workflow succeeds."],
|
|
51
|
+
persona: "Platform engineer",
|
|
52
|
+
agent: ["codex:gpt-5-4", "claude-code:claude-opus-4-8"],
|
|
53
|
+
clis: [savedCli, inlineCli],
|
|
54
|
+
mcps: [savedMcp, inlineMcp],
|
|
55
|
+
runs: 2,
|
|
56
|
+
tags: ["smoke-test"],
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
assert.deepEqual(parsed.clis, [savedCli, inlineCli]);
|
|
60
|
+
assert.deepEqual(parsed.mcps, [savedMcp, inlineMcp]);
|
|
61
|
+
assert.equal(parsed.runs, 2);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("direct eval run accepts the same canonical simulation fields", () => {
|
|
65
|
+
const parsed = CreateV2EvalRunRequest$zodSchema.parse({
|
|
66
|
+
prompts: ["Use the CLI.", "Use the MCP server."],
|
|
67
|
+
judges: ["The workflow succeeds."],
|
|
68
|
+
agent: ["codex:gpt-5-4", "claude-code:claude-opus-4-8"],
|
|
69
|
+
persona: "Platform engineer",
|
|
70
|
+
clis: [savedCli, inlineCli],
|
|
71
|
+
mcps: [savedMcp, inlineMcp],
|
|
72
|
+
env: { LIGHTSAGE_API_KEY: "runtime-only" },
|
|
73
|
+
skills: ["resend/resend-skills"],
|
|
74
|
+
runs: 3,
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
assert.deepEqual(parsed.agent, [
|
|
78
|
+
"codex:gpt-5-4",
|
|
79
|
+
"claude-code:claude-opus-4-8",
|
|
80
|
+
]);
|
|
81
|
+
assert.deepEqual(parsed.clis, [savedCli, inlineCli]);
|
|
82
|
+
assert.deepEqual(parsed.mcps, [savedMcp, inlineMcp]);
|
|
83
|
+
assert.deepEqual(parsed.skills, ["resend/resend-skills"]);
|
|
84
|
+
assert.equal(parsed.runs, 3);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("simulation tools reject non-object CLI and MCP entries", () => {
|
|
88
|
+
const result = CreateV2EvalRunRequest$zodSchema.safeParse({
|
|
89
|
+
prompts: ["Use the CLI."],
|
|
90
|
+
judges: ["The workflow succeeds."],
|
|
91
|
+
agent: ["codex:gpt-5-4"],
|
|
92
|
+
clis: ["brew install lightsage"],
|
|
93
|
+
mcps: ["https://mcp.lightsage.com/mcp"],
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
assert.equal(result.success, false);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("MCP tools forward canonical eval resources unchanged", async () => {
|
|
100
|
+
const requests = [];
|
|
101
|
+
const server = createSimulationServer(requests);
|
|
102
|
+
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
103
|
+
const address = server.address();
|
|
104
|
+
const client = new LightsageMcpCore({
|
|
105
|
+
serverURL: `http://127.0.0.1:${address.port}`,
|
|
106
|
+
security: { LightsageApiKeyAuth: "ls_simulation_test" },
|
|
107
|
+
});
|
|
108
|
+
const ctx = { signal: new AbortController().signal };
|
|
109
|
+
const createRequest = {
|
|
110
|
+
name: "Lightsage interface simulation",
|
|
111
|
+
prompts: ["Use the CLI.", "Use the MCP server."],
|
|
112
|
+
judges: ["The workflow succeeds."],
|
|
113
|
+
agent: ["codex:gpt-5-4"],
|
|
114
|
+
clis: [savedCli, inlineCli],
|
|
115
|
+
mcps: [savedMcp, inlineMcp],
|
|
116
|
+
runs: 2,
|
|
117
|
+
tags: ["smoke-test"],
|
|
118
|
+
};
|
|
119
|
+
const runRequest = {
|
|
120
|
+
prompts: ["Use the CLI.", "Use the MCP server."],
|
|
121
|
+
judges: ["The workflow succeeds."],
|
|
122
|
+
agent: ["codex:gpt-5-4"],
|
|
123
|
+
clis: [savedCli, inlineCli],
|
|
124
|
+
mcps: [savedMcp, inlineMcp],
|
|
125
|
+
env: { LIGHTSAGE_API_KEY: "runtime-only" },
|
|
126
|
+
skills: ["resend/resend-skills"],
|
|
127
|
+
runs: 3,
|
|
128
|
+
};
|
|
129
|
+
|
|
130
|
+
try {
|
|
131
|
+
const createResult = await tool$evalsV2EvalsCreate.tool(
|
|
132
|
+
client,
|
|
133
|
+
{ request: createRequest },
|
|
134
|
+
ctx,
|
|
135
|
+
);
|
|
136
|
+
assert.equal(createResult.isError, undefined);
|
|
137
|
+
|
|
138
|
+
const runResult = await tool$evalsRun.tool(
|
|
139
|
+
client,
|
|
140
|
+
{ request: runRequest },
|
|
141
|
+
ctx,
|
|
142
|
+
);
|
|
143
|
+
assert.equal(runResult.isError, undefined);
|
|
144
|
+
|
|
145
|
+
assert.deepEqual(requests, [
|
|
146
|
+
{
|
|
147
|
+
method: "POST",
|
|
148
|
+
url: "/v2/evals",
|
|
149
|
+
apiKey: "ls_simulation_test",
|
|
150
|
+
body: createRequest,
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
method: "POST",
|
|
154
|
+
url: "/v2/eval-runs",
|
|
155
|
+
apiKey: "ls_simulation_test",
|
|
156
|
+
body: runRequest,
|
|
157
|
+
},
|
|
158
|
+
]);
|
|
159
|
+
} finally {
|
|
160
|
+
server.close();
|
|
161
|
+
}
|
|
162
|
+
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"v2evaljudge.d.ts","sourceRoot":"","sources":["../../src/models/v2evaljudge.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAEzB,MAAM,MAAM,WAAW,GAAG;IAAE,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAE/C,eAAO,MAAM,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,WAAW,CAEvD,CAAC"}
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
-
* @generated-id: ffd5279bc045
|
|
4
|
-
*/
|
|
5
|
-
import * as z from "zod";
|
|
6
|
-
export const V2EvalJudge$zodSchema = z.object({
|
|
7
|
-
criteria: z.string().describe("The outcome that counts as success."),
|
|
8
|
-
});
|
|
9
|
-
//# sourceMappingURL=v2evaljudge.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"v2evaljudge.js","sourceRoot":"","sources":["../../src/models/v2evaljudge.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC;AAIzB,MAAM,CAAC,MAAM,qBAAqB,GAA2B,CAAC,CAAC,MAAM,CAAC;IACpE,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;CACrE,CAAC,CAAC"}
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
/*
|
|
2
|
-
* Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
|
|
3
|
-
* @generated-id: ffd5279bc045
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
import * as z from "zod";
|
|
7
|
-
|
|
8
|
-
export type V2EvalJudge = { criteria: string };
|
|
9
|
-
|
|
10
|
-
export const V2EvalJudge$zodSchema: z.ZodType<V2EvalJudge> = z.object({
|
|
11
|
-
criteria: z.string().describe("The outcome that counts as success."),
|
|
12
|
-
});
|