@agentv/eval 3.14.6 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +29 -16
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +681 -138
- package/dist/index.d.ts +681 -138
- package/dist/index.js +25 -16
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -4,6 +4,22 @@ export { z } from 'zod';
|
|
|
4
4
|
/**
|
|
5
5
|
* Zod schemas for code grader input/output validation.
|
|
6
6
|
* Provides both compile-time types and runtime validation.
|
|
7
|
+
*
|
|
8
|
+
* ## Content model
|
|
9
|
+
*
|
|
10
|
+
* `Message.content` accepts `string | Content[]`:
|
|
11
|
+
* - `string` — backward-compatible plain text (most common case)
|
|
12
|
+
* - `Content[]` — typed content blocks for multimodal messages
|
|
13
|
+
*
|
|
14
|
+
* Content variants:
|
|
15
|
+
* - `ContentText` — `{ type: 'text', text: string }`
|
|
16
|
+
* - `ContentImage` — `{ type: 'image', media_type: string, path: string }` (file path, not base64)
|
|
17
|
+
* - `ContentFile` — `{ type: 'file', media_type: string, path: string }`
|
|
18
|
+
*
|
|
19
|
+
* To add a new content variant:
|
|
20
|
+
* 1. Define a new Zod schema with a unique `type` literal
|
|
21
|
+
* 2. Add it to `ContentSchema` discriminated union
|
|
22
|
+
* 3. Re-export from `index.ts`
|
|
7
23
|
*/
|
|
8
24
|
|
|
9
25
|
/**
|
|
@@ -72,12 +88,125 @@ declare const ToolCallSchema: z.ZodObject<{
|
|
|
72
88
|
endTime?: string | undefined;
|
|
73
89
|
durationMs?: number | undefined;
|
|
74
90
|
}>;
|
|
91
|
+
/** Text content block. */
|
|
92
|
+
declare const ContentTextSchema: z.ZodObject<{
|
|
93
|
+
type: z.ZodLiteral<"text">;
|
|
94
|
+
text: z.ZodString;
|
|
95
|
+
}, "strip", z.ZodTypeAny, {
|
|
96
|
+
type: "text";
|
|
97
|
+
text: string;
|
|
98
|
+
}, {
|
|
99
|
+
type: "text";
|
|
100
|
+
text: string;
|
|
101
|
+
}>;
|
|
102
|
+
/**
|
|
103
|
+
* Image content block.
|
|
104
|
+
* `path` is a filesystem path — never inline base64.
|
|
105
|
+
*/
|
|
106
|
+
declare const ContentImageSchema: z.ZodObject<{
|
|
107
|
+
type: z.ZodLiteral<"image">;
|
|
108
|
+
media_type: z.ZodString;
|
|
109
|
+
path: z.ZodString;
|
|
110
|
+
}, "strip", z.ZodTypeAny, {
|
|
111
|
+
path: string;
|
|
112
|
+
type: "image";
|
|
113
|
+
media_type: string;
|
|
114
|
+
}, {
|
|
115
|
+
path: string;
|
|
116
|
+
type: "image";
|
|
117
|
+
media_type: string;
|
|
118
|
+
}>;
|
|
119
|
+
/** File content block. */
|
|
120
|
+
declare const ContentFileSchema: z.ZodObject<{
|
|
121
|
+
type: z.ZodLiteral<"file">;
|
|
122
|
+
media_type: z.ZodString;
|
|
123
|
+
path: z.ZodString;
|
|
124
|
+
}, "strip", z.ZodTypeAny, {
|
|
125
|
+
path: string;
|
|
126
|
+
type: "file";
|
|
127
|
+
media_type: string;
|
|
128
|
+
}, {
|
|
129
|
+
path: string;
|
|
130
|
+
type: "file";
|
|
131
|
+
media_type: string;
|
|
132
|
+
}>;
|
|
133
|
+
/** Discriminated union of all content block types. */
|
|
134
|
+
declare const ContentSchema: z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
135
|
+
type: z.ZodLiteral<"text">;
|
|
136
|
+
text: z.ZodString;
|
|
137
|
+
}, "strip", z.ZodTypeAny, {
|
|
138
|
+
type: "text";
|
|
139
|
+
text: string;
|
|
140
|
+
}, {
|
|
141
|
+
type: "text";
|
|
142
|
+
text: string;
|
|
143
|
+
}>, z.ZodObject<{
|
|
144
|
+
type: z.ZodLiteral<"image">;
|
|
145
|
+
media_type: z.ZodString;
|
|
146
|
+
path: z.ZodString;
|
|
147
|
+
}, "strip", z.ZodTypeAny, {
|
|
148
|
+
path: string;
|
|
149
|
+
type: "image";
|
|
150
|
+
media_type: string;
|
|
151
|
+
}, {
|
|
152
|
+
path: string;
|
|
153
|
+
type: "image";
|
|
154
|
+
media_type: string;
|
|
155
|
+
}>, z.ZodObject<{
|
|
156
|
+
type: z.ZodLiteral<"file">;
|
|
157
|
+
media_type: z.ZodString;
|
|
158
|
+
path: z.ZodString;
|
|
159
|
+
}, "strip", z.ZodTypeAny, {
|
|
160
|
+
path: string;
|
|
161
|
+
type: "file";
|
|
162
|
+
media_type: string;
|
|
163
|
+
}, {
|
|
164
|
+
path: string;
|
|
165
|
+
type: "file";
|
|
166
|
+
media_type: string;
|
|
167
|
+
}>]>;
|
|
75
168
|
/**
|
|
76
169
|
* Unified message schema for input, expected, and output messages.
|
|
170
|
+
*
|
|
171
|
+
* `content` is either a plain string or a `Content[]` array of typed blocks.
|
|
172
|
+
* Use `getTextContent()` from `@agentv/core` to extract plain text from either form.
|
|
77
173
|
*/
|
|
78
174
|
declare const MessageSchema: z.ZodObject<{
|
|
79
175
|
role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
|
|
80
|
-
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.
|
|
176
|
+
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
177
|
+
type: z.ZodLiteral<"text">;
|
|
178
|
+
text: z.ZodString;
|
|
179
|
+
}, "strip", z.ZodTypeAny, {
|
|
180
|
+
type: "text";
|
|
181
|
+
text: string;
|
|
182
|
+
}, {
|
|
183
|
+
type: "text";
|
|
184
|
+
text: string;
|
|
185
|
+
}>, z.ZodObject<{
|
|
186
|
+
type: z.ZodLiteral<"image">;
|
|
187
|
+
media_type: z.ZodString;
|
|
188
|
+
path: z.ZodString;
|
|
189
|
+
}, "strip", z.ZodTypeAny, {
|
|
190
|
+
path: string;
|
|
191
|
+
type: "image";
|
|
192
|
+
media_type: string;
|
|
193
|
+
}, {
|
|
194
|
+
path: string;
|
|
195
|
+
type: "image";
|
|
196
|
+
media_type: string;
|
|
197
|
+
}>, z.ZodObject<{
|
|
198
|
+
type: z.ZodLiteral<"file">;
|
|
199
|
+
media_type: z.ZodString;
|
|
200
|
+
path: z.ZodString;
|
|
201
|
+
}, "strip", z.ZodTypeAny, {
|
|
202
|
+
path: string;
|
|
203
|
+
type: "file";
|
|
204
|
+
media_type: string;
|
|
205
|
+
}, {
|
|
206
|
+
path: string;
|
|
207
|
+
type: "file";
|
|
208
|
+
media_type: string;
|
|
209
|
+
}>]>, "many">]>>;
|
|
81
210
|
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
82
211
|
tool: z.ZodString;
|
|
83
212
|
input: z.ZodOptional<z.ZodUnknown>;
|
|
@@ -122,7 +251,18 @@ declare const MessageSchema: z.ZodObject<{
|
|
|
122
251
|
startTime?: string | undefined;
|
|
123
252
|
endTime?: string | undefined;
|
|
124
253
|
durationMs?: number | undefined;
|
|
125
|
-
content?: string |
|
|
254
|
+
content?: string | ({
|
|
255
|
+
type: "text";
|
|
256
|
+
text: string;
|
|
257
|
+
} | {
|
|
258
|
+
path: string;
|
|
259
|
+
type: "image";
|
|
260
|
+
media_type: string;
|
|
261
|
+
} | {
|
|
262
|
+
path: string;
|
|
263
|
+
type: "file";
|
|
264
|
+
media_type: string;
|
|
265
|
+
})[] | undefined;
|
|
126
266
|
name?: string | undefined;
|
|
127
267
|
metadata?: Record<string, unknown> | undefined;
|
|
128
268
|
}, {
|
|
@@ -139,21 +279,65 @@ declare const MessageSchema: z.ZodObject<{
|
|
|
139
279
|
startTime?: string | undefined;
|
|
140
280
|
endTime?: string | undefined;
|
|
141
281
|
durationMs?: number | undefined;
|
|
142
|
-
content?: string |
|
|
282
|
+
content?: string | ({
|
|
283
|
+
type: "text";
|
|
284
|
+
text: string;
|
|
285
|
+
} | {
|
|
286
|
+
path: string;
|
|
287
|
+
type: "image";
|
|
288
|
+
media_type: string;
|
|
289
|
+
} | {
|
|
290
|
+
path: string;
|
|
291
|
+
type: "file";
|
|
292
|
+
media_type: string;
|
|
293
|
+
})[] | undefined;
|
|
143
294
|
name?: string | undefined;
|
|
144
295
|
metadata?: Record<string, unknown> | undefined;
|
|
145
296
|
}>;
|
|
146
297
|
/**
|
|
147
298
|
* Code grader input schema (camelCase, converted from snake_case wire format).
|
|
148
299
|
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
300
|
+
* Structured fields (`input`, `output`, `expectedOutput`) are always `Message[]`.
|
|
301
|
+
* To extract plain text from message content, use `getTextContent()` from `@agentv/core`.
|
|
151
302
|
*/
|
|
152
303
|
declare const CodeGraderInputSchema: z.ZodObject<{
|
|
153
304
|
criteria: z.ZodString;
|
|
154
305
|
expectedOutput: z.ZodArray<z.ZodObject<{
|
|
155
306
|
role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
|
|
156
|
-
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.
|
|
307
|
+
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
308
|
+
type: z.ZodLiteral<"text">;
|
|
309
|
+
text: z.ZodString;
|
|
310
|
+
}, "strip", z.ZodTypeAny, {
|
|
311
|
+
type: "text";
|
|
312
|
+
text: string;
|
|
313
|
+
}, {
|
|
314
|
+
type: "text";
|
|
315
|
+
text: string;
|
|
316
|
+
}>, z.ZodObject<{
|
|
317
|
+
type: z.ZodLiteral<"image">;
|
|
318
|
+
media_type: z.ZodString;
|
|
319
|
+
path: z.ZodString;
|
|
320
|
+
}, "strip", z.ZodTypeAny, {
|
|
321
|
+
path: string;
|
|
322
|
+
type: "image";
|
|
323
|
+
media_type: string;
|
|
324
|
+
}, {
|
|
325
|
+
path: string;
|
|
326
|
+
type: "image";
|
|
327
|
+
media_type: string;
|
|
328
|
+
}>, z.ZodObject<{
|
|
329
|
+
type: z.ZodLiteral<"file">;
|
|
330
|
+
media_type: z.ZodString;
|
|
331
|
+
path: z.ZodString;
|
|
332
|
+
}, "strip", z.ZodTypeAny, {
|
|
333
|
+
path: string;
|
|
334
|
+
type: "file";
|
|
335
|
+
media_type: string;
|
|
336
|
+
}, {
|
|
337
|
+
path: string;
|
|
338
|
+
type: "file";
|
|
339
|
+
media_type: string;
|
|
340
|
+
}>]>, "many">]>>;
|
|
157
341
|
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
158
342
|
tool: z.ZodString;
|
|
159
343
|
input: z.ZodOptional<z.ZodUnknown>;
|
|
@@ -198,7 +382,18 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
198
382
|
startTime?: string | undefined;
|
|
199
383
|
endTime?: string | undefined;
|
|
200
384
|
durationMs?: number | undefined;
|
|
201
|
-
content?: string |
|
|
385
|
+
content?: string | ({
|
|
386
|
+
type: "text";
|
|
387
|
+
text: string;
|
|
388
|
+
} | {
|
|
389
|
+
path: string;
|
|
390
|
+
type: "image";
|
|
391
|
+
media_type: string;
|
|
392
|
+
} | {
|
|
393
|
+
path: string;
|
|
394
|
+
type: "file";
|
|
395
|
+
media_type: string;
|
|
396
|
+
})[] | undefined;
|
|
202
397
|
name?: string | undefined;
|
|
203
398
|
metadata?: Record<string, unknown> | undefined;
|
|
204
399
|
}, {
|
|
@@ -215,15 +410,57 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
215
410
|
startTime?: string | undefined;
|
|
216
411
|
endTime?: string | undefined;
|
|
217
412
|
durationMs?: number | undefined;
|
|
218
|
-
content?: string |
|
|
413
|
+
content?: string | ({
|
|
414
|
+
type: "text";
|
|
415
|
+
text: string;
|
|
416
|
+
} | {
|
|
417
|
+
path: string;
|
|
418
|
+
type: "image";
|
|
419
|
+
media_type: string;
|
|
420
|
+
} | {
|
|
421
|
+
path: string;
|
|
422
|
+
type: "file";
|
|
423
|
+
media_type: string;
|
|
424
|
+
})[] | undefined;
|
|
219
425
|
name?: string | undefined;
|
|
220
426
|
metadata?: Record<string, unknown> | undefined;
|
|
221
427
|
}>, "many">;
|
|
222
|
-
/** Last assistant message content as string. */
|
|
223
|
-
outputText: z.ZodString;
|
|
224
428
|
output: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
|
|
225
429
|
role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
|
|
226
|
-
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.
|
|
430
|
+
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
431
|
+
type: z.ZodLiteral<"text">;
|
|
432
|
+
text: z.ZodString;
|
|
433
|
+
}, "strip", z.ZodTypeAny, {
|
|
434
|
+
type: "text";
|
|
435
|
+
text: string;
|
|
436
|
+
}, {
|
|
437
|
+
type: "text";
|
|
438
|
+
text: string;
|
|
439
|
+
}>, z.ZodObject<{
|
|
440
|
+
type: z.ZodLiteral<"image">;
|
|
441
|
+
media_type: z.ZodString;
|
|
442
|
+
path: z.ZodString;
|
|
443
|
+
}, "strip", z.ZodTypeAny, {
|
|
444
|
+
path: string;
|
|
445
|
+
type: "image";
|
|
446
|
+
media_type: string;
|
|
447
|
+
}, {
|
|
448
|
+
path: string;
|
|
449
|
+
type: "image";
|
|
450
|
+
media_type: string;
|
|
451
|
+
}>, z.ZodObject<{
|
|
452
|
+
type: z.ZodLiteral<"file">;
|
|
453
|
+
media_type: z.ZodString;
|
|
454
|
+
path: z.ZodString;
|
|
455
|
+
}, "strip", z.ZodTypeAny, {
|
|
456
|
+
path: string;
|
|
457
|
+
type: "file";
|
|
458
|
+
media_type: string;
|
|
459
|
+
}, {
|
|
460
|
+
path: string;
|
|
461
|
+
type: "file";
|
|
462
|
+
media_type: string;
|
|
463
|
+
}>]>, "many">]>>;
|
|
227
464
|
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
228
465
|
tool: z.ZodString;
|
|
229
466
|
input: z.ZodOptional<z.ZodUnknown>;
|
|
@@ -268,7 +505,18 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
268
505
|
startTime?: string | undefined;
|
|
269
506
|
endTime?: string | undefined;
|
|
270
507
|
durationMs?: number | undefined;
|
|
271
|
-
content?: string |
|
|
508
|
+
content?: string | ({
|
|
509
|
+
type: "text";
|
|
510
|
+
text: string;
|
|
511
|
+
} | {
|
|
512
|
+
path: string;
|
|
513
|
+
type: "image";
|
|
514
|
+
media_type: string;
|
|
515
|
+
} | {
|
|
516
|
+
path: string;
|
|
517
|
+
type: "file";
|
|
518
|
+
media_type: string;
|
|
519
|
+
})[] | undefined;
|
|
272
520
|
name?: string | undefined;
|
|
273
521
|
metadata?: Record<string, unknown> | undefined;
|
|
274
522
|
}, {
|
|
@@ -285,7 +533,18 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
285
533
|
startTime?: string | undefined;
|
|
286
534
|
endTime?: string | undefined;
|
|
287
535
|
durationMs?: number | undefined;
|
|
288
|
-
content?: string |
|
|
536
|
+
content?: string | ({
|
|
537
|
+
type: "text";
|
|
538
|
+
text: string;
|
|
539
|
+
} | {
|
|
540
|
+
path: string;
|
|
541
|
+
type: "image";
|
|
542
|
+
media_type: string;
|
|
543
|
+
} | {
|
|
544
|
+
path: string;
|
|
545
|
+
type: "file";
|
|
546
|
+
media_type: string;
|
|
547
|
+
})[] | undefined;
|
|
289
548
|
name?: string | undefined;
|
|
290
549
|
metadata?: Record<string, unknown> | undefined;
|
|
291
550
|
}>, "many">>>;
|
|
@@ -294,7 +553,40 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
294
553
|
inputFiles: z.ZodArray<z.ZodString, "many">;
|
|
295
554
|
input: z.ZodArray<z.ZodObject<{
|
|
296
555
|
role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
|
|
297
|
-
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.
|
|
556
|
+
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
557
|
+
type: z.ZodLiteral<"text">;
|
|
558
|
+
text: z.ZodString;
|
|
559
|
+
}, "strip", z.ZodTypeAny, {
|
|
560
|
+
type: "text";
|
|
561
|
+
text: string;
|
|
562
|
+
}, {
|
|
563
|
+
type: "text";
|
|
564
|
+
text: string;
|
|
565
|
+
}>, z.ZodObject<{
|
|
566
|
+
type: z.ZodLiteral<"image">;
|
|
567
|
+
media_type: z.ZodString;
|
|
568
|
+
path: z.ZodString;
|
|
569
|
+
}, "strip", z.ZodTypeAny, {
|
|
570
|
+
path: string;
|
|
571
|
+
type: "image";
|
|
572
|
+
media_type: string;
|
|
573
|
+
}, {
|
|
574
|
+
path: string;
|
|
575
|
+
type: "image";
|
|
576
|
+
media_type: string;
|
|
577
|
+
}>, z.ZodObject<{
|
|
578
|
+
type: z.ZodLiteral<"file">;
|
|
579
|
+
media_type: z.ZodString;
|
|
580
|
+
path: z.ZodString;
|
|
581
|
+
}, "strip", z.ZodTypeAny, {
|
|
582
|
+
path: string;
|
|
583
|
+
type: "file";
|
|
584
|
+
media_type: string;
|
|
585
|
+
}, {
|
|
586
|
+
path: string;
|
|
587
|
+
type: "file";
|
|
588
|
+
media_type: string;
|
|
589
|
+
}>]>, "many">]>>;
|
|
298
590
|
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
299
591
|
tool: z.ZodString;
|
|
300
592
|
input: z.ZodOptional<z.ZodUnknown>;
|
|
@@ -339,7 +631,18 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
339
631
|
startTime?: string | undefined;
|
|
340
632
|
endTime?: string | undefined;
|
|
341
633
|
durationMs?: number | undefined;
|
|
342
|
-
content?: string |
|
|
634
|
+
content?: string | ({
|
|
635
|
+
type: "text";
|
|
636
|
+
text: string;
|
|
637
|
+
} | {
|
|
638
|
+
path: string;
|
|
639
|
+
type: "image";
|
|
640
|
+
media_type: string;
|
|
641
|
+
} | {
|
|
642
|
+
path: string;
|
|
643
|
+
type: "file";
|
|
644
|
+
media_type: string;
|
|
645
|
+
})[] | undefined;
|
|
343
646
|
name?: string | undefined;
|
|
344
647
|
metadata?: Record<string, unknown> | undefined;
|
|
345
648
|
}, {
|
|
@@ -356,7 +659,18 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
356
659
|
startTime?: string | undefined;
|
|
357
660
|
endTime?: string | undefined;
|
|
358
661
|
durationMs?: number | undefined;
|
|
359
|
-
content?: string |
|
|
662
|
+
content?: string | ({
|
|
663
|
+
type: "text";
|
|
664
|
+
text: string;
|
|
665
|
+
} | {
|
|
666
|
+
path: string;
|
|
667
|
+
type: "image";
|
|
668
|
+
media_type: string;
|
|
669
|
+
} | {
|
|
670
|
+
path: string;
|
|
671
|
+
type: "file";
|
|
672
|
+
media_type: string;
|
|
673
|
+
})[] | undefined;
|
|
360
674
|
name?: string | undefined;
|
|
361
675
|
metadata?: Record<string, unknown> | undefined;
|
|
362
676
|
}>, "many">;
|
|
@@ -399,10 +713,6 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
399
713
|
fileChanges: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
400
714
|
workspacePath: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
401
715
|
config: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
402
|
-
/** First user message content as string. */
|
|
403
|
-
inputText: z.ZodString;
|
|
404
|
-
/** Expected output content as string. */
|
|
405
|
-
expectedOutputText: z.ZodOptional<z.ZodString>;
|
|
406
716
|
}, "strip", z.ZodTypeAny, {
|
|
407
717
|
input: {
|
|
408
718
|
role: "tool" | "assistant" | "user" | "system";
|
|
@@ -418,7 +728,18 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
418
728
|
startTime?: string | undefined;
|
|
419
729
|
endTime?: string | undefined;
|
|
420
730
|
durationMs?: number | undefined;
|
|
421
|
-
content?: string |
|
|
731
|
+
content?: string | ({
|
|
732
|
+
type: "text";
|
|
733
|
+
text: string;
|
|
734
|
+
} | {
|
|
735
|
+
path: string;
|
|
736
|
+
type: "image";
|
|
737
|
+
media_type: string;
|
|
738
|
+
} | {
|
|
739
|
+
path: string;
|
|
740
|
+
type: "file";
|
|
741
|
+
media_type: string;
|
|
742
|
+
})[] | undefined;
|
|
422
743
|
name?: string | undefined;
|
|
423
744
|
metadata?: Record<string, unknown> | undefined;
|
|
424
745
|
}[];
|
|
@@ -437,13 +758,22 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
437
758
|
startTime?: string | undefined;
|
|
438
759
|
endTime?: string | undefined;
|
|
439
760
|
durationMs?: number | undefined;
|
|
440
|
-
content?: string |
|
|
761
|
+
content?: string | ({
|
|
762
|
+
type: "text";
|
|
763
|
+
text: string;
|
|
764
|
+
} | {
|
|
765
|
+
path: string;
|
|
766
|
+
type: "image";
|
|
767
|
+
media_type: string;
|
|
768
|
+
} | {
|
|
769
|
+
path: string;
|
|
770
|
+
type: "file";
|
|
771
|
+
media_type: string;
|
|
772
|
+
})[] | undefined;
|
|
441
773
|
name?: string | undefined;
|
|
442
774
|
metadata?: Record<string, unknown> | undefined;
|
|
443
775
|
}[];
|
|
444
|
-
outputText: string;
|
|
445
776
|
inputFiles: string[];
|
|
446
|
-
inputText: string;
|
|
447
777
|
output?: {
|
|
448
778
|
role: "tool" | "assistant" | "user" | "system";
|
|
449
779
|
toolCalls?: {
|
|
@@ -458,7 +788,18 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
458
788
|
startTime?: string | undefined;
|
|
459
789
|
endTime?: string | undefined;
|
|
460
790
|
durationMs?: number | undefined;
|
|
461
|
-
content?: string |
|
|
791
|
+
content?: string | ({
|
|
792
|
+
type: "text";
|
|
793
|
+
text: string;
|
|
794
|
+
} | {
|
|
795
|
+
path: string;
|
|
796
|
+
type: "image";
|
|
797
|
+
media_type: string;
|
|
798
|
+
} | {
|
|
799
|
+
path: string;
|
|
800
|
+
type: "file";
|
|
801
|
+
media_type: string;
|
|
802
|
+
})[] | undefined;
|
|
462
803
|
name?: string | undefined;
|
|
463
804
|
metadata?: Record<string, unknown> | undefined;
|
|
464
805
|
}[] | null | undefined;
|
|
@@ -482,7 +823,6 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
482
823
|
fileChanges?: string | null | undefined;
|
|
483
824
|
workspacePath?: string | null | undefined;
|
|
484
825
|
config?: Record<string, unknown> | null | undefined;
|
|
485
|
-
expectedOutputText?: string | undefined;
|
|
486
826
|
}, {
|
|
487
827
|
input: {
|
|
488
828
|
role: "tool" | "assistant" | "user" | "system";
|
|
@@ -498,7 +838,18 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
498
838
|
startTime?: string | undefined;
|
|
499
839
|
endTime?: string | undefined;
|
|
500
840
|
durationMs?: number | undefined;
|
|
501
|
-
content?: string |
|
|
841
|
+
content?: string | ({
|
|
842
|
+
type: "text";
|
|
843
|
+
text: string;
|
|
844
|
+
} | {
|
|
845
|
+
path: string;
|
|
846
|
+
type: "image";
|
|
847
|
+
media_type: string;
|
|
848
|
+
} | {
|
|
849
|
+
path: string;
|
|
850
|
+
type: "file";
|
|
851
|
+
media_type: string;
|
|
852
|
+
})[] | undefined;
|
|
502
853
|
name?: string | undefined;
|
|
503
854
|
metadata?: Record<string, unknown> | undefined;
|
|
504
855
|
}[];
|
|
@@ -517,13 +868,22 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
517
868
|
startTime?: string | undefined;
|
|
518
869
|
endTime?: string | undefined;
|
|
519
870
|
durationMs?: number | undefined;
|
|
520
|
-
content?: string |
|
|
871
|
+
content?: string | ({
|
|
872
|
+
type: "text";
|
|
873
|
+
text: string;
|
|
874
|
+
} | {
|
|
875
|
+
path: string;
|
|
876
|
+
type: "image";
|
|
877
|
+
media_type: string;
|
|
878
|
+
} | {
|
|
879
|
+
path: string;
|
|
880
|
+
type: "file";
|
|
881
|
+
media_type: string;
|
|
882
|
+
})[] | undefined;
|
|
521
883
|
name?: string | undefined;
|
|
522
884
|
metadata?: Record<string, unknown> | undefined;
|
|
523
885
|
}[];
|
|
524
|
-
outputText: string;
|
|
525
886
|
inputFiles: string[];
|
|
526
|
-
inputText: string;
|
|
527
887
|
output?: {
|
|
528
888
|
role: "tool" | "assistant" | "user" | "system";
|
|
529
889
|
toolCalls?: {
|
|
@@ -538,7 +898,18 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
538
898
|
startTime?: string | undefined;
|
|
539
899
|
endTime?: string | undefined;
|
|
540
900
|
durationMs?: number | undefined;
|
|
541
|
-
content?: string |
|
|
901
|
+
content?: string | ({
|
|
902
|
+
type: "text";
|
|
903
|
+
text: string;
|
|
904
|
+
} | {
|
|
905
|
+
path: string;
|
|
906
|
+
type: "image";
|
|
907
|
+
media_type: string;
|
|
908
|
+
} | {
|
|
909
|
+
path: string;
|
|
910
|
+
type: "file";
|
|
911
|
+
media_type: string;
|
|
912
|
+
})[] | undefined;
|
|
542
913
|
name?: string | undefined;
|
|
543
914
|
metadata?: Record<string, unknown> | undefined;
|
|
544
915
|
}[] | null | undefined;
|
|
@@ -562,7 +933,6 @@ declare const CodeGraderInputSchema: z.ZodObject<{
|
|
|
562
933
|
fileChanges?: string | null | undefined;
|
|
563
934
|
workspacePath?: string | null | undefined;
|
|
564
935
|
config?: Record<string, unknown> | null | undefined;
|
|
565
|
-
expectedOutputText?: string | undefined;
|
|
566
936
|
}>;
|
|
567
937
|
/**
|
|
568
938
|
* Code grader result schema (validated before output).
|
|
@@ -606,24 +976,14 @@ declare const CodeGraderResultSchema: z.ZodObject<{
|
|
|
606
976
|
*/
|
|
607
977
|
type CodeGraderInput = z.infer<typeof CodeGraderInputSchema>;
|
|
608
978
|
type CodeGraderResult = z.infer<typeof CodeGraderResultSchema>;
|
|
609
|
-
/**
|
|
610
|
-
* CodeGraderInput after `enrichInput()` has run.
|
|
611
|
-
*
|
|
612
|
-
* The text accessors (`inputText`, `outputText`, `expectedOutputText`)
|
|
613
|
-
* are always populated by the runtime before the handler is called, so they are
|
|
614
|
-
* guaranteed to be `string` (never `undefined`).
|
|
615
|
-
*
|
|
616
|
-
* Handler function signatures (`CodeGraderHandler`, `AssertionHandler`) use this
|
|
617
|
-
* type so that user code can destructure `{ outputText }` without null-checks.
|
|
618
|
-
*/
|
|
619
|
-
type EnrichedCodeGraderInput = Omit<CodeGraderInput, 'expectedOutputText'> & {
|
|
620
|
-
/** Expected output content as string. */
|
|
621
|
-
readonly expectedOutputText: string;
|
|
622
|
-
};
|
|
623
979
|
type TraceSummary = z.infer<typeof TraceSummarySchema>;
|
|
624
980
|
type Message = z.infer<typeof MessageSchema>;
|
|
625
981
|
type ToolCall = z.infer<typeof ToolCallSchema>;
|
|
626
982
|
type TokenUsage = z.infer<typeof TokenUsageSchema>;
|
|
983
|
+
type ContentText = z.infer<typeof ContentTextSchema>;
|
|
984
|
+
type ContentImage = z.infer<typeof ContentImageSchema>;
|
|
985
|
+
type ContentFile = z.infer<typeof ContentFileSchema>;
|
|
986
|
+
type Content = z.infer<typeof ContentSchema>;
|
|
627
987
|
/**
|
|
628
988
|
* Prompt template input schema (camelCase, converted from snake_case wire format).
|
|
629
989
|
* Uses the same schema as CodeGraderInput since the orchestrator sends identical payloads.
|
|
@@ -632,7 +992,40 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
632
992
|
criteria: z.ZodString;
|
|
633
993
|
expectedOutput: z.ZodArray<z.ZodObject<{
|
|
634
994
|
role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
|
|
635
|
-
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.
|
|
995
|
+
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
996
|
+
type: z.ZodLiteral<"text">;
|
|
997
|
+
text: z.ZodString;
|
|
998
|
+
}, "strip", z.ZodTypeAny, {
|
|
999
|
+
type: "text";
|
|
1000
|
+
text: string;
|
|
1001
|
+
}, {
|
|
1002
|
+
type: "text";
|
|
1003
|
+
text: string;
|
|
1004
|
+
}>, z.ZodObject<{
|
|
1005
|
+
type: z.ZodLiteral<"image">;
|
|
1006
|
+
media_type: z.ZodString;
|
|
1007
|
+
path: z.ZodString;
|
|
1008
|
+
}, "strip", z.ZodTypeAny, {
|
|
1009
|
+
path: string;
|
|
1010
|
+
type: "image";
|
|
1011
|
+
media_type: string;
|
|
1012
|
+
}, {
|
|
1013
|
+
path: string;
|
|
1014
|
+
type: "image";
|
|
1015
|
+
media_type: string;
|
|
1016
|
+
}>, z.ZodObject<{
|
|
1017
|
+
type: z.ZodLiteral<"file">;
|
|
1018
|
+
media_type: z.ZodString;
|
|
1019
|
+
path: z.ZodString;
|
|
1020
|
+
}, "strip", z.ZodTypeAny, {
|
|
1021
|
+
path: string;
|
|
1022
|
+
type: "file";
|
|
1023
|
+
media_type: string;
|
|
1024
|
+
}, {
|
|
1025
|
+
path: string;
|
|
1026
|
+
type: "file";
|
|
1027
|
+
media_type: string;
|
|
1028
|
+
}>]>, "many">]>>;
|
|
636
1029
|
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
637
1030
|
tool: z.ZodString;
|
|
638
1031
|
input: z.ZodOptional<z.ZodUnknown>;
|
|
@@ -677,7 +1070,18 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
677
1070
|
startTime?: string | undefined;
|
|
678
1071
|
endTime?: string | undefined;
|
|
679
1072
|
durationMs?: number | undefined;
|
|
680
|
-
content?: string |
|
|
1073
|
+
content?: string | ({
|
|
1074
|
+
type: "text";
|
|
1075
|
+
text: string;
|
|
1076
|
+
} | {
|
|
1077
|
+
path: string;
|
|
1078
|
+
type: "image";
|
|
1079
|
+
media_type: string;
|
|
1080
|
+
} | {
|
|
1081
|
+
path: string;
|
|
1082
|
+
type: "file";
|
|
1083
|
+
media_type: string;
|
|
1084
|
+
})[] | undefined;
|
|
681
1085
|
name?: string | undefined;
|
|
682
1086
|
metadata?: Record<string, unknown> | undefined;
|
|
683
1087
|
}, {
|
|
@@ -694,15 +1098,57 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
694
1098
|
startTime?: string | undefined;
|
|
695
1099
|
endTime?: string | undefined;
|
|
696
1100
|
durationMs?: number | undefined;
|
|
697
|
-
content?: string |
|
|
1101
|
+
content?: string | ({
|
|
1102
|
+
type: "text";
|
|
1103
|
+
text: string;
|
|
1104
|
+
} | {
|
|
1105
|
+
path: string;
|
|
1106
|
+
type: "image";
|
|
1107
|
+
media_type: string;
|
|
1108
|
+
} | {
|
|
1109
|
+
path: string;
|
|
1110
|
+
type: "file";
|
|
1111
|
+
media_type: string;
|
|
1112
|
+
})[] | undefined;
|
|
698
1113
|
name?: string | undefined;
|
|
699
1114
|
metadata?: Record<string, unknown> | undefined;
|
|
700
1115
|
}>, "many">;
|
|
701
|
-
/** Last assistant message content as string. */
|
|
702
|
-
outputText: z.ZodString;
|
|
703
1116
|
output: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodObject<{
|
|
704
1117
|
role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
|
|
705
|
-
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.
|
|
1118
|
+
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
1119
|
+
type: z.ZodLiteral<"text">;
|
|
1120
|
+
text: z.ZodString;
|
|
1121
|
+
}, "strip", z.ZodTypeAny, {
|
|
1122
|
+
type: "text";
|
|
1123
|
+
text: string;
|
|
1124
|
+
}, {
|
|
1125
|
+
type: "text";
|
|
1126
|
+
text: string;
|
|
1127
|
+
}>, z.ZodObject<{
|
|
1128
|
+
type: z.ZodLiteral<"image">;
|
|
1129
|
+
media_type: z.ZodString;
|
|
1130
|
+
path: z.ZodString;
|
|
1131
|
+
}, "strip", z.ZodTypeAny, {
|
|
1132
|
+
path: string;
|
|
1133
|
+
type: "image";
|
|
1134
|
+
media_type: string;
|
|
1135
|
+
}, {
|
|
1136
|
+
path: string;
|
|
1137
|
+
type: "image";
|
|
1138
|
+
media_type: string;
|
|
1139
|
+
}>, z.ZodObject<{
|
|
1140
|
+
type: z.ZodLiteral<"file">;
|
|
1141
|
+
media_type: z.ZodString;
|
|
1142
|
+
path: z.ZodString;
|
|
1143
|
+
}, "strip", z.ZodTypeAny, {
|
|
1144
|
+
path: string;
|
|
1145
|
+
type: "file";
|
|
1146
|
+
media_type: string;
|
|
1147
|
+
}, {
|
|
1148
|
+
path: string;
|
|
1149
|
+
type: "file";
|
|
1150
|
+
media_type: string;
|
|
1151
|
+
}>]>, "many">]>>;
|
|
706
1152
|
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
707
1153
|
tool: z.ZodString;
|
|
708
1154
|
input: z.ZodOptional<z.ZodUnknown>;
|
|
@@ -747,7 +1193,18 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
747
1193
|
startTime?: string | undefined;
|
|
748
1194
|
endTime?: string | undefined;
|
|
749
1195
|
durationMs?: number | undefined;
|
|
750
|
-
content?: string |
|
|
1196
|
+
content?: string | ({
|
|
1197
|
+
type: "text";
|
|
1198
|
+
text: string;
|
|
1199
|
+
} | {
|
|
1200
|
+
path: string;
|
|
1201
|
+
type: "image";
|
|
1202
|
+
media_type: string;
|
|
1203
|
+
} | {
|
|
1204
|
+
path: string;
|
|
1205
|
+
type: "file";
|
|
1206
|
+
media_type: string;
|
|
1207
|
+
})[] | undefined;
|
|
751
1208
|
name?: string | undefined;
|
|
752
1209
|
metadata?: Record<string, unknown> | undefined;
|
|
753
1210
|
}, {
|
|
@@ -764,7 +1221,18 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
764
1221
|
startTime?: string | undefined;
|
|
765
1222
|
endTime?: string | undefined;
|
|
766
1223
|
durationMs?: number | undefined;
|
|
767
|
-
content?: string |
|
|
1224
|
+
content?: string | ({
|
|
1225
|
+
type: "text";
|
|
1226
|
+
text: string;
|
|
1227
|
+
} | {
|
|
1228
|
+
path: string;
|
|
1229
|
+
type: "image";
|
|
1230
|
+
media_type: string;
|
|
1231
|
+
} | {
|
|
1232
|
+
path: string;
|
|
1233
|
+
type: "file";
|
|
1234
|
+
media_type: string;
|
|
1235
|
+
})[] | undefined;
|
|
768
1236
|
name?: string | undefined;
|
|
769
1237
|
metadata?: Record<string, unknown> | undefined;
|
|
770
1238
|
}>, "many">>>;
|
|
@@ -773,7 +1241,40 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
773
1241
|
inputFiles: z.ZodArray<z.ZodString, "many">;
|
|
774
1242
|
input: z.ZodArray<z.ZodObject<{
|
|
775
1243
|
role: z.ZodEnum<["assistant", "user", "system", "tool"]>;
|
|
776
|
-
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.
|
|
1244
|
+
content: z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodDiscriminatedUnion<"type", [z.ZodObject<{
|
|
1245
|
+
type: z.ZodLiteral<"text">;
|
|
1246
|
+
text: z.ZodString;
|
|
1247
|
+
}, "strip", z.ZodTypeAny, {
|
|
1248
|
+
type: "text";
|
|
1249
|
+
text: string;
|
|
1250
|
+
}, {
|
|
1251
|
+
type: "text";
|
|
1252
|
+
text: string;
|
|
1253
|
+
}>, z.ZodObject<{
|
|
1254
|
+
type: z.ZodLiteral<"image">;
|
|
1255
|
+
media_type: z.ZodString;
|
|
1256
|
+
path: z.ZodString;
|
|
1257
|
+
}, "strip", z.ZodTypeAny, {
|
|
1258
|
+
path: string;
|
|
1259
|
+
type: "image";
|
|
1260
|
+
media_type: string;
|
|
1261
|
+
}, {
|
|
1262
|
+
path: string;
|
|
1263
|
+
type: "image";
|
|
1264
|
+
media_type: string;
|
|
1265
|
+
}>, z.ZodObject<{
|
|
1266
|
+
type: z.ZodLiteral<"file">;
|
|
1267
|
+
media_type: z.ZodString;
|
|
1268
|
+
path: z.ZodString;
|
|
1269
|
+
}, "strip", z.ZodTypeAny, {
|
|
1270
|
+
path: string;
|
|
1271
|
+
type: "file";
|
|
1272
|
+
media_type: string;
|
|
1273
|
+
}, {
|
|
1274
|
+
path: string;
|
|
1275
|
+
type: "file";
|
|
1276
|
+
media_type: string;
|
|
1277
|
+
}>]>, "many">]>>;
|
|
777
1278
|
toolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
778
1279
|
tool: z.ZodString;
|
|
779
1280
|
input: z.ZodOptional<z.ZodUnknown>;
|
|
@@ -818,7 +1319,18 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
818
1319
|
startTime?: string | undefined;
|
|
819
1320
|
endTime?: string | undefined;
|
|
820
1321
|
durationMs?: number | undefined;
|
|
821
|
-
content?: string |
|
|
1322
|
+
content?: string | ({
|
|
1323
|
+
type: "text";
|
|
1324
|
+
text: string;
|
|
1325
|
+
} | {
|
|
1326
|
+
path: string;
|
|
1327
|
+
type: "image";
|
|
1328
|
+
media_type: string;
|
|
1329
|
+
} | {
|
|
1330
|
+
path: string;
|
|
1331
|
+
type: "file";
|
|
1332
|
+
media_type: string;
|
|
1333
|
+
})[] | undefined;
|
|
822
1334
|
name?: string | undefined;
|
|
823
1335
|
metadata?: Record<string, unknown> | undefined;
|
|
824
1336
|
}, {
|
|
@@ -835,7 +1347,18 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
835
1347
|
startTime?: string | undefined;
|
|
836
1348
|
endTime?: string | undefined;
|
|
837
1349
|
durationMs?: number | undefined;
|
|
838
|
-
content?: string |
|
|
1350
|
+
content?: string | ({
|
|
1351
|
+
type: "text";
|
|
1352
|
+
text: string;
|
|
1353
|
+
} | {
|
|
1354
|
+
path: string;
|
|
1355
|
+
type: "image";
|
|
1356
|
+
media_type: string;
|
|
1357
|
+
} | {
|
|
1358
|
+
path: string;
|
|
1359
|
+
type: "file";
|
|
1360
|
+
media_type: string;
|
|
1361
|
+
})[] | undefined;
|
|
839
1362
|
name?: string | undefined;
|
|
840
1363
|
metadata?: Record<string, unknown> | undefined;
|
|
841
1364
|
}>, "many">;
|
|
@@ -878,10 +1401,6 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
878
1401
|
fileChanges: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
879
1402
|
workspacePath: z.ZodOptional<z.ZodNullable<z.ZodString>>;
|
|
880
1403
|
config: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
881
|
-
/** First user message content as string. */
|
|
882
|
-
inputText: z.ZodString;
|
|
883
|
-
/** Expected output content as string. */
|
|
884
|
-
expectedOutputText: z.ZodOptional<z.ZodString>;
|
|
885
1404
|
}, "strip", z.ZodTypeAny, {
|
|
886
1405
|
input: {
|
|
887
1406
|
role: "tool" | "assistant" | "user" | "system";
|
|
@@ -897,7 +1416,18 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
897
1416
|
startTime?: string | undefined;
|
|
898
1417
|
endTime?: string | undefined;
|
|
899
1418
|
durationMs?: number | undefined;
|
|
900
|
-
content?: string |
|
|
1419
|
+
content?: string | ({
|
|
1420
|
+
type: "text";
|
|
1421
|
+
text: string;
|
|
1422
|
+
} | {
|
|
1423
|
+
path: string;
|
|
1424
|
+
type: "image";
|
|
1425
|
+
media_type: string;
|
|
1426
|
+
} | {
|
|
1427
|
+
path: string;
|
|
1428
|
+
type: "file";
|
|
1429
|
+
media_type: string;
|
|
1430
|
+
})[] | undefined;
|
|
901
1431
|
name?: string | undefined;
|
|
902
1432
|
metadata?: Record<string, unknown> | undefined;
|
|
903
1433
|
}[];
|
|
@@ -916,13 +1446,22 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
916
1446
|
startTime?: string | undefined;
|
|
917
1447
|
endTime?: string | undefined;
|
|
918
1448
|
durationMs?: number | undefined;
|
|
919
|
-
content?: string |
|
|
1449
|
+
content?: string | ({
|
|
1450
|
+
type: "text";
|
|
1451
|
+
text: string;
|
|
1452
|
+
} | {
|
|
1453
|
+
path: string;
|
|
1454
|
+
type: "image";
|
|
1455
|
+
media_type: string;
|
|
1456
|
+
} | {
|
|
1457
|
+
path: string;
|
|
1458
|
+
type: "file";
|
|
1459
|
+
media_type: string;
|
|
1460
|
+
})[] | undefined;
|
|
920
1461
|
name?: string | undefined;
|
|
921
1462
|
metadata?: Record<string, unknown> | undefined;
|
|
922
1463
|
}[];
|
|
923
|
-
outputText: string;
|
|
924
1464
|
inputFiles: string[];
|
|
925
|
-
inputText: string;
|
|
926
1465
|
output?: {
|
|
927
1466
|
role: "tool" | "assistant" | "user" | "system";
|
|
928
1467
|
toolCalls?: {
|
|
@@ -937,7 +1476,18 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
937
1476
|
startTime?: string | undefined;
|
|
938
1477
|
endTime?: string | undefined;
|
|
939
1478
|
durationMs?: number | undefined;
|
|
940
|
-
content?: string |
|
|
1479
|
+
content?: string | ({
|
|
1480
|
+
type: "text";
|
|
1481
|
+
text: string;
|
|
1482
|
+
} | {
|
|
1483
|
+
path: string;
|
|
1484
|
+
type: "image";
|
|
1485
|
+
media_type: string;
|
|
1486
|
+
} | {
|
|
1487
|
+
path: string;
|
|
1488
|
+
type: "file";
|
|
1489
|
+
media_type: string;
|
|
1490
|
+
})[] | undefined;
|
|
941
1491
|
name?: string | undefined;
|
|
942
1492
|
metadata?: Record<string, unknown> | undefined;
|
|
943
1493
|
}[] | null | undefined;
|
|
@@ -961,7 +1511,6 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
961
1511
|
fileChanges?: string | null | undefined;
|
|
962
1512
|
workspacePath?: string | null | undefined;
|
|
963
1513
|
config?: Record<string, unknown> | null | undefined;
|
|
964
|
-
expectedOutputText?: string | undefined;
|
|
965
1514
|
}, {
|
|
966
1515
|
input: {
|
|
967
1516
|
role: "tool" | "assistant" | "user" | "system";
|
|
@@ -977,7 +1526,18 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
977
1526
|
startTime?: string | undefined;
|
|
978
1527
|
endTime?: string | undefined;
|
|
979
1528
|
durationMs?: number | undefined;
|
|
980
|
-
content?: string |
|
|
1529
|
+
content?: string | ({
|
|
1530
|
+
type: "text";
|
|
1531
|
+
text: string;
|
|
1532
|
+
} | {
|
|
1533
|
+
path: string;
|
|
1534
|
+
type: "image";
|
|
1535
|
+
media_type: string;
|
|
1536
|
+
} | {
|
|
1537
|
+
path: string;
|
|
1538
|
+
type: "file";
|
|
1539
|
+
media_type: string;
|
|
1540
|
+
})[] | undefined;
|
|
981
1541
|
name?: string | undefined;
|
|
982
1542
|
metadata?: Record<string, unknown> | undefined;
|
|
983
1543
|
}[];
|
|
@@ -996,13 +1556,22 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
996
1556
|
startTime?: string | undefined;
|
|
997
1557
|
endTime?: string | undefined;
|
|
998
1558
|
durationMs?: number | undefined;
|
|
999
|
-
content?: string |
|
|
1559
|
+
content?: string | ({
|
|
1560
|
+
type: "text";
|
|
1561
|
+
text: string;
|
|
1562
|
+
} | {
|
|
1563
|
+
path: string;
|
|
1564
|
+
type: "image";
|
|
1565
|
+
media_type: string;
|
|
1566
|
+
} | {
|
|
1567
|
+
path: string;
|
|
1568
|
+
type: "file";
|
|
1569
|
+
media_type: string;
|
|
1570
|
+
})[] | undefined;
|
|
1000
1571
|
name?: string | undefined;
|
|
1001
1572
|
metadata?: Record<string, unknown> | undefined;
|
|
1002
1573
|
}[];
|
|
1003
|
-
outputText: string;
|
|
1004
1574
|
inputFiles: string[];
|
|
1005
|
-
inputText: string;
|
|
1006
1575
|
output?: {
|
|
1007
1576
|
role: "tool" | "assistant" | "user" | "system";
|
|
1008
1577
|
toolCalls?: {
|
|
@@ -1017,7 +1586,18 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
1017
1586
|
startTime?: string | undefined;
|
|
1018
1587
|
endTime?: string | undefined;
|
|
1019
1588
|
durationMs?: number | undefined;
|
|
1020
|
-
content?: string |
|
|
1589
|
+
content?: string | ({
|
|
1590
|
+
type: "text";
|
|
1591
|
+
text: string;
|
|
1592
|
+
} | {
|
|
1593
|
+
path: string;
|
|
1594
|
+
type: "image";
|
|
1595
|
+
media_type: string;
|
|
1596
|
+
} | {
|
|
1597
|
+
path: string;
|
|
1598
|
+
type: "file";
|
|
1599
|
+
media_type: string;
|
|
1600
|
+
})[] | undefined;
|
|
1021
1601
|
name?: string | undefined;
|
|
1022
1602
|
metadata?: Record<string, unknown> | undefined;
|
|
1023
1603
|
}[] | null | undefined;
|
|
@@ -1041,7 +1621,6 @@ declare const PromptTemplateInputSchema: z.ZodObject<{
|
|
|
1041
1621
|
fileChanges?: string | null | undefined;
|
|
1042
1622
|
workspacePath?: string | null | undefined;
|
|
1043
1623
|
config?: Record<string, unknown> | null | undefined;
|
|
1044
|
-
expectedOutputText?: string | undefined;
|
|
1045
1624
|
}>;
|
|
1046
1625
|
type PromptTemplateInput = CodeGraderInput;
|
|
1047
1626
|
|
|
@@ -1157,12 +1736,8 @@ declare function createTargetClient(): TargetClient | undefined;
|
|
|
1157
1736
|
|
|
1158
1737
|
/**
|
|
1159
1738
|
* Context provided to assertion handlers.
|
|
1160
|
-
*
|
|
1161
|
-
* Same shape as CodeGraderInput but with `inputText`, `outputText`, and
|
|
1162
|
-
* `expectedOutputText` guaranteed to be strings (populated by the runtime
|
|
1163
|
-
* before the handler is called).
|
|
1164
1739
|
*/
|
|
1165
|
-
type AssertionContext =
|
|
1740
|
+
type AssertionContext = CodeGraderInput;
|
|
1166
1741
|
/**
|
|
1167
1742
|
* Known built-in assertion types. Custom types are extensible via string.
|
|
1168
1743
|
*
|
|
@@ -1222,19 +1797,13 @@ type AssertionHandler = (ctx: AssertionContext) => AssertionScore | Promise<Asse
|
|
|
1222
1797
|
/**
|
|
1223
1798
|
* Handler function type for prompt templates.
|
|
1224
1799
|
* Returns the prompt string to use for evaluation.
|
|
1225
|
-
*
|
|
1226
|
-
* The input is enriched at runtime: `inputText`, `outputText`, and
|
|
1227
|
-
* `expectedOutputText` are always populated before the handler is called.
|
|
1228
1800
|
*/
|
|
1229
|
-
type PromptTemplateHandler = (input:
|
|
1801
|
+
type PromptTemplateHandler = (input: CodeGraderInput) => string | Promise<string>;
|
|
1230
1802
|
|
|
1231
1803
|
/**
|
|
1232
1804
|
* Handler function type for code graders.
|
|
1233
|
-
*
|
|
1234
|
-
* The input is enriched at runtime: `inputText`, `outputText`, and
|
|
1235
|
-
* `expectedOutputText` are always populated before the handler is called.
|
|
1236
1805
|
*/
|
|
1237
|
-
type CodeGraderHandler = (input:
|
|
1806
|
+
type CodeGraderHandler = (input: CodeGraderInput) => CodeGraderResult | Promise<CodeGraderResult>;
|
|
1238
1807
|
|
|
1239
1808
|
/**
|
|
1240
1809
|
* AgentV Evaluation SDK
|
|
@@ -1246,9 +1815,12 @@ type CodeGraderHandler = (input: EnrichedCodeGraderInput) => CodeGraderResult |
|
|
|
1246
1815
|
* #!/usr/bin/env bun
|
|
1247
1816
|
* import { defineAssertion } from '@agentv/eval';
|
|
1248
1817
|
*
|
|
1249
|
-
* export default defineAssertion(({
|
|
1250
|
-
*
|
|
1251
|
-
*
|
|
1818
|
+
* export default defineAssertion(({ output, criteria }) => {
|
|
1819
|
+
* const text = output?.map(m => String(m.content ?? '')).join(' ') ?? '';
|
|
1820
|
+
* return {
|
|
1821
|
+
* pass: text.includes('hello'),
|
|
1822
|
+
* assertions: [{ text: 'Checks greeting', passed: text.includes('hello') }],
|
|
1823
|
+
* };
|
|
1252
1824
|
* }));
|
|
1253
1825
|
* ```
|
|
1254
1826
|
*
|
|
@@ -1257,33 +1829,15 @@ type CodeGraderHandler = (input: EnrichedCodeGraderInput) => CodeGraderResult |
|
|
|
1257
1829
|
* #!/usr/bin/env bun
|
|
1258
1830
|
* import { defineCodeGrader } from '@agentv/eval';
|
|
1259
1831
|
*
|
|
1260
|
-
* export default defineCodeGrader(({ trace,
|
|
1261
|
-
*
|
|
1262
|
-
*
|
|
1832
|
+
* export default defineCodeGrader(({ trace, output }) => {
|
|
1833
|
+
* const text = output?.map(m => String(m.content ?? '')).join(' ') ?? '';
|
|
1834
|
+
* return {
|
|
1835
|
+
* score: trace?.eventCount <= 5 ? 1.0 : 0.5,
|
|
1836
|
+
* assertions: [{ text: 'Efficient tool usage', passed: trace?.eventCount <= 5 }],
|
|
1837
|
+
* };
|
|
1263
1838
|
* }));
|
|
1264
1839
|
* ```
|
|
1265
1840
|
*
|
|
1266
|
-
* @example Code grader with target access (requires `target` config in YAML)
|
|
1267
|
-
* ```typescript
|
|
1268
|
-
* #!/usr/bin/env bun
|
|
1269
|
-
* import { defineCodeGrader, createTargetClient } from '@agentv/eval';
|
|
1270
|
-
*
|
|
1271
|
-
* export default defineCodeGrader(async ({ inputText }) => {
|
|
1272
|
-
* const target = createTargetClient();
|
|
1273
|
-
* if (!target) {
|
|
1274
|
-
* return { score: 0, assertions: [{ text: 'Target not available', passed: false }] };
|
|
1275
|
-
* }
|
|
1276
|
-
*
|
|
1277
|
-
* const response = await target.invoke({
|
|
1278
|
-
* question: `Evaluate: ${inputText}`,
|
|
1279
|
-
* systemPrompt: 'Respond with JSON: { "score": 0-1 }'
|
|
1280
|
-
* });
|
|
1281
|
-
*
|
|
1282
|
-
* const result = JSON.parse(response.rawText ?? '{}');
|
|
1283
|
-
* return { score: result.score ?? 0 };
|
|
1284
|
-
* });
|
|
1285
|
-
* ```
|
|
1286
|
-
*
|
|
1287
1841
|
* @packageDocumentation
|
|
1288
1842
|
*/
|
|
1289
1843
|
|
|
@@ -1347,25 +1901,10 @@ declare function defineCodeGrader(handler: CodeGraderHandler): void;
|
|
|
1347
1901
|
* ```typescript
|
|
1348
1902
|
* import { definePromptTemplate } from '@agentv/eval';
|
|
1349
1903
|
*
|
|
1350
|
-
* export default definePromptTemplate((ctx) => `
|
|
1351
|
-
* Question: ${ctx.inputText}
|
|
1352
|
-
* Answer: ${ctx.outputText}
|
|
1353
|
-
*
|
|
1354
|
-
* ${ctx.expectedOutputText ? `Reference: ${ctx.expectedOutputText}` : ''}
|
|
1355
|
-
* `);
|
|
1356
|
-
* ```
|
|
1357
|
-
*
|
|
1358
|
-
* @example With conditional logic
|
|
1359
|
-
* ```typescript
|
|
1360
|
-
* import { definePromptTemplate } from '@agentv/eval';
|
|
1361
|
-
*
|
|
1362
1904
|
* export default definePromptTemplate((ctx) => {
|
|
1363
|
-
* const
|
|
1364
|
-
*
|
|
1365
|
-
*
|
|
1366
|
-
* Candidate Answer: ${ctx.outputText}
|
|
1367
|
-
* ${rubric ? `\nEvaluation Criteria:\n${rubric}` : ''}
|
|
1368
|
-
* `;
|
|
1905
|
+
* const question = ctx.input.map(m => String(m.content ?? '')).join('\n');
|
|
1906
|
+
* const answer = ctx.output?.map(m => String(m.content ?? '')).join('\n') ?? '';
|
|
1907
|
+
* return `Question: ${question}\nAnswer: ${answer}`;
|
|
1369
1908
|
* });
|
|
1370
1909
|
* ```
|
|
1371
1910
|
*/
|
|
@@ -1391,9 +1930,12 @@ declare function definePromptTemplate(handler: PromptTemplateHandler): void;
|
|
|
1391
1930
|
* ```typescript
|
|
1392
1931
|
* import { defineAssertion } from '@agentv/eval';
|
|
1393
1932
|
*
|
|
1394
|
-
* export default defineAssertion(({
|
|
1395
|
-
*
|
|
1396
|
-
*
|
|
1933
|
+
* export default defineAssertion(({ output }) => {
|
|
1934
|
+
* const text = output?.map(m => String(m.content ?? '')).join(' ') ?? '';
|
|
1935
|
+
* return {
|
|
1936
|
+
* pass: text.toLowerCase().includes('hello'),
|
|
1937
|
+
* assertions: [{ text: 'Checks for greeting', passed: text.toLowerCase().includes('hello') }],
|
|
1938
|
+
* };
|
|
1397
1939
|
* }));
|
|
1398
1940
|
* ```
|
|
1399
1941
|
*
|
|
@@ -1401,8 +1943,9 @@ declare function definePromptTemplate(handler: PromptTemplateHandler): void;
|
|
|
1401
1943
|
* ```typescript
|
|
1402
1944
|
* import { defineAssertion } from '@agentv/eval';
|
|
1403
1945
|
*
|
|
1404
|
-
* export default defineAssertion(({
|
|
1405
|
-
* const
|
|
1946
|
+
* export default defineAssertion(({ output, trace }) => {
|
|
1947
|
+
* const text = output?.map(m => String(m.content ?? '')).join(' ') ?? '';
|
|
1948
|
+
* const hasContent = text.length > 0 ? 0.5 : 0;
|
|
1406
1949
|
* const isEfficient = (trace?.eventCount ?? 0) <= 5 ? 0.5 : 0;
|
|
1407
1950
|
* return {
|
|
1408
1951
|
* score: hasContent + isEfficient,
|
|
@@ -1411,9 +1954,9 @@ declare function definePromptTemplate(handler: PromptTemplateHandler): void;
|
|
|
1411
1954
|
* { text: 'Efficient', passed: !!isEfficient },
|
|
1412
1955
|
* ],
|
|
1413
1956
|
* };
|
|
1414
|
-
* });
|
|
1957
|
+
* }));
|
|
1415
1958
|
* ```
|
|
1416
1959
|
*/
|
|
1417
1960
|
declare function defineAssertion(handler: AssertionHandler): void;
|
|
1418
1961
|
|
|
1419
|
-
export { type AssertionContext, type AssertionHandler, type AssertionScore, type AssertionType, type CodeGraderHandler, type CodeGraderInput, CodeGraderInputSchema, type CodeGraderResult, CodeGraderResultSchema, type
|
|
1962
|
+
export { type AssertionContext, type AssertionHandler, type AssertionScore, type AssertionType, type CodeGraderHandler, type CodeGraderInput, CodeGraderInputSchema, type CodeGraderResult, CodeGraderResultSchema, type Content, type ContentFile, ContentFileSchema, type ContentImage, ContentImageSchema, ContentSchema, type ContentText, ContentTextSchema, type Message, MessageSchema, type PromptTemplateHandler, type PromptTemplateInput, PromptTemplateInputSchema, type TargetClient, type TargetInfo, TargetInvocationError, type TargetInvokeRequest, type TargetInvokeResponse, TargetNotAvailableError, type TokenUsage, TokenUsageSchema, type ToolCall, ToolCallSchema, type TraceSummary, TraceSummarySchema, createTargetClient, defineAssertion, defineCodeGrader, definePromptTemplate };
|