@kortyx/agent 0.31.1 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/evals.d.ts +199 -16
- package/dist/evals.js +417 -270
- package/dist/evals.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +602 -421
- package/dist/index.js.map +1 -1
- package/dist/{studio-judge-CG86eozt.d.ts → studio-judge-DpRLvf8j.d.ts} +700 -57
- package/package.json +3 -3
|
@@ -199,6 +199,35 @@ interface Agent extends ReturnType<typeof createInterruptDiscovery> {
|
|
|
199
199
|
}
|
|
200
200
|
declare function createAgent(args: CreateAgentArgs): Agent;
|
|
201
201
|
|
|
202
|
+
/** Public events retained after streaming noise has been removed. */
|
|
203
|
+
declare const EVAL_EVIDENCE_EVENT_TYPES: readonly ["tool-call-start", "tool-call-result", "tool-call-error", "tool-result", "structured-data", "structured-data-invalidated", "interrupt", "error", "cancelled", "limit-reached"];
|
|
204
|
+
declare const EvalEvidencePolicySchema: z.ZodObject<{
|
|
205
|
+
history: z.ZodOptional<z.ZodBoolean>;
|
|
206
|
+
events: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodEnum<{
|
|
207
|
+
"tool-call-start": "tool-call-start";
|
|
208
|
+
"tool-call-result": "tool-call-result";
|
|
209
|
+
"tool-call-error": "tool-call-error";
|
|
210
|
+
"tool-result": "tool-result";
|
|
211
|
+
"structured-data": "structured-data";
|
|
212
|
+
"structured-data-invalidated": "structured-data-invalidated";
|
|
213
|
+
interrupt: "interrupt";
|
|
214
|
+
error: "error";
|
|
215
|
+
cancelled: "cancelled";
|
|
216
|
+
"limit-reached": "limit-reached";
|
|
217
|
+
}>>, z.ZodObject<{
|
|
218
|
+
using: z.ZodString;
|
|
219
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
220
|
+
}, z.core.$strict>]>>;
|
|
221
|
+
outputs: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodObject<{
|
|
222
|
+
dataType: z.ZodOptional<z.ZodString>;
|
|
223
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
224
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
225
|
+
}, z.core.$strict>>, z.ZodObject<{
|
|
226
|
+
using: z.ZodString;
|
|
227
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
228
|
+
}, z.core.$strict>]>>;
|
|
229
|
+
}, z.core.$strict>;
|
|
230
|
+
|
|
202
231
|
type EvalJson = null | boolean | number | string | readonly EvalJson[] | {
|
|
203
232
|
readonly [key: string]: EvalJson;
|
|
204
233
|
};
|
|
@@ -239,10 +268,38 @@ type EvalCase<Params = EvalJson> = {
|
|
|
239
268
|
workflowId?: string;
|
|
240
269
|
steps: readonly EvalStep[];
|
|
241
270
|
};
|
|
271
|
+
/** Serializable selection; custom predicates are registered on createEvals. */
|
|
272
|
+
type EvalEvidencePolicy = {
|
|
273
|
+
history?: boolean;
|
|
274
|
+
events?: false | readonly EvalEvidenceEventType[] | EvalHandlerRef;
|
|
275
|
+
outputs?: false | readonly EvalOutputSelector[] | EvalHandlerRef;
|
|
276
|
+
};
|
|
277
|
+
type EvalEvidenceEventType = (typeof EVAL_EVIDENCE_EVENT_TYPES)[number];
|
|
278
|
+
/** Sanitized public event. structured-data events have kind=final. */
|
|
279
|
+
type EvalEvidenceEvent = {
|
|
280
|
+
readonly type: EvalEvidenceEventType;
|
|
281
|
+
readonly [key: string]: EvalJson;
|
|
282
|
+
};
|
|
283
|
+
/** Match every supplied field; dataType or schemaId is required at runtime. */
|
|
284
|
+
type EvalOutputSelector = {
|
|
285
|
+
readonly dataType?: string;
|
|
286
|
+
readonly schemaId?: string;
|
|
287
|
+
readonly schemaVersion?: string;
|
|
288
|
+
};
|
|
289
|
+
type EvalEvidenceFilters = {
|
|
290
|
+
events?: Record<string, (event: EvalEvidenceEvent, params?: EvalJson) => boolean>;
|
|
291
|
+
outputs?: Record<string, (output: EvalJson, params?: EvalJson) => boolean>;
|
|
292
|
+
};
|
|
293
|
+
type EvalJudgeEvidence = {
|
|
294
|
+
version: "compact-v1";
|
|
295
|
+
history: boolean;
|
|
296
|
+
observation: EvalObservation;
|
|
297
|
+
};
|
|
242
298
|
type EvalSuite<Params = EvalJson> = {
|
|
243
299
|
id: string;
|
|
244
300
|
name?: string;
|
|
245
301
|
cases: readonly EvalCase<Params>[];
|
|
302
|
+
evidence?: EvalEvidencePolicy;
|
|
246
303
|
};
|
|
247
304
|
type EvalInterrupt = {
|
|
248
305
|
requestId: string;
|
|
@@ -307,6 +364,8 @@ type EvalCriterionResult = EvalVerdict & {
|
|
|
307
364
|
};
|
|
308
365
|
type EvalJudgeUsage = z.infer<typeof EvalJudgeUsageSchema>;
|
|
309
366
|
type EvalStepResult = {
|
|
367
|
+
/** Exact compact observation selected by the app; full observation remains unchanged. */
|
|
368
|
+
evidence?: EvalJudgeEvidence;
|
|
310
369
|
judgeCalls?: number;
|
|
311
370
|
judgeUsage?: EvalJudgeUsage[];
|
|
312
371
|
index: number;
|
|
@@ -402,6 +461,7 @@ type EvalResponder<Params, Prepared> = ((input: EvalContext<Params, Prepared> &
|
|
|
402
461
|
}) => ResumeResponse | Promise<ResumeResponse>;
|
|
403
462
|
};
|
|
404
463
|
type EvalDefaults = {
|
|
464
|
+
evidence?: EvalEvidencePolicy;
|
|
405
465
|
repetitions?: number;
|
|
406
466
|
concurrency?: number;
|
|
407
467
|
caseTimeoutMs?: number;
|
|
@@ -430,6 +490,7 @@ type CreateEvalsOptions<Params = EvalJson | undefined, Prepared = undefined> = {
|
|
|
430
490
|
args?: EvalJson;
|
|
431
491
|
}) => EvalJson | Promise<EvalJson>>;
|
|
432
492
|
judge?: EvalJudge;
|
|
493
|
+
evidenceFilters?: EvalEvidenceFilters;
|
|
433
494
|
defaults?: EvalDefaults;
|
|
434
495
|
};
|
|
435
496
|
type EvalProgress = {
|
|
@@ -485,6 +546,32 @@ declare const EvalResumeResponseSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
485
546
|
type: z.ZodLiteral<"cancel">;
|
|
486
547
|
}, z.core.$strict>], "type">;
|
|
487
548
|
declare const EvalSuiteSchema: z.ZodObject<{
|
|
549
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
550
|
+
history: z.ZodOptional<z.ZodBoolean>;
|
|
551
|
+
events: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodEnum<{
|
|
552
|
+
"tool-call-start": "tool-call-start";
|
|
553
|
+
"tool-call-result": "tool-call-result";
|
|
554
|
+
"tool-call-error": "tool-call-error";
|
|
555
|
+
"tool-result": "tool-result";
|
|
556
|
+
"structured-data": "structured-data";
|
|
557
|
+
"structured-data-invalidated": "structured-data-invalidated";
|
|
558
|
+
interrupt: "interrupt";
|
|
559
|
+
error: "error";
|
|
560
|
+
cancelled: "cancelled";
|
|
561
|
+
"limit-reached": "limit-reached";
|
|
562
|
+
}>>, z.ZodObject<{
|
|
563
|
+
using: z.ZodString;
|
|
564
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
565
|
+
}, z.core.$strict>]>>;
|
|
566
|
+
outputs: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodObject<{
|
|
567
|
+
dataType: z.ZodOptional<z.ZodString>;
|
|
568
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
569
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
570
|
+
}, z.core.$strict>>, z.ZodObject<{
|
|
571
|
+
using: z.ZodString;
|
|
572
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
573
|
+
}, z.core.$strict>]>>;
|
|
574
|
+
}, z.core.$strict>>;
|
|
488
575
|
id: z.ZodString;
|
|
489
576
|
name: z.ZodOptional<z.ZodString>;
|
|
490
577
|
cases: z.ZodArray<z.ZodObject<{
|
|
@@ -554,9 +641,9 @@ declare const EvalVerdictSchema: z.ZodObject<{
|
|
|
554
641
|
}, z.core.$strict>;
|
|
555
642
|
declare const EvalObservationSchema: z.ZodObject<{
|
|
556
643
|
type: z.ZodEnum<{
|
|
557
|
-
cancelled: "cancelled";
|
|
558
644
|
interrupt: "interrupt";
|
|
559
645
|
error: "error";
|
|
646
|
+
cancelled: "cancelled";
|
|
560
647
|
answer: "answer";
|
|
561
648
|
}>;
|
|
562
649
|
text: z.ZodString;
|
|
@@ -567,8 +654,8 @@ declare const EvalObservationSchema: z.ZodObject<{
|
|
|
567
654
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
568
655
|
requestId: z.ZodString;
|
|
569
656
|
kind: z.ZodEnum<{
|
|
570
|
-
text: "text";
|
|
571
657
|
custom: "custom";
|
|
658
|
+
text: "text";
|
|
572
659
|
"multi-choice": "multi-choice";
|
|
573
660
|
choice: "choice";
|
|
574
661
|
}>;
|
|
@@ -612,6 +699,41 @@ declare const EvalJudgeUsageSchema: z.ZodObject<{
|
|
|
612
699
|
}, z.core.$strict>>;
|
|
613
700
|
}, z.core.$strict>;
|
|
614
701
|
declare const EvalStepResultSchema: z.ZodObject<{
|
|
702
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
703
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
704
|
+
history: z.ZodBoolean;
|
|
705
|
+
observation: z.ZodObject<{
|
|
706
|
+
type: z.ZodEnum<{
|
|
707
|
+
interrupt: "interrupt";
|
|
708
|
+
error: "error";
|
|
709
|
+
cancelled: "cancelled";
|
|
710
|
+
answer: "answer";
|
|
711
|
+
}>;
|
|
712
|
+
text: z.ZodString;
|
|
713
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
714
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
715
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
716
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
717
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
718
|
+
requestId: z.ZodString;
|
|
719
|
+
kind: z.ZodEnum<{
|
|
720
|
+
custom: "custom";
|
|
721
|
+
text: "text";
|
|
722
|
+
"multi-choice": "multi-choice";
|
|
723
|
+
choice: "choice";
|
|
724
|
+
}>;
|
|
725
|
+
question: z.ZodOptional<z.ZodString>;
|
|
726
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
727
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
728
|
+
options: z.ZodArray<z.ZodObject<{
|
|
729
|
+
id: z.ZodString;
|
|
730
|
+
label: z.ZodString;
|
|
731
|
+
description: z.ZodOptional<z.ZodString>;
|
|
732
|
+
}, z.core.$strict>>;
|
|
733
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
734
|
+
}, z.core.$strict>>;
|
|
735
|
+
}, z.core.$strict>;
|
|
736
|
+
}, z.core.$strict>>;
|
|
615
737
|
index: z.ZodNumber;
|
|
616
738
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
617
739
|
message: z.ZodString;
|
|
@@ -648,9 +770,9 @@ declare const EvalStepResultSchema: z.ZodObject<{
|
|
|
648
770
|
}, z.core.$strict>;
|
|
649
771
|
observation: z.ZodObject<{
|
|
650
772
|
type: z.ZodEnum<{
|
|
651
|
-
cancelled: "cancelled";
|
|
652
773
|
interrupt: "interrupt";
|
|
653
774
|
error: "error";
|
|
775
|
+
cancelled: "cancelled";
|
|
654
776
|
answer: "answer";
|
|
655
777
|
}>;
|
|
656
778
|
text: z.ZodString;
|
|
@@ -661,8 +783,8 @@ declare const EvalStepResultSchema: z.ZodObject<{
|
|
|
661
783
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
662
784
|
requestId: z.ZodString;
|
|
663
785
|
kind: z.ZodEnum<{
|
|
664
|
-
text: "text";
|
|
665
786
|
custom: "custom";
|
|
787
|
+
text: "text";
|
|
666
788
|
"multi-choice": "multi-choice";
|
|
667
789
|
choice: "choice";
|
|
668
790
|
}>;
|
|
@@ -679,9 +801,9 @@ declare const EvalStepResultSchema: z.ZodObject<{
|
|
|
679
801
|
}, z.core.$strict>;
|
|
680
802
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
681
803
|
status: z.ZodEnum<{
|
|
804
|
+
error: "error";
|
|
682
805
|
cancelled: "cancelled";
|
|
683
806
|
failed: "failed";
|
|
684
|
-
error: "error";
|
|
685
807
|
passed: "passed";
|
|
686
808
|
ungraded: "ungraded";
|
|
687
809
|
}>;
|
|
@@ -727,14 +849,49 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
727
849
|
repetition: z.ZodNumber;
|
|
728
850
|
sessionId: z.ZodString;
|
|
729
851
|
status: z.ZodEnum<{
|
|
852
|
+
error: "error";
|
|
730
853
|
cancelled: "cancelled";
|
|
731
854
|
failed: "failed";
|
|
732
|
-
error: "error";
|
|
733
855
|
passed: "passed";
|
|
734
856
|
ungraded: "ungraded";
|
|
735
857
|
}>;
|
|
736
858
|
durationMs: z.ZodNumber;
|
|
737
859
|
steps: z.ZodArray<z.ZodObject<{
|
|
860
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
861
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
862
|
+
history: z.ZodBoolean;
|
|
863
|
+
observation: z.ZodObject<{
|
|
864
|
+
type: z.ZodEnum<{
|
|
865
|
+
interrupt: "interrupt";
|
|
866
|
+
error: "error";
|
|
867
|
+
cancelled: "cancelled";
|
|
868
|
+
answer: "answer";
|
|
869
|
+
}>;
|
|
870
|
+
text: z.ZodString;
|
|
871
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
872
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
873
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
874
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
875
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
876
|
+
requestId: z.ZodString;
|
|
877
|
+
kind: z.ZodEnum<{
|
|
878
|
+
custom: "custom";
|
|
879
|
+
text: "text";
|
|
880
|
+
"multi-choice": "multi-choice";
|
|
881
|
+
choice: "choice";
|
|
882
|
+
}>;
|
|
883
|
+
question: z.ZodOptional<z.ZodString>;
|
|
884
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
885
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
886
|
+
options: z.ZodArray<z.ZodObject<{
|
|
887
|
+
id: z.ZodString;
|
|
888
|
+
label: z.ZodString;
|
|
889
|
+
description: z.ZodOptional<z.ZodString>;
|
|
890
|
+
}, z.core.$strict>>;
|
|
891
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
892
|
+
}, z.core.$strict>>;
|
|
893
|
+
}, z.core.$strict>;
|
|
894
|
+
}, z.core.$strict>>;
|
|
738
895
|
index: z.ZodNumber;
|
|
739
896
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
740
897
|
message: z.ZodString;
|
|
@@ -771,9 +928,9 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
771
928
|
}, z.core.$strict>;
|
|
772
929
|
observation: z.ZodObject<{
|
|
773
930
|
type: z.ZodEnum<{
|
|
774
|
-
cancelled: "cancelled";
|
|
775
931
|
interrupt: "interrupt";
|
|
776
932
|
error: "error";
|
|
933
|
+
cancelled: "cancelled";
|
|
777
934
|
answer: "answer";
|
|
778
935
|
}>;
|
|
779
936
|
text: z.ZodString;
|
|
@@ -784,8 +941,8 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
784
941
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
785
942
|
requestId: z.ZodString;
|
|
786
943
|
kind: z.ZodEnum<{
|
|
787
|
-
text: "text";
|
|
788
944
|
custom: "custom";
|
|
945
|
+
text: "text";
|
|
789
946
|
"multi-choice": "multi-choice";
|
|
790
947
|
choice: "choice";
|
|
791
948
|
}>;
|
|
@@ -802,9 +959,9 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
802
959
|
}, z.core.$strict>;
|
|
803
960
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
804
961
|
status: z.ZodEnum<{
|
|
962
|
+
error: "error";
|
|
805
963
|
cancelled: "cancelled";
|
|
806
964
|
failed: "failed";
|
|
807
|
-
error: "error";
|
|
808
965
|
passed: "passed";
|
|
809
966
|
ungraded: "ungraded";
|
|
810
967
|
}>;
|
|
@@ -900,9 +1057,9 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
900
1057
|
}, z.core.$strict>]>;
|
|
901
1058
|
observation: z.ZodObject<{
|
|
902
1059
|
type: z.ZodEnum<{
|
|
903
|
-
cancelled: "cancelled";
|
|
904
1060
|
interrupt: "interrupt";
|
|
905
1061
|
error: "error";
|
|
1062
|
+
cancelled: "cancelled";
|
|
906
1063
|
answer: "answer";
|
|
907
1064
|
}>;
|
|
908
1065
|
text: z.ZodString;
|
|
@@ -913,8 +1070,8 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
913
1070
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
914
1071
|
requestId: z.ZodString;
|
|
915
1072
|
kind: z.ZodEnum<{
|
|
916
|
-
text: "text";
|
|
917
1073
|
custom: "custom";
|
|
1074
|
+
text: "text";
|
|
918
1075
|
"multi-choice": "multi-choice";
|
|
919
1076
|
choice: "choice";
|
|
920
1077
|
}>;
|
|
@@ -931,6 +1088,41 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
931
1088
|
}, z.core.$strict>;
|
|
932
1089
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
933
1090
|
conversation: z.ZodArray<z.ZodObject<{
|
|
1091
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
1092
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
1093
|
+
history: z.ZodBoolean;
|
|
1094
|
+
observation: z.ZodObject<{
|
|
1095
|
+
type: z.ZodEnum<{
|
|
1096
|
+
interrupt: "interrupt";
|
|
1097
|
+
error: "error";
|
|
1098
|
+
cancelled: "cancelled";
|
|
1099
|
+
answer: "answer";
|
|
1100
|
+
}>;
|
|
1101
|
+
text: z.ZodString;
|
|
1102
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
1103
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
1104
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
1105
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
1106
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
1107
|
+
requestId: z.ZodString;
|
|
1108
|
+
kind: z.ZodEnum<{
|
|
1109
|
+
custom: "custom";
|
|
1110
|
+
text: "text";
|
|
1111
|
+
"multi-choice": "multi-choice";
|
|
1112
|
+
choice: "choice";
|
|
1113
|
+
}>;
|
|
1114
|
+
question: z.ZodOptional<z.ZodString>;
|
|
1115
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
1116
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
1117
|
+
options: z.ZodArray<z.ZodObject<{
|
|
1118
|
+
id: z.ZodString;
|
|
1119
|
+
label: z.ZodString;
|
|
1120
|
+
description: z.ZodOptional<z.ZodString>;
|
|
1121
|
+
}, z.core.$strict>>;
|
|
1122
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
1123
|
+
}, z.core.$strict>>;
|
|
1124
|
+
}, z.core.$strict>;
|
|
1125
|
+
}, z.core.$strict>>;
|
|
934
1126
|
index: z.ZodNumber;
|
|
935
1127
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
936
1128
|
message: z.ZodString;
|
|
@@ -967,9 +1159,9 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
967
1159
|
}, z.core.$strict>;
|
|
968
1160
|
observation: z.ZodObject<{
|
|
969
1161
|
type: z.ZodEnum<{
|
|
970
|
-
cancelled: "cancelled";
|
|
971
1162
|
interrupt: "interrupt";
|
|
972
1163
|
error: "error";
|
|
1164
|
+
cancelled: "cancelled";
|
|
973
1165
|
answer: "answer";
|
|
974
1166
|
}>;
|
|
975
1167
|
text: z.ZodString;
|
|
@@ -980,8 +1172,8 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
980
1172
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
981
1173
|
requestId: z.ZodString;
|
|
982
1174
|
kind: z.ZodEnum<{
|
|
983
|
-
text: "text";
|
|
984
1175
|
custom: "custom";
|
|
1176
|
+
text: "text";
|
|
985
1177
|
"multi-choice": "multi-choice";
|
|
986
1178
|
choice: "choice";
|
|
987
1179
|
}>;
|
|
@@ -998,9 +1190,9 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
998
1190
|
}, z.core.$strict>;
|
|
999
1191
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1000
1192
|
status: z.ZodEnum<{
|
|
1193
|
+
error: "error";
|
|
1001
1194
|
cancelled: "cancelled";
|
|
1002
1195
|
failed: "failed";
|
|
1003
|
-
error: "error";
|
|
1004
1196
|
passed: "passed";
|
|
1005
1197
|
ungraded: "ungraded";
|
|
1006
1198
|
}>;
|
|
@@ -1089,6 +1281,32 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1089
1281
|
suiteId: z.ZodString;
|
|
1090
1282
|
suiteRevision: z.ZodString;
|
|
1091
1283
|
suite: z.ZodObject<{
|
|
1284
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
1285
|
+
history: z.ZodOptional<z.ZodBoolean>;
|
|
1286
|
+
events: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodEnum<{
|
|
1287
|
+
"tool-call-start": "tool-call-start";
|
|
1288
|
+
"tool-call-result": "tool-call-result";
|
|
1289
|
+
"tool-call-error": "tool-call-error";
|
|
1290
|
+
"tool-result": "tool-result";
|
|
1291
|
+
"structured-data": "structured-data";
|
|
1292
|
+
"structured-data-invalidated": "structured-data-invalidated";
|
|
1293
|
+
interrupt: "interrupt";
|
|
1294
|
+
error: "error";
|
|
1295
|
+
cancelled: "cancelled";
|
|
1296
|
+
"limit-reached": "limit-reached";
|
|
1297
|
+
}>>, z.ZodObject<{
|
|
1298
|
+
using: z.ZodString;
|
|
1299
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
1300
|
+
}, z.core.$strict>]>>;
|
|
1301
|
+
outputs: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodObject<{
|
|
1302
|
+
dataType: z.ZodOptional<z.ZodString>;
|
|
1303
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
1304
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
1305
|
+
}, z.core.$strict>>, z.ZodObject<{
|
|
1306
|
+
using: z.ZodString;
|
|
1307
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
1308
|
+
}, z.core.$strict>]>>;
|
|
1309
|
+
}, z.core.$strict>>;
|
|
1092
1310
|
id: z.ZodString;
|
|
1093
1311
|
name: z.ZodOptional<z.ZodString>;
|
|
1094
1312
|
cases: z.ZodArray<z.ZodObject<{
|
|
@@ -1162,9 +1380,9 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1162
1380
|
startedAt: z.ZodISODateTime;
|
|
1163
1381
|
durationMs: z.ZodNumber;
|
|
1164
1382
|
status: z.ZodEnum<{
|
|
1383
|
+
error: "error";
|
|
1165
1384
|
cancelled: "cancelled";
|
|
1166
1385
|
failed: "failed";
|
|
1167
|
-
error: "error";
|
|
1168
1386
|
passed: "passed";
|
|
1169
1387
|
ungraded: "ungraded";
|
|
1170
1388
|
}>;
|
|
@@ -1180,14 +1398,49 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1180
1398
|
repetition: z.ZodNumber;
|
|
1181
1399
|
sessionId: z.ZodString;
|
|
1182
1400
|
status: z.ZodEnum<{
|
|
1401
|
+
error: "error";
|
|
1183
1402
|
cancelled: "cancelled";
|
|
1184
1403
|
failed: "failed";
|
|
1185
|
-
error: "error";
|
|
1186
1404
|
passed: "passed";
|
|
1187
1405
|
ungraded: "ungraded";
|
|
1188
1406
|
}>;
|
|
1189
1407
|
durationMs: z.ZodNumber;
|
|
1190
1408
|
steps: z.ZodArray<z.ZodObject<{
|
|
1409
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
1410
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
1411
|
+
history: z.ZodBoolean;
|
|
1412
|
+
observation: z.ZodObject<{
|
|
1413
|
+
type: z.ZodEnum<{
|
|
1414
|
+
interrupt: "interrupt";
|
|
1415
|
+
error: "error";
|
|
1416
|
+
cancelled: "cancelled";
|
|
1417
|
+
answer: "answer";
|
|
1418
|
+
}>;
|
|
1419
|
+
text: z.ZodString;
|
|
1420
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
1421
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
1422
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
1423
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
1424
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
1425
|
+
requestId: z.ZodString;
|
|
1426
|
+
kind: z.ZodEnum<{
|
|
1427
|
+
custom: "custom";
|
|
1428
|
+
text: "text";
|
|
1429
|
+
"multi-choice": "multi-choice";
|
|
1430
|
+
choice: "choice";
|
|
1431
|
+
}>;
|
|
1432
|
+
question: z.ZodOptional<z.ZodString>;
|
|
1433
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
1434
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
1435
|
+
options: z.ZodArray<z.ZodObject<{
|
|
1436
|
+
id: z.ZodString;
|
|
1437
|
+
label: z.ZodString;
|
|
1438
|
+
description: z.ZodOptional<z.ZodString>;
|
|
1439
|
+
}, z.core.$strict>>;
|
|
1440
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
1441
|
+
}, z.core.$strict>>;
|
|
1442
|
+
}, z.core.$strict>;
|
|
1443
|
+
}, z.core.$strict>>;
|
|
1191
1444
|
index: z.ZodNumber;
|
|
1192
1445
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
1193
1446
|
message: z.ZodString;
|
|
@@ -1224,9 +1477,9 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1224
1477
|
}, z.core.$strict>;
|
|
1225
1478
|
observation: z.ZodObject<{
|
|
1226
1479
|
type: z.ZodEnum<{
|
|
1227
|
-
cancelled: "cancelled";
|
|
1228
1480
|
interrupt: "interrupt";
|
|
1229
1481
|
error: "error";
|
|
1482
|
+
cancelled: "cancelled";
|
|
1230
1483
|
answer: "answer";
|
|
1231
1484
|
}>;
|
|
1232
1485
|
text: z.ZodString;
|
|
@@ -1237,8 +1490,8 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1237
1490
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
1238
1491
|
requestId: z.ZodString;
|
|
1239
1492
|
kind: z.ZodEnum<{
|
|
1240
|
-
text: "text";
|
|
1241
1493
|
custom: "custom";
|
|
1494
|
+
text: "text";
|
|
1242
1495
|
"multi-choice": "multi-choice";
|
|
1243
1496
|
choice: "choice";
|
|
1244
1497
|
}>;
|
|
@@ -1255,9 +1508,9 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1255
1508
|
}, z.core.$strict>;
|
|
1256
1509
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1257
1510
|
status: z.ZodEnum<{
|
|
1511
|
+
error: "error";
|
|
1258
1512
|
cancelled: "cancelled";
|
|
1259
1513
|
failed: "failed";
|
|
1260
|
-
error: "error";
|
|
1261
1514
|
passed: "passed";
|
|
1262
1515
|
ungraded: "ungraded";
|
|
1263
1516
|
}>;
|
|
@@ -1332,6 +1585,32 @@ declare const EvalManifestSchema: z.ZodObject<{
|
|
|
1332
1585
|
schemaVersion: z.ZodLiteral<1>;
|
|
1333
1586
|
studioJudging: z.ZodOptional<z.ZodLiteral<true>>;
|
|
1334
1587
|
suites: z.ZodArray<z.ZodObject<{
|
|
1588
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
1589
|
+
history: z.ZodOptional<z.ZodBoolean>;
|
|
1590
|
+
events: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodEnum<{
|
|
1591
|
+
"tool-call-start": "tool-call-start";
|
|
1592
|
+
"tool-call-result": "tool-call-result";
|
|
1593
|
+
"tool-call-error": "tool-call-error";
|
|
1594
|
+
"tool-result": "tool-result";
|
|
1595
|
+
"structured-data": "structured-data";
|
|
1596
|
+
"structured-data-invalidated": "structured-data-invalidated";
|
|
1597
|
+
interrupt: "interrupt";
|
|
1598
|
+
error: "error";
|
|
1599
|
+
cancelled: "cancelled";
|
|
1600
|
+
"limit-reached": "limit-reached";
|
|
1601
|
+
}>>, z.ZodObject<{
|
|
1602
|
+
using: z.ZodString;
|
|
1603
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
1604
|
+
}, z.core.$strict>]>>;
|
|
1605
|
+
outputs: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodObject<{
|
|
1606
|
+
dataType: z.ZodOptional<z.ZodString>;
|
|
1607
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
1608
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
1609
|
+
}, z.core.$strict>>, z.ZodObject<{
|
|
1610
|
+
using: z.ZodString;
|
|
1611
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
1612
|
+
}, z.core.$strict>]>>;
|
|
1613
|
+
}, z.core.$strict>>;
|
|
1335
1614
|
id: z.ZodString;
|
|
1336
1615
|
name: z.ZodOptional<z.ZodString>;
|
|
1337
1616
|
cases: z.ZodArray<z.ZodObject<{
|
|
@@ -1440,6 +1719,41 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1440
1719
|
caseId: z.ZodString;
|
|
1441
1720
|
repetition: z.ZodNumber;
|
|
1442
1721
|
step: z.ZodObject<{
|
|
1722
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
1723
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
1724
|
+
history: z.ZodBoolean;
|
|
1725
|
+
observation: z.ZodObject<{
|
|
1726
|
+
type: z.ZodEnum<{
|
|
1727
|
+
interrupt: "interrupt";
|
|
1728
|
+
error: "error";
|
|
1729
|
+
cancelled: "cancelled";
|
|
1730
|
+
answer: "answer";
|
|
1731
|
+
}>;
|
|
1732
|
+
text: z.ZodString;
|
|
1733
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
1734
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
1735
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
1736
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
1737
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
1738
|
+
requestId: z.ZodString;
|
|
1739
|
+
kind: z.ZodEnum<{
|
|
1740
|
+
custom: "custom";
|
|
1741
|
+
text: "text";
|
|
1742
|
+
"multi-choice": "multi-choice";
|
|
1743
|
+
choice: "choice";
|
|
1744
|
+
}>;
|
|
1745
|
+
question: z.ZodOptional<z.ZodString>;
|
|
1746
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
1747
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
1748
|
+
options: z.ZodArray<z.ZodObject<{
|
|
1749
|
+
id: z.ZodString;
|
|
1750
|
+
label: z.ZodString;
|
|
1751
|
+
description: z.ZodOptional<z.ZodString>;
|
|
1752
|
+
}, z.core.$strict>>;
|
|
1753
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
1754
|
+
}, z.core.$strict>>;
|
|
1755
|
+
}, z.core.$strict>;
|
|
1756
|
+
}, z.core.$strict>>;
|
|
1443
1757
|
index: z.ZodNumber;
|
|
1444
1758
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
1445
1759
|
message: z.ZodString;
|
|
@@ -1476,9 +1790,9 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1476
1790
|
}, z.core.$strict>;
|
|
1477
1791
|
observation: z.ZodObject<{
|
|
1478
1792
|
type: z.ZodEnum<{
|
|
1479
|
-
cancelled: "cancelled";
|
|
1480
1793
|
interrupt: "interrupt";
|
|
1481
1794
|
error: "error";
|
|
1795
|
+
cancelled: "cancelled";
|
|
1482
1796
|
answer: "answer";
|
|
1483
1797
|
}>;
|
|
1484
1798
|
text: z.ZodString;
|
|
@@ -1489,8 +1803,8 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1489
1803
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
1490
1804
|
requestId: z.ZodString;
|
|
1491
1805
|
kind: z.ZodEnum<{
|
|
1492
|
-
text: "text";
|
|
1493
1806
|
custom: "custom";
|
|
1807
|
+
text: "text";
|
|
1494
1808
|
"multi-choice": "multi-choice";
|
|
1495
1809
|
choice: "choice";
|
|
1496
1810
|
}>;
|
|
@@ -1507,9 +1821,9 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1507
1821
|
}, z.core.$strict>;
|
|
1508
1822
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1509
1823
|
status: z.ZodEnum<{
|
|
1824
|
+
error: "error";
|
|
1510
1825
|
cancelled: "cancelled";
|
|
1511
1826
|
failed: "failed";
|
|
1512
|
-
error: "error";
|
|
1513
1827
|
passed: "passed";
|
|
1514
1828
|
ungraded: "ungraded";
|
|
1515
1829
|
}>;
|
|
@@ -1557,14 +1871,49 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1557
1871
|
repetition: z.ZodNumber;
|
|
1558
1872
|
sessionId: z.ZodString;
|
|
1559
1873
|
status: z.ZodEnum<{
|
|
1874
|
+
error: "error";
|
|
1560
1875
|
cancelled: "cancelled";
|
|
1561
1876
|
failed: "failed";
|
|
1562
|
-
error: "error";
|
|
1563
1877
|
passed: "passed";
|
|
1564
1878
|
ungraded: "ungraded";
|
|
1565
1879
|
}>;
|
|
1566
1880
|
durationMs: z.ZodNumber;
|
|
1567
1881
|
steps: z.ZodArray<z.ZodObject<{
|
|
1882
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
1883
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
1884
|
+
history: z.ZodBoolean;
|
|
1885
|
+
observation: z.ZodObject<{
|
|
1886
|
+
type: z.ZodEnum<{
|
|
1887
|
+
interrupt: "interrupt";
|
|
1888
|
+
error: "error";
|
|
1889
|
+
cancelled: "cancelled";
|
|
1890
|
+
answer: "answer";
|
|
1891
|
+
}>;
|
|
1892
|
+
text: z.ZodString;
|
|
1893
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
1894
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
1895
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
1896
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
1897
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
1898
|
+
requestId: z.ZodString;
|
|
1899
|
+
kind: z.ZodEnum<{
|
|
1900
|
+
custom: "custom";
|
|
1901
|
+
text: "text";
|
|
1902
|
+
"multi-choice": "multi-choice";
|
|
1903
|
+
choice: "choice";
|
|
1904
|
+
}>;
|
|
1905
|
+
question: z.ZodOptional<z.ZodString>;
|
|
1906
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
1907
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
1908
|
+
options: z.ZodArray<z.ZodObject<{
|
|
1909
|
+
id: z.ZodString;
|
|
1910
|
+
label: z.ZodString;
|
|
1911
|
+
description: z.ZodOptional<z.ZodString>;
|
|
1912
|
+
}, z.core.$strict>>;
|
|
1913
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
1914
|
+
}, z.core.$strict>>;
|
|
1915
|
+
}, z.core.$strict>;
|
|
1916
|
+
}, z.core.$strict>>;
|
|
1568
1917
|
index: z.ZodNumber;
|
|
1569
1918
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
1570
1919
|
message: z.ZodString;
|
|
@@ -1601,9 +1950,9 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1601
1950
|
}, z.core.$strict>;
|
|
1602
1951
|
observation: z.ZodObject<{
|
|
1603
1952
|
type: z.ZodEnum<{
|
|
1604
|
-
cancelled: "cancelled";
|
|
1605
1953
|
interrupt: "interrupt";
|
|
1606
1954
|
error: "error";
|
|
1955
|
+
cancelled: "cancelled";
|
|
1607
1956
|
answer: "answer";
|
|
1608
1957
|
}>;
|
|
1609
1958
|
text: z.ZodString;
|
|
@@ -1614,8 +1963,8 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1614
1963
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
1615
1964
|
requestId: z.ZodString;
|
|
1616
1965
|
kind: z.ZodEnum<{
|
|
1617
|
-
text: "text";
|
|
1618
1966
|
custom: "custom";
|
|
1967
|
+
text: "text";
|
|
1619
1968
|
"multi-choice": "multi-choice";
|
|
1620
1969
|
choice: "choice";
|
|
1621
1970
|
}>;
|
|
@@ -1632,9 +1981,9 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1632
1981
|
}, z.core.$strict>;
|
|
1633
1982
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1634
1983
|
status: z.ZodEnum<{
|
|
1984
|
+
error: "error";
|
|
1635
1985
|
cancelled: "cancelled";
|
|
1636
1986
|
failed: "failed";
|
|
1637
|
-
error: "error";
|
|
1638
1987
|
passed: "passed";
|
|
1639
1988
|
ungraded: "ungraded";
|
|
1640
1989
|
}>;
|
|
@@ -1754,6 +2103,41 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1754
2103
|
caseId: z.ZodString;
|
|
1755
2104
|
repetition: z.ZodNumber;
|
|
1756
2105
|
step: z.ZodObject<{
|
|
2106
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
2107
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
2108
|
+
history: z.ZodBoolean;
|
|
2109
|
+
observation: z.ZodObject<{
|
|
2110
|
+
type: z.ZodEnum<{
|
|
2111
|
+
interrupt: "interrupt";
|
|
2112
|
+
error: "error";
|
|
2113
|
+
cancelled: "cancelled";
|
|
2114
|
+
answer: "answer";
|
|
2115
|
+
}>;
|
|
2116
|
+
text: z.ZodString;
|
|
2117
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
2118
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
2119
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
2120
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
2121
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
2122
|
+
requestId: z.ZodString;
|
|
2123
|
+
kind: z.ZodEnum<{
|
|
2124
|
+
custom: "custom";
|
|
2125
|
+
text: "text";
|
|
2126
|
+
"multi-choice": "multi-choice";
|
|
2127
|
+
choice: "choice";
|
|
2128
|
+
}>;
|
|
2129
|
+
question: z.ZodOptional<z.ZodString>;
|
|
2130
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
2131
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
2132
|
+
options: z.ZodArray<z.ZodObject<{
|
|
2133
|
+
id: z.ZodString;
|
|
2134
|
+
label: z.ZodString;
|
|
2135
|
+
description: z.ZodOptional<z.ZodString>;
|
|
2136
|
+
}, z.core.$strict>>;
|
|
2137
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
2138
|
+
}, z.core.$strict>>;
|
|
2139
|
+
}, z.core.$strict>;
|
|
2140
|
+
}, z.core.$strict>>;
|
|
1757
2141
|
index: z.ZodNumber;
|
|
1758
2142
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
1759
2143
|
message: z.ZodString;
|
|
@@ -1790,9 +2174,9 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1790
2174
|
}, z.core.$strict>;
|
|
1791
2175
|
observation: z.ZodObject<{
|
|
1792
2176
|
type: z.ZodEnum<{
|
|
1793
|
-
cancelled: "cancelled";
|
|
1794
2177
|
interrupt: "interrupt";
|
|
1795
2178
|
error: "error";
|
|
2179
|
+
cancelled: "cancelled";
|
|
1796
2180
|
answer: "answer";
|
|
1797
2181
|
}>;
|
|
1798
2182
|
text: z.ZodString;
|
|
@@ -1803,8 +2187,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1803
2187
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
1804
2188
|
requestId: z.ZodString;
|
|
1805
2189
|
kind: z.ZodEnum<{
|
|
1806
|
-
text: "text";
|
|
1807
2190
|
custom: "custom";
|
|
2191
|
+
text: "text";
|
|
1808
2192
|
"multi-choice": "multi-choice";
|
|
1809
2193
|
choice: "choice";
|
|
1810
2194
|
}>;
|
|
@@ -1821,9 +2205,9 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1821
2205
|
}, z.core.$strict>;
|
|
1822
2206
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1823
2207
|
status: z.ZodEnum<{
|
|
2208
|
+
error: "error";
|
|
1824
2209
|
cancelled: "cancelled";
|
|
1825
2210
|
failed: "failed";
|
|
1826
|
-
error: "error";
|
|
1827
2211
|
passed: "passed";
|
|
1828
2212
|
ungraded: "ungraded";
|
|
1829
2213
|
}>;
|
|
@@ -1871,14 +2255,49 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1871
2255
|
repetition: z.ZodNumber;
|
|
1872
2256
|
sessionId: z.ZodString;
|
|
1873
2257
|
status: z.ZodEnum<{
|
|
2258
|
+
error: "error";
|
|
1874
2259
|
cancelled: "cancelled";
|
|
1875
2260
|
failed: "failed";
|
|
1876
|
-
error: "error";
|
|
1877
2261
|
passed: "passed";
|
|
1878
2262
|
ungraded: "ungraded";
|
|
1879
2263
|
}>;
|
|
1880
2264
|
durationMs: z.ZodNumber;
|
|
1881
2265
|
steps: z.ZodArray<z.ZodObject<{
|
|
2266
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
2267
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
2268
|
+
history: z.ZodBoolean;
|
|
2269
|
+
observation: z.ZodObject<{
|
|
2270
|
+
type: z.ZodEnum<{
|
|
2271
|
+
interrupt: "interrupt";
|
|
2272
|
+
error: "error";
|
|
2273
|
+
cancelled: "cancelled";
|
|
2274
|
+
answer: "answer";
|
|
2275
|
+
}>;
|
|
2276
|
+
text: z.ZodString;
|
|
2277
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
2278
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
2279
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
2280
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
2281
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
2282
|
+
requestId: z.ZodString;
|
|
2283
|
+
kind: z.ZodEnum<{
|
|
2284
|
+
custom: "custom";
|
|
2285
|
+
text: "text";
|
|
2286
|
+
"multi-choice": "multi-choice";
|
|
2287
|
+
choice: "choice";
|
|
2288
|
+
}>;
|
|
2289
|
+
question: z.ZodOptional<z.ZodString>;
|
|
2290
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
2291
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
2292
|
+
options: z.ZodArray<z.ZodObject<{
|
|
2293
|
+
id: z.ZodString;
|
|
2294
|
+
label: z.ZodString;
|
|
2295
|
+
description: z.ZodOptional<z.ZodString>;
|
|
2296
|
+
}, z.core.$strict>>;
|
|
2297
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
2298
|
+
}, z.core.$strict>>;
|
|
2299
|
+
}, z.core.$strict>;
|
|
2300
|
+
}, z.core.$strict>>;
|
|
1882
2301
|
index: z.ZodNumber;
|
|
1883
2302
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
1884
2303
|
message: z.ZodString;
|
|
@@ -1915,9 +2334,9 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1915
2334
|
}, z.core.$strict>;
|
|
1916
2335
|
observation: z.ZodObject<{
|
|
1917
2336
|
type: z.ZodEnum<{
|
|
1918
|
-
cancelled: "cancelled";
|
|
1919
2337
|
interrupt: "interrupt";
|
|
1920
2338
|
error: "error";
|
|
2339
|
+
cancelled: "cancelled";
|
|
1921
2340
|
answer: "answer";
|
|
1922
2341
|
}>;
|
|
1923
2342
|
text: z.ZodString;
|
|
@@ -1928,8 +2347,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1928
2347
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
1929
2348
|
requestId: z.ZodString;
|
|
1930
2349
|
kind: z.ZodEnum<{
|
|
1931
|
-
text: "text";
|
|
1932
2350
|
custom: "custom";
|
|
2351
|
+
text: "text";
|
|
1933
2352
|
"multi-choice": "multi-choice";
|
|
1934
2353
|
choice: "choice";
|
|
1935
2354
|
}>;
|
|
@@ -1946,9 +2365,9 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1946
2365
|
}, z.core.$strict>;
|
|
1947
2366
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1948
2367
|
status: z.ZodEnum<{
|
|
2368
|
+
error: "error";
|
|
1949
2369
|
cancelled: "cancelled";
|
|
1950
2370
|
failed: "failed";
|
|
1951
|
-
error: "error";
|
|
1952
2371
|
passed: "passed";
|
|
1953
2372
|
ungraded: "ungraded";
|
|
1954
2373
|
}>;
|
|
@@ -2012,6 +2431,32 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2012
2431
|
suiteId: z.ZodString;
|
|
2013
2432
|
suiteRevision: z.ZodString;
|
|
2014
2433
|
suite: z.ZodObject<{
|
|
2434
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
2435
|
+
history: z.ZodOptional<z.ZodBoolean>;
|
|
2436
|
+
events: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodEnum<{
|
|
2437
|
+
"tool-call-start": "tool-call-start";
|
|
2438
|
+
"tool-call-result": "tool-call-result";
|
|
2439
|
+
"tool-call-error": "tool-call-error";
|
|
2440
|
+
"tool-result": "tool-result";
|
|
2441
|
+
"structured-data": "structured-data";
|
|
2442
|
+
"structured-data-invalidated": "structured-data-invalidated";
|
|
2443
|
+
interrupt: "interrupt";
|
|
2444
|
+
error: "error";
|
|
2445
|
+
cancelled: "cancelled";
|
|
2446
|
+
"limit-reached": "limit-reached";
|
|
2447
|
+
}>>, z.ZodObject<{
|
|
2448
|
+
using: z.ZodString;
|
|
2449
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
2450
|
+
}, z.core.$strict>]>>;
|
|
2451
|
+
outputs: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodObject<{
|
|
2452
|
+
dataType: z.ZodOptional<z.ZodString>;
|
|
2453
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
2454
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
2455
|
+
}, z.core.$strict>>, z.ZodObject<{
|
|
2456
|
+
using: z.ZodString;
|
|
2457
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
2458
|
+
}, z.core.$strict>]>>;
|
|
2459
|
+
}, z.core.$strict>>;
|
|
2015
2460
|
id: z.ZodString;
|
|
2016
2461
|
name: z.ZodOptional<z.ZodString>;
|
|
2017
2462
|
cases: z.ZodArray<z.ZodObject<{
|
|
@@ -2085,9 +2530,9 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2085
2530
|
startedAt: z.ZodISODateTime;
|
|
2086
2531
|
durationMs: z.ZodNumber;
|
|
2087
2532
|
status: z.ZodEnum<{
|
|
2533
|
+
error: "error";
|
|
2088
2534
|
cancelled: "cancelled";
|
|
2089
2535
|
failed: "failed";
|
|
2090
|
-
error: "error";
|
|
2091
2536
|
passed: "passed";
|
|
2092
2537
|
ungraded: "ungraded";
|
|
2093
2538
|
}>;
|
|
@@ -2103,14 +2548,49 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2103
2548
|
repetition: z.ZodNumber;
|
|
2104
2549
|
sessionId: z.ZodString;
|
|
2105
2550
|
status: z.ZodEnum<{
|
|
2551
|
+
error: "error";
|
|
2106
2552
|
cancelled: "cancelled";
|
|
2107
2553
|
failed: "failed";
|
|
2108
|
-
error: "error";
|
|
2109
2554
|
passed: "passed";
|
|
2110
2555
|
ungraded: "ungraded";
|
|
2111
2556
|
}>;
|
|
2112
2557
|
durationMs: z.ZodNumber;
|
|
2113
2558
|
steps: z.ZodArray<z.ZodObject<{
|
|
2559
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
2560
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
2561
|
+
history: z.ZodBoolean;
|
|
2562
|
+
observation: z.ZodObject<{
|
|
2563
|
+
type: z.ZodEnum<{
|
|
2564
|
+
interrupt: "interrupt";
|
|
2565
|
+
error: "error";
|
|
2566
|
+
cancelled: "cancelled";
|
|
2567
|
+
answer: "answer";
|
|
2568
|
+
}>;
|
|
2569
|
+
text: z.ZodString;
|
|
2570
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
2571
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
2572
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
2573
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
2574
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
2575
|
+
requestId: z.ZodString;
|
|
2576
|
+
kind: z.ZodEnum<{
|
|
2577
|
+
custom: "custom";
|
|
2578
|
+
text: "text";
|
|
2579
|
+
"multi-choice": "multi-choice";
|
|
2580
|
+
choice: "choice";
|
|
2581
|
+
}>;
|
|
2582
|
+
question: z.ZodOptional<z.ZodString>;
|
|
2583
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
2584
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
2585
|
+
options: z.ZodArray<z.ZodObject<{
|
|
2586
|
+
id: z.ZodString;
|
|
2587
|
+
label: z.ZodString;
|
|
2588
|
+
description: z.ZodOptional<z.ZodString>;
|
|
2589
|
+
}, z.core.$strict>>;
|
|
2590
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
2591
|
+
}, z.core.$strict>>;
|
|
2592
|
+
}, z.core.$strict>;
|
|
2593
|
+
}, z.core.$strict>>;
|
|
2114
2594
|
index: z.ZodNumber;
|
|
2115
2595
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
2116
2596
|
message: z.ZodString;
|
|
@@ -2147,9 +2627,9 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2147
2627
|
}, z.core.$strict>;
|
|
2148
2628
|
observation: z.ZodObject<{
|
|
2149
2629
|
type: z.ZodEnum<{
|
|
2150
|
-
cancelled: "cancelled";
|
|
2151
2630
|
interrupt: "interrupt";
|
|
2152
2631
|
error: "error";
|
|
2632
|
+
cancelled: "cancelled";
|
|
2153
2633
|
answer: "answer";
|
|
2154
2634
|
}>;
|
|
2155
2635
|
text: z.ZodString;
|
|
@@ -2160,8 +2640,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2160
2640
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
2161
2641
|
requestId: z.ZodString;
|
|
2162
2642
|
kind: z.ZodEnum<{
|
|
2163
|
-
text: "text";
|
|
2164
2643
|
custom: "custom";
|
|
2644
|
+
text: "text";
|
|
2165
2645
|
"multi-choice": "multi-choice";
|
|
2166
2646
|
choice: "choice";
|
|
2167
2647
|
}>;
|
|
@@ -2178,9 +2658,9 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2178
2658
|
}, z.core.$strict>;
|
|
2179
2659
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
2180
2660
|
status: z.ZodEnum<{
|
|
2661
|
+
error: "error";
|
|
2181
2662
|
cancelled: "cancelled";
|
|
2182
2663
|
failed: "failed";
|
|
2183
|
-
error: "error";
|
|
2184
2664
|
passed: "passed";
|
|
2185
2665
|
ungraded: "ungraded";
|
|
2186
2666
|
}>;
|
|
@@ -2343,9 +2823,9 @@ declare const StudioEvaluationSummarySchema: z.ZodObject<{
|
|
|
2343
2823
|
}>>;
|
|
2344
2824
|
}, z.core.$strict>>;
|
|
2345
2825
|
status: z.ZodEnum<{
|
|
2826
|
+
error: "error";
|
|
2346
2827
|
cancelled: "cancelled";
|
|
2347
2828
|
failed: "failed";
|
|
2348
|
-
error: "error";
|
|
2349
2829
|
passed: "passed";
|
|
2350
2830
|
queued: "queued";
|
|
2351
2831
|
running: "running";
|
|
@@ -2434,9 +2914,9 @@ declare const StudioEvaluationHistorySchema: z.ZodObject<{
|
|
|
2434
2914
|
}>>;
|
|
2435
2915
|
}, z.core.$strict>>;
|
|
2436
2916
|
status: z.ZodEnum<{
|
|
2917
|
+
error: "error";
|
|
2437
2918
|
cancelled: "cancelled";
|
|
2438
2919
|
failed: "failed";
|
|
2439
|
-
error: "error";
|
|
2440
2920
|
passed: "passed";
|
|
2441
2921
|
queued: "queued";
|
|
2442
2922
|
running: "running";
|
|
@@ -2526,9 +3006,9 @@ declare const StudioEvaluationDetailSchema: z.ZodObject<{
|
|
|
2526
3006
|
}>>;
|
|
2527
3007
|
}, z.core.$strict>>;
|
|
2528
3008
|
status: z.ZodEnum<{
|
|
3009
|
+
error: "error";
|
|
2529
3010
|
cancelled: "cancelled";
|
|
2530
3011
|
failed: "failed";
|
|
2531
|
-
error: "error";
|
|
2532
3012
|
passed: "passed";
|
|
2533
3013
|
queued: "queued";
|
|
2534
3014
|
running: "running";
|
|
@@ -2595,9 +3075,9 @@ declare const StudioEvaluationDetailSchema: z.ZodObject<{
|
|
|
2595
3075
|
suiteId: z.ZodString;
|
|
2596
3076
|
suiteRevision: z.ZodString;
|
|
2597
3077
|
status: z.ZodEnum<{
|
|
3078
|
+
error: "error";
|
|
2598
3079
|
cancelled: "cancelled";
|
|
2599
3080
|
failed: "failed";
|
|
2600
|
-
error: "error";
|
|
2601
3081
|
passed: "passed";
|
|
2602
3082
|
queued: "queued";
|
|
2603
3083
|
running: "running";
|
|
@@ -2688,9 +3168,9 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
2688
3168
|
}>>;
|
|
2689
3169
|
}, z.core.$strict>>;
|
|
2690
3170
|
status: z.ZodEnum<{
|
|
3171
|
+
error: "error";
|
|
2691
3172
|
cancelled: "cancelled";
|
|
2692
3173
|
failed: "failed";
|
|
2693
|
-
error: "error";
|
|
2694
3174
|
passed: "passed";
|
|
2695
3175
|
queued: "queued";
|
|
2696
3176
|
running: "running";
|
|
@@ -2757,9 +3237,9 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
2757
3237
|
suiteId: z.ZodString;
|
|
2758
3238
|
suiteRevision: z.ZodString;
|
|
2759
3239
|
status: z.ZodEnum<{
|
|
3240
|
+
error: "error";
|
|
2760
3241
|
cancelled: "cancelled";
|
|
2761
3242
|
failed: "failed";
|
|
2762
|
-
error: "error";
|
|
2763
3243
|
passed: "passed";
|
|
2764
3244
|
queued: "queued";
|
|
2765
3245
|
running: "running";
|
|
@@ -2873,6 +3353,32 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
2873
3353
|
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
2874
3354
|
}, z.core.$strip>>;
|
|
2875
3355
|
suite: z.ZodObject<{
|
|
3356
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
3357
|
+
history: z.ZodOptional<z.ZodBoolean>;
|
|
3358
|
+
events: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodEnum<{
|
|
3359
|
+
"tool-call-start": "tool-call-start";
|
|
3360
|
+
"tool-call-result": "tool-call-result";
|
|
3361
|
+
"tool-call-error": "tool-call-error";
|
|
3362
|
+
"tool-result": "tool-result";
|
|
3363
|
+
"structured-data": "structured-data";
|
|
3364
|
+
"structured-data-invalidated": "structured-data-invalidated";
|
|
3365
|
+
interrupt: "interrupt";
|
|
3366
|
+
error: "error";
|
|
3367
|
+
cancelled: "cancelled";
|
|
3368
|
+
"limit-reached": "limit-reached";
|
|
3369
|
+
}>>, z.ZodObject<{
|
|
3370
|
+
using: z.ZodString;
|
|
3371
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
3372
|
+
}, z.core.$strict>]>>;
|
|
3373
|
+
outputs: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodObject<{
|
|
3374
|
+
dataType: z.ZodOptional<z.ZodString>;
|
|
3375
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
3376
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
3377
|
+
}, z.core.$strict>>, z.ZodObject<{
|
|
3378
|
+
using: z.ZodString;
|
|
3379
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
3380
|
+
}, z.core.$strict>]>>;
|
|
3381
|
+
}, z.core.$strict>>;
|
|
2876
3382
|
id: z.ZodString;
|
|
2877
3383
|
name: z.ZodOptional<z.ZodString>;
|
|
2878
3384
|
cases: z.ZodArray<z.ZodObject<{
|
|
@@ -2940,6 +3446,32 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
2940
3446
|
suiteId: z.ZodString;
|
|
2941
3447
|
suiteRevision: z.ZodString;
|
|
2942
3448
|
suite: z.ZodObject<{
|
|
3449
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
3450
|
+
history: z.ZodOptional<z.ZodBoolean>;
|
|
3451
|
+
events: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodEnum<{
|
|
3452
|
+
"tool-call-start": "tool-call-start";
|
|
3453
|
+
"tool-call-result": "tool-call-result";
|
|
3454
|
+
"tool-call-error": "tool-call-error";
|
|
3455
|
+
"tool-result": "tool-result";
|
|
3456
|
+
"structured-data": "structured-data";
|
|
3457
|
+
"structured-data-invalidated": "structured-data-invalidated";
|
|
3458
|
+
interrupt: "interrupt";
|
|
3459
|
+
error: "error";
|
|
3460
|
+
cancelled: "cancelled";
|
|
3461
|
+
"limit-reached": "limit-reached";
|
|
3462
|
+
}>>, z.ZodObject<{
|
|
3463
|
+
using: z.ZodString;
|
|
3464
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
3465
|
+
}, z.core.$strict>]>>;
|
|
3466
|
+
outputs: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<false>, z.ZodArray<z.ZodObject<{
|
|
3467
|
+
dataType: z.ZodOptional<z.ZodString>;
|
|
3468
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
3469
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
3470
|
+
}, z.core.$strict>>, z.ZodObject<{
|
|
3471
|
+
using: z.ZodString;
|
|
3472
|
+
params: z.ZodOptional<z.ZodJSONSchema>;
|
|
3473
|
+
}, z.core.$strict>]>>;
|
|
3474
|
+
}, z.core.$strict>>;
|
|
2943
3475
|
id: z.ZodString;
|
|
2944
3476
|
name: z.ZodOptional<z.ZodString>;
|
|
2945
3477
|
cases: z.ZodArray<z.ZodObject<{
|
|
@@ -3013,9 +3545,9 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3013
3545
|
startedAt: z.ZodISODateTime;
|
|
3014
3546
|
durationMs: z.ZodNumber;
|
|
3015
3547
|
status: z.ZodEnum<{
|
|
3548
|
+
error: "error";
|
|
3016
3549
|
cancelled: "cancelled";
|
|
3017
3550
|
failed: "failed";
|
|
3018
|
-
error: "error";
|
|
3019
3551
|
passed: "passed";
|
|
3020
3552
|
ungraded: "ungraded";
|
|
3021
3553
|
}>;
|
|
@@ -3031,14 +3563,49 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3031
3563
|
repetition: z.ZodNumber;
|
|
3032
3564
|
sessionId: z.ZodString;
|
|
3033
3565
|
status: z.ZodEnum<{
|
|
3566
|
+
error: "error";
|
|
3034
3567
|
cancelled: "cancelled";
|
|
3035
3568
|
failed: "failed";
|
|
3036
|
-
error: "error";
|
|
3037
3569
|
passed: "passed";
|
|
3038
3570
|
ungraded: "ungraded";
|
|
3039
3571
|
}>;
|
|
3040
3572
|
durationMs: z.ZodNumber;
|
|
3041
3573
|
steps: z.ZodArray<z.ZodObject<{
|
|
3574
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
3575
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
3576
|
+
history: z.ZodBoolean;
|
|
3577
|
+
observation: z.ZodObject<{
|
|
3578
|
+
type: z.ZodEnum<{
|
|
3579
|
+
interrupt: "interrupt";
|
|
3580
|
+
error: "error";
|
|
3581
|
+
cancelled: "cancelled";
|
|
3582
|
+
answer: "answer";
|
|
3583
|
+
}>;
|
|
3584
|
+
text: z.ZodString;
|
|
3585
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
3586
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
3587
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
3588
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
3589
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
3590
|
+
requestId: z.ZodString;
|
|
3591
|
+
kind: z.ZodEnum<{
|
|
3592
|
+
custom: "custom";
|
|
3593
|
+
text: "text";
|
|
3594
|
+
"multi-choice": "multi-choice";
|
|
3595
|
+
choice: "choice";
|
|
3596
|
+
}>;
|
|
3597
|
+
question: z.ZodOptional<z.ZodString>;
|
|
3598
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
3599
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
3600
|
+
options: z.ZodArray<z.ZodObject<{
|
|
3601
|
+
id: z.ZodString;
|
|
3602
|
+
label: z.ZodString;
|
|
3603
|
+
description: z.ZodOptional<z.ZodString>;
|
|
3604
|
+
}, z.core.$strict>>;
|
|
3605
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
3606
|
+
}, z.core.$strict>>;
|
|
3607
|
+
}, z.core.$strict>;
|
|
3608
|
+
}, z.core.$strict>>;
|
|
3042
3609
|
index: z.ZodNumber;
|
|
3043
3610
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
3044
3611
|
message: z.ZodString;
|
|
@@ -3075,9 +3642,9 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3075
3642
|
}, z.core.$strict>;
|
|
3076
3643
|
observation: z.ZodObject<{
|
|
3077
3644
|
type: z.ZodEnum<{
|
|
3078
|
-
cancelled: "cancelled";
|
|
3079
3645
|
interrupt: "interrupt";
|
|
3080
3646
|
error: "error";
|
|
3647
|
+
cancelled: "cancelled";
|
|
3081
3648
|
answer: "answer";
|
|
3082
3649
|
}>;
|
|
3083
3650
|
text: z.ZodString;
|
|
@@ -3088,8 +3655,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3088
3655
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
3089
3656
|
requestId: z.ZodString;
|
|
3090
3657
|
kind: z.ZodEnum<{
|
|
3091
|
-
text: "text";
|
|
3092
3658
|
custom: "custom";
|
|
3659
|
+
text: "text";
|
|
3093
3660
|
"multi-choice": "multi-choice";
|
|
3094
3661
|
choice: "choice";
|
|
3095
3662
|
}>;
|
|
@@ -3106,9 +3673,9 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3106
3673
|
}, z.core.$strict>;
|
|
3107
3674
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
3108
3675
|
status: z.ZodEnum<{
|
|
3676
|
+
error: "error";
|
|
3109
3677
|
cancelled: "cancelled";
|
|
3110
3678
|
failed: "failed";
|
|
3111
|
-
error: "error";
|
|
3112
3679
|
passed: "passed";
|
|
3113
3680
|
ungraded: "ungraded";
|
|
3114
3681
|
}>;
|
|
@@ -3211,6 +3778,41 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3211
3778
|
caseId: z.ZodString;
|
|
3212
3779
|
repetition: z.ZodNumber;
|
|
3213
3780
|
step: z.ZodObject<{
|
|
3781
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
3782
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
3783
|
+
history: z.ZodBoolean;
|
|
3784
|
+
observation: z.ZodObject<{
|
|
3785
|
+
type: z.ZodEnum<{
|
|
3786
|
+
interrupt: "interrupt";
|
|
3787
|
+
error: "error";
|
|
3788
|
+
cancelled: "cancelled";
|
|
3789
|
+
answer: "answer";
|
|
3790
|
+
}>;
|
|
3791
|
+
text: z.ZodString;
|
|
3792
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
3793
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
3794
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
3795
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
3796
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
3797
|
+
requestId: z.ZodString;
|
|
3798
|
+
kind: z.ZodEnum<{
|
|
3799
|
+
custom: "custom";
|
|
3800
|
+
text: "text";
|
|
3801
|
+
"multi-choice": "multi-choice";
|
|
3802
|
+
choice: "choice";
|
|
3803
|
+
}>;
|
|
3804
|
+
question: z.ZodOptional<z.ZodString>;
|
|
3805
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
3806
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
3807
|
+
options: z.ZodArray<z.ZodObject<{
|
|
3808
|
+
id: z.ZodString;
|
|
3809
|
+
label: z.ZodString;
|
|
3810
|
+
description: z.ZodOptional<z.ZodString>;
|
|
3811
|
+
}, z.core.$strict>>;
|
|
3812
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
3813
|
+
}, z.core.$strict>>;
|
|
3814
|
+
}, z.core.$strict>;
|
|
3815
|
+
}, z.core.$strict>>;
|
|
3214
3816
|
index: z.ZodNumber;
|
|
3215
3817
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
3216
3818
|
message: z.ZodString;
|
|
@@ -3247,9 +3849,9 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3247
3849
|
}, z.core.$strict>;
|
|
3248
3850
|
observation: z.ZodObject<{
|
|
3249
3851
|
type: z.ZodEnum<{
|
|
3250
|
-
cancelled: "cancelled";
|
|
3251
3852
|
interrupt: "interrupt";
|
|
3252
3853
|
error: "error";
|
|
3854
|
+
cancelled: "cancelled";
|
|
3253
3855
|
answer: "answer";
|
|
3254
3856
|
}>;
|
|
3255
3857
|
text: z.ZodString;
|
|
@@ -3260,8 +3862,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3260
3862
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
3261
3863
|
requestId: z.ZodString;
|
|
3262
3864
|
kind: z.ZodEnum<{
|
|
3263
|
-
text: "text";
|
|
3264
3865
|
custom: "custom";
|
|
3866
|
+
text: "text";
|
|
3265
3867
|
"multi-choice": "multi-choice";
|
|
3266
3868
|
choice: "choice";
|
|
3267
3869
|
}>;
|
|
@@ -3278,9 +3880,9 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3278
3880
|
}, z.core.$strict>;
|
|
3279
3881
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
3280
3882
|
status: z.ZodEnum<{
|
|
3883
|
+
error: "error";
|
|
3281
3884
|
cancelled: "cancelled";
|
|
3282
3885
|
failed: "failed";
|
|
3283
|
-
error: "error";
|
|
3284
3886
|
passed: "passed";
|
|
3285
3887
|
ungraded: "ungraded";
|
|
3286
3888
|
}>;
|
|
@@ -3328,14 +3930,49 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3328
3930
|
repetition: z.ZodNumber;
|
|
3329
3931
|
sessionId: z.ZodString;
|
|
3330
3932
|
status: z.ZodEnum<{
|
|
3933
|
+
error: "error";
|
|
3331
3934
|
cancelled: "cancelled";
|
|
3332
3935
|
failed: "failed";
|
|
3333
|
-
error: "error";
|
|
3334
3936
|
passed: "passed";
|
|
3335
3937
|
ungraded: "ungraded";
|
|
3336
3938
|
}>;
|
|
3337
3939
|
durationMs: z.ZodNumber;
|
|
3338
3940
|
steps: z.ZodArray<z.ZodObject<{
|
|
3941
|
+
evidence: z.ZodOptional<z.ZodObject<{
|
|
3942
|
+
version: z.ZodLiteral<"compact-v1">;
|
|
3943
|
+
history: z.ZodBoolean;
|
|
3944
|
+
observation: z.ZodObject<{
|
|
3945
|
+
type: z.ZodEnum<{
|
|
3946
|
+
interrupt: "interrupt";
|
|
3947
|
+
error: "error";
|
|
3948
|
+
cancelled: "cancelled";
|
|
3949
|
+
answer: "answer";
|
|
3950
|
+
}>;
|
|
3951
|
+
text: z.ZodString;
|
|
3952
|
+
structured: z.ZodArray<z.ZodJSONSchema>;
|
|
3953
|
+
events: z.ZodOptional<z.ZodArray<z.ZodJSONSchema>>;
|
|
3954
|
+
runId: z.ZodOptional<z.ZodString>;
|
|
3955
|
+
checkpointId: z.ZodOptional<z.ZodString>;
|
|
3956
|
+
interrupt: z.ZodOptional<z.ZodObject<{
|
|
3957
|
+
requestId: z.ZodString;
|
|
3958
|
+
kind: z.ZodEnum<{
|
|
3959
|
+
custom: "custom";
|
|
3960
|
+
text: "text";
|
|
3961
|
+
"multi-choice": "multi-choice";
|
|
3962
|
+
choice: "choice";
|
|
3963
|
+
}>;
|
|
3964
|
+
question: z.ZodOptional<z.ZodString>;
|
|
3965
|
+
schemaId: z.ZodOptional<z.ZodString>;
|
|
3966
|
+
schemaVersion: z.ZodOptional<z.ZodString>;
|
|
3967
|
+
options: z.ZodArray<z.ZodObject<{
|
|
3968
|
+
id: z.ZodString;
|
|
3969
|
+
label: z.ZodString;
|
|
3970
|
+
description: z.ZodOptional<z.ZodString>;
|
|
3971
|
+
}, z.core.$strict>>;
|
|
3972
|
+
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
3973
|
+
}, z.core.$strict>>;
|
|
3974
|
+
}, z.core.$strict>;
|
|
3975
|
+
}, z.core.$strict>>;
|
|
3339
3976
|
index: z.ZodNumber;
|
|
3340
3977
|
input: z.ZodUnion<readonly [z.ZodObject<{
|
|
3341
3978
|
message: z.ZodString;
|
|
@@ -3372,9 +4009,9 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3372
4009
|
}, z.core.$strict>;
|
|
3373
4010
|
observation: z.ZodObject<{
|
|
3374
4011
|
type: z.ZodEnum<{
|
|
3375
|
-
cancelled: "cancelled";
|
|
3376
4012
|
interrupt: "interrupt";
|
|
3377
4013
|
error: "error";
|
|
4014
|
+
cancelled: "cancelled";
|
|
3378
4015
|
answer: "answer";
|
|
3379
4016
|
}>;
|
|
3380
4017
|
text: z.ZodString;
|
|
@@ -3385,8 +4022,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3385
4022
|
interrupt: z.ZodOptional<z.ZodObject<{
|
|
3386
4023
|
requestId: z.ZodString;
|
|
3387
4024
|
kind: z.ZodEnum<{
|
|
3388
|
-
text: "text";
|
|
3389
4025
|
custom: "custom";
|
|
4026
|
+
text: "text";
|
|
3390
4027
|
"multi-choice": "multi-choice";
|
|
3391
4028
|
choice: "choice";
|
|
3392
4029
|
}>;
|
|
@@ -3403,9 +4040,9 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3403
4040
|
}, z.core.$strict>;
|
|
3404
4041
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
3405
4042
|
status: z.ZodEnum<{
|
|
4043
|
+
error: "error";
|
|
3406
4044
|
cancelled: "cancelled";
|
|
3407
4045
|
failed: "failed";
|
|
3408
|
-
error: "error";
|
|
3409
4046
|
passed: "passed";
|
|
3410
4047
|
ungraded: "ungraded";
|
|
3411
4048
|
}>;
|
|
@@ -3469,7 +4106,13 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3469
4106
|
type StudioEvaluationSummary = z.infer<typeof StudioEvaluationSummarySchema>;
|
|
3470
4107
|
type StudioEvaluationDetail = z.infer<typeof StudioEvaluationDetailSchema>["run"];
|
|
3471
4108
|
|
|
4109
|
+
/** Shared by local grading and Studio replay. Never expose the original evidence via history. */
|
|
4110
|
+
declare function getEvalGradeEvidence(step: EvalStepResult, previous: readonly EvalStepResult[]): {
|
|
4111
|
+
observation: EvalObservation;
|
|
4112
|
+
conversation: EvalStepResult[];
|
|
4113
|
+
};
|
|
4114
|
+
|
|
3472
4115
|
/** Discover and pin the Studio backend's judge; grading never runs in the browser. */
|
|
3473
4116
|
declare function createStudioEvalJudge(options: StudioEvalJudgeOptions): Promise<EvalJudge>;
|
|
3474
4117
|
|
|
3475
|
-
export {
|
|
4118
|
+
export { type EvalVerdict as $, type EvalJudgeUsage as A, EvalJudgeUsageSchema as B, type CreateEvalsOptions as C, type EvalManifest as D, type EvalCase as E, EvalManifestSchema as F, type EvalObservation as G, EvalObservationSchema as H, type EvalOutputExpectation as I, type EvalOutputSelector as J, type EvalPhase as K, type EvalProgress as L, EvalProgressSchema as M, type EvalRemoteRunRequest as N, EvalRemoteRunRequestSchema as O, type EvalResponder as P, EvalResumeResponseSchema as Q, type EvalRunOptions as R, type EvalRunResult as S, EvalRunResultSchema as T, type EvalSetupContext as U, type EvalStatus as V, type EvalStep as W, type EvalStepResult as X, EvalStepResultSchema as Y, type EvalSuite as Z, EvalSuiteSchema as _, type EvalCaseResult as a, EvalVerdictSchema as a0, type EvalWireEvent as a1, EvalWireEventSchema as a2, EvaluationMetadataSchema as a3, type StudioEvalJudgeOptions as a4, StudioEvalJudgeRequestSchema as a5, StudioEvalJudgeResponseSchema as a6, StudioEvalStartRequestSchema as a7, type StudioEvaluationDetail as a8, StudioEvaluationDetailSchema as a9, type PendingInterrupt as aA, type ResumableInterrupt as aB, type ResumeHandle as aC, type ResumeOptions as aD, type ResumeResponse as aE, createAgent as aF, projectWorkflowTopology as aG, StudioEvaluationHistorySchema as aa, StudioEvaluationResultsSchema as ab, type StudioEvaluationStartRequest as ac, StudioEvaluationStartRequestSchema as ad, type StudioEvaluationSummary as ae, StudioEvaluationSummarySchema as af, createStudioEvalJudge as ag, defineSuite as ah, getEvalGradeEvidence as ai, parseEvalSuite as aj, type ChatLifecyclePhase as ak, ChatLifecycleHookError as al, type Agent as am, type ChatResponseFinalized as an, type AgentProcessOptions as ao, type AgentProjectTopologyOptions as ap, type ChatResponseStatus as aq, type CreateAgentArgs as ar, type ExecutableWorkflow as as, type ExecuteOptions as at, type ExecutionInfo as au, type ExecutionInterrupt as av, ExecutionRequestError as aw, type ExecutionResult as ax, type InterruptScope as ay, type ListInterruptsOptions as az, EvalCaseResultSchema as b, type EvalCommand as c, EvalConfigurationError as d, type EvalContext as e, type EvalCriterion as f, type EvalCriterionResult as g, type EvalDefaults as h, type EvalEvidenceEvent as i, type EvalEvidenceEventType as j, type EvalEvidenceFilters as k, type EvalEvidencePolicy as l, EvalEvidencePolicySchema as m, type EvalExecution as n, type EvalExecutionStatus as o, type EvalExpectation as p, type EvalGradeInput as q, type EvalHandlerRef as r, type EvalInterrupt as s, type EvalIssue as t, type EvalJson as u, type EvalJudge as v, type EvalJudgeEvidence as w, type EvalJudgeIdentity as x, EvalJudgeIdentitySchema as y, type EvalJudgeOptions as z };
|