@kortyx/agent 0.34.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -0
- package/dist/evals.d.ts +134 -12
- package/dist/evals.js +57 -3
- package/dist/evals.js.map +1 -1
- package/dist/index.d.ts +19 -5
- package/dist/index.js +1709 -692
- package/dist/index.js.map +1 -1
- package/dist/{studio-judge-DpRLvf8j.d.ts → studio-judge-Brk9R5oB.d.ts} +464 -41
- package/package.json +10 -8
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import * as _kortyx_prompts from '@kortyx/prompts';
|
|
2
3
|
import { GetProviderFn, ProviderModelRef } from '@kortyx/providers';
|
|
3
4
|
import { KortyxError, FailureDescriptor } from '@kortyx/core/errors';
|
|
4
5
|
import { FinalizedChatMessage, StreamChunk } from '@kortyx/stream';
|
|
@@ -161,6 +162,7 @@ interface AgentProjectTopologyOptions {
|
|
|
161
162
|
service: AgentTopologySnapshot["service"];
|
|
162
163
|
}
|
|
163
164
|
interface CreateAgentArgs {
|
|
165
|
+
prompts?: _kortyx_prompts.PromptManager | undefined;
|
|
164
166
|
limits?: ExecutionLimits | undefined;
|
|
165
167
|
getProvider?: GetProviderFn | undefined;
|
|
166
168
|
workflows?: WorkflowDefinition[];
|
|
@@ -171,6 +173,12 @@ interface CreateAgentArgs {
|
|
|
171
173
|
telemetry?: ExecutionRuntimeConfig["telemetry"];
|
|
172
174
|
}
|
|
173
175
|
interface Agent extends ReturnType<typeof createInterruptDiscovery> {
|
|
176
|
+
describePromptContracts?: () => {
|
|
177
|
+
id: string;
|
|
178
|
+
format: "system-user" | "chat";
|
|
179
|
+
variablesSchema: Record<string, unknown>;
|
|
180
|
+
configSchema: Record<string, unknown>;
|
|
181
|
+
}[];
|
|
174
182
|
execute<W extends ExecutableWorkflow>(args: ExecuteOptions<W>): Promise<ExecutionResult<z.output<W["outputSchema"]>>>;
|
|
175
183
|
execute(args: {
|
|
176
184
|
limits?: ExecutionLimits;
|
|
@@ -316,6 +324,7 @@ type EvalInterrupt = {
|
|
|
316
324
|
};
|
|
317
325
|
/** Public output and emitted execution evidence; private runtime state is excluded. */
|
|
318
326
|
type EvalObservation = {
|
|
327
|
+
promptUsage?: _kortyx_prompts.PromptUsageReceipt[];
|
|
319
328
|
type: "answer" | "interrupt" | "error" | "cancelled";
|
|
320
329
|
text: string;
|
|
321
330
|
structured: readonly EvalJson[];
|
|
@@ -381,7 +390,14 @@ type EvalStepResult = {
|
|
|
381
390
|
reason?: string;
|
|
382
391
|
criteria: EvalCriterionResult[];
|
|
383
392
|
};
|
|
393
|
+
type EvalRuntimeExecution = {
|
|
394
|
+
sessionId: string;
|
|
395
|
+
runId: string;
|
|
396
|
+
};
|
|
384
397
|
type EvalCaseResult = {
|
|
398
|
+
/** SDK-owned identity and execution associations for automatic cost attribution. */
|
|
399
|
+
attemptId?: string;
|
|
400
|
+
runtimeExecutions?: EvalRuntimeExecution[];
|
|
385
401
|
caseId: string;
|
|
386
402
|
repetition: number;
|
|
387
403
|
sessionId: string;
|
|
@@ -468,7 +484,7 @@ type EvalDefaults = {
|
|
|
468
484
|
cleanupTimeoutMs?: number;
|
|
469
485
|
};
|
|
470
486
|
type CreateEvalsOptions<Params = EvalJson | undefined, Prepared = undefined> = {
|
|
471
|
-
agent: Pick<Agent, "streamChat">;
|
|
487
|
+
agent: Pick<Agent, "streamChat" | "describePromptContracts">;
|
|
472
488
|
suites: readonly EvalSuite[];
|
|
473
489
|
paramsSchema?: z.ZodType<Params>;
|
|
474
490
|
setup?: (input: EvalSetupContext<Params>) => Prepared | Promise<Prepared>;
|
|
@@ -504,8 +520,15 @@ type EvalProgress = {
|
|
|
504
520
|
repetition: number;
|
|
505
521
|
phase: EvalPhase;
|
|
506
522
|
stepIndex?: number;
|
|
523
|
+
} | {
|
|
524
|
+
type: "case-runtime-associated";
|
|
525
|
+
caseId: string;
|
|
526
|
+
repetition: number;
|
|
527
|
+
attemptId: string;
|
|
528
|
+
execution: EvalRuntimeExecution;
|
|
507
529
|
} | {
|
|
508
530
|
type: "case-started";
|
|
531
|
+
attemptId?: string;
|
|
509
532
|
caseId: string;
|
|
510
533
|
repetition: number;
|
|
511
534
|
sessionId: string;
|
|
@@ -519,11 +542,17 @@ type EvalProgress = {
|
|
|
519
542
|
result: EvalCaseResult;
|
|
520
543
|
};
|
|
521
544
|
type EvalRunOptions = {
|
|
545
|
+
promptSnapshot?: _kortyx_prompts.PromptSnapshot;
|
|
522
546
|
/** Studio captures execution without invoking the code judge. Local runs default to app. */
|
|
523
547
|
grading?: "app" | "studio";
|
|
524
548
|
/** Selected identity pinned by the Studio server when enqueuing a run. */
|
|
525
549
|
judgeIdentity?: EvalJudgeIdentity;
|
|
526
550
|
suiteId: string;
|
|
551
|
+
/** Execute one scheduled attempt, preserving its original repetition identity. */
|
|
552
|
+
attempt?: {
|
|
553
|
+
caseId: string;
|
|
554
|
+
repetition: number;
|
|
555
|
+
};
|
|
527
556
|
caseIds?: readonly string[];
|
|
528
557
|
repetitions?: number;
|
|
529
558
|
concurrency?: number;
|
|
@@ -640,6 +669,14 @@ declare const EvalVerdictSchema: z.ZodObject<{
|
|
|
640
669
|
evidence: z.ZodArray<z.ZodString>;
|
|
641
670
|
}, z.core.$strict>;
|
|
642
671
|
declare const EvalObservationSchema: z.ZodObject<{
|
|
672
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
673
|
+
id: z.ZodString;
|
|
674
|
+
version: z.ZodNumber;
|
|
675
|
+
hash: z.ZodString;
|
|
676
|
+
environment: z.ZodString;
|
|
677
|
+
snapshotRevision: z.ZodString;
|
|
678
|
+
model: z.ZodOptional<z.ZodString>;
|
|
679
|
+
}, z.core.$strict>>>;
|
|
643
680
|
type: z.ZodEnum<{
|
|
644
681
|
interrupt: "interrupt";
|
|
645
682
|
error: "error";
|
|
@@ -703,6 +740,14 @@ declare const EvalStepResultSchema: z.ZodObject<{
|
|
|
703
740
|
version: z.ZodLiteral<"compact-v1">;
|
|
704
741
|
history: z.ZodBoolean;
|
|
705
742
|
observation: z.ZodObject<{
|
|
743
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
744
|
+
id: z.ZodString;
|
|
745
|
+
version: z.ZodNumber;
|
|
746
|
+
hash: z.ZodString;
|
|
747
|
+
environment: z.ZodString;
|
|
748
|
+
snapshotRevision: z.ZodString;
|
|
749
|
+
model: z.ZodOptional<z.ZodString>;
|
|
750
|
+
}, z.core.$strict>>>;
|
|
706
751
|
type: z.ZodEnum<{
|
|
707
752
|
interrupt: "interrupt";
|
|
708
753
|
error: "error";
|
|
@@ -769,6 +814,14 @@ declare const EvalStepResultSchema: z.ZodObject<{
|
|
|
769
814
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
770
815
|
}, z.core.$strict>;
|
|
771
816
|
observation: z.ZodObject<{
|
|
817
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
818
|
+
id: z.ZodString;
|
|
819
|
+
version: z.ZodNumber;
|
|
820
|
+
hash: z.ZodString;
|
|
821
|
+
environment: z.ZodString;
|
|
822
|
+
snapshotRevision: z.ZodString;
|
|
823
|
+
model: z.ZodOptional<z.ZodString>;
|
|
824
|
+
}, z.core.$strict>>>;
|
|
772
825
|
type: z.ZodEnum<{
|
|
773
826
|
interrupt: "interrupt";
|
|
774
827
|
error: "error";
|
|
@@ -804,8 +857,8 @@ declare const EvalStepResultSchema: z.ZodObject<{
|
|
|
804
857
|
error: "error";
|
|
805
858
|
cancelled: "cancelled";
|
|
806
859
|
failed: "failed";
|
|
807
|
-
passed: "passed";
|
|
808
860
|
ungraded: "ungraded";
|
|
861
|
+
passed: "passed";
|
|
809
862
|
}>;
|
|
810
863
|
reason: z.ZodOptional<z.ZodString>;
|
|
811
864
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -847,13 +900,18 @@ declare const EvalStepResultSchema: z.ZodObject<{
|
|
|
847
900
|
declare const EvalCaseResultSchema: z.ZodObject<{
|
|
848
901
|
caseId: z.ZodString;
|
|
849
902
|
repetition: z.ZodNumber;
|
|
903
|
+
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
904
|
+
runtimeExecutions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
905
|
+
sessionId: z.ZodString;
|
|
906
|
+
runId: z.ZodString;
|
|
907
|
+
}, z.core.$strict>>>;
|
|
850
908
|
sessionId: z.ZodString;
|
|
851
909
|
status: z.ZodEnum<{
|
|
852
910
|
error: "error";
|
|
853
911
|
cancelled: "cancelled";
|
|
854
912
|
failed: "failed";
|
|
855
|
-
passed: "passed";
|
|
856
913
|
ungraded: "ungraded";
|
|
914
|
+
passed: "passed";
|
|
857
915
|
}>;
|
|
858
916
|
durationMs: z.ZodNumber;
|
|
859
917
|
steps: z.ZodArray<z.ZodObject<{
|
|
@@ -861,6 +919,14 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
861
919
|
version: z.ZodLiteral<"compact-v1">;
|
|
862
920
|
history: z.ZodBoolean;
|
|
863
921
|
observation: z.ZodObject<{
|
|
922
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
923
|
+
id: z.ZodString;
|
|
924
|
+
version: z.ZodNumber;
|
|
925
|
+
hash: z.ZodString;
|
|
926
|
+
environment: z.ZodString;
|
|
927
|
+
snapshotRevision: z.ZodString;
|
|
928
|
+
model: z.ZodOptional<z.ZodString>;
|
|
929
|
+
}, z.core.$strict>>>;
|
|
864
930
|
type: z.ZodEnum<{
|
|
865
931
|
interrupt: "interrupt";
|
|
866
932
|
error: "error";
|
|
@@ -927,6 +993,14 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
927
993
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
928
994
|
}, z.core.$strict>;
|
|
929
995
|
observation: z.ZodObject<{
|
|
996
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
997
|
+
id: z.ZodString;
|
|
998
|
+
version: z.ZodNumber;
|
|
999
|
+
hash: z.ZodString;
|
|
1000
|
+
environment: z.ZodString;
|
|
1001
|
+
snapshotRevision: z.ZodString;
|
|
1002
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1003
|
+
}, z.core.$strict>>>;
|
|
930
1004
|
type: z.ZodEnum<{
|
|
931
1005
|
interrupt: "interrupt";
|
|
932
1006
|
error: "error";
|
|
@@ -962,8 +1036,8 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
962
1036
|
error: "error";
|
|
963
1037
|
cancelled: "cancelled";
|
|
964
1038
|
failed: "failed";
|
|
965
|
-
passed: "passed";
|
|
966
1039
|
ungraded: "ungraded";
|
|
1040
|
+
passed: "passed";
|
|
967
1041
|
}>;
|
|
968
1042
|
reason: z.ZodOptional<z.ZodString>;
|
|
969
1043
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -1021,8 +1095,8 @@ declare const EvalJudgeIdentitySchema: z.ZodObject<{
|
|
|
1021
1095
|
id: z.ZodString;
|
|
1022
1096
|
version: z.ZodString;
|
|
1023
1097
|
location: z.ZodOptional<z.ZodEnum<{
|
|
1024
|
-
app: "app";
|
|
1025
1098
|
studio: "studio";
|
|
1099
|
+
app: "app";
|
|
1026
1100
|
}>>;
|
|
1027
1101
|
}, z.core.$strict>;
|
|
1028
1102
|
declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
@@ -1031,8 +1105,8 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
1031
1105
|
id: z.ZodString;
|
|
1032
1106
|
version: z.ZodString;
|
|
1033
1107
|
location: z.ZodOptional<z.ZodEnum<{
|
|
1034
|
-
app: "app";
|
|
1035
1108
|
studio: "studio";
|
|
1109
|
+
app: "app";
|
|
1036
1110
|
}>>;
|
|
1037
1111
|
}, z.core.$strict>;
|
|
1038
1112
|
criterion: z.ZodObject<{
|
|
@@ -1056,6 +1130,14 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
1056
1130
|
}, z.core.$strict>], "type">;
|
|
1057
1131
|
}, z.core.$strict>]>;
|
|
1058
1132
|
observation: z.ZodObject<{
|
|
1133
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1134
|
+
id: z.ZodString;
|
|
1135
|
+
version: z.ZodNumber;
|
|
1136
|
+
hash: z.ZodString;
|
|
1137
|
+
environment: z.ZodString;
|
|
1138
|
+
snapshotRevision: z.ZodString;
|
|
1139
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1140
|
+
}, z.core.$strict>>>;
|
|
1059
1141
|
type: z.ZodEnum<{
|
|
1060
1142
|
interrupt: "interrupt";
|
|
1061
1143
|
error: "error";
|
|
@@ -1092,6 +1174,14 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
1092
1174
|
version: z.ZodLiteral<"compact-v1">;
|
|
1093
1175
|
history: z.ZodBoolean;
|
|
1094
1176
|
observation: z.ZodObject<{
|
|
1177
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1178
|
+
id: z.ZodString;
|
|
1179
|
+
version: z.ZodNumber;
|
|
1180
|
+
hash: z.ZodString;
|
|
1181
|
+
environment: z.ZodString;
|
|
1182
|
+
snapshotRevision: z.ZodString;
|
|
1183
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1184
|
+
}, z.core.$strict>>>;
|
|
1095
1185
|
type: z.ZodEnum<{
|
|
1096
1186
|
interrupt: "interrupt";
|
|
1097
1187
|
error: "error";
|
|
@@ -1158,6 +1248,14 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
1158
1248
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1159
1249
|
}, z.core.$strict>;
|
|
1160
1250
|
observation: z.ZodObject<{
|
|
1251
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1252
|
+
id: z.ZodString;
|
|
1253
|
+
version: z.ZodNumber;
|
|
1254
|
+
hash: z.ZodString;
|
|
1255
|
+
environment: z.ZodString;
|
|
1256
|
+
snapshotRevision: z.ZodString;
|
|
1257
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1258
|
+
}, z.core.$strict>>>;
|
|
1161
1259
|
type: z.ZodEnum<{
|
|
1162
1260
|
interrupt: "interrupt";
|
|
1163
1261
|
error: "error";
|
|
@@ -1193,8 +1291,8 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
1193
1291
|
error: "error";
|
|
1194
1292
|
cancelled: "cancelled";
|
|
1195
1293
|
failed: "failed";
|
|
1196
|
-
passed: "passed";
|
|
1197
1294
|
ungraded: "ungraded";
|
|
1295
|
+
passed: "passed";
|
|
1198
1296
|
}>;
|
|
1199
1297
|
reason: z.ZodOptional<z.ZodString>;
|
|
1200
1298
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -1239,8 +1337,8 @@ declare const StudioEvalJudgeResponseSchema: z.ZodObject<{
|
|
|
1239
1337
|
id: z.ZodString;
|
|
1240
1338
|
version: z.ZodString;
|
|
1241
1339
|
location: z.ZodOptional<z.ZodEnum<{
|
|
1242
|
-
app: "app";
|
|
1243
1340
|
studio: "studio";
|
|
1341
|
+
app: "app";
|
|
1244
1342
|
}>>;
|
|
1245
1343
|
}, z.core.$strict>;
|
|
1246
1344
|
verdict: z.ZodObject<{
|
|
@@ -1373,8 +1471,8 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1373
1471
|
id: z.ZodString;
|
|
1374
1472
|
version: z.ZodString;
|
|
1375
1473
|
location: z.ZodOptional<z.ZodEnum<{
|
|
1376
|
-
app: "app";
|
|
1377
1474
|
studio: "studio";
|
|
1475
|
+
app: "app";
|
|
1378
1476
|
}>>;
|
|
1379
1477
|
}, z.core.$strict>>;
|
|
1380
1478
|
startedAt: z.ZodISODateTime;
|
|
@@ -1383,8 +1481,8 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1383
1481
|
error: "error";
|
|
1384
1482
|
cancelled: "cancelled";
|
|
1385
1483
|
failed: "failed";
|
|
1386
|
-
passed: "passed";
|
|
1387
1484
|
ungraded: "ungraded";
|
|
1485
|
+
passed: "passed";
|
|
1388
1486
|
}>;
|
|
1389
1487
|
counts: z.ZodObject<{
|
|
1390
1488
|
passed: z.ZodNumber;
|
|
@@ -1396,13 +1494,18 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1396
1494
|
cases: z.ZodArray<z.ZodObject<{
|
|
1397
1495
|
caseId: z.ZodString;
|
|
1398
1496
|
repetition: z.ZodNumber;
|
|
1497
|
+
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
1498
|
+
runtimeExecutions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1499
|
+
sessionId: z.ZodString;
|
|
1500
|
+
runId: z.ZodString;
|
|
1501
|
+
}, z.core.$strict>>>;
|
|
1399
1502
|
sessionId: z.ZodString;
|
|
1400
1503
|
status: z.ZodEnum<{
|
|
1401
1504
|
error: "error";
|
|
1402
1505
|
cancelled: "cancelled";
|
|
1403
1506
|
failed: "failed";
|
|
1404
|
-
passed: "passed";
|
|
1405
1507
|
ungraded: "ungraded";
|
|
1508
|
+
passed: "passed";
|
|
1406
1509
|
}>;
|
|
1407
1510
|
durationMs: z.ZodNumber;
|
|
1408
1511
|
steps: z.ZodArray<z.ZodObject<{
|
|
@@ -1410,6 +1513,14 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1410
1513
|
version: z.ZodLiteral<"compact-v1">;
|
|
1411
1514
|
history: z.ZodBoolean;
|
|
1412
1515
|
observation: z.ZodObject<{
|
|
1516
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1517
|
+
id: z.ZodString;
|
|
1518
|
+
version: z.ZodNumber;
|
|
1519
|
+
hash: z.ZodString;
|
|
1520
|
+
environment: z.ZodString;
|
|
1521
|
+
snapshotRevision: z.ZodString;
|
|
1522
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1523
|
+
}, z.core.$strict>>>;
|
|
1413
1524
|
type: z.ZodEnum<{
|
|
1414
1525
|
interrupt: "interrupt";
|
|
1415
1526
|
error: "error";
|
|
@@ -1476,6 +1587,14 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1476
1587
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1477
1588
|
}, z.core.$strict>;
|
|
1478
1589
|
observation: z.ZodObject<{
|
|
1590
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1591
|
+
id: z.ZodString;
|
|
1592
|
+
version: z.ZodNumber;
|
|
1593
|
+
hash: z.ZodString;
|
|
1594
|
+
environment: z.ZodString;
|
|
1595
|
+
snapshotRevision: z.ZodString;
|
|
1596
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1597
|
+
}, z.core.$strict>>>;
|
|
1479
1598
|
type: z.ZodEnum<{
|
|
1480
1599
|
interrupt: "interrupt";
|
|
1481
1600
|
error: "error";
|
|
@@ -1511,8 +1630,8 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1511
1630
|
error: "error";
|
|
1512
1631
|
cancelled: "cancelled";
|
|
1513
1632
|
failed: "failed";
|
|
1514
|
-
passed: "passed";
|
|
1515
1633
|
ungraded: "ungraded";
|
|
1634
|
+
passed: "passed";
|
|
1516
1635
|
}>;
|
|
1517
1636
|
reason: z.ZodOptional<z.ZodString>;
|
|
1518
1637
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -1584,6 +1703,16 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1584
1703
|
declare const EvalManifestSchema: z.ZodObject<{
|
|
1585
1704
|
schemaVersion: z.ZodLiteral<1>;
|
|
1586
1705
|
studioJudging: z.ZodOptional<z.ZodLiteral<true>>;
|
|
1706
|
+
promptContracts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1707
|
+
id: z.ZodString;
|
|
1708
|
+
format: z.ZodEnum<{
|
|
1709
|
+
"system-user": "system-user";
|
|
1710
|
+
chat: "chat";
|
|
1711
|
+
}>;
|
|
1712
|
+
variablesSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
1713
|
+
configSchema: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
1714
|
+
}, z.core.$strict>>>;
|
|
1715
|
+
attemptScheduling: z.ZodOptional<z.ZodLiteral<true>>;
|
|
1587
1716
|
suites: z.ZodArray<z.ZodObject<{
|
|
1588
1717
|
evidence: z.ZodOptional<z.ZodObject<{
|
|
1589
1718
|
history: z.ZodOptional<z.ZodBoolean>;
|
|
@@ -1684,12 +1813,21 @@ declare const EvalManifestSchema: z.ZodObject<{
|
|
|
1684
1813
|
id: z.ZodString;
|
|
1685
1814
|
version: z.ZodString;
|
|
1686
1815
|
location: z.ZodOptional<z.ZodEnum<{
|
|
1687
|
-
app: "app";
|
|
1688
1816
|
studio: "studio";
|
|
1817
|
+
app: "app";
|
|
1689
1818
|
}>>;
|
|
1690
1819
|
}, z.core.$strict>>;
|
|
1691
1820
|
}, z.core.$strict>;
|
|
1692
1821
|
declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
1822
|
+
type: z.ZodLiteral<"case-runtime-associated">;
|
|
1823
|
+
caseId: z.ZodString;
|
|
1824
|
+
repetition: z.ZodNumber;
|
|
1825
|
+
attemptId: z.ZodUUID;
|
|
1826
|
+
execution: z.ZodObject<{
|
|
1827
|
+
sessionId: z.ZodString;
|
|
1828
|
+
runId: z.ZodString;
|
|
1829
|
+
}, z.core.$strict>;
|
|
1830
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1693
1831
|
type: z.ZodLiteral<"run-started">;
|
|
1694
1832
|
caseIds: z.ZodArray<z.ZodString>;
|
|
1695
1833
|
repetitions: z.ZodNumber;
|
|
@@ -1711,6 +1849,7 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1711
1849
|
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
1712
1850
|
}, z.core.$strict>, z.ZodObject<{
|
|
1713
1851
|
type: z.ZodLiteral<"case-started">;
|
|
1852
|
+
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
1714
1853
|
caseId: z.ZodString;
|
|
1715
1854
|
repetition: z.ZodNumber;
|
|
1716
1855
|
sessionId: z.ZodString;
|
|
@@ -1723,6 +1862,14 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1723
1862
|
version: z.ZodLiteral<"compact-v1">;
|
|
1724
1863
|
history: z.ZodBoolean;
|
|
1725
1864
|
observation: z.ZodObject<{
|
|
1865
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1866
|
+
id: z.ZodString;
|
|
1867
|
+
version: z.ZodNumber;
|
|
1868
|
+
hash: z.ZodString;
|
|
1869
|
+
environment: z.ZodString;
|
|
1870
|
+
snapshotRevision: z.ZodString;
|
|
1871
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1872
|
+
}, z.core.$strict>>>;
|
|
1726
1873
|
type: z.ZodEnum<{
|
|
1727
1874
|
interrupt: "interrupt";
|
|
1728
1875
|
error: "error";
|
|
@@ -1789,6 +1936,14 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1789
1936
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1790
1937
|
}, z.core.$strict>;
|
|
1791
1938
|
observation: z.ZodObject<{
|
|
1939
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1940
|
+
id: z.ZodString;
|
|
1941
|
+
version: z.ZodNumber;
|
|
1942
|
+
hash: z.ZodString;
|
|
1943
|
+
environment: z.ZodString;
|
|
1944
|
+
snapshotRevision: z.ZodString;
|
|
1945
|
+
model: z.ZodOptional<z.ZodString>;
|
|
1946
|
+
}, z.core.$strict>>>;
|
|
1792
1947
|
type: z.ZodEnum<{
|
|
1793
1948
|
interrupt: "interrupt";
|
|
1794
1949
|
error: "error";
|
|
@@ -1824,8 +1979,8 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1824
1979
|
error: "error";
|
|
1825
1980
|
cancelled: "cancelled";
|
|
1826
1981
|
failed: "failed";
|
|
1827
|
-
passed: "passed";
|
|
1828
1982
|
ungraded: "ungraded";
|
|
1983
|
+
passed: "passed";
|
|
1829
1984
|
}>;
|
|
1830
1985
|
reason: z.ZodOptional<z.ZodString>;
|
|
1831
1986
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -1869,13 +2024,18 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1869
2024
|
result: z.ZodObject<{
|
|
1870
2025
|
caseId: z.ZodString;
|
|
1871
2026
|
repetition: z.ZodNumber;
|
|
2027
|
+
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
2028
|
+
runtimeExecutions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2029
|
+
sessionId: z.ZodString;
|
|
2030
|
+
runId: z.ZodString;
|
|
2031
|
+
}, z.core.$strict>>>;
|
|
1872
2032
|
sessionId: z.ZodString;
|
|
1873
2033
|
status: z.ZodEnum<{
|
|
1874
2034
|
error: "error";
|
|
1875
2035
|
cancelled: "cancelled";
|
|
1876
2036
|
failed: "failed";
|
|
1877
|
-
passed: "passed";
|
|
1878
2037
|
ungraded: "ungraded";
|
|
2038
|
+
passed: "passed";
|
|
1879
2039
|
}>;
|
|
1880
2040
|
durationMs: z.ZodNumber;
|
|
1881
2041
|
steps: z.ZodArray<z.ZodObject<{
|
|
@@ -1883,6 +2043,14 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1883
2043
|
version: z.ZodLiteral<"compact-v1">;
|
|
1884
2044
|
history: z.ZodBoolean;
|
|
1885
2045
|
observation: z.ZodObject<{
|
|
2046
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2047
|
+
id: z.ZodString;
|
|
2048
|
+
version: z.ZodNumber;
|
|
2049
|
+
hash: z.ZodString;
|
|
2050
|
+
environment: z.ZodString;
|
|
2051
|
+
snapshotRevision: z.ZodString;
|
|
2052
|
+
model: z.ZodOptional<z.ZodString>;
|
|
2053
|
+
}, z.core.$strict>>>;
|
|
1886
2054
|
type: z.ZodEnum<{
|
|
1887
2055
|
interrupt: "interrupt";
|
|
1888
2056
|
error: "error";
|
|
@@ -1949,6 +2117,14 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1949
2117
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1950
2118
|
}, z.core.$strict>;
|
|
1951
2119
|
observation: z.ZodObject<{
|
|
2120
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2121
|
+
id: z.ZodString;
|
|
2122
|
+
version: z.ZodNumber;
|
|
2123
|
+
hash: z.ZodString;
|
|
2124
|
+
environment: z.ZodString;
|
|
2125
|
+
snapshotRevision: z.ZodString;
|
|
2126
|
+
model: z.ZodOptional<z.ZodString>;
|
|
2127
|
+
}, z.core.$strict>>>;
|
|
1952
2128
|
type: z.ZodEnum<{
|
|
1953
2129
|
interrupt: "interrupt";
|
|
1954
2130
|
error: "error";
|
|
@@ -1984,8 +2160,8 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1984
2160
|
error: "error";
|
|
1985
2161
|
cancelled: "cancelled";
|
|
1986
2162
|
failed: "failed";
|
|
1987
|
-
passed: "passed";
|
|
1988
2163
|
ungraded: "ungraded";
|
|
2164
|
+
passed: "passed";
|
|
1989
2165
|
}>;
|
|
1990
2166
|
reason: z.ZodOptional<z.ZodString>;
|
|
1991
2167
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -2041,23 +2217,71 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2041
2217
|
}, z.core.$strict>;
|
|
2042
2218
|
}, z.core.$strict>], "type">;
|
|
2043
2219
|
declare const EvalRemoteRunRequestSchema: z.ZodObject<{
|
|
2220
|
+
promptSnapshot: z.ZodOptional<z.ZodObject<{
|
|
2221
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
2222
|
+
environment: z.ZodString;
|
|
2223
|
+
tag: z.ZodOptional<z.ZodString>;
|
|
2224
|
+
revision: z.ZodString;
|
|
2225
|
+
versions: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
2226
|
+
id: z.ZodString;
|
|
2227
|
+
version: z.ZodNumber;
|
|
2228
|
+
hash: z.ZodString;
|
|
2229
|
+
content: z.ZodObject<{
|
|
2230
|
+
format: z.ZodEnum<{
|
|
2231
|
+
"system-user": "system-user";
|
|
2232
|
+
chat: "chat";
|
|
2233
|
+
}>;
|
|
2234
|
+
messages: z.ZodArray<z.ZodObject<{
|
|
2235
|
+
role: z.ZodEnum<{
|
|
2236
|
+
system: "system";
|
|
2237
|
+
user: "user";
|
|
2238
|
+
assistant: "assistant";
|
|
2239
|
+
}>;
|
|
2240
|
+
content: z.ZodString;
|
|
2241
|
+
}, z.core.$strict>>;
|
|
2242
|
+
variablesSchema: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
|
|
2243
|
+
configSchema: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
|
|
2244
|
+
config: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
|
|
2245
|
+
dependencies: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
2246
|
+
id: z.ZodString;
|
|
2247
|
+
version: z.ZodNumber;
|
|
2248
|
+
hash: z.ZodString;
|
|
2249
|
+
}, z.core.$strict>>>;
|
|
2250
|
+
}, z.core.$strict>;
|
|
2251
|
+
}, z.core.$strip>>;
|
|
2252
|
+
source: z.ZodEnum<{
|
|
2253
|
+
studio: "studio";
|
|
2254
|
+
local: "local";
|
|
2255
|
+
cache: "cache";
|
|
2256
|
+
fallback: "fallback";
|
|
2257
|
+
eval: "eval";
|
|
2258
|
+
}>;
|
|
2259
|
+
resolvedAt: z.ZodISODateTime;
|
|
2260
|
+
requestedIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
2261
|
+
}, z.core.$strict>>;
|
|
2262
|
+
promptGroupName: z.ZodOptional<z.ZodString>;
|
|
2044
2263
|
suiteId: z.ZodString;
|
|
2045
2264
|
suiteRevision: z.ZodString;
|
|
2046
2265
|
grading: z.ZodOptional<z.ZodEnum<{
|
|
2047
|
-
app: "app";
|
|
2048
2266
|
studio: "studio";
|
|
2267
|
+
app: "app";
|
|
2049
2268
|
}>>;
|
|
2050
2269
|
judge: z.ZodOptional<z.ZodObject<{
|
|
2051
2270
|
id: z.ZodString;
|
|
2052
2271
|
version: z.ZodString;
|
|
2053
2272
|
location: z.ZodOptional<z.ZodEnum<{
|
|
2054
|
-
app: "app";
|
|
2055
2273
|
studio: "studio";
|
|
2274
|
+
app: "app";
|
|
2056
2275
|
}>>;
|
|
2057
2276
|
}, z.core.$strict>>;
|
|
2058
2277
|
caseIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
2059
2278
|
repetitions: z.ZodDefault<z.ZodNumber>;
|
|
2060
2279
|
concurrency: z.ZodDefault<z.ZodNumber>;
|
|
2280
|
+
attempt: z.ZodOptional<z.ZodObject<{
|
|
2281
|
+
caseId: z.ZodString;
|
|
2282
|
+
repetition: z.ZodNumber;
|
|
2283
|
+
evaluationId: z.ZodUUID;
|
|
2284
|
+
}, z.core.$strict>>;
|
|
2061
2285
|
}, z.core.$strict>;
|
|
2062
2286
|
declare const StudioEvalStartRequestSchema: z.ZodObject<{
|
|
2063
2287
|
suiteId: z.ZodString;
|
|
@@ -2065,15 +2289,34 @@ declare const StudioEvalStartRequestSchema: z.ZodObject<{
|
|
|
2065
2289
|
caseIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
2066
2290
|
repetitions: z.ZodDefault<z.ZodNumber>;
|
|
2067
2291
|
concurrency: z.ZodDefault<z.ZodNumber>;
|
|
2292
|
+
promptSelection: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
2293
|
+
type: z.ZodLiteral<"live">;
|
|
2294
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
2295
|
+
type: z.ZodLiteral<"single">;
|
|
2296
|
+
id: z.ZodString;
|
|
2297
|
+
version: z.ZodNumber;
|
|
2298
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
2299
|
+
type: z.ZodLiteral<"group">;
|
|
2300
|
+
groupId: z.ZodUUID;
|
|
2301
|
+
}, z.core.$strict>], "type">>;
|
|
2068
2302
|
targetId: z.ZodString;
|
|
2069
2303
|
judge: z.ZodDefault<z.ZodEnum<{
|
|
2070
|
-
app: "app";
|
|
2071
2304
|
studio: "studio";
|
|
2305
|
+
app: "app";
|
|
2072
2306
|
}>>;
|
|
2073
2307
|
}, z.core.$strict>;
|
|
2074
2308
|
declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
2075
2309
|
type: z.ZodLiteral<"progress">;
|
|
2076
2310
|
event: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
2311
|
+
type: z.ZodLiteral<"case-runtime-associated">;
|
|
2312
|
+
caseId: z.ZodString;
|
|
2313
|
+
repetition: z.ZodNumber;
|
|
2314
|
+
attemptId: z.ZodUUID;
|
|
2315
|
+
execution: z.ZodObject<{
|
|
2316
|
+
sessionId: z.ZodString;
|
|
2317
|
+
runId: z.ZodString;
|
|
2318
|
+
}, z.core.$strict>;
|
|
2319
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
2077
2320
|
type: z.ZodLiteral<"run-started">;
|
|
2078
2321
|
caseIds: z.ZodArray<z.ZodString>;
|
|
2079
2322
|
repetitions: z.ZodNumber;
|
|
@@ -2095,6 +2338,7 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2095
2338
|
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
2096
2339
|
}, z.core.$strict>, z.ZodObject<{
|
|
2097
2340
|
type: z.ZodLiteral<"case-started">;
|
|
2341
|
+
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
2098
2342
|
caseId: z.ZodString;
|
|
2099
2343
|
repetition: z.ZodNumber;
|
|
2100
2344
|
sessionId: z.ZodString;
|
|
@@ -2107,6 +2351,14 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2107
2351
|
version: z.ZodLiteral<"compact-v1">;
|
|
2108
2352
|
history: z.ZodBoolean;
|
|
2109
2353
|
observation: z.ZodObject<{
|
|
2354
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2355
|
+
id: z.ZodString;
|
|
2356
|
+
version: z.ZodNumber;
|
|
2357
|
+
hash: z.ZodString;
|
|
2358
|
+
environment: z.ZodString;
|
|
2359
|
+
snapshotRevision: z.ZodString;
|
|
2360
|
+
model: z.ZodOptional<z.ZodString>;
|
|
2361
|
+
}, z.core.$strict>>>;
|
|
2110
2362
|
type: z.ZodEnum<{
|
|
2111
2363
|
interrupt: "interrupt";
|
|
2112
2364
|
error: "error";
|
|
@@ -2173,6 +2425,14 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2173
2425
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
2174
2426
|
}, z.core.$strict>;
|
|
2175
2427
|
observation: z.ZodObject<{
|
|
2428
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2429
|
+
id: z.ZodString;
|
|
2430
|
+
version: z.ZodNumber;
|
|
2431
|
+
hash: z.ZodString;
|
|
2432
|
+
environment: z.ZodString;
|
|
2433
|
+
snapshotRevision: z.ZodString;
|
|
2434
|
+
model: z.ZodOptional<z.ZodString>;
|
|
2435
|
+
}, z.core.$strict>>>;
|
|
2176
2436
|
type: z.ZodEnum<{
|
|
2177
2437
|
interrupt: "interrupt";
|
|
2178
2438
|
error: "error";
|
|
@@ -2208,8 +2468,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2208
2468
|
error: "error";
|
|
2209
2469
|
cancelled: "cancelled";
|
|
2210
2470
|
failed: "failed";
|
|
2211
|
-
passed: "passed";
|
|
2212
2471
|
ungraded: "ungraded";
|
|
2472
|
+
passed: "passed";
|
|
2213
2473
|
}>;
|
|
2214
2474
|
reason: z.ZodOptional<z.ZodString>;
|
|
2215
2475
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -2253,13 +2513,18 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2253
2513
|
result: z.ZodObject<{
|
|
2254
2514
|
caseId: z.ZodString;
|
|
2255
2515
|
repetition: z.ZodNumber;
|
|
2516
|
+
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
2517
|
+
runtimeExecutions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2518
|
+
sessionId: z.ZodString;
|
|
2519
|
+
runId: z.ZodString;
|
|
2520
|
+
}, z.core.$strict>>>;
|
|
2256
2521
|
sessionId: z.ZodString;
|
|
2257
2522
|
status: z.ZodEnum<{
|
|
2258
2523
|
error: "error";
|
|
2259
2524
|
cancelled: "cancelled";
|
|
2260
2525
|
failed: "failed";
|
|
2261
|
-
passed: "passed";
|
|
2262
2526
|
ungraded: "ungraded";
|
|
2527
|
+
passed: "passed";
|
|
2263
2528
|
}>;
|
|
2264
2529
|
durationMs: z.ZodNumber;
|
|
2265
2530
|
steps: z.ZodArray<z.ZodObject<{
|
|
@@ -2267,6 +2532,14 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2267
2532
|
version: z.ZodLiteral<"compact-v1">;
|
|
2268
2533
|
history: z.ZodBoolean;
|
|
2269
2534
|
observation: z.ZodObject<{
|
|
2535
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2536
|
+
id: z.ZodString;
|
|
2537
|
+
version: z.ZodNumber;
|
|
2538
|
+
hash: z.ZodString;
|
|
2539
|
+
environment: z.ZodString;
|
|
2540
|
+
snapshotRevision: z.ZodString;
|
|
2541
|
+
model: z.ZodOptional<z.ZodString>;
|
|
2542
|
+
}, z.core.$strict>>>;
|
|
2270
2543
|
type: z.ZodEnum<{
|
|
2271
2544
|
interrupt: "interrupt";
|
|
2272
2545
|
error: "error";
|
|
@@ -2333,6 +2606,14 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2333
2606
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
2334
2607
|
}, z.core.$strict>;
|
|
2335
2608
|
observation: z.ZodObject<{
|
|
2609
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2610
|
+
id: z.ZodString;
|
|
2611
|
+
version: z.ZodNumber;
|
|
2612
|
+
hash: z.ZodString;
|
|
2613
|
+
environment: z.ZodString;
|
|
2614
|
+
snapshotRevision: z.ZodString;
|
|
2615
|
+
model: z.ZodOptional<z.ZodString>;
|
|
2616
|
+
}, z.core.$strict>>>;
|
|
2336
2617
|
type: z.ZodEnum<{
|
|
2337
2618
|
interrupt: "interrupt";
|
|
2338
2619
|
error: "error";
|
|
@@ -2368,8 +2649,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2368
2649
|
error: "error";
|
|
2369
2650
|
cancelled: "cancelled";
|
|
2370
2651
|
failed: "failed";
|
|
2371
|
-
passed: "passed";
|
|
2372
2652
|
ungraded: "ungraded";
|
|
2653
|
+
passed: "passed";
|
|
2373
2654
|
}>;
|
|
2374
2655
|
reason: z.ZodOptional<z.ZodString>;
|
|
2375
2656
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -2523,8 +2804,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2523
2804
|
id: z.ZodString;
|
|
2524
2805
|
version: z.ZodString;
|
|
2525
2806
|
location: z.ZodOptional<z.ZodEnum<{
|
|
2526
|
-
app: "app";
|
|
2527
2807
|
studio: "studio";
|
|
2808
|
+
app: "app";
|
|
2528
2809
|
}>>;
|
|
2529
2810
|
}, z.core.$strict>>;
|
|
2530
2811
|
startedAt: z.ZodISODateTime;
|
|
@@ -2533,8 +2814,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2533
2814
|
error: "error";
|
|
2534
2815
|
cancelled: "cancelled";
|
|
2535
2816
|
failed: "failed";
|
|
2536
|
-
passed: "passed";
|
|
2537
2817
|
ungraded: "ungraded";
|
|
2818
|
+
passed: "passed";
|
|
2538
2819
|
}>;
|
|
2539
2820
|
counts: z.ZodObject<{
|
|
2540
2821
|
passed: z.ZodNumber;
|
|
@@ -2546,13 +2827,18 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2546
2827
|
cases: z.ZodArray<z.ZodObject<{
|
|
2547
2828
|
caseId: z.ZodString;
|
|
2548
2829
|
repetition: z.ZodNumber;
|
|
2830
|
+
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
2831
|
+
runtimeExecutions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2832
|
+
sessionId: z.ZodString;
|
|
2833
|
+
runId: z.ZodString;
|
|
2834
|
+
}, z.core.$strict>>>;
|
|
2549
2835
|
sessionId: z.ZodString;
|
|
2550
2836
|
status: z.ZodEnum<{
|
|
2551
2837
|
error: "error";
|
|
2552
2838
|
cancelled: "cancelled";
|
|
2553
2839
|
failed: "failed";
|
|
2554
|
-
passed: "passed";
|
|
2555
2840
|
ungraded: "ungraded";
|
|
2841
|
+
passed: "passed";
|
|
2556
2842
|
}>;
|
|
2557
2843
|
durationMs: z.ZodNumber;
|
|
2558
2844
|
steps: z.ZodArray<z.ZodObject<{
|
|
@@ -2560,6 +2846,14 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2560
2846
|
version: z.ZodLiteral<"compact-v1">;
|
|
2561
2847
|
history: z.ZodBoolean;
|
|
2562
2848
|
observation: z.ZodObject<{
|
|
2849
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2850
|
+
id: z.ZodString;
|
|
2851
|
+
version: z.ZodNumber;
|
|
2852
|
+
hash: z.ZodString;
|
|
2853
|
+
environment: z.ZodString;
|
|
2854
|
+
snapshotRevision: z.ZodString;
|
|
2855
|
+
model: z.ZodOptional<z.ZodString>;
|
|
2856
|
+
}, z.core.$strict>>>;
|
|
2563
2857
|
type: z.ZodEnum<{
|
|
2564
2858
|
interrupt: "interrupt";
|
|
2565
2859
|
error: "error";
|
|
@@ -2626,6 +2920,14 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2626
2920
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
2627
2921
|
}, z.core.$strict>;
|
|
2628
2922
|
observation: z.ZodObject<{
|
|
2923
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2924
|
+
id: z.ZodString;
|
|
2925
|
+
version: z.ZodNumber;
|
|
2926
|
+
hash: z.ZodString;
|
|
2927
|
+
environment: z.ZodString;
|
|
2928
|
+
snapshotRevision: z.ZodString;
|
|
2929
|
+
model: z.ZodOptional<z.ZodString>;
|
|
2930
|
+
}, z.core.$strict>>>;
|
|
2629
2931
|
type: z.ZodEnum<{
|
|
2630
2932
|
interrupt: "interrupt";
|
|
2631
2933
|
error: "error";
|
|
@@ -2661,8 +2963,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2661
2963
|
error: "error";
|
|
2662
2964
|
cancelled: "cancelled";
|
|
2663
2965
|
failed: "failed";
|
|
2664
|
-
passed: "passed";
|
|
2665
2966
|
ungraded: "ungraded";
|
|
2967
|
+
passed: "passed";
|
|
2666
2968
|
}>;
|
|
2667
2969
|
reason: z.ZodOptional<z.ZodString>;
|
|
2668
2970
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -2766,6 +3068,16 @@ declare const EvaluationMetadataSchema: z.ZodObject<{
|
|
|
2766
3068
|
declare const StudioEvaluationStartRequestSchema: z.ZodObject<{
|
|
2767
3069
|
targetId: z.ZodString;
|
|
2768
3070
|
name: z.ZodOptional<z.ZodString>;
|
|
3071
|
+
promptSelection: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
3072
|
+
type: z.ZodLiteral<"live">;
|
|
3073
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
3074
|
+
type: z.ZodLiteral<"single">;
|
|
3075
|
+
id: z.ZodString;
|
|
3076
|
+
version: z.ZodNumber;
|
|
3077
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
3078
|
+
type: z.ZodLiteral<"group">;
|
|
3079
|
+
groupId: z.ZodUUID;
|
|
3080
|
+
}, z.core.$strict>], "type">>;
|
|
2769
3081
|
selection: z.ZodEnum<{
|
|
2770
3082
|
selected: "selected";
|
|
2771
3083
|
all: "all";
|
|
@@ -2776,8 +3088,8 @@ declare const StudioEvaluationStartRequestSchema: z.ZodObject<{
|
|
|
2776
3088
|
caseIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
2777
3089
|
}, z.core.$strict>>;
|
|
2778
3090
|
judge: z.ZodDefault<z.ZodEnum<{
|
|
2779
|
-
app: "app";
|
|
2780
3091
|
studio: "studio";
|
|
3092
|
+
app: "app";
|
|
2781
3093
|
}>>;
|
|
2782
3094
|
repetitions: z.ZodDefault<z.ZodNumber>;
|
|
2783
3095
|
concurrency: z.ZodDefault<z.ZodNumber>;
|
|
@@ -2818,8 +3130,8 @@ declare const StudioEvaluationSummarySchema: z.ZodObject<{
|
|
|
2818
3130
|
id: z.ZodString;
|
|
2819
3131
|
version: z.ZodString;
|
|
2820
3132
|
location: z.ZodOptional<z.ZodEnum<{
|
|
2821
|
-
app: "app";
|
|
2822
3133
|
studio: "studio";
|
|
3134
|
+
app: "app";
|
|
2823
3135
|
}>>;
|
|
2824
3136
|
}, z.core.$strict>>;
|
|
2825
3137
|
status: z.ZodEnum<{
|
|
@@ -2909,8 +3221,8 @@ declare const StudioEvaluationHistorySchema: z.ZodObject<{
|
|
|
2909
3221
|
id: z.ZodString;
|
|
2910
3222
|
version: z.ZodString;
|
|
2911
3223
|
location: z.ZodOptional<z.ZodEnum<{
|
|
2912
|
-
app: "app";
|
|
2913
3224
|
studio: "studio";
|
|
3225
|
+
app: "app";
|
|
2914
3226
|
}>>;
|
|
2915
3227
|
}, z.core.$strict>>;
|
|
2916
3228
|
status: z.ZodEnum<{
|
|
@@ -3001,8 +3313,8 @@ declare const StudioEvaluationDetailSchema: z.ZodObject<{
|
|
|
3001
3313
|
id: z.ZodString;
|
|
3002
3314
|
version: z.ZodString;
|
|
3003
3315
|
location: z.ZodOptional<z.ZodEnum<{
|
|
3004
|
-
app: "app";
|
|
3005
3316
|
studio: "studio";
|
|
3317
|
+
app: "app";
|
|
3006
3318
|
}>>;
|
|
3007
3319
|
}, z.core.$strict>>;
|
|
3008
3320
|
status: z.ZodEnum<{
|
|
@@ -3163,8 +3475,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3163
3475
|
id: z.ZodString;
|
|
3164
3476
|
version: z.ZodString;
|
|
3165
3477
|
location: z.ZodOptional<z.ZodEnum<{
|
|
3166
|
-
app: "app";
|
|
3167
3478
|
studio: "studio";
|
|
3479
|
+
app: "app";
|
|
3168
3480
|
}>>;
|
|
3169
3481
|
}, z.core.$strict>>;
|
|
3170
3482
|
status: z.ZodEnum<{
|
|
@@ -3338,19 +3650,62 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3338
3650
|
request: z.ZodOptional<z.ZodObject<{
|
|
3339
3651
|
caseIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
3340
3652
|
grading: z.ZodOptional<z.ZodEnum<{
|
|
3341
|
-
app: "app";
|
|
3342
3653
|
studio: "studio";
|
|
3654
|
+
app: "app";
|
|
3343
3655
|
}>>;
|
|
3344
3656
|
judge: z.ZodOptional<z.ZodObject<{
|
|
3345
3657
|
id: z.ZodString;
|
|
3346
3658
|
version: z.ZodString;
|
|
3347
3659
|
location: z.ZodOptional<z.ZodEnum<{
|
|
3348
|
-
app: "app";
|
|
3349
3660
|
studio: "studio";
|
|
3661
|
+
app: "app";
|
|
3350
3662
|
}>>;
|
|
3351
3663
|
}, z.core.$strict>>;
|
|
3352
3664
|
repetitions: z.ZodNumber;
|
|
3353
3665
|
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
3666
|
+
promptSnapshot: z.ZodOptional<z.ZodObject<{
|
|
3667
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
3668
|
+
environment: z.ZodString;
|
|
3669
|
+
tag: z.ZodOptional<z.ZodString>;
|
|
3670
|
+
revision: z.ZodString;
|
|
3671
|
+
versions: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
3672
|
+
id: z.ZodString;
|
|
3673
|
+
version: z.ZodNumber;
|
|
3674
|
+
hash: z.ZodString;
|
|
3675
|
+
content: z.ZodObject<{
|
|
3676
|
+
format: z.ZodEnum<{
|
|
3677
|
+
"system-user": "system-user";
|
|
3678
|
+
chat: "chat";
|
|
3679
|
+
}>;
|
|
3680
|
+
messages: z.ZodArray<z.ZodObject<{
|
|
3681
|
+
role: z.ZodEnum<{
|
|
3682
|
+
system: "system";
|
|
3683
|
+
user: "user";
|
|
3684
|
+
assistant: "assistant";
|
|
3685
|
+
}>;
|
|
3686
|
+
content: z.ZodString;
|
|
3687
|
+
}, z.core.$strict>>;
|
|
3688
|
+
variablesSchema: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
|
|
3689
|
+
configSchema: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
|
|
3690
|
+
config: z.ZodRecord<z.ZodString, z.ZodJSONSchema>;
|
|
3691
|
+
dependencies: z.ZodDefault<z.ZodArray<z.ZodObject<{
|
|
3692
|
+
id: z.ZodString;
|
|
3693
|
+
version: z.ZodNumber;
|
|
3694
|
+
hash: z.ZodString;
|
|
3695
|
+
}, z.core.$strict>>>;
|
|
3696
|
+
}, z.core.$strict>;
|
|
3697
|
+
}, z.core.$strip>>;
|
|
3698
|
+
source: z.ZodEnum<{
|
|
3699
|
+
studio: "studio";
|
|
3700
|
+
local: "local";
|
|
3701
|
+
cache: "cache";
|
|
3702
|
+
fallback: "fallback";
|
|
3703
|
+
eval: "eval";
|
|
3704
|
+
}>;
|
|
3705
|
+
resolvedAt: z.ZodISODateTime;
|
|
3706
|
+
requestedIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
3707
|
+
}, z.core.$strict>>;
|
|
3708
|
+
promptGroupName: z.ZodOptional<z.ZodString>;
|
|
3354
3709
|
}, z.core.$strip>>;
|
|
3355
3710
|
suite: z.ZodObject<{
|
|
3356
3711
|
evidence: z.ZodOptional<z.ZodObject<{
|
|
@@ -3538,8 +3893,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3538
3893
|
id: z.ZodString;
|
|
3539
3894
|
version: z.ZodString;
|
|
3540
3895
|
location: z.ZodOptional<z.ZodEnum<{
|
|
3541
|
-
app: "app";
|
|
3542
3896
|
studio: "studio";
|
|
3897
|
+
app: "app";
|
|
3543
3898
|
}>>;
|
|
3544
3899
|
}, z.core.$strict>>;
|
|
3545
3900
|
startedAt: z.ZodISODateTime;
|
|
@@ -3548,8 +3903,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3548
3903
|
error: "error";
|
|
3549
3904
|
cancelled: "cancelled";
|
|
3550
3905
|
failed: "failed";
|
|
3551
|
-
passed: "passed";
|
|
3552
3906
|
ungraded: "ungraded";
|
|
3907
|
+
passed: "passed";
|
|
3553
3908
|
}>;
|
|
3554
3909
|
counts: z.ZodObject<{
|
|
3555
3910
|
passed: z.ZodNumber;
|
|
@@ -3561,13 +3916,18 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3561
3916
|
cases: z.ZodArray<z.ZodObject<{
|
|
3562
3917
|
caseId: z.ZodString;
|
|
3563
3918
|
repetition: z.ZodNumber;
|
|
3919
|
+
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
3920
|
+
runtimeExecutions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
3921
|
+
sessionId: z.ZodString;
|
|
3922
|
+
runId: z.ZodString;
|
|
3923
|
+
}, z.core.$strict>>>;
|
|
3564
3924
|
sessionId: z.ZodString;
|
|
3565
3925
|
status: z.ZodEnum<{
|
|
3566
3926
|
error: "error";
|
|
3567
3927
|
cancelled: "cancelled";
|
|
3568
3928
|
failed: "failed";
|
|
3569
|
-
passed: "passed";
|
|
3570
3929
|
ungraded: "ungraded";
|
|
3930
|
+
passed: "passed";
|
|
3571
3931
|
}>;
|
|
3572
3932
|
durationMs: z.ZodNumber;
|
|
3573
3933
|
steps: z.ZodArray<z.ZodObject<{
|
|
@@ -3575,6 +3935,14 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3575
3935
|
version: z.ZodLiteral<"compact-v1">;
|
|
3576
3936
|
history: z.ZodBoolean;
|
|
3577
3937
|
observation: z.ZodObject<{
|
|
3938
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
3939
|
+
id: z.ZodString;
|
|
3940
|
+
version: z.ZodNumber;
|
|
3941
|
+
hash: z.ZodString;
|
|
3942
|
+
environment: z.ZodString;
|
|
3943
|
+
snapshotRevision: z.ZodString;
|
|
3944
|
+
model: z.ZodOptional<z.ZodString>;
|
|
3945
|
+
}, z.core.$strict>>>;
|
|
3578
3946
|
type: z.ZodEnum<{
|
|
3579
3947
|
interrupt: "interrupt";
|
|
3580
3948
|
error: "error";
|
|
@@ -3641,6 +4009,14 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3641
4009
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
3642
4010
|
}, z.core.$strict>;
|
|
3643
4011
|
observation: z.ZodObject<{
|
|
4012
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4013
|
+
id: z.ZodString;
|
|
4014
|
+
version: z.ZodNumber;
|
|
4015
|
+
hash: z.ZodString;
|
|
4016
|
+
environment: z.ZodString;
|
|
4017
|
+
snapshotRevision: z.ZodString;
|
|
4018
|
+
model: z.ZodOptional<z.ZodString>;
|
|
4019
|
+
}, z.core.$strict>>>;
|
|
3644
4020
|
type: z.ZodEnum<{
|
|
3645
4021
|
interrupt: "interrupt";
|
|
3646
4022
|
error: "error";
|
|
@@ -3676,8 +4052,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3676
4052
|
error: "error";
|
|
3677
4053
|
cancelled: "cancelled";
|
|
3678
4054
|
failed: "failed";
|
|
3679
|
-
passed: "passed";
|
|
3680
4055
|
ungraded: "ungraded";
|
|
4056
|
+
passed: "passed";
|
|
3681
4057
|
}>;
|
|
3682
4058
|
reason: z.ZodOptional<z.ZodString>;
|
|
3683
4059
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -3749,6 +4125,15 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3749
4125
|
events: z.ZodArray<z.ZodObject<{
|
|
3750
4126
|
id: z.ZodNumber;
|
|
3751
4127
|
event: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
4128
|
+
type: z.ZodLiteral<"case-runtime-associated">;
|
|
4129
|
+
caseId: z.ZodString;
|
|
4130
|
+
repetition: z.ZodNumber;
|
|
4131
|
+
attemptId: z.ZodUUID;
|
|
4132
|
+
execution: z.ZodObject<{
|
|
4133
|
+
sessionId: z.ZodString;
|
|
4134
|
+
runId: z.ZodString;
|
|
4135
|
+
}, z.core.$strict>;
|
|
4136
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
3752
4137
|
type: z.ZodLiteral<"run-started">;
|
|
3753
4138
|
caseIds: z.ZodArray<z.ZodString>;
|
|
3754
4139
|
repetitions: z.ZodNumber;
|
|
@@ -3770,6 +4155,7 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3770
4155
|
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
3771
4156
|
}, z.core.$strict>, z.ZodObject<{
|
|
3772
4157
|
type: z.ZodLiteral<"case-started">;
|
|
4158
|
+
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
3773
4159
|
caseId: z.ZodString;
|
|
3774
4160
|
repetition: z.ZodNumber;
|
|
3775
4161
|
sessionId: z.ZodString;
|
|
@@ -3782,6 +4168,14 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3782
4168
|
version: z.ZodLiteral<"compact-v1">;
|
|
3783
4169
|
history: z.ZodBoolean;
|
|
3784
4170
|
observation: z.ZodObject<{
|
|
4171
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4172
|
+
id: z.ZodString;
|
|
4173
|
+
version: z.ZodNumber;
|
|
4174
|
+
hash: z.ZodString;
|
|
4175
|
+
environment: z.ZodString;
|
|
4176
|
+
snapshotRevision: z.ZodString;
|
|
4177
|
+
model: z.ZodOptional<z.ZodString>;
|
|
4178
|
+
}, z.core.$strict>>>;
|
|
3785
4179
|
type: z.ZodEnum<{
|
|
3786
4180
|
interrupt: "interrupt";
|
|
3787
4181
|
error: "error";
|
|
@@ -3848,6 +4242,14 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3848
4242
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
3849
4243
|
}, z.core.$strict>;
|
|
3850
4244
|
observation: z.ZodObject<{
|
|
4245
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4246
|
+
id: z.ZodString;
|
|
4247
|
+
version: z.ZodNumber;
|
|
4248
|
+
hash: z.ZodString;
|
|
4249
|
+
environment: z.ZodString;
|
|
4250
|
+
snapshotRevision: z.ZodString;
|
|
4251
|
+
model: z.ZodOptional<z.ZodString>;
|
|
4252
|
+
}, z.core.$strict>>>;
|
|
3851
4253
|
type: z.ZodEnum<{
|
|
3852
4254
|
interrupt: "interrupt";
|
|
3853
4255
|
error: "error";
|
|
@@ -3883,8 +4285,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3883
4285
|
error: "error";
|
|
3884
4286
|
cancelled: "cancelled";
|
|
3885
4287
|
failed: "failed";
|
|
3886
|
-
passed: "passed";
|
|
3887
4288
|
ungraded: "ungraded";
|
|
4289
|
+
passed: "passed";
|
|
3888
4290
|
}>;
|
|
3889
4291
|
reason: z.ZodOptional<z.ZodString>;
|
|
3890
4292
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -3928,13 +4330,18 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3928
4330
|
result: z.ZodObject<{
|
|
3929
4331
|
caseId: z.ZodString;
|
|
3930
4332
|
repetition: z.ZodNumber;
|
|
4333
|
+
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
4334
|
+
runtimeExecutions: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4335
|
+
sessionId: z.ZodString;
|
|
4336
|
+
runId: z.ZodString;
|
|
4337
|
+
}, z.core.$strict>>>;
|
|
3931
4338
|
sessionId: z.ZodString;
|
|
3932
4339
|
status: z.ZodEnum<{
|
|
3933
4340
|
error: "error";
|
|
3934
4341
|
cancelled: "cancelled";
|
|
3935
4342
|
failed: "failed";
|
|
3936
|
-
passed: "passed";
|
|
3937
4343
|
ungraded: "ungraded";
|
|
4344
|
+
passed: "passed";
|
|
3938
4345
|
}>;
|
|
3939
4346
|
durationMs: z.ZodNumber;
|
|
3940
4347
|
steps: z.ZodArray<z.ZodObject<{
|
|
@@ -3942,6 +4349,14 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3942
4349
|
version: z.ZodLiteral<"compact-v1">;
|
|
3943
4350
|
history: z.ZodBoolean;
|
|
3944
4351
|
observation: z.ZodObject<{
|
|
4352
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4353
|
+
id: z.ZodString;
|
|
4354
|
+
version: z.ZodNumber;
|
|
4355
|
+
hash: z.ZodString;
|
|
4356
|
+
environment: z.ZodString;
|
|
4357
|
+
snapshotRevision: z.ZodString;
|
|
4358
|
+
model: z.ZodOptional<z.ZodString>;
|
|
4359
|
+
}, z.core.$strict>>>;
|
|
3945
4360
|
type: z.ZodEnum<{
|
|
3946
4361
|
interrupt: "interrupt";
|
|
3947
4362
|
error: "error";
|
|
@@ -4008,6 +4423,14 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4008
4423
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
4009
4424
|
}, z.core.$strict>;
|
|
4010
4425
|
observation: z.ZodObject<{
|
|
4426
|
+
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4427
|
+
id: z.ZodString;
|
|
4428
|
+
version: z.ZodNumber;
|
|
4429
|
+
hash: z.ZodString;
|
|
4430
|
+
environment: z.ZodString;
|
|
4431
|
+
snapshotRevision: z.ZodString;
|
|
4432
|
+
model: z.ZodOptional<z.ZodString>;
|
|
4433
|
+
}, z.core.$strict>>>;
|
|
4011
4434
|
type: z.ZodEnum<{
|
|
4012
4435
|
interrupt: "interrupt";
|
|
4013
4436
|
error: "error";
|
|
@@ -4043,8 +4466,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4043
4466
|
error: "error";
|
|
4044
4467
|
cancelled: "cancelled";
|
|
4045
4468
|
failed: "failed";
|
|
4046
|
-
passed: "passed";
|
|
4047
4469
|
ungraded: "ungraded";
|
|
4470
|
+
passed: "passed";
|
|
4048
4471
|
}>;
|
|
4049
4472
|
reason: z.ZodOptional<z.ZodString>;
|
|
4050
4473
|
judgeCalls: z.ZodOptional<z.ZodNumber>;
|
|
@@ -4115,4 +4538,4 @@ declare function getEvalGradeEvidence(step: EvalStepResult, previous: readonly E
|
|
|
4115
4538
|
/** Discover and pin the Studio backend's judge; grading never runs in the browser. */
|
|
4116
4539
|
declare function createStudioEvalJudge(options: StudioEvalJudgeOptions): Promise<EvalJudge>;
|
|
4117
4540
|
|
|
4118
|
-
export {
|
|
4541
|
+
export { EvalSuiteSchema as $, type EvalJudgeUsage as A, EvalJudgeUsageSchema as B, type CreateEvalsOptions as C, type EvalManifest as D, type EvalCase as E, EvalManifestSchema as F, type EvalObservation as G, EvalObservationSchema as H, type EvalOutputExpectation as I, type EvalOutputSelector as J, type EvalPhase as K, type EvalProgress as L, EvalProgressSchema as M, type EvalRemoteRunRequest as N, EvalRemoteRunRequestSchema as O, type EvalResponder as P, EvalResumeResponseSchema as Q, type EvalRunOptions as R, type EvalRunResult as S, EvalRunResultSchema as T, type EvalRuntimeExecution as U, type EvalSetupContext as V, type EvalStatus as W, type EvalStep as X, type EvalStepResult as Y, EvalStepResultSchema as Z, type EvalSuite as _, type EvalCaseResult as a, type EvalVerdict as a0, EvalVerdictSchema as a1, type EvalWireEvent as a2, EvalWireEventSchema as a3, EvaluationMetadataSchema as a4, type StudioEvalJudgeOptions as a5, StudioEvalJudgeRequestSchema as a6, StudioEvalJudgeResponseSchema as a7, StudioEvalStartRequestSchema as a8, type StudioEvaluationDetail as a9, type ListInterruptsOptions as aA, type PendingInterrupt as aB, type ResumableInterrupt as aC, type ResumeHandle as aD, type ResumeOptions as aE, type ResumeResponse as aF, createAgent as aG, projectWorkflowTopology as aH, StudioEvaluationDetailSchema as aa, StudioEvaluationHistorySchema as ab, StudioEvaluationResultsSchema as ac, type StudioEvaluationStartRequest as ad, StudioEvaluationStartRequestSchema as ae, type StudioEvaluationSummary as af, StudioEvaluationSummarySchema as ag, createStudioEvalJudge as ah, defineSuite as ai, getEvalGradeEvidence as aj, parseEvalSuite as ak, type ChatLifecyclePhase as al, ChatLifecycleHookError as am, type Agent as an, type ChatResponseFinalized as ao, type AgentProcessOptions as ap, type AgentProjectTopologyOptions as aq, type ChatResponseStatus as ar, type CreateAgentArgs as as, type ExecutableWorkflow as at, type ExecuteOptions as au, type ExecutionInfo as av, type ExecutionInterrupt as aw, ExecutionRequestError as ax, type ExecutionResult as ay, type InterruptScope as az, EvalCaseResultSchema as b, type EvalCommand as c, EvalConfigurationError as d, type EvalContext as e, type EvalCriterion as f, type EvalCriterionResult as g, type EvalDefaults as h, type EvalEvidenceEvent as i, type EvalEvidenceEventType as j, type EvalEvidenceFilters as k, type EvalEvidencePolicy as l, EvalEvidencePolicySchema as m, type EvalExecution as n, type EvalExecutionStatus as o, type EvalExpectation as p, type EvalGradeInput as q, type EvalHandlerRef as r, type EvalInterrupt as s, type EvalIssue as t, type EvalJson as u, type EvalJudge as v, type EvalJudgeEvidence as w, type EvalJudgeIdentity as x, EvalJudgeIdentitySchema as y, type EvalJudgeOptions as z };
|