@kortyx/agent 0.37.1 → 0.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/README.md +16 -0
- package/dist/evals.d.ts +70 -3
- package/dist/evals.js +188 -110
- package/dist/evals.js.map +1 -1
- package/dist/index.d.ts +11 -5
- package/dist/index.js +1219 -647
- package/dist/index.js.map +1 -1
- package/dist/{studio-judge-Dsz6KDY2.d.ts → studio-judge-DCAw3v60.d.ts} +310 -6
- package/package.json +10 -10
|
@@ -1,10 +1,12 @@
|
|
|
1
|
+
import * as _kortyx_core from '@kortyx/core';
|
|
2
|
+
import { WorkflowDefinition, ExecutionLimits, TokenUsage, ExecutionLimitReached } from '@kortyx/core';
|
|
1
3
|
import { z } from 'zod';
|
|
2
4
|
import * as _kortyx_prompts from '@kortyx/prompts';
|
|
5
|
+
import * as _kortyx_core_errors from '@kortyx/core/errors';
|
|
6
|
+
import { KortyxError, FailureDescriptor, SerializedFailureDescriptor } from '@kortyx/core/errors';
|
|
3
7
|
import { GetProviderFn, ProviderModelRef } from '@kortyx/providers';
|
|
4
|
-
import { KortyxError, FailureDescriptor } from '@kortyx/core/errors';
|
|
5
8
|
import { FinalizedChatMessage, StreamChunk } from '@kortyx/stream';
|
|
6
|
-
import {
|
|
7
|
-
import { PendingRequestRecord, PendingRequestStore, CheckpointSummary, CheckpointId, SessionCheckpointRecord, RollbackSessionCheckpointResult, ForkSessionCheckpointResult, WorkflowRegistry, FrameworkAdapter, ExecutionRuntimeConfig } from '@kortyx/runtime';
|
|
9
|
+
import { PendingRequestRecord, PendingRequestStore, ExecutionRuntimeConfig, CheckpointSummary, CheckpointId, SessionCheckpointRecord, RollbackSessionCheckpointResult, ForkSessionCheckpointResult, WorkflowRegistry, FrameworkAdapter } from '@kortyx/runtime';
|
|
8
10
|
import { KortyxTelemetryConfig, EnsureWorkflowTopologyRequest } from '@kortyx/hooks';
|
|
9
11
|
import { C as ChatMessage } from './chat-message-D4EsFcr1.js';
|
|
10
12
|
|
|
@@ -173,6 +175,8 @@ interface CreateAgentArgs {
|
|
|
173
175
|
telemetry?: ExecutionRuntimeConfig["telemetry"];
|
|
174
176
|
}
|
|
175
177
|
interface Agent extends ReturnType<typeof createInterruptDiscovery> {
|
|
178
|
+
/** Report private exception evidence through the configured native telemetry adapter. */
|
|
179
|
+
reportError?: NonNullable<NonNullable<ExecutionRuntimeConfig["telemetry"]>["trace"]>["reportError"];
|
|
176
180
|
describePromptContracts?: () => {
|
|
177
181
|
id: string;
|
|
178
182
|
format: "system-user" | "chat";
|
|
@@ -324,6 +328,9 @@ type EvalInterrupt = {
|
|
|
324
328
|
};
|
|
325
329
|
/** Public output and emitted execution evidence; private runtime state is excluded. */
|
|
326
330
|
type EvalObservation = {
|
|
331
|
+
/** Safe terminal failure; raw exception evidence belongs in the private diagnostic. */
|
|
332
|
+
failure?: SerializedFailureDescriptor;
|
|
333
|
+
diagnosticId?: string;
|
|
327
334
|
promptUsage?: _kortyx_prompts.PromptUsageReceipt[];
|
|
328
335
|
type: "answer" | "interrupt" | "error" | "cancelled";
|
|
329
336
|
text: string;
|
|
@@ -356,11 +363,13 @@ type EvalJudgeIdentity = {
|
|
|
356
363
|
version: string;
|
|
357
364
|
location?: "app" | "studio" | undefined;
|
|
358
365
|
};
|
|
359
|
-
type EvalPhase = "params" | "setup" | "execute" | "responder" | "reference" | "grading" | "cleanup" | "reporting";
|
|
366
|
+
type EvalPhase = "params" | "setup" | "execute" | "responder" | "reference" | "grading" | "verification" | "cleanup" | "reporting";
|
|
360
367
|
type EvalIssue = {
|
|
361
368
|
phase: EvalPhase;
|
|
362
369
|
code: string;
|
|
363
370
|
message: string;
|
|
371
|
+
failure?: SerializedFailureDescriptor;
|
|
372
|
+
diagnosticId?: string;
|
|
364
373
|
};
|
|
365
374
|
type EvalVerdict = {
|
|
366
375
|
passed: boolean;
|
|
@@ -395,6 +404,8 @@ type EvalRuntimeExecution = {
|
|
|
395
404
|
runId: string;
|
|
396
405
|
};
|
|
397
406
|
type EvalCaseResult = {
|
|
407
|
+
verification?: EvalVerdict;
|
|
408
|
+
phaseDurationsMs?: Partial<Record<EvalPhase, number>>;
|
|
398
409
|
/** SDK-owned identity and execution associations for automatic cost attribution. */
|
|
399
410
|
attemptId?: string;
|
|
400
411
|
runtimeExecutions?: EvalRuntimeExecution[];
|
|
@@ -481,10 +492,15 @@ type EvalDefaults = {
|
|
|
481
492
|
repetitions?: number;
|
|
482
493
|
concurrency?: number;
|
|
483
494
|
caseTimeoutMs?: number;
|
|
495
|
+
/** Params/setup admission allowance, independent of the execution deadline. */
|
|
496
|
+
setupTimeoutMs?: number;
|
|
497
|
+
verificationTimeoutMs?: number;
|
|
484
498
|
cleanupTimeoutMs?: number;
|
|
485
499
|
};
|
|
486
500
|
type CreateEvalsOptions<Params = EvalJson | undefined, Prepared = undefined> = {
|
|
487
|
-
agent: Pick<Agent, "streamChat" | "describePromptContracts">;
|
|
501
|
+
agent: Pick<Agent, "streamChat" | "describePromptContracts" | "reportError">;
|
|
502
|
+
/** Optional reporter for custom agents; defaults to the real agent's telemetry. */
|
|
503
|
+
reportError?: Agent["reportError"];
|
|
488
504
|
suites: readonly EvalSuite[];
|
|
489
505
|
paramsSchema?: z.ZodType<Params>;
|
|
490
506
|
setup?: (input: EvalSetupContext<Params>) => Prepared | Promise<Prepared>;
|
|
@@ -500,6 +516,10 @@ type CreateEvalsOptions<Params = EvalJson | undefined, Prepared = undefined> = {
|
|
|
500
516
|
teardown?: (input: EvalContext<Params, NoInfer<Prepared>> & {
|
|
501
517
|
result: EvalCaseResult;
|
|
502
518
|
}) => void | Promise<void>;
|
|
519
|
+
/** Assert postconditions before teardown. False is Failed; thrown infrastructure faults are Error. */
|
|
520
|
+
verify?: (input: EvalContext<Params, NoInfer<Prepared>> & {
|
|
521
|
+
result: EvalCaseResult;
|
|
522
|
+
}) => EvalVerdict | Promise<EvalVerdict>;
|
|
503
523
|
responders?: Record<string, EvalResponder<Params, NoInfer<Prepared>>>;
|
|
504
524
|
references?: Record<string, (input: EvalContext<Params, NoInfer<Prepared>> & {
|
|
505
525
|
observation: EvalObservation;
|
|
@@ -510,6 +530,15 @@ type CreateEvalsOptions<Params = EvalJson | undefined, Prepared = undefined> = {
|
|
|
510
530
|
defaults?: EvalDefaults;
|
|
511
531
|
};
|
|
512
532
|
type EvalProgress = {
|
|
533
|
+
type: "case-dispatch";
|
|
534
|
+
caseId: string;
|
|
535
|
+
repetition: number;
|
|
536
|
+
status: "queued" | "retrying" | "reconnecting";
|
|
537
|
+
retryAt?: string;
|
|
538
|
+
httpStatus?: number;
|
|
539
|
+
executionId?: string;
|
|
540
|
+
generation?: string;
|
|
541
|
+
} | {
|
|
513
542
|
type: "run-started";
|
|
514
543
|
caseIds: string[];
|
|
515
544
|
repetitions: number;
|
|
@@ -669,6 +698,8 @@ declare const EvalVerdictSchema: z.ZodObject<{
|
|
|
669
698
|
evidence: z.ZodArray<z.ZodString>;
|
|
670
699
|
}, z.core.$strict>;
|
|
671
700
|
declare const EvalObservationSchema: z.ZodObject<{
|
|
701
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
702
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
672
703
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
673
704
|
id: z.ZodString;
|
|
674
705
|
version: z.ZodNumber;
|
|
@@ -707,6 +738,23 @@ declare const EvalObservationSchema: z.ZodObject<{
|
|
|
707
738
|
request: z.ZodOptional<z.ZodJSONSchema>;
|
|
708
739
|
}, z.core.$strict>>;
|
|
709
740
|
}, z.core.$strict>;
|
|
741
|
+
declare const EvalIssueSchema: z.ZodObject<{
|
|
742
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
743
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
744
|
+
phase: z.ZodEnum<{
|
|
745
|
+
params: "params";
|
|
746
|
+
execute: "execute";
|
|
747
|
+
setup: "setup";
|
|
748
|
+
responder: "responder";
|
|
749
|
+
reference: "reference";
|
|
750
|
+
grading: "grading";
|
|
751
|
+
verification: "verification";
|
|
752
|
+
cleanup: "cleanup";
|
|
753
|
+
reporting: "reporting";
|
|
754
|
+
}>;
|
|
755
|
+
code: z.ZodString;
|
|
756
|
+
message: z.ZodString;
|
|
757
|
+
}, z.core.$strict>;
|
|
710
758
|
/** Billing evidence only: no prompts, provider raw responses, or credentials. */
|
|
711
759
|
declare const EvalJudgeUsageSchema: z.ZodObject<{
|
|
712
760
|
provider: z.ZodString;
|
|
@@ -740,6 +788,8 @@ declare const EvalStepResultSchema: z.ZodObject<{
|
|
|
740
788
|
version: z.ZodLiteral<"compact-v1">;
|
|
741
789
|
history: z.ZodBoolean;
|
|
742
790
|
observation: z.ZodObject<{
|
|
791
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
792
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
743
793
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
744
794
|
id: z.ZodString;
|
|
745
795
|
version: z.ZodNumber;
|
|
@@ -814,6 +864,8 @@ declare const EvalStepResultSchema: z.ZodObject<{
|
|
|
814
864
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
815
865
|
}, z.core.$strict>;
|
|
816
866
|
observation: z.ZodObject<{
|
|
867
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
868
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
817
869
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
818
870
|
id: z.ZodString;
|
|
819
871
|
version: z.ZodNumber;
|
|
@@ -898,6 +950,22 @@ declare const EvalStepResultSchema: z.ZodObject<{
|
|
|
898
950
|
}, z.core.$strict>>;
|
|
899
951
|
}, z.core.$strict>;
|
|
900
952
|
declare const EvalCaseResultSchema: z.ZodObject<{
|
|
953
|
+
verification: z.ZodOptional<z.ZodObject<{
|
|
954
|
+
passed: z.ZodBoolean;
|
|
955
|
+
reason: z.ZodString;
|
|
956
|
+
evidence: z.ZodArray<z.ZodString>;
|
|
957
|
+
}, z.core.$strict>>;
|
|
958
|
+
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
959
|
+
params: "params";
|
|
960
|
+
execute: "execute";
|
|
961
|
+
setup: "setup";
|
|
962
|
+
responder: "responder";
|
|
963
|
+
reference: "reference";
|
|
964
|
+
grading: "grading";
|
|
965
|
+
verification: "verification";
|
|
966
|
+
cleanup: "cleanup";
|
|
967
|
+
reporting: "reporting";
|
|
968
|
+
}> & z.core.$partial, z.ZodNumber>>;
|
|
901
969
|
caseId: z.ZodString;
|
|
902
970
|
repetition: z.ZodNumber;
|
|
903
971
|
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
@@ -919,6 +987,8 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
919
987
|
version: z.ZodLiteral<"compact-v1">;
|
|
920
988
|
history: z.ZodBoolean;
|
|
921
989
|
observation: z.ZodObject<{
|
|
990
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
991
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
922
992
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
923
993
|
id: z.ZodString;
|
|
924
994
|
version: z.ZodNumber;
|
|
@@ -993,6 +1063,8 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
993
1063
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
994
1064
|
}, z.core.$strict>;
|
|
995
1065
|
observation: z.ZodObject<{
|
|
1066
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1067
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
996
1068
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
997
1069
|
id: z.ZodString;
|
|
998
1070
|
version: z.ZodNumber;
|
|
@@ -1077,6 +1149,8 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
1077
1149
|
}, z.core.$strict>>;
|
|
1078
1150
|
}, z.core.$strict>>;
|
|
1079
1151
|
errors: z.ZodArray<z.ZodObject<{
|
|
1152
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1153
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1080
1154
|
phase: z.ZodEnum<{
|
|
1081
1155
|
params: "params";
|
|
1082
1156
|
execute: "execute";
|
|
@@ -1084,6 +1158,7 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
1084
1158
|
responder: "responder";
|
|
1085
1159
|
reference: "reference";
|
|
1086
1160
|
grading: "grading";
|
|
1161
|
+
verification: "verification";
|
|
1087
1162
|
cleanup: "cleanup";
|
|
1088
1163
|
reporting: "reporting";
|
|
1089
1164
|
}>;
|
|
@@ -1130,6 +1205,8 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
1130
1205
|
}, z.core.$strict>], "type">;
|
|
1131
1206
|
}, z.core.$strict>]>;
|
|
1132
1207
|
observation: z.ZodObject<{
|
|
1208
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1209
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1133
1210
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1134
1211
|
id: z.ZodString;
|
|
1135
1212
|
version: z.ZodNumber;
|
|
@@ -1174,6 +1251,8 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
1174
1251
|
version: z.ZodLiteral<"compact-v1">;
|
|
1175
1252
|
history: z.ZodBoolean;
|
|
1176
1253
|
observation: z.ZodObject<{
|
|
1254
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1255
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1177
1256
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1178
1257
|
id: z.ZodString;
|
|
1179
1258
|
version: z.ZodNumber;
|
|
@@ -1248,6 +1327,8 @@ declare const StudioEvalJudgeRequestSchema: z.ZodObject<{
|
|
|
1248
1327
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1249
1328
|
}, z.core.$strict>;
|
|
1250
1329
|
observation: z.ZodObject<{
|
|
1330
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1331
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1251
1332
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1252
1333
|
id: z.ZodString;
|
|
1253
1334
|
version: z.ZodNumber;
|
|
@@ -1492,6 +1573,22 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1492
1573
|
ungraded: z.ZodOptional<z.ZodNumber>;
|
|
1493
1574
|
}, z.core.$strict>;
|
|
1494
1575
|
cases: z.ZodArray<z.ZodObject<{
|
|
1576
|
+
verification: z.ZodOptional<z.ZodObject<{
|
|
1577
|
+
passed: z.ZodBoolean;
|
|
1578
|
+
reason: z.ZodString;
|
|
1579
|
+
evidence: z.ZodArray<z.ZodString>;
|
|
1580
|
+
}, z.core.$strict>>;
|
|
1581
|
+
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
1582
|
+
params: "params";
|
|
1583
|
+
execute: "execute";
|
|
1584
|
+
setup: "setup";
|
|
1585
|
+
responder: "responder";
|
|
1586
|
+
reference: "reference";
|
|
1587
|
+
grading: "grading";
|
|
1588
|
+
verification: "verification";
|
|
1589
|
+
cleanup: "cleanup";
|
|
1590
|
+
reporting: "reporting";
|
|
1591
|
+
}> & z.core.$partial, z.ZodNumber>>;
|
|
1495
1592
|
caseId: z.ZodString;
|
|
1496
1593
|
repetition: z.ZodNumber;
|
|
1497
1594
|
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
@@ -1513,6 +1610,8 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1513
1610
|
version: z.ZodLiteral<"compact-v1">;
|
|
1514
1611
|
history: z.ZodBoolean;
|
|
1515
1612
|
observation: z.ZodObject<{
|
|
1613
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1614
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1516
1615
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1517
1616
|
id: z.ZodString;
|
|
1518
1617
|
version: z.ZodNumber;
|
|
@@ -1587,6 +1686,8 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1587
1686
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1588
1687
|
}, z.core.$strict>;
|
|
1589
1688
|
observation: z.ZodObject<{
|
|
1689
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1690
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1590
1691
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1591
1692
|
id: z.ZodString;
|
|
1592
1693
|
version: z.ZodNumber;
|
|
@@ -1671,6 +1772,8 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1671
1772
|
}, z.core.$strict>>;
|
|
1672
1773
|
}, z.core.$strict>>;
|
|
1673
1774
|
errors: z.ZodArray<z.ZodObject<{
|
|
1775
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1776
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1674
1777
|
phase: z.ZodEnum<{
|
|
1675
1778
|
params: "params";
|
|
1676
1779
|
execute: "execute";
|
|
@@ -1678,6 +1781,7 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1678
1781
|
responder: "responder";
|
|
1679
1782
|
reference: "reference";
|
|
1680
1783
|
grading: "grading";
|
|
1784
|
+
verification: "verification";
|
|
1681
1785
|
cleanup: "cleanup";
|
|
1682
1786
|
reporting: "reporting";
|
|
1683
1787
|
}>;
|
|
@@ -1686,6 +1790,8 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1686
1790
|
}, z.core.$strict>>;
|
|
1687
1791
|
}, z.core.$strict>>;
|
|
1688
1792
|
errors: z.ZodArray<z.ZodObject<{
|
|
1793
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1794
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1689
1795
|
phase: z.ZodEnum<{
|
|
1690
1796
|
params: "params";
|
|
1691
1797
|
execute: "execute";
|
|
@@ -1693,6 +1799,7 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1693
1799
|
responder: "responder";
|
|
1694
1800
|
reference: "reference";
|
|
1695
1801
|
grading: "grading";
|
|
1802
|
+
verification: "verification";
|
|
1696
1803
|
cleanup: "cleanup";
|
|
1697
1804
|
reporting: "reporting";
|
|
1698
1805
|
}>;
|
|
@@ -1819,6 +1926,19 @@ declare const EvalManifestSchema: z.ZodObject<{
|
|
|
1819
1926
|
}, z.core.$strict>>;
|
|
1820
1927
|
}, z.core.$strict>;
|
|
1821
1928
|
declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
1929
|
+
type: z.ZodLiteral<"case-dispatch">;
|
|
1930
|
+
caseId: z.ZodString;
|
|
1931
|
+
repetition: z.ZodNumber;
|
|
1932
|
+
status: z.ZodEnum<{
|
|
1933
|
+
queued: "queued";
|
|
1934
|
+
retrying: "retrying";
|
|
1935
|
+
reconnecting: "reconnecting";
|
|
1936
|
+
}>;
|
|
1937
|
+
retryAt: z.ZodOptional<z.ZodISODateTime>;
|
|
1938
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
1939
|
+
executionId: z.ZodOptional<z.ZodUUID>;
|
|
1940
|
+
generation: z.ZodOptional<z.ZodUUID>;
|
|
1941
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1822
1942
|
type: z.ZodLiteral<"case-runtime-associated">;
|
|
1823
1943
|
caseId: z.ZodString;
|
|
1824
1944
|
repetition: z.ZodNumber;
|
|
@@ -1843,6 +1963,7 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1843
1963
|
responder: "responder";
|
|
1844
1964
|
reference: "reference";
|
|
1845
1965
|
grading: "grading";
|
|
1966
|
+
verification: "verification";
|
|
1846
1967
|
cleanup: "cleanup";
|
|
1847
1968
|
reporting: "reporting";
|
|
1848
1969
|
}>;
|
|
@@ -1862,6 +1983,8 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1862
1983
|
version: z.ZodLiteral<"compact-v1">;
|
|
1863
1984
|
history: z.ZodBoolean;
|
|
1864
1985
|
observation: z.ZodObject<{
|
|
1986
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1987
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1865
1988
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1866
1989
|
id: z.ZodString;
|
|
1867
1990
|
version: z.ZodNumber;
|
|
@@ -1936,6 +2059,8 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1936
2059
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
1937
2060
|
}, z.core.$strict>;
|
|
1938
2061
|
observation: z.ZodObject<{
|
|
2062
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2063
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1939
2064
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
1940
2065
|
id: z.ZodString;
|
|
1941
2066
|
version: z.ZodNumber;
|
|
@@ -2022,6 +2147,22 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2022
2147
|
}, z.core.$strict>, z.ZodObject<{
|
|
2023
2148
|
type: z.ZodLiteral<"case-completed">;
|
|
2024
2149
|
result: z.ZodObject<{
|
|
2150
|
+
verification: z.ZodOptional<z.ZodObject<{
|
|
2151
|
+
passed: z.ZodBoolean;
|
|
2152
|
+
reason: z.ZodString;
|
|
2153
|
+
evidence: z.ZodArray<z.ZodString>;
|
|
2154
|
+
}, z.core.$strict>>;
|
|
2155
|
+
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
2156
|
+
params: "params";
|
|
2157
|
+
execute: "execute";
|
|
2158
|
+
setup: "setup";
|
|
2159
|
+
responder: "responder";
|
|
2160
|
+
reference: "reference";
|
|
2161
|
+
grading: "grading";
|
|
2162
|
+
verification: "verification";
|
|
2163
|
+
cleanup: "cleanup";
|
|
2164
|
+
reporting: "reporting";
|
|
2165
|
+
}> & z.core.$partial, z.ZodNumber>>;
|
|
2025
2166
|
caseId: z.ZodString;
|
|
2026
2167
|
repetition: z.ZodNumber;
|
|
2027
2168
|
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
@@ -2043,6 +2184,8 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2043
2184
|
version: z.ZodLiteral<"compact-v1">;
|
|
2044
2185
|
history: z.ZodBoolean;
|
|
2045
2186
|
observation: z.ZodObject<{
|
|
2187
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2188
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2046
2189
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2047
2190
|
id: z.ZodString;
|
|
2048
2191
|
version: z.ZodNumber;
|
|
@@ -2117,6 +2260,8 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2117
2260
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
2118
2261
|
}, z.core.$strict>;
|
|
2119
2262
|
observation: z.ZodObject<{
|
|
2263
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2264
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2120
2265
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2121
2266
|
id: z.ZodString;
|
|
2122
2267
|
version: z.ZodNumber;
|
|
@@ -2201,6 +2346,8 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2201
2346
|
}, z.core.$strict>>;
|
|
2202
2347
|
}, z.core.$strict>>;
|
|
2203
2348
|
errors: z.ZodArray<z.ZodObject<{
|
|
2349
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2350
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2204
2351
|
phase: z.ZodEnum<{
|
|
2205
2352
|
params: "params";
|
|
2206
2353
|
execute: "execute";
|
|
@@ -2208,6 +2355,7 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2208
2355
|
responder: "responder";
|
|
2209
2356
|
reference: "reference";
|
|
2210
2357
|
grading: "grading";
|
|
2358
|
+
verification: "verification";
|
|
2211
2359
|
cleanup: "cleanup";
|
|
2212
2360
|
reporting: "reporting";
|
|
2213
2361
|
}>;
|
|
@@ -2306,8 +2454,30 @@ declare const StudioEvalStartRequestSchema: z.ZodObject<{
|
|
|
2306
2454
|
}>>;
|
|
2307
2455
|
}, z.core.$strict>;
|
|
2308
2456
|
declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
2457
|
+
type: z.ZodLiteral<"execution-status">;
|
|
2458
|
+
state: z.ZodEnum<{
|
|
2459
|
+
cancelled: "cancelled";
|
|
2460
|
+
unknown: "unknown";
|
|
2461
|
+
completed: "completed";
|
|
2462
|
+
queued: "queued";
|
|
2463
|
+
running: "running";
|
|
2464
|
+
}>;
|
|
2465
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
2309
2466
|
type: z.ZodLiteral<"progress">;
|
|
2310
2467
|
event: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
2468
|
+
type: z.ZodLiteral<"case-dispatch">;
|
|
2469
|
+
caseId: z.ZodString;
|
|
2470
|
+
repetition: z.ZodNumber;
|
|
2471
|
+
status: z.ZodEnum<{
|
|
2472
|
+
queued: "queued";
|
|
2473
|
+
retrying: "retrying";
|
|
2474
|
+
reconnecting: "reconnecting";
|
|
2475
|
+
}>;
|
|
2476
|
+
retryAt: z.ZodOptional<z.ZodISODateTime>;
|
|
2477
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
2478
|
+
executionId: z.ZodOptional<z.ZodUUID>;
|
|
2479
|
+
generation: z.ZodOptional<z.ZodUUID>;
|
|
2480
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
2311
2481
|
type: z.ZodLiteral<"case-runtime-associated">;
|
|
2312
2482
|
caseId: z.ZodString;
|
|
2313
2483
|
repetition: z.ZodNumber;
|
|
@@ -2332,6 +2502,7 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2332
2502
|
responder: "responder";
|
|
2333
2503
|
reference: "reference";
|
|
2334
2504
|
grading: "grading";
|
|
2505
|
+
verification: "verification";
|
|
2335
2506
|
cleanup: "cleanup";
|
|
2336
2507
|
reporting: "reporting";
|
|
2337
2508
|
}>;
|
|
@@ -2351,6 +2522,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2351
2522
|
version: z.ZodLiteral<"compact-v1">;
|
|
2352
2523
|
history: z.ZodBoolean;
|
|
2353
2524
|
observation: z.ZodObject<{
|
|
2525
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2526
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2354
2527
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2355
2528
|
id: z.ZodString;
|
|
2356
2529
|
version: z.ZodNumber;
|
|
@@ -2425,6 +2598,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2425
2598
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
2426
2599
|
}, z.core.$strict>;
|
|
2427
2600
|
observation: z.ZodObject<{
|
|
2601
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2602
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2428
2603
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2429
2604
|
id: z.ZodString;
|
|
2430
2605
|
version: z.ZodNumber;
|
|
@@ -2511,6 +2686,22 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2511
2686
|
}, z.core.$strict>, z.ZodObject<{
|
|
2512
2687
|
type: z.ZodLiteral<"case-completed">;
|
|
2513
2688
|
result: z.ZodObject<{
|
|
2689
|
+
verification: z.ZodOptional<z.ZodObject<{
|
|
2690
|
+
passed: z.ZodBoolean;
|
|
2691
|
+
reason: z.ZodString;
|
|
2692
|
+
evidence: z.ZodArray<z.ZodString>;
|
|
2693
|
+
}, z.core.$strict>>;
|
|
2694
|
+
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
2695
|
+
params: "params";
|
|
2696
|
+
execute: "execute";
|
|
2697
|
+
setup: "setup";
|
|
2698
|
+
responder: "responder";
|
|
2699
|
+
reference: "reference";
|
|
2700
|
+
grading: "grading";
|
|
2701
|
+
verification: "verification";
|
|
2702
|
+
cleanup: "cleanup";
|
|
2703
|
+
reporting: "reporting";
|
|
2704
|
+
}> & z.core.$partial, z.ZodNumber>>;
|
|
2514
2705
|
caseId: z.ZodString;
|
|
2515
2706
|
repetition: z.ZodNumber;
|
|
2516
2707
|
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
@@ -2532,6 +2723,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2532
2723
|
version: z.ZodLiteral<"compact-v1">;
|
|
2533
2724
|
history: z.ZodBoolean;
|
|
2534
2725
|
observation: z.ZodObject<{
|
|
2726
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2727
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2535
2728
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2536
2729
|
id: z.ZodString;
|
|
2537
2730
|
version: z.ZodNumber;
|
|
@@ -2606,6 +2799,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2606
2799
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
2607
2800
|
}, z.core.$strict>;
|
|
2608
2801
|
observation: z.ZodObject<{
|
|
2802
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2803
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2609
2804
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2610
2805
|
id: z.ZodString;
|
|
2611
2806
|
version: z.ZodNumber;
|
|
@@ -2690,6 +2885,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2690
2885
|
}, z.core.$strict>>;
|
|
2691
2886
|
}, z.core.$strict>>;
|
|
2692
2887
|
errors: z.ZodArray<z.ZodObject<{
|
|
2888
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2889
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2693
2890
|
phase: z.ZodEnum<{
|
|
2694
2891
|
params: "params";
|
|
2695
2892
|
execute: "execute";
|
|
@@ -2697,6 +2894,7 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2697
2894
|
responder: "responder";
|
|
2698
2895
|
reference: "reference";
|
|
2699
2896
|
grading: "grading";
|
|
2897
|
+
verification: "verification";
|
|
2700
2898
|
cleanup: "cleanup";
|
|
2701
2899
|
reporting: "reporting";
|
|
2702
2900
|
}>;
|
|
@@ -2705,6 +2903,7 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2705
2903
|
}, z.core.$strict>>;
|
|
2706
2904
|
}, z.core.$strict>;
|
|
2707
2905
|
}, z.core.$strict>], "type">;
|
|
2906
|
+
sequence: z.ZodOptional<z.ZodNumber>;
|
|
2708
2907
|
}, z.core.$strict>, z.ZodObject<{
|
|
2709
2908
|
type: z.ZodLiteral<"result">;
|
|
2710
2909
|
result: z.ZodObject<{
|
|
@@ -2825,6 +3024,22 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2825
3024
|
ungraded: z.ZodOptional<z.ZodNumber>;
|
|
2826
3025
|
}, z.core.$strict>;
|
|
2827
3026
|
cases: z.ZodArray<z.ZodObject<{
|
|
3027
|
+
verification: z.ZodOptional<z.ZodObject<{
|
|
3028
|
+
passed: z.ZodBoolean;
|
|
3029
|
+
reason: z.ZodString;
|
|
3030
|
+
evidence: z.ZodArray<z.ZodString>;
|
|
3031
|
+
}, z.core.$strict>>;
|
|
3032
|
+
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
3033
|
+
params: "params";
|
|
3034
|
+
execute: "execute";
|
|
3035
|
+
setup: "setup";
|
|
3036
|
+
responder: "responder";
|
|
3037
|
+
reference: "reference";
|
|
3038
|
+
grading: "grading";
|
|
3039
|
+
verification: "verification";
|
|
3040
|
+
cleanup: "cleanup";
|
|
3041
|
+
reporting: "reporting";
|
|
3042
|
+
}> & z.core.$partial, z.ZodNumber>>;
|
|
2828
3043
|
caseId: z.ZodString;
|
|
2829
3044
|
repetition: z.ZodNumber;
|
|
2830
3045
|
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
@@ -2846,6 +3061,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2846
3061
|
version: z.ZodLiteral<"compact-v1">;
|
|
2847
3062
|
history: z.ZodBoolean;
|
|
2848
3063
|
observation: z.ZodObject<{
|
|
3064
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
3065
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2849
3066
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2850
3067
|
id: z.ZodString;
|
|
2851
3068
|
version: z.ZodNumber;
|
|
@@ -2920,6 +3137,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2920
3137
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
2921
3138
|
}, z.core.$strict>;
|
|
2922
3139
|
observation: z.ZodObject<{
|
|
3140
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
3141
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2923
3142
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
2924
3143
|
id: z.ZodString;
|
|
2925
3144
|
version: z.ZodNumber;
|
|
@@ -3004,6 +3223,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
3004
3223
|
}, z.core.$strict>>;
|
|
3005
3224
|
}, z.core.$strict>>;
|
|
3006
3225
|
errors: z.ZodArray<z.ZodObject<{
|
|
3226
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
3227
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
3007
3228
|
phase: z.ZodEnum<{
|
|
3008
3229
|
params: "params";
|
|
3009
3230
|
execute: "execute";
|
|
@@ -3011,6 +3232,7 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
3011
3232
|
responder: "responder";
|
|
3012
3233
|
reference: "reference";
|
|
3013
3234
|
grading: "grading";
|
|
3235
|
+
verification: "verification";
|
|
3014
3236
|
cleanup: "cleanup";
|
|
3015
3237
|
reporting: "reporting";
|
|
3016
3238
|
}>;
|
|
@@ -3019,6 +3241,8 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
3019
3241
|
}, z.core.$strict>>;
|
|
3020
3242
|
}, z.core.$strict>>;
|
|
3021
3243
|
errors: z.ZodArray<z.ZodObject<{
|
|
3244
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
3245
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
3022
3246
|
phase: z.ZodEnum<{
|
|
3023
3247
|
params: "params";
|
|
3024
3248
|
execute: "execute";
|
|
@@ -3026,6 +3250,7 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
3026
3250
|
responder: "responder";
|
|
3027
3251
|
reference: "reference";
|
|
3028
3252
|
grading: "grading";
|
|
3253
|
+
verification: "verification";
|
|
3029
3254
|
cleanup: "cleanup";
|
|
3030
3255
|
reporting: "reporting";
|
|
3031
3256
|
}>;
|
|
@@ -3033,9 +3258,13 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
3033
3258
|
message: z.ZodString;
|
|
3034
3259
|
}, z.core.$strict>>;
|
|
3035
3260
|
}, z.core.$strict>;
|
|
3261
|
+
sequence: z.ZodOptional<z.ZodNumber>;
|
|
3036
3262
|
}, z.core.$strict>, z.ZodObject<{
|
|
3037
3263
|
type: z.ZodLiteral<"error">;
|
|
3038
3264
|
message: z.ZodString;
|
|
3265
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
3266
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
3267
|
+
sequence: z.ZodOptional<z.ZodNumber>;
|
|
3039
3268
|
}, z.core.$strict>], "type">;
|
|
3040
3269
|
type EvalManifest = z.infer<typeof EvalManifestSchema>;
|
|
3041
3270
|
type EvalRemoteRunRequest = z.infer<typeof EvalRemoteRunRequestSchema>;
|
|
@@ -3055,6 +3284,14 @@ type DefinedSuite<Params> = Omit<EvalSuite<Params>, "cases"> & {
|
|
|
3055
3284
|
/** Author a suite with typed case parameters. Runtime validation belongs to createEvals. */
|
|
3056
3285
|
declare function defineSuite<Params = EvalJson | undefined>(suite: DefinedSuite<Params>): DefinedSuite<Params>;
|
|
3057
3286
|
|
|
3287
|
+
/** Public eval evidence contains bounded safe descriptors and opaque references only. */
|
|
3288
|
+
declare function createEvalIssue(phase: EvalPhase, error: unknown, options?: {
|
|
3289
|
+
code?: string;
|
|
3290
|
+
diagnosticId?: string;
|
|
3291
|
+
reportError?: Agent["reportError"];
|
|
3292
|
+
correlation?: Parameters<NonNullable<Agent["reportError"]>>[1];
|
|
3293
|
+
}): EvalIssue;
|
|
3294
|
+
|
|
3058
3295
|
declare const EvaluationMetadataSchema: z.ZodObject<{
|
|
3059
3296
|
source: z.ZodDefault<z.ZodEnum<{
|
|
3060
3297
|
manual: "manual";
|
|
@@ -4061,6 +4298,22 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4061
4298
|
ungraded: z.ZodOptional<z.ZodNumber>;
|
|
4062
4299
|
}, z.core.$strict>;
|
|
4063
4300
|
cases: z.ZodArray<z.ZodObject<{
|
|
4301
|
+
verification: z.ZodOptional<z.ZodObject<{
|
|
4302
|
+
passed: z.ZodBoolean;
|
|
4303
|
+
reason: z.ZodString;
|
|
4304
|
+
evidence: z.ZodArray<z.ZodString>;
|
|
4305
|
+
}, z.core.$strict>>;
|
|
4306
|
+
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
4307
|
+
params: "params";
|
|
4308
|
+
execute: "execute";
|
|
4309
|
+
setup: "setup";
|
|
4310
|
+
responder: "responder";
|
|
4311
|
+
reference: "reference";
|
|
4312
|
+
grading: "grading";
|
|
4313
|
+
verification: "verification";
|
|
4314
|
+
cleanup: "cleanup";
|
|
4315
|
+
reporting: "reporting";
|
|
4316
|
+
}> & z.core.$partial, z.ZodNumber>>;
|
|
4064
4317
|
caseId: z.ZodString;
|
|
4065
4318
|
repetition: z.ZodNumber;
|
|
4066
4319
|
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
@@ -4082,6 +4335,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4082
4335
|
version: z.ZodLiteral<"compact-v1">;
|
|
4083
4336
|
history: z.ZodBoolean;
|
|
4084
4337
|
observation: z.ZodObject<{
|
|
4338
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4339
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4085
4340
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4086
4341
|
id: z.ZodString;
|
|
4087
4342
|
version: z.ZodNumber;
|
|
@@ -4156,6 +4411,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4156
4411
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
4157
4412
|
}, z.core.$strict>;
|
|
4158
4413
|
observation: z.ZodObject<{
|
|
4414
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4415
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4159
4416
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4160
4417
|
id: z.ZodString;
|
|
4161
4418
|
version: z.ZodNumber;
|
|
@@ -4240,6 +4497,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4240
4497
|
}, z.core.$strict>>;
|
|
4241
4498
|
}, z.core.$strict>>;
|
|
4242
4499
|
errors: z.ZodArray<z.ZodObject<{
|
|
4500
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4501
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4243
4502
|
phase: z.ZodEnum<{
|
|
4244
4503
|
params: "params";
|
|
4245
4504
|
execute: "execute";
|
|
@@ -4247,6 +4506,7 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4247
4506
|
responder: "responder";
|
|
4248
4507
|
reference: "reference";
|
|
4249
4508
|
grading: "grading";
|
|
4509
|
+
verification: "verification";
|
|
4250
4510
|
cleanup: "cleanup";
|
|
4251
4511
|
reporting: "reporting";
|
|
4252
4512
|
}>;
|
|
@@ -4255,6 +4515,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4255
4515
|
}, z.core.$strict>>;
|
|
4256
4516
|
}, z.core.$strict>>;
|
|
4257
4517
|
errors: z.ZodArray<z.ZodObject<{
|
|
4518
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4519
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4258
4520
|
phase: z.ZodEnum<{
|
|
4259
4521
|
params: "params";
|
|
4260
4522
|
execute: "execute";
|
|
@@ -4262,6 +4524,7 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4262
4524
|
responder: "responder";
|
|
4263
4525
|
reference: "reference";
|
|
4264
4526
|
grading: "grading";
|
|
4527
|
+
verification: "verification";
|
|
4265
4528
|
cleanup: "cleanup";
|
|
4266
4529
|
reporting: "reporting";
|
|
4267
4530
|
}>;
|
|
@@ -4272,6 +4535,19 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4272
4535
|
events: z.ZodArray<z.ZodObject<{
|
|
4273
4536
|
id: z.ZodNumber;
|
|
4274
4537
|
event: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
4538
|
+
type: z.ZodLiteral<"case-dispatch">;
|
|
4539
|
+
caseId: z.ZodString;
|
|
4540
|
+
repetition: z.ZodNumber;
|
|
4541
|
+
status: z.ZodEnum<{
|
|
4542
|
+
queued: "queued";
|
|
4543
|
+
retrying: "retrying";
|
|
4544
|
+
reconnecting: "reconnecting";
|
|
4545
|
+
}>;
|
|
4546
|
+
retryAt: z.ZodOptional<z.ZodISODateTime>;
|
|
4547
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
4548
|
+
executionId: z.ZodOptional<z.ZodUUID>;
|
|
4549
|
+
generation: z.ZodOptional<z.ZodUUID>;
|
|
4550
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
4275
4551
|
type: z.ZodLiteral<"case-runtime-associated">;
|
|
4276
4552
|
caseId: z.ZodString;
|
|
4277
4553
|
repetition: z.ZodNumber;
|
|
@@ -4296,6 +4572,7 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4296
4572
|
responder: "responder";
|
|
4297
4573
|
reference: "reference";
|
|
4298
4574
|
grading: "grading";
|
|
4575
|
+
verification: "verification";
|
|
4299
4576
|
cleanup: "cleanup";
|
|
4300
4577
|
reporting: "reporting";
|
|
4301
4578
|
}>;
|
|
@@ -4315,6 +4592,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4315
4592
|
version: z.ZodLiteral<"compact-v1">;
|
|
4316
4593
|
history: z.ZodBoolean;
|
|
4317
4594
|
observation: z.ZodObject<{
|
|
4595
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4596
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4318
4597
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4319
4598
|
id: z.ZodString;
|
|
4320
4599
|
version: z.ZodNumber;
|
|
@@ -4389,6 +4668,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4389
4668
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
4390
4669
|
}, z.core.$strict>;
|
|
4391
4670
|
observation: z.ZodObject<{
|
|
4671
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4672
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4392
4673
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4393
4674
|
id: z.ZodString;
|
|
4394
4675
|
version: z.ZodNumber;
|
|
@@ -4475,6 +4756,22 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4475
4756
|
}, z.core.$strict>, z.ZodObject<{
|
|
4476
4757
|
type: z.ZodLiteral<"case-completed">;
|
|
4477
4758
|
result: z.ZodObject<{
|
|
4759
|
+
verification: z.ZodOptional<z.ZodObject<{
|
|
4760
|
+
passed: z.ZodBoolean;
|
|
4761
|
+
reason: z.ZodString;
|
|
4762
|
+
evidence: z.ZodArray<z.ZodString>;
|
|
4763
|
+
}, z.core.$strict>>;
|
|
4764
|
+
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
4765
|
+
params: "params";
|
|
4766
|
+
execute: "execute";
|
|
4767
|
+
setup: "setup";
|
|
4768
|
+
responder: "responder";
|
|
4769
|
+
reference: "reference";
|
|
4770
|
+
grading: "grading";
|
|
4771
|
+
verification: "verification";
|
|
4772
|
+
cleanup: "cleanup";
|
|
4773
|
+
reporting: "reporting";
|
|
4774
|
+
}> & z.core.$partial, z.ZodNumber>>;
|
|
4478
4775
|
caseId: z.ZodString;
|
|
4479
4776
|
repetition: z.ZodNumber;
|
|
4480
4777
|
attemptId: z.ZodOptional<z.ZodUUID>;
|
|
@@ -4496,6 +4793,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4496
4793
|
version: z.ZodLiteral<"compact-v1">;
|
|
4497
4794
|
history: z.ZodBoolean;
|
|
4498
4795
|
observation: z.ZodObject<{
|
|
4796
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4797
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4499
4798
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4500
4799
|
id: z.ZodString;
|
|
4501
4800
|
version: z.ZodNumber;
|
|
@@ -4570,6 +4869,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4570
4869
|
reference: z.ZodOptional<z.ZodJSONSchema>;
|
|
4571
4870
|
}, z.core.$strict>;
|
|
4572
4871
|
observation: z.ZodObject<{
|
|
4872
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4873
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4573
4874
|
promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4574
4875
|
id: z.ZodString;
|
|
4575
4876
|
version: z.ZodNumber;
|
|
@@ -4654,6 +4955,8 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4654
4955
|
}, z.core.$strict>>;
|
|
4655
4956
|
}, z.core.$strict>>;
|
|
4656
4957
|
errors: z.ZodArray<z.ZodObject<{
|
|
4958
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4959
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4657
4960
|
phase: z.ZodEnum<{
|
|
4658
4961
|
params: "params";
|
|
4659
4962
|
execute: "execute";
|
|
@@ -4661,6 +4964,7 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4661
4964
|
responder: "responder";
|
|
4662
4965
|
reference: "reference";
|
|
4663
4966
|
grading: "grading";
|
|
4967
|
+
verification: "verification";
|
|
4664
4968
|
cleanup: "cleanup";
|
|
4665
4969
|
reporting: "reporting";
|
|
4666
4970
|
}>;
|
|
@@ -4685,4 +4989,4 @@ declare function getEvalGradeEvidence(step: EvalStepResult, previous: readonly E
|
|
|
4685
4989
|
/** Discover and pin the Studio backend's judge; grading never runs in the browser. */
|
|
4686
4990
|
declare function createStudioEvalJudge(options: StudioEvalJudgeOptions): Promise<EvalJudge>;
|
|
4687
4991
|
|
|
4688
|
-
export {
|
|
4992
|
+
export { type EvalSuite as $, type EvalJudgeOptions as A, type EvalJudgeUsage as B, type CreateEvalsOptions as C, EvalJudgeUsageSchema as D, type EvalCase as E, type EvalManifest as F, EvalManifestSchema as G, type EvalObservation as H, EvalObservationSchema as I, type EvalOutputExpectation as J, type EvalOutputSelector as K, type EvalPhase as L, type EvalProgress as M, EvalProgressSchema as N, type EvalRemoteRunRequest as O, EvalRemoteRunRequestSchema as P, type EvalResponder as Q, EvalResumeResponseSchema as R, type EvalRunOptions as S, type EvalRunResult as T, EvalRunResultSchema as U, type EvalRuntimeExecution as V, type EvalSetupContext as W, type EvalStatus as X, type EvalStep as Y, type EvalStepResult as Z, EvalStepResultSchema as _, type EvalCaseResult as a, EvalSuiteSchema as a0, type EvalVerdict as a1, EvalVerdictSchema as a2, type EvalWireEvent as a3, EvalWireEventSchema as a4, EvaluationMetadataSchema as a5, type StudioEvalJudgeOptions as a6, StudioEvalJudgeRequestSchema as a7, StudioEvalJudgeResponseSchema as a8, StudioEvalStartRequestSchema as a9, type ExecutionResult as aA, type InterruptScope as aB, type ListInterruptsOptions as aC, type PendingInterrupt as aD, type ResumableInterrupt as aE, type ResumeHandle as aF, type ResumeOptions as aG, type ResumeResponse as aH, createAgent as aI, projectWorkflowTopology as aJ, type StudioEvaluationDetail as aa, StudioEvaluationDetailSchema as ab, StudioEvaluationHistorySchema as ac, StudioEvaluationResultsSchema as ad, type StudioEvaluationStartRequest as ae, StudioEvaluationStartRequestSchema as af, type StudioEvaluationSummary as ag, StudioEvaluationSummarySchema as ah, createEvalIssue as ai, createStudioEvalJudge as aj, defineSuite as ak, getEvalGradeEvidence as al, parseEvalSuite as am, type ChatLifecyclePhase as an, ChatLifecycleHookError as ao, type Agent as ap, type ChatResponseFinalized as aq, type AgentProcessOptions as ar, type AgentProjectTopologyOptions as as, type ChatResponseStatus as at, type CreateAgentArgs as au, type ExecutableWorkflow as av, type ExecuteOptions as aw, type ExecutionInfo as ax, type ExecutionInterrupt as ay, ExecutionRequestError as az, EvalCaseResultSchema as b, type EvalCommand as c, EvalConfigurationError as d, type EvalContext as e, type EvalCriterion as f, type EvalCriterionResult as g, type EvalDefaults as h, type EvalEvidenceEvent as i, type EvalEvidenceEventType as j, type EvalEvidenceFilters as k, type EvalEvidencePolicy as l, EvalEvidencePolicySchema as m, type EvalExecution as n, type EvalExecutionStatus as o, type EvalExpectation as p, type EvalGradeInput as q, type EvalHandlerRef as r, type EvalInterrupt as s, type EvalIssue as t, EvalIssueSchema as u, type EvalJson as v, type EvalJudge as w, type EvalJudgeEvidence as x, type EvalJudgeIdentity as y, EvalJudgeIdentitySchema as z };
|