@kortyx/agent 0.38.1 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +11 -0
- package/dist/evals.d.ts +414 -7
- package/dist/evals.js +495 -330
- package/dist/evals.js.map +1 -1
- package/dist/index.d.ts +18 -3
- package/dist/index.js +1128 -2940
- package/dist/index.js.map +1 -1
- package/dist/{studio-judge-DCAw3v60.d.ts → studio-judge-caDvI8By.d.ts} +973 -43
- package/package.json +11 -10
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as _kortyx_core from '@kortyx/core';
|
|
2
2
|
import { WorkflowDefinition, ExecutionLimits, TokenUsage, ExecutionLimitReached } from '@kortyx/core';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
+
import * as _kortyx_telemetry_contracts from '@kortyx/telemetry-contracts';
|
|
4
5
|
import * as _kortyx_prompts from '@kortyx/prompts';
|
|
5
6
|
import * as _kortyx_core_errors from '@kortyx/core/errors';
|
|
6
7
|
import { KortyxError, FailureDescriptor, SerializedFailureDescriptor } from '@kortyx/core/errors';
|
|
@@ -365,11 +366,17 @@ type EvalJudgeIdentity = {
|
|
|
365
366
|
};
|
|
366
367
|
type EvalPhase = "params" | "setup" | "execute" | "responder" | "reference" | "grading" | "verification" | "cleanup" | "reporting";
|
|
367
368
|
type EvalIssue = {
|
|
369
|
+
/** Original exception, including stack, causes and response details; only credentials are redacted. */
|
|
370
|
+
exception?: _kortyx_telemetry_contracts.DiagnosticContent;
|
|
368
371
|
phase: EvalPhase;
|
|
369
372
|
code: string;
|
|
370
373
|
message: string;
|
|
371
374
|
failure?: SerializedFailureDescriptor;
|
|
372
375
|
diagnosticId?: string;
|
|
376
|
+
details?: {
|
|
377
|
+
timeoutMs?: number;
|
|
378
|
+
httpStatus?: number;
|
|
379
|
+
};
|
|
373
380
|
};
|
|
374
381
|
type EvalVerdict = {
|
|
375
382
|
passed: boolean;
|
|
@@ -528,6 +535,11 @@ type CreateEvalsOptions<Params = EvalJson | undefined, Prepared = undefined> = {
|
|
|
528
535
|
judge?: EvalJudge;
|
|
529
536
|
evidenceFilters?: EvalEvidenceFilters;
|
|
530
537
|
defaults?: EvalDefaults;
|
|
538
|
+
/** Application-enforced ceilings for local and remote runs. */
|
|
539
|
+
limits?: {
|
|
540
|
+
concurrency?: number;
|
|
541
|
+
caseTimeoutMs?: number;
|
|
542
|
+
};
|
|
531
543
|
};
|
|
532
544
|
type EvalProgress = {
|
|
533
545
|
type: "case-dispatch";
|
|
@@ -571,6 +583,7 @@ type EvalProgress = {
|
|
|
571
583
|
result: EvalCaseResult;
|
|
572
584
|
};
|
|
573
585
|
type EvalRunOptions = {
|
|
586
|
+
caseTimeoutMs?: number;
|
|
574
587
|
promptSnapshot?: _kortyx_prompts.PromptSnapshot;
|
|
575
588
|
/** Studio captures execution without invoking the code judge. Local runs default to app. */
|
|
576
589
|
grading?: "app" | "studio";
|
|
@@ -739,17 +752,44 @@ declare const EvalObservationSchema: z.ZodObject<{
|
|
|
739
752
|
}, z.core.$strict>>;
|
|
740
753
|
}, z.core.$strict>;
|
|
741
754
|
declare const EvalIssueSchema: z.ZodObject<{
|
|
755
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
756
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
757
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
758
|
+
capture: z.ZodObject<{
|
|
759
|
+
status: z.ZodEnum<{
|
|
760
|
+
complete: "complete";
|
|
761
|
+
failed: "failed";
|
|
762
|
+
partial: "partial";
|
|
763
|
+
}>;
|
|
764
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
765
|
+
path: z.ZodString;
|
|
766
|
+
reason: z.ZodString;
|
|
767
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
768
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
769
|
+
}, z.core.$strict>>;
|
|
770
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
771
|
+
path: z.ZodString;
|
|
772
|
+
reason: z.ZodString;
|
|
773
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
774
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
775
|
+
}, z.core.$strict>>;
|
|
776
|
+
}, z.core.$strict>;
|
|
777
|
+
}, z.core.$strict>>;
|
|
778
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
779
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
780
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
781
|
+
}, z.core.$strict>>;
|
|
742
782
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
743
783
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
744
784
|
phase: z.ZodEnum<{
|
|
745
785
|
params: "params";
|
|
786
|
+
cleanup: "cleanup";
|
|
746
787
|
execute: "execute";
|
|
747
788
|
setup: "setup";
|
|
748
789
|
responder: "responder";
|
|
749
790
|
reference: "reference";
|
|
750
791
|
grading: "grading";
|
|
751
792
|
verification: "verification";
|
|
752
|
-
cleanup: "cleanup";
|
|
753
793
|
reporting: "reporting";
|
|
754
794
|
}>;
|
|
755
795
|
code: z.ZodString;
|
|
@@ -957,13 +997,13 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
957
997
|
}, z.core.$strict>>;
|
|
958
998
|
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
959
999
|
params: "params";
|
|
1000
|
+
cleanup: "cleanup";
|
|
960
1001
|
execute: "execute";
|
|
961
1002
|
setup: "setup";
|
|
962
1003
|
responder: "responder";
|
|
963
1004
|
reference: "reference";
|
|
964
1005
|
grading: "grading";
|
|
965
1006
|
verification: "verification";
|
|
966
|
-
cleanup: "cleanup";
|
|
967
1007
|
reporting: "reporting";
|
|
968
1008
|
}> & z.core.$partial, z.ZodNumber>>;
|
|
969
1009
|
caseId: z.ZodString;
|
|
@@ -1149,17 +1189,44 @@ declare const EvalCaseResultSchema: z.ZodObject<{
|
|
|
1149
1189
|
}, z.core.$strict>>;
|
|
1150
1190
|
}, z.core.$strict>>;
|
|
1151
1191
|
errors: z.ZodArray<z.ZodObject<{
|
|
1192
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
1193
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
1194
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
1195
|
+
capture: z.ZodObject<{
|
|
1196
|
+
status: z.ZodEnum<{
|
|
1197
|
+
complete: "complete";
|
|
1198
|
+
failed: "failed";
|
|
1199
|
+
partial: "partial";
|
|
1200
|
+
}>;
|
|
1201
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
1202
|
+
path: z.ZodString;
|
|
1203
|
+
reason: z.ZodString;
|
|
1204
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1205
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1206
|
+
}, z.core.$strict>>;
|
|
1207
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
1208
|
+
path: z.ZodString;
|
|
1209
|
+
reason: z.ZodString;
|
|
1210
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1211
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1212
|
+
}, z.core.$strict>>;
|
|
1213
|
+
}, z.core.$strict>;
|
|
1214
|
+
}, z.core.$strict>>;
|
|
1215
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
1216
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
1217
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
1218
|
+
}, z.core.$strict>>;
|
|
1152
1219
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1153
1220
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1154
1221
|
phase: z.ZodEnum<{
|
|
1155
1222
|
params: "params";
|
|
1223
|
+
cleanup: "cleanup";
|
|
1156
1224
|
execute: "execute";
|
|
1157
1225
|
setup: "setup";
|
|
1158
1226
|
responder: "responder";
|
|
1159
1227
|
reference: "reference";
|
|
1160
1228
|
grading: "grading";
|
|
1161
1229
|
verification: "verification";
|
|
1162
|
-
cleanup: "cleanup";
|
|
1163
1230
|
reporting: "reporting";
|
|
1164
1231
|
}>;
|
|
1165
1232
|
code: z.ZodString;
|
|
@@ -1580,13 +1647,13 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1580
1647
|
}, z.core.$strict>>;
|
|
1581
1648
|
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
1582
1649
|
params: "params";
|
|
1650
|
+
cleanup: "cleanup";
|
|
1583
1651
|
execute: "execute";
|
|
1584
1652
|
setup: "setup";
|
|
1585
1653
|
responder: "responder";
|
|
1586
1654
|
reference: "reference";
|
|
1587
1655
|
grading: "grading";
|
|
1588
1656
|
verification: "verification";
|
|
1589
|
-
cleanup: "cleanup";
|
|
1590
1657
|
reporting: "reporting";
|
|
1591
1658
|
}> & z.core.$partial, z.ZodNumber>>;
|
|
1592
1659
|
caseId: z.ZodString;
|
|
@@ -1772,17 +1839,44 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1772
1839
|
}, z.core.$strict>>;
|
|
1773
1840
|
}, z.core.$strict>>;
|
|
1774
1841
|
errors: z.ZodArray<z.ZodObject<{
|
|
1842
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
1843
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
1844
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
1845
|
+
capture: z.ZodObject<{
|
|
1846
|
+
status: z.ZodEnum<{
|
|
1847
|
+
complete: "complete";
|
|
1848
|
+
failed: "failed";
|
|
1849
|
+
partial: "partial";
|
|
1850
|
+
}>;
|
|
1851
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
1852
|
+
path: z.ZodString;
|
|
1853
|
+
reason: z.ZodString;
|
|
1854
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1855
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1856
|
+
}, z.core.$strict>>;
|
|
1857
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
1858
|
+
path: z.ZodString;
|
|
1859
|
+
reason: z.ZodString;
|
|
1860
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1861
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1862
|
+
}, z.core.$strict>>;
|
|
1863
|
+
}, z.core.$strict>;
|
|
1864
|
+
}, z.core.$strict>>;
|
|
1865
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
1866
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
1867
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
1868
|
+
}, z.core.$strict>>;
|
|
1775
1869
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1776
1870
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1777
1871
|
phase: z.ZodEnum<{
|
|
1778
1872
|
params: "params";
|
|
1873
|
+
cleanup: "cleanup";
|
|
1779
1874
|
execute: "execute";
|
|
1780
1875
|
setup: "setup";
|
|
1781
1876
|
responder: "responder";
|
|
1782
1877
|
reference: "reference";
|
|
1783
1878
|
grading: "grading";
|
|
1784
1879
|
verification: "verification";
|
|
1785
|
-
cleanup: "cleanup";
|
|
1786
1880
|
reporting: "reporting";
|
|
1787
1881
|
}>;
|
|
1788
1882
|
code: z.ZodString;
|
|
@@ -1790,17 +1884,44 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1790
1884
|
}, z.core.$strict>>;
|
|
1791
1885
|
}, z.core.$strict>>;
|
|
1792
1886
|
errors: z.ZodArray<z.ZodObject<{
|
|
1887
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
1888
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
1889
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
1890
|
+
capture: z.ZodObject<{
|
|
1891
|
+
status: z.ZodEnum<{
|
|
1892
|
+
complete: "complete";
|
|
1893
|
+
failed: "failed";
|
|
1894
|
+
partial: "partial";
|
|
1895
|
+
}>;
|
|
1896
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
1897
|
+
path: z.ZodString;
|
|
1898
|
+
reason: z.ZodString;
|
|
1899
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1900
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1901
|
+
}, z.core.$strict>>;
|
|
1902
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
1903
|
+
path: z.ZodString;
|
|
1904
|
+
reason: z.ZodString;
|
|
1905
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1906
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1907
|
+
}, z.core.$strict>>;
|
|
1908
|
+
}, z.core.$strict>;
|
|
1909
|
+
}, z.core.$strict>>;
|
|
1910
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
1911
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
1912
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
1913
|
+
}, z.core.$strict>>;
|
|
1793
1914
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
1794
1915
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1795
1916
|
phase: z.ZodEnum<{
|
|
1796
1917
|
params: "params";
|
|
1918
|
+
cleanup: "cleanup";
|
|
1797
1919
|
execute: "execute";
|
|
1798
1920
|
setup: "setup";
|
|
1799
1921
|
responder: "responder";
|
|
1800
1922
|
reference: "reference";
|
|
1801
1923
|
grading: "grading";
|
|
1802
1924
|
verification: "verification";
|
|
1803
|
-
cleanup: "cleanup";
|
|
1804
1925
|
reporting: "reporting";
|
|
1805
1926
|
}>;
|
|
1806
1927
|
code: z.ZodString;
|
|
@@ -1808,6 +1929,22 @@ declare const EvalRunResultSchema: z.ZodObject<{
|
|
|
1808
1929
|
}, z.core.$strict>>;
|
|
1809
1930
|
}, z.core.$strict>;
|
|
1810
1931
|
declare const EvalManifestSchema: z.ZodObject<{
|
|
1932
|
+
execution: z.ZodOptional<z.ZodObject<{
|
|
1933
|
+
defaults: z.ZodObject<{
|
|
1934
|
+
repetitions: z.ZodOptional<z.ZodNumber>;
|
|
1935
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
1936
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
1937
|
+
}, z.core.$strict>;
|
|
1938
|
+
limits: z.ZodObject<{
|
|
1939
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
1940
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
1941
|
+
}, z.core.$strict>;
|
|
1942
|
+
overrides: z.ZodArray<z.ZodEnum<{
|
|
1943
|
+
repetitions: "repetitions";
|
|
1944
|
+
concurrency: "concurrency";
|
|
1945
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
1946
|
+
}>>;
|
|
1947
|
+
}, z.core.$strict>>;
|
|
1811
1948
|
schemaVersion: z.ZodLiteral<1>;
|
|
1812
1949
|
studioJudging: z.ZodOptional<z.ZodLiteral<true>>;
|
|
1813
1950
|
promptContracts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
@@ -1958,13 +2095,13 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
1958
2095
|
repetition: z.ZodNumber;
|
|
1959
2096
|
phase: z.ZodEnum<{
|
|
1960
2097
|
params: "params";
|
|
2098
|
+
cleanup: "cleanup";
|
|
1961
2099
|
execute: "execute";
|
|
1962
2100
|
setup: "setup";
|
|
1963
2101
|
responder: "responder";
|
|
1964
2102
|
reference: "reference";
|
|
1965
2103
|
grading: "grading";
|
|
1966
2104
|
verification: "verification";
|
|
1967
|
-
cleanup: "cleanup";
|
|
1968
2105
|
reporting: "reporting";
|
|
1969
2106
|
}>;
|
|
1970
2107
|
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
@@ -2154,13 +2291,13 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2154
2291
|
}, z.core.$strict>>;
|
|
2155
2292
|
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
2156
2293
|
params: "params";
|
|
2294
|
+
cleanup: "cleanup";
|
|
2157
2295
|
execute: "execute";
|
|
2158
2296
|
setup: "setup";
|
|
2159
2297
|
responder: "responder";
|
|
2160
2298
|
reference: "reference";
|
|
2161
2299
|
grading: "grading";
|
|
2162
2300
|
verification: "verification";
|
|
2163
|
-
cleanup: "cleanup";
|
|
2164
2301
|
reporting: "reporting";
|
|
2165
2302
|
}> & z.core.$partial, z.ZodNumber>>;
|
|
2166
2303
|
caseId: z.ZodString;
|
|
@@ -2346,17 +2483,44 @@ declare const EvalProgressSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2346
2483
|
}, z.core.$strict>>;
|
|
2347
2484
|
}, z.core.$strict>>;
|
|
2348
2485
|
errors: z.ZodArray<z.ZodObject<{
|
|
2486
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
2487
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
2488
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
2489
|
+
capture: z.ZodObject<{
|
|
2490
|
+
status: z.ZodEnum<{
|
|
2491
|
+
complete: "complete";
|
|
2492
|
+
failed: "failed";
|
|
2493
|
+
partial: "partial";
|
|
2494
|
+
}>;
|
|
2495
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
2496
|
+
path: z.ZodString;
|
|
2497
|
+
reason: z.ZodString;
|
|
2498
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
2499
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
2500
|
+
}, z.core.$strict>>;
|
|
2501
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
2502
|
+
path: z.ZodString;
|
|
2503
|
+
reason: z.ZodString;
|
|
2504
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
2505
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
2506
|
+
}, z.core.$strict>>;
|
|
2507
|
+
}, z.core.$strict>;
|
|
2508
|
+
}, z.core.$strict>>;
|
|
2509
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
2510
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
2511
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
2512
|
+
}, z.core.$strict>>;
|
|
2349
2513
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2350
2514
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2351
2515
|
phase: z.ZodEnum<{
|
|
2352
2516
|
params: "params";
|
|
2517
|
+
cleanup: "cleanup";
|
|
2353
2518
|
execute: "execute";
|
|
2354
2519
|
setup: "setup";
|
|
2355
2520
|
responder: "responder";
|
|
2356
2521
|
reference: "reference";
|
|
2357
2522
|
grading: "grading";
|
|
2358
2523
|
verification: "verification";
|
|
2359
|
-
cleanup: "cleanup";
|
|
2360
2524
|
reporting: "reporting";
|
|
2361
2525
|
}>;
|
|
2362
2526
|
code: z.ZodString;
|
|
@@ -2408,6 +2572,33 @@ declare const EvalRemoteRunRequestSchema: z.ZodObject<{
|
|
|
2408
2572
|
requestedIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
2409
2573
|
}, z.core.$strict>>;
|
|
2410
2574
|
promptGroupName: z.ZodOptional<z.ZodString>;
|
|
2575
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
2576
|
+
runTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
2577
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
2578
|
+
values: z.ZodObject<{
|
|
2579
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
2580
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
2581
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
2582
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
2583
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
2584
|
+
studio: "studio";
|
|
2585
|
+
app: "app";
|
|
2586
|
+
}>>>;
|
|
2587
|
+
}, z.core.$strict>;
|
|
2588
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
2589
|
+
repetitions: "repetitions";
|
|
2590
|
+
concurrency: "concurrency";
|
|
2591
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
2592
|
+
runTimeoutMs: "runTimeoutMs";
|
|
2593
|
+
judge: "judge";
|
|
2594
|
+
}>, z.ZodEnum<{
|
|
2595
|
+
studio: "studio";
|
|
2596
|
+
run: "run";
|
|
2597
|
+
sdk: "sdk";
|
|
2598
|
+
"sdk-default": "sdk-default";
|
|
2599
|
+
"studio-default": "studio-default";
|
|
2600
|
+
}>>;
|
|
2601
|
+
}, z.core.$strict>>;
|
|
2411
2602
|
suiteId: z.ZodString;
|
|
2412
2603
|
suiteRevision: z.ZodString;
|
|
2413
2604
|
grading: z.ZodOptional<z.ZodEnum<{
|
|
@@ -2432,11 +2623,36 @@ declare const EvalRemoteRunRequestSchema: z.ZodObject<{
|
|
|
2432
2623
|
}, z.core.$strict>>;
|
|
2433
2624
|
}, z.core.$strict>;
|
|
2434
2625
|
declare const StudioEvalStartRequestSchema: z.ZodObject<{
|
|
2626
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
2627
|
+
runTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
2435
2628
|
suiteId: z.ZodString;
|
|
2436
2629
|
suiteRevision: z.ZodString;
|
|
2437
2630
|
caseIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
2438
|
-
|
|
2439
|
-
|
|
2631
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
2632
|
+
values: z.ZodObject<{
|
|
2633
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
2634
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
2635
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
2636
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
2637
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
2638
|
+
studio: "studio";
|
|
2639
|
+
app: "app";
|
|
2640
|
+
}>>>;
|
|
2641
|
+
}, z.core.$strict>;
|
|
2642
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
2643
|
+
repetitions: "repetitions";
|
|
2644
|
+
concurrency: "concurrency";
|
|
2645
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
2646
|
+
runTimeoutMs: "runTimeoutMs";
|
|
2647
|
+
judge: "judge";
|
|
2648
|
+
}>, z.ZodEnum<{
|
|
2649
|
+
studio: "studio";
|
|
2650
|
+
run: "run";
|
|
2651
|
+
sdk: "sdk";
|
|
2652
|
+
"sdk-default": "sdk-default";
|
|
2653
|
+
"studio-default": "studio-default";
|
|
2654
|
+
}>>;
|
|
2655
|
+
}, z.core.$strict>>;
|
|
2440
2656
|
promptSelection: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
2441
2657
|
type: z.ZodLiteral<"live">;
|
|
2442
2658
|
}, z.core.$strict>, z.ZodObject<{
|
|
@@ -2448,10 +2664,12 @@ declare const StudioEvalStartRequestSchema: z.ZodObject<{
|
|
|
2448
2664
|
groupId: z.ZodUUID;
|
|
2449
2665
|
}, z.core.$strict>], "type">>;
|
|
2450
2666
|
targetId: z.ZodString;
|
|
2451
|
-
judge: z.
|
|
2667
|
+
judge: z.ZodOptional<z.ZodEnum<{
|
|
2452
2668
|
studio: "studio";
|
|
2453
2669
|
app: "app";
|
|
2454
2670
|
}>>;
|
|
2671
|
+
repetitions: z.ZodOptional<z.ZodNumber>;
|
|
2672
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
2455
2673
|
}, z.core.$strict>;
|
|
2456
2674
|
declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
2457
2675
|
type: z.ZodLiteral<"execution-status">;
|
|
@@ -2497,13 +2715,13 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2497
2715
|
repetition: z.ZodNumber;
|
|
2498
2716
|
phase: z.ZodEnum<{
|
|
2499
2717
|
params: "params";
|
|
2718
|
+
cleanup: "cleanup";
|
|
2500
2719
|
execute: "execute";
|
|
2501
2720
|
setup: "setup";
|
|
2502
2721
|
responder: "responder";
|
|
2503
2722
|
reference: "reference";
|
|
2504
2723
|
grading: "grading";
|
|
2505
2724
|
verification: "verification";
|
|
2506
|
-
cleanup: "cleanup";
|
|
2507
2725
|
reporting: "reporting";
|
|
2508
2726
|
}>;
|
|
2509
2727
|
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
@@ -2693,13 +2911,13 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2693
2911
|
}, z.core.$strict>>;
|
|
2694
2912
|
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
2695
2913
|
params: "params";
|
|
2914
|
+
cleanup: "cleanup";
|
|
2696
2915
|
execute: "execute";
|
|
2697
2916
|
setup: "setup";
|
|
2698
2917
|
responder: "responder";
|
|
2699
2918
|
reference: "reference";
|
|
2700
2919
|
grading: "grading";
|
|
2701
2920
|
verification: "verification";
|
|
2702
|
-
cleanup: "cleanup";
|
|
2703
2921
|
reporting: "reporting";
|
|
2704
2922
|
}> & z.core.$partial, z.ZodNumber>>;
|
|
2705
2923
|
caseId: z.ZodString;
|
|
@@ -2885,17 +3103,44 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
2885
3103
|
}, z.core.$strict>>;
|
|
2886
3104
|
}, z.core.$strict>>;
|
|
2887
3105
|
errors: z.ZodArray<z.ZodObject<{
|
|
3106
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
3107
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
3108
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
3109
|
+
capture: z.ZodObject<{
|
|
3110
|
+
status: z.ZodEnum<{
|
|
3111
|
+
complete: "complete";
|
|
3112
|
+
failed: "failed";
|
|
3113
|
+
partial: "partial";
|
|
3114
|
+
}>;
|
|
3115
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
3116
|
+
path: z.ZodString;
|
|
3117
|
+
reason: z.ZodString;
|
|
3118
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
3119
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
3120
|
+
}, z.core.$strict>>;
|
|
3121
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
3122
|
+
path: z.ZodString;
|
|
3123
|
+
reason: z.ZodString;
|
|
3124
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
3125
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
3126
|
+
}, z.core.$strict>>;
|
|
3127
|
+
}, z.core.$strict>;
|
|
3128
|
+
}, z.core.$strict>>;
|
|
3129
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
3130
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
3131
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
3132
|
+
}, z.core.$strict>>;
|
|
2888
3133
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
2889
3134
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
2890
3135
|
phase: z.ZodEnum<{
|
|
2891
3136
|
params: "params";
|
|
3137
|
+
cleanup: "cleanup";
|
|
2892
3138
|
execute: "execute";
|
|
2893
3139
|
setup: "setup";
|
|
2894
3140
|
responder: "responder";
|
|
2895
3141
|
reference: "reference";
|
|
2896
3142
|
grading: "grading";
|
|
2897
3143
|
verification: "verification";
|
|
2898
|
-
cleanup: "cleanup";
|
|
2899
3144
|
reporting: "reporting";
|
|
2900
3145
|
}>;
|
|
2901
3146
|
code: z.ZodString;
|
|
@@ -3031,13 +3276,13 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
3031
3276
|
}, z.core.$strict>>;
|
|
3032
3277
|
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
3033
3278
|
params: "params";
|
|
3279
|
+
cleanup: "cleanup";
|
|
3034
3280
|
execute: "execute";
|
|
3035
3281
|
setup: "setup";
|
|
3036
3282
|
responder: "responder";
|
|
3037
3283
|
reference: "reference";
|
|
3038
3284
|
grading: "grading";
|
|
3039
3285
|
verification: "verification";
|
|
3040
|
-
cleanup: "cleanup";
|
|
3041
3286
|
reporting: "reporting";
|
|
3042
3287
|
}> & z.core.$partial, z.ZodNumber>>;
|
|
3043
3288
|
caseId: z.ZodString;
|
|
@@ -3223,17 +3468,44 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
3223
3468
|
}, z.core.$strict>>;
|
|
3224
3469
|
}, z.core.$strict>>;
|
|
3225
3470
|
errors: z.ZodArray<z.ZodObject<{
|
|
3471
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
3472
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
3473
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
3474
|
+
capture: z.ZodObject<{
|
|
3475
|
+
status: z.ZodEnum<{
|
|
3476
|
+
complete: "complete";
|
|
3477
|
+
failed: "failed";
|
|
3478
|
+
partial: "partial";
|
|
3479
|
+
}>;
|
|
3480
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
3481
|
+
path: z.ZodString;
|
|
3482
|
+
reason: z.ZodString;
|
|
3483
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
3484
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
3485
|
+
}, z.core.$strict>>;
|
|
3486
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
3487
|
+
path: z.ZodString;
|
|
3488
|
+
reason: z.ZodString;
|
|
3489
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
3490
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
3491
|
+
}, z.core.$strict>>;
|
|
3492
|
+
}, z.core.$strict>;
|
|
3493
|
+
}, z.core.$strict>>;
|
|
3494
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
3495
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
3496
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
3497
|
+
}, z.core.$strict>>;
|
|
3226
3498
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
3227
3499
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
3228
3500
|
phase: z.ZodEnum<{
|
|
3229
3501
|
params: "params";
|
|
3502
|
+
cleanup: "cleanup";
|
|
3230
3503
|
execute: "execute";
|
|
3231
3504
|
setup: "setup";
|
|
3232
3505
|
responder: "responder";
|
|
3233
3506
|
reference: "reference";
|
|
3234
3507
|
grading: "grading";
|
|
3235
3508
|
verification: "verification";
|
|
3236
|
-
cleanup: "cleanup";
|
|
3237
3509
|
reporting: "reporting";
|
|
3238
3510
|
}>;
|
|
3239
3511
|
code: z.ZodString;
|
|
@@ -3241,17 +3513,44 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
3241
3513
|
}, z.core.$strict>>;
|
|
3242
3514
|
}, z.core.$strict>>;
|
|
3243
3515
|
errors: z.ZodArray<z.ZodObject<{
|
|
3516
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
3517
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
3518
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
3519
|
+
capture: z.ZodObject<{
|
|
3520
|
+
status: z.ZodEnum<{
|
|
3521
|
+
complete: "complete";
|
|
3522
|
+
failed: "failed";
|
|
3523
|
+
partial: "partial";
|
|
3524
|
+
}>;
|
|
3525
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
3526
|
+
path: z.ZodString;
|
|
3527
|
+
reason: z.ZodString;
|
|
3528
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
3529
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
3530
|
+
}, z.core.$strict>>;
|
|
3531
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
3532
|
+
path: z.ZodString;
|
|
3533
|
+
reason: z.ZodString;
|
|
3534
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
3535
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
3536
|
+
}, z.core.$strict>>;
|
|
3537
|
+
}, z.core.$strict>;
|
|
3538
|
+
}, z.core.$strict>>;
|
|
3539
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
3540
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
3541
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
3542
|
+
}, z.core.$strict>>;
|
|
3244
3543
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
3245
3544
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
3246
3545
|
phase: z.ZodEnum<{
|
|
3247
3546
|
params: "params";
|
|
3547
|
+
cleanup: "cleanup";
|
|
3248
3548
|
execute: "execute";
|
|
3249
3549
|
setup: "setup";
|
|
3250
3550
|
responder: "responder";
|
|
3251
3551
|
reference: "reference";
|
|
3252
3552
|
grading: "grading";
|
|
3253
3553
|
verification: "verification";
|
|
3254
|
-
cleanup: "cleanup";
|
|
3255
3554
|
reporting: "reporting";
|
|
3256
3555
|
}>;
|
|
3257
3556
|
code: z.ZodString;
|
|
@@ -3261,6 +3560,29 @@ declare const EvalWireEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
3261
3560
|
sequence: z.ZodOptional<z.ZodNumber>;
|
|
3262
3561
|
}, z.core.$strict>, z.ZodObject<{
|
|
3263
3562
|
type: z.ZodLiteral<"error">;
|
|
3563
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
3564
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
3565
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
3566
|
+
capture: z.ZodObject<{
|
|
3567
|
+
status: z.ZodEnum<{
|
|
3568
|
+
complete: "complete";
|
|
3569
|
+
failed: "failed";
|
|
3570
|
+
partial: "partial";
|
|
3571
|
+
}>;
|
|
3572
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
3573
|
+
path: z.ZodString;
|
|
3574
|
+
reason: z.ZodString;
|
|
3575
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
3576
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
3577
|
+
}, z.core.$strict>>;
|
|
3578
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
3579
|
+
path: z.ZodString;
|
|
3580
|
+
reason: z.ZodString;
|
|
3581
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
3582
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
3583
|
+
}, z.core.$strict>>;
|
|
3584
|
+
}, z.core.$strict>;
|
|
3585
|
+
}, z.core.$strict>>;
|
|
3264
3586
|
message: z.ZodString;
|
|
3265
3587
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core_errors.FailureDescriptor, _kortyx_core_errors.FailureDescriptor>, z.ZodTransform<_kortyx_core_errors.SerializedFailureDescriptor, _kortyx_core_errors.FailureDescriptor>>>;
|
|
3266
3588
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
@@ -3284,7 +3606,7 @@ type DefinedSuite<Params> = Omit<EvalSuite<Params>, "cases"> & {
|
|
|
3284
3606
|
/** Author a suite with typed case parameters. Runtime validation belongs to createEvals. */
|
|
3285
3607
|
declare function defineSuite<Params = EvalJson | undefined>(suite: DefinedSuite<Params>): DefinedSuite<Params>;
|
|
3286
3608
|
|
|
3287
|
-
/**
|
|
3609
|
+
/** Authenticated eval evidence retains original exceptions with credential-only redaction. */
|
|
3288
3610
|
declare function createEvalIssue(phase: EvalPhase, error: unknown, options?: {
|
|
3289
3611
|
code?: string;
|
|
3290
3612
|
diagnosticId?: string;
|
|
@@ -3303,6 +3625,25 @@ declare const EvaluationMetadataSchema: z.ZodObject<{
|
|
|
3303
3625
|
deploymentUrl: z.ZodOptional<z.ZodURL>;
|
|
3304
3626
|
}, z.core.$strict>;
|
|
3305
3627
|
declare const StudioEvaluationStartRequestSchema: z.ZodObject<{
|
|
3628
|
+
metadata: z.ZodDefault<z.ZodObject<{
|
|
3629
|
+
source: z.ZodDefault<z.ZodEnum<{
|
|
3630
|
+
manual: "manual";
|
|
3631
|
+
deployment: "deployment";
|
|
3632
|
+
schedule: "schedule";
|
|
3633
|
+
ci: "ci";
|
|
3634
|
+
}>>;
|
|
3635
|
+
commit: z.ZodOptional<z.ZodString>;
|
|
3636
|
+
deploymentUrl: z.ZodOptional<z.ZodURL>;
|
|
3637
|
+
}, z.core.$strict>>;
|
|
3638
|
+
idempotencyKey: z.ZodOptional<z.ZodString>;
|
|
3639
|
+
repetitions: z.ZodOptional<z.ZodNumber>;
|
|
3640
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
3641
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
3642
|
+
runTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
3643
|
+
judge: z.ZodOptional<z.ZodEnum<{
|
|
3644
|
+
studio: "studio";
|
|
3645
|
+
app: "app";
|
|
3646
|
+
}>>;
|
|
3306
3647
|
targetId: z.ZodString;
|
|
3307
3648
|
name: z.ZodOptional<z.ZodString>;
|
|
3308
3649
|
promptSelection: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
@@ -3324,26 +3665,66 @@ declare const StudioEvaluationStartRequestSchema: z.ZodObject<{
|
|
|
3324
3665
|
suiteRevision: z.ZodString;
|
|
3325
3666
|
caseIds: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
3326
3667
|
}, z.core.$strict>>;
|
|
3327
|
-
judge: z.ZodDefault<z.ZodEnum<{
|
|
3328
|
-
studio: "studio";
|
|
3329
|
-
app: "app";
|
|
3330
|
-
}>>;
|
|
3331
|
-
repetitions: z.ZodDefault<z.ZodNumber>;
|
|
3332
|
-
concurrency: z.ZodDefault<z.ZodNumber>;
|
|
3333
|
-
metadata: z.ZodDefault<z.ZodObject<{
|
|
3334
|
-
source: z.ZodDefault<z.ZodEnum<{
|
|
3335
|
-
manual: "manual";
|
|
3336
|
-
deployment: "deployment";
|
|
3337
|
-
schedule: "schedule";
|
|
3338
|
-
ci: "ci";
|
|
3339
|
-
}>>;
|
|
3340
|
-
commit: z.ZodOptional<z.ZodString>;
|
|
3341
|
-
deploymentUrl: z.ZodOptional<z.ZodURL>;
|
|
3342
|
-
}, z.core.$strict>>;
|
|
3343
|
-
idempotencyKey: z.ZodOptional<z.ZodString>;
|
|
3344
3668
|
}, z.core.$strict>;
|
|
3345
3669
|
type StudioEvaluationStartRequest = z.input<typeof StudioEvaluationStartRequestSchema>;
|
|
3346
3670
|
declare const StudioEvaluationSummarySchema: z.ZodObject<{
|
|
3671
|
+
termination: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
3672
|
+
code: z.ZodEnum<{
|
|
3673
|
+
EVAL_RUN_TIMEOUT: "EVAL_RUN_TIMEOUT";
|
|
3674
|
+
EVAL_CANCELLED: "EVAL_CANCELLED";
|
|
3675
|
+
EVAL_LEASE_LOST: "EVAL_LEASE_LOST";
|
|
3676
|
+
EVAL_EXECUTION_UNKNOWN: "EVAL_EXECUTION_UNKNOWN";
|
|
3677
|
+
EVAL_EXECUTOR_FAILED: "EVAL_EXECUTOR_FAILED";
|
|
3678
|
+
}>;
|
|
3679
|
+
message: z.ZodString;
|
|
3680
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
3681
|
+
phase: z.ZodOptional<z.ZodString>;
|
|
3682
|
+
cleanup: z.ZodOptional<z.ZodEnum<{
|
|
3683
|
+
unknown: "unknown";
|
|
3684
|
+
signalled: "signalled";
|
|
3685
|
+
}>>;
|
|
3686
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
3687
|
+
}, z.core.$strict>>>;
|
|
3688
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
3689
|
+
values: z.ZodObject<{
|
|
3690
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
3691
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
3692
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
3693
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
3694
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
3695
|
+
studio: "studio";
|
|
3696
|
+
app: "app";
|
|
3697
|
+
}>>>;
|
|
3698
|
+
}, z.core.$strict>;
|
|
3699
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
3700
|
+
repetitions: "repetitions";
|
|
3701
|
+
concurrency: "concurrency";
|
|
3702
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
3703
|
+
runTimeoutMs: "runTimeoutMs";
|
|
3704
|
+
judge: "judge";
|
|
3705
|
+
}>, z.ZodEnum<{
|
|
3706
|
+
studio: "studio";
|
|
3707
|
+
run: "run";
|
|
3708
|
+
sdk: "sdk";
|
|
3709
|
+
"sdk-default": "sdk-default";
|
|
3710
|
+
"studio-default": "studio-default";
|
|
3711
|
+
}>>;
|
|
3712
|
+
}, z.core.$strict>>;
|
|
3713
|
+
interruptedSuites: z.ZodOptional<z.ZodNumber>;
|
|
3714
|
+
failures: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
3715
|
+
suiteRunId: z.ZodUUID;
|
|
3716
|
+
caseId: z.ZodOptional<z.ZodString>;
|
|
3717
|
+
repetition: z.ZodOptional<z.ZodNumber>;
|
|
3718
|
+
phase: z.ZodString;
|
|
3719
|
+
code: z.ZodString;
|
|
3720
|
+
message: z.ZodString;
|
|
3721
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
3722
|
+
}, z.core.$strip>>>;
|
|
3723
|
+
pendingAttempts: z.ZodOptional<z.ZodObject<{
|
|
3724
|
+
notStarted: z.ZodNumber;
|
|
3725
|
+
executing: z.ZodNumber;
|
|
3726
|
+
awaitingGrading: z.ZodNumber;
|
|
3727
|
+
}, z.core.$strip>>;
|
|
3347
3728
|
id: z.ZodUUID;
|
|
3348
3729
|
name: z.ZodString;
|
|
3349
3730
|
targetId: z.ZodString;
|
|
@@ -3456,6 +3837,63 @@ declare const StudioEvaluationSummarySchema: z.ZodObject<{
|
|
|
3456
3837
|
}, z.core.$strip>;
|
|
3457
3838
|
declare const StudioEvaluationHistorySchema: z.ZodObject<{
|
|
3458
3839
|
runs: z.ZodArray<z.ZodObject<{
|
|
3840
|
+
termination: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
3841
|
+
code: z.ZodEnum<{
|
|
3842
|
+
EVAL_RUN_TIMEOUT: "EVAL_RUN_TIMEOUT";
|
|
3843
|
+
EVAL_CANCELLED: "EVAL_CANCELLED";
|
|
3844
|
+
EVAL_LEASE_LOST: "EVAL_LEASE_LOST";
|
|
3845
|
+
EVAL_EXECUTION_UNKNOWN: "EVAL_EXECUTION_UNKNOWN";
|
|
3846
|
+
EVAL_EXECUTOR_FAILED: "EVAL_EXECUTOR_FAILED";
|
|
3847
|
+
}>;
|
|
3848
|
+
message: z.ZodString;
|
|
3849
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
3850
|
+
phase: z.ZodOptional<z.ZodString>;
|
|
3851
|
+
cleanup: z.ZodOptional<z.ZodEnum<{
|
|
3852
|
+
unknown: "unknown";
|
|
3853
|
+
signalled: "signalled";
|
|
3854
|
+
}>>;
|
|
3855
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
3856
|
+
}, z.core.$strict>>>;
|
|
3857
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
3858
|
+
values: z.ZodObject<{
|
|
3859
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
3860
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
3861
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
3862
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
3863
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
3864
|
+
studio: "studio";
|
|
3865
|
+
app: "app";
|
|
3866
|
+
}>>>;
|
|
3867
|
+
}, z.core.$strict>;
|
|
3868
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
3869
|
+
repetitions: "repetitions";
|
|
3870
|
+
concurrency: "concurrency";
|
|
3871
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
3872
|
+
runTimeoutMs: "runTimeoutMs";
|
|
3873
|
+
judge: "judge";
|
|
3874
|
+
}>, z.ZodEnum<{
|
|
3875
|
+
studio: "studio";
|
|
3876
|
+
run: "run";
|
|
3877
|
+
sdk: "sdk";
|
|
3878
|
+
"sdk-default": "sdk-default";
|
|
3879
|
+
"studio-default": "studio-default";
|
|
3880
|
+
}>>;
|
|
3881
|
+
}, z.core.$strict>>;
|
|
3882
|
+
interruptedSuites: z.ZodOptional<z.ZodNumber>;
|
|
3883
|
+
failures: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
3884
|
+
suiteRunId: z.ZodUUID;
|
|
3885
|
+
caseId: z.ZodOptional<z.ZodString>;
|
|
3886
|
+
repetition: z.ZodOptional<z.ZodNumber>;
|
|
3887
|
+
phase: z.ZodString;
|
|
3888
|
+
code: z.ZodString;
|
|
3889
|
+
message: z.ZodString;
|
|
3890
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
3891
|
+
}, z.core.$strip>>>;
|
|
3892
|
+
pendingAttempts: z.ZodOptional<z.ZodObject<{
|
|
3893
|
+
notStarted: z.ZodNumber;
|
|
3894
|
+
executing: z.ZodNumber;
|
|
3895
|
+
awaitingGrading: z.ZodNumber;
|
|
3896
|
+
}, z.core.$strip>>;
|
|
3459
3897
|
id: z.ZodUUID;
|
|
3460
3898
|
name: z.ZodString;
|
|
3461
3899
|
targetId: z.ZodString;
|
|
@@ -3569,6 +4007,63 @@ declare const StudioEvaluationHistorySchema: z.ZodObject<{
|
|
|
3569
4007
|
}, z.core.$strip>;
|
|
3570
4008
|
declare const StudioEvaluationDetailSchema: z.ZodObject<{
|
|
3571
4009
|
run: z.ZodObject<{
|
|
4010
|
+
termination: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
4011
|
+
code: z.ZodEnum<{
|
|
4012
|
+
EVAL_RUN_TIMEOUT: "EVAL_RUN_TIMEOUT";
|
|
4013
|
+
EVAL_CANCELLED: "EVAL_CANCELLED";
|
|
4014
|
+
EVAL_LEASE_LOST: "EVAL_LEASE_LOST";
|
|
4015
|
+
EVAL_EXECUTION_UNKNOWN: "EVAL_EXECUTION_UNKNOWN";
|
|
4016
|
+
EVAL_EXECUTOR_FAILED: "EVAL_EXECUTOR_FAILED";
|
|
4017
|
+
}>;
|
|
4018
|
+
message: z.ZodString;
|
|
4019
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
4020
|
+
phase: z.ZodOptional<z.ZodString>;
|
|
4021
|
+
cleanup: z.ZodOptional<z.ZodEnum<{
|
|
4022
|
+
unknown: "unknown";
|
|
4023
|
+
signalled: "signalled";
|
|
4024
|
+
}>>;
|
|
4025
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
4026
|
+
}, z.core.$strict>>>;
|
|
4027
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
4028
|
+
values: z.ZodObject<{
|
|
4029
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4030
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4031
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4032
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4033
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
4034
|
+
studio: "studio";
|
|
4035
|
+
app: "app";
|
|
4036
|
+
}>>>;
|
|
4037
|
+
}, z.core.$strict>;
|
|
4038
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
4039
|
+
repetitions: "repetitions";
|
|
4040
|
+
concurrency: "concurrency";
|
|
4041
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
4042
|
+
runTimeoutMs: "runTimeoutMs";
|
|
4043
|
+
judge: "judge";
|
|
4044
|
+
}>, z.ZodEnum<{
|
|
4045
|
+
studio: "studio";
|
|
4046
|
+
run: "run";
|
|
4047
|
+
sdk: "sdk";
|
|
4048
|
+
"sdk-default": "sdk-default";
|
|
4049
|
+
"studio-default": "studio-default";
|
|
4050
|
+
}>>;
|
|
4051
|
+
}, z.core.$strict>>;
|
|
4052
|
+
interruptedSuites: z.ZodOptional<z.ZodNumber>;
|
|
4053
|
+
failures: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4054
|
+
suiteRunId: z.ZodUUID;
|
|
4055
|
+
caseId: z.ZodOptional<z.ZodString>;
|
|
4056
|
+
repetition: z.ZodOptional<z.ZodNumber>;
|
|
4057
|
+
phase: z.ZodString;
|
|
4058
|
+
code: z.ZodString;
|
|
4059
|
+
message: z.ZodString;
|
|
4060
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
4061
|
+
}, z.core.$strip>>>;
|
|
4062
|
+
pendingAttempts: z.ZodOptional<z.ZodObject<{
|
|
4063
|
+
notStarted: z.ZodNumber;
|
|
4064
|
+
executing: z.ZodNumber;
|
|
4065
|
+
awaitingGrading: z.ZodNumber;
|
|
4066
|
+
}, z.core.$strip>>;
|
|
3572
4067
|
id: z.ZodUUID;
|
|
3573
4068
|
name: z.ZodString;
|
|
3574
4069
|
targetId: z.ZodString;
|
|
@@ -3679,6 +4174,92 @@ declare const StudioEvaluationDetailSchema: z.ZodObject<{
|
|
|
3679
4174
|
}, z.core.$strip>>;
|
|
3680
4175
|
legacy: z.ZodOptional<z.ZodBoolean>;
|
|
3681
4176
|
suites: z.ZodArray<z.ZodObject<{
|
|
4177
|
+
issue: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
4178
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
4179
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
4180
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
4181
|
+
capture: z.ZodObject<{
|
|
4182
|
+
status: z.ZodEnum<{
|
|
4183
|
+
complete: "complete";
|
|
4184
|
+
failed: "failed";
|
|
4185
|
+
partial: "partial";
|
|
4186
|
+
}>;
|
|
4187
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
4188
|
+
path: z.ZodString;
|
|
4189
|
+
reason: z.ZodString;
|
|
4190
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
4191
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
4192
|
+
}, z.core.$strict>>;
|
|
4193
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
4194
|
+
path: z.ZodString;
|
|
4195
|
+
reason: z.ZodString;
|
|
4196
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
4197
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
4198
|
+
}, z.core.$strict>>;
|
|
4199
|
+
}, z.core.$strict>;
|
|
4200
|
+
}, z.core.$strict>>;
|
|
4201
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
4202
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
4203
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
4204
|
+
}, z.core.$strict>>;
|
|
4205
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4206
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4207
|
+
phase: z.ZodEnum<{
|
|
4208
|
+
params: "params";
|
|
4209
|
+
cleanup: "cleanup";
|
|
4210
|
+
execute: "execute";
|
|
4211
|
+
setup: "setup";
|
|
4212
|
+
responder: "responder";
|
|
4213
|
+
reference: "reference";
|
|
4214
|
+
grading: "grading";
|
|
4215
|
+
verification: "verification";
|
|
4216
|
+
reporting: "reporting";
|
|
4217
|
+
}>;
|
|
4218
|
+
code: z.ZodString;
|
|
4219
|
+
message: z.ZodString;
|
|
4220
|
+
}, z.core.$strict>>>;
|
|
4221
|
+
termination: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
4222
|
+
code: z.ZodEnum<{
|
|
4223
|
+
EVAL_RUN_TIMEOUT: "EVAL_RUN_TIMEOUT";
|
|
4224
|
+
EVAL_CANCELLED: "EVAL_CANCELLED";
|
|
4225
|
+
EVAL_LEASE_LOST: "EVAL_LEASE_LOST";
|
|
4226
|
+
EVAL_EXECUTION_UNKNOWN: "EVAL_EXECUTION_UNKNOWN";
|
|
4227
|
+
EVAL_EXECUTOR_FAILED: "EVAL_EXECUTOR_FAILED";
|
|
4228
|
+
}>;
|
|
4229
|
+
message: z.ZodString;
|
|
4230
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
4231
|
+
phase: z.ZodOptional<z.ZodString>;
|
|
4232
|
+
cleanup: z.ZodOptional<z.ZodEnum<{
|
|
4233
|
+
unknown: "unknown";
|
|
4234
|
+
signalled: "signalled";
|
|
4235
|
+
}>>;
|
|
4236
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
4237
|
+
}, z.core.$strict>>>;
|
|
4238
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
4239
|
+
values: z.ZodObject<{
|
|
4240
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4241
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4242
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4243
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4244
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
4245
|
+
studio: "studio";
|
|
4246
|
+
app: "app";
|
|
4247
|
+
}>>>;
|
|
4248
|
+
}, z.core.$strict>;
|
|
4249
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
4250
|
+
repetitions: "repetitions";
|
|
4251
|
+
concurrency: "concurrency";
|
|
4252
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
4253
|
+
runTimeoutMs: "runTimeoutMs";
|
|
4254
|
+
judge: "judge";
|
|
4255
|
+
}>, z.ZodEnum<{
|
|
4256
|
+
studio: "studio";
|
|
4257
|
+
run: "run";
|
|
4258
|
+
sdk: "sdk";
|
|
4259
|
+
"sdk-default": "sdk-default";
|
|
4260
|
+
"studio-default": "studio-default";
|
|
4261
|
+
}>>;
|
|
4262
|
+
}, z.core.$strict>>;
|
|
3682
4263
|
id: z.ZodUUID;
|
|
3683
4264
|
evaluationId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
|
|
3684
4265
|
targetId: z.ZodString;
|
|
@@ -3773,6 +4354,63 @@ declare const StudioEvaluationDetailSchema: z.ZodObject<{
|
|
|
3773
4354
|
}, z.core.$strip>;
|
|
3774
4355
|
declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
3775
4356
|
run: z.ZodObject<{
|
|
4357
|
+
termination: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
4358
|
+
code: z.ZodEnum<{
|
|
4359
|
+
EVAL_RUN_TIMEOUT: "EVAL_RUN_TIMEOUT";
|
|
4360
|
+
EVAL_CANCELLED: "EVAL_CANCELLED";
|
|
4361
|
+
EVAL_LEASE_LOST: "EVAL_LEASE_LOST";
|
|
4362
|
+
EVAL_EXECUTION_UNKNOWN: "EVAL_EXECUTION_UNKNOWN";
|
|
4363
|
+
EVAL_EXECUTOR_FAILED: "EVAL_EXECUTOR_FAILED";
|
|
4364
|
+
}>;
|
|
4365
|
+
message: z.ZodString;
|
|
4366
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
4367
|
+
phase: z.ZodOptional<z.ZodString>;
|
|
4368
|
+
cleanup: z.ZodOptional<z.ZodEnum<{
|
|
4369
|
+
unknown: "unknown";
|
|
4370
|
+
signalled: "signalled";
|
|
4371
|
+
}>>;
|
|
4372
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
4373
|
+
}, z.core.$strict>>>;
|
|
4374
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
4375
|
+
values: z.ZodObject<{
|
|
4376
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4377
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4378
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4379
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4380
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
4381
|
+
studio: "studio";
|
|
4382
|
+
app: "app";
|
|
4383
|
+
}>>>;
|
|
4384
|
+
}, z.core.$strict>;
|
|
4385
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
4386
|
+
repetitions: "repetitions";
|
|
4387
|
+
concurrency: "concurrency";
|
|
4388
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
4389
|
+
runTimeoutMs: "runTimeoutMs";
|
|
4390
|
+
judge: "judge";
|
|
4391
|
+
}>, z.ZodEnum<{
|
|
4392
|
+
studio: "studio";
|
|
4393
|
+
run: "run";
|
|
4394
|
+
sdk: "sdk";
|
|
4395
|
+
"sdk-default": "sdk-default";
|
|
4396
|
+
"studio-default": "studio-default";
|
|
4397
|
+
}>>;
|
|
4398
|
+
}, z.core.$strict>>;
|
|
4399
|
+
interruptedSuites: z.ZodOptional<z.ZodNumber>;
|
|
4400
|
+
failures: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
4401
|
+
suiteRunId: z.ZodUUID;
|
|
4402
|
+
caseId: z.ZodOptional<z.ZodString>;
|
|
4403
|
+
repetition: z.ZodOptional<z.ZodNumber>;
|
|
4404
|
+
phase: z.ZodString;
|
|
4405
|
+
code: z.ZodString;
|
|
4406
|
+
message: z.ZodString;
|
|
4407
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
4408
|
+
}, z.core.$strip>>>;
|
|
4409
|
+
pendingAttempts: z.ZodOptional<z.ZodObject<{
|
|
4410
|
+
notStarted: z.ZodNumber;
|
|
4411
|
+
executing: z.ZodNumber;
|
|
4412
|
+
awaitingGrading: z.ZodNumber;
|
|
4413
|
+
}, z.core.$strip>>;
|
|
3776
4414
|
id: z.ZodUUID;
|
|
3777
4415
|
name: z.ZodString;
|
|
3778
4416
|
targetId: z.ZodString;
|
|
@@ -3883,6 +4521,92 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
3883
4521
|
}, z.core.$strip>>;
|
|
3884
4522
|
legacy: z.ZodOptional<z.ZodBoolean>;
|
|
3885
4523
|
suites: z.ZodArray<z.ZodObject<{
|
|
4524
|
+
issue: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
4525
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
4526
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
4527
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
4528
|
+
capture: z.ZodObject<{
|
|
4529
|
+
status: z.ZodEnum<{
|
|
4530
|
+
complete: "complete";
|
|
4531
|
+
failed: "failed";
|
|
4532
|
+
partial: "partial";
|
|
4533
|
+
}>;
|
|
4534
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
4535
|
+
path: z.ZodString;
|
|
4536
|
+
reason: z.ZodString;
|
|
4537
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
4538
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
4539
|
+
}, z.core.$strict>>;
|
|
4540
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
4541
|
+
path: z.ZodString;
|
|
4542
|
+
reason: z.ZodString;
|
|
4543
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
4544
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
4545
|
+
}, z.core.$strict>>;
|
|
4546
|
+
}, z.core.$strict>;
|
|
4547
|
+
}, z.core.$strict>>;
|
|
4548
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
4549
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
4550
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
4551
|
+
}, z.core.$strict>>;
|
|
4552
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4553
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4554
|
+
phase: z.ZodEnum<{
|
|
4555
|
+
params: "params";
|
|
4556
|
+
cleanup: "cleanup";
|
|
4557
|
+
execute: "execute";
|
|
4558
|
+
setup: "setup";
|
|
4559
|
+
responder: "responder";
|
|
4560
|
+
reference: "reference";
|
|
4561
|
+
grading: "grading";
|
|
4562
|
+
verification: "verification";
|
|
4563
|
+
reporting: "reporting";
|
|
4564
|
+
}>;
|
|
4565
|
+
code: z.ZodString;
|
|
4566
|
+
message: z.ZodString;
|
|
4567
|
+
}, z.core.$strict>>>;
|
|
4568
|
+
termination: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
4569
|
+
code: z.ZodEnum<{
|
|
4570
|
+
EVAL_RUN_TIMEOUT: "EVAL_RUN_TIMEOUT";
|
|
4571
|
+
EVAL_CANCELLED: "EVAL_CANCELLED";
|
|
4572
|
+
EVAL_LEASE_LOST: "EVAL_LEASE_LOST";
|
|
4573
|
+
EVAL_EXECUTION_UNKNOWN: "EVAL_EXECUTION_UNKNOWN";
|
|
4574
|
+
EVAL_EXECUTOR_FAILED: "EVAL_EXECUTOR_FAILED";
|
|
4575
|
+
}>;
|
|
4576
|
+
message: z.ZodString;
|
|
4577
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
4578
|
+
phase: z.ZodOptional<z.ZodString>;
|
|
4579
|
+
cleanup: z.ZodOptional<z.ZodEnum<{
|
|
4580
|
+
unknown: "unknown";
|
|
4581
|
+
signalled: "signalled";
|
|
4582
|
+
}>>;
|
|
4583
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
4584
|
+
}, z.core.$strict>>>;
|
|
4585
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
4586
|
+
values: z.ZodObject<{
|
|
4587
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4588
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4589
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4590
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4591
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
4592
|
+
studio: "studio";
|
|
4593
|
+
app: "app";
|
|
4594
|
+
}>>>;
|
|
4595
|
+
}, z.core.$strict>;
|
|
4596
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
4597
|
+
repetitions: "repetitions";
|
|
4598
|
+
concurrency: "concurrency";
|
|
4599
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
4600
|
+
runTimeoutMs: "runTimeoutMs";
|
|
4601
|
+
judge: "judge";
|
|
4602
|
+
}>, z.ZodEnum<{
|
|
4603
|
+
studio: "studio";
|
|
4604
|
+
run: "run";
|
|
4605
|
+
sdk: "sdk";
|
|
4606
|
+
"sdk-default": "sdk-default";
|
|
4607
|
+
"studio-default": "studio-default";
|
|
4608
|
+
}>>;
|
|
4609
|
+
}, z.core.$strict>>;
|
|
3886
4610
|
id: z.ZodUUID;
|
|
3887
4611
|
evaluationId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
|
|
3888
4612
|
targetId: z.ZodString;
|
|
@@ -4047,6 +4771,33 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4047
4771
|
}, z.core.$strict>>;
|
|
4048
4772
|
repetitions: z.ZodNumber;
|
|
4049
4773
|
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
4774
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
4775
|
+
runTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
4776
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
4777
|
+
values: z.ZodObject<{
|
|
4778
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4779
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4780
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4781
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
4782
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
4783
|
+
studio: "studio";
|
|
4784
|
+
app: "app";
|
|
4785
|
+
}>>>;
|
|
4786
|
+
}, z.core.$strict>;
|
|
4787
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
4788
|
+
repetitions: "repetitions";
|
|
4789
|
+
concurrency: "concurrency";
|
|
4790
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
4791
|
+
runTimeoutMs: "runTimeoutMs";
|
|
4792
|
+
judge: "judge";
|
|
4793
|
+
}>, z.ZodEnum<{
|
|
4794
|
+
studio: "studio";
|
|
4795
|
+
run: "run";
|
|
4796
|
+
sdk: "sdk";
|
|
4797
|
+
"sdk-default": "sdk-default";
|
|
4798
|
+
"studio-default": "studio-default";
|
|
4799
|
+
}>>;
|
|
4800
|
+
}, z.core.$strict>>;
|
|
4050
4801
|
promptSnapshot: z.ZodOptional<z.ZodObject<{
|
|
4051
4802
|
schemaVersion: z.ZodLiteral<1>;
|
|
4052
4803
|
environment: z.ZodString;
|
|
@@ -4305,13 +5056,13 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4305
5056
|
}, z.core.$strict>>;
|
|
4306
5057
|
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
4307
5058
|
params: "params";
|
|
5059
|
+
cleanup: "cleanup";
|
|
4308
5060
|
execute: "execute";
|
|
4309
5061
|
setup: "setup";
|
|
4310
5062
|
responder: "responder";
|
|
4311
5063
|
reference: "reference";
|
|
4312
5064
|
grading: "grading";
|
|
4313
5065
|
verification: "verification";
|
|
4314
|
-
cleanup: "cleanup";
|
|
4315
5066
|
reporting: "reporting";
|
|
4316
5067
|
}> & z.core.$partial, z.ZodNumber>>;
|
|
4317
5068
|
caseId: z.ZodString;
|
|
@@ -4497,17 +5248,44 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4497
5248
|
}, z.core.$strict>>;
|
|
4498
5249
|
}, z.core.$strict>>;
|
|
4499
5250
|
errors: z.ZodArray<z.ZodObject<{
|
|
5251
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
5252
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
5253
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
5254
|
+
capture: z.ZodObject<{
|
|
5255
|
+
status: z.ZodEnum<{
|
|
5256
|
+
complete: "complete";
|
|
5257
|
+
failed: "failed";
|
|
5258
|
+
partial: "partial";
|
|
5259
|
+
}>;
|
|
5260
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
5261
|
+
path: z.ZodString;
|
|
5262
|
+
reason: z.ZodString;
|
|
5263
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
5264
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
5265
|
+
}, z.core.$strict>>;
|
|
5266
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
5267
|
+
path: z.ZodString;
|
|
5268
|
+
reason: z.ZodString;
|
|
5269
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
5270
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
5271
|
+
}, z.core.$strict>>;
|
|
5272
|
+
}, z.core.$strict>;
|
|
5273
|
+
}, z.core.$strict>>;
|
|
5274
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
5275
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5276
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
5277
|
+
}, z.core.$strict>>;
|
|
4500
5278
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4501
5279
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4502
5280
|
phase: z.ZodEnum<{
|
|
4503
5281
|
params: "params";
|
|
5282
|
+
cleanup: "cleanup";
|
|
4504
5283
|
execute: "execute";
|
|
4505
5284
|
setup: "setup";
|
|
4506
5285
|
responder: "responder";
|
|
4507
5286
|
reference: "reference";
|
|
4508
5287
|
grading: "grading";
|
|
4509
5288
|
verification: "verification";
|
|
4510
|
-
cleanup: "cleanup";
|
|
4511
5289
|
reporting: "reporting";
|
|
4512
5290
|
}>;
|
|
4513
5291
|
code: z.ZodString;
|
|
@@ -4515,17 +5293,44 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4515
5293
|
}, z.core.$strict>>;
|
|
4516
5294
|
}, z.core.$strict>>;
|
|
4517
5295
|
errors: z.ZodArray<z.ZodObject<{
|
|
5296
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
5297
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
5298
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
5299
|
+
capture: z.ZodObject<{
|
|
5300
|
+
status: z.ZodEnum<{
|
|
5301
|
+
complete: "complete";
|
|
5302
|
+
failed: "failed";
|
|
5303
|
+
partial: "partial";
|
|
5304
|
+
}>;
|
|
5305
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
5306
|
+
path: z.ZodString;
|
|
5307
|
+
reason: z.ZodString;
|
|
5308
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
5309
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
5310
|
+
}, z.core.$strict>>;
|
|
5311
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
5312
|
+
path: z.ZodString;
|
|
5313
|
+
reason: z.ZodString;
|
|
5314
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
5315
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
5316
|
+
}, z.core.$strict>>;
|
|
5317
|
+
}, z.core.$strict>;
|
|
5318
|
+
}, z.core.$strict>>;
|
|
5319
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
5320
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5321
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
5322
|
+
}, z.core.$strict>>;
|
|
4518
5323
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4519
5324
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4520
5325
|
phase: z.ZodEnum<{
|
|
4521
5326
|
params: "params";
|
|
5327
|
+
cleanup: "cleanup";
|
|
4522
5328
|
execute: "execute";
|
|
4523
5329
|
setup: "setup";
|
|
4524
5330
|
responder: "responder";
|
|
4525
5331
|
reference: "reference";
|
|
4526
5332
|
grading: "grading";
|
|
4527
5333
|
verification: "verification";
|
|
4528
|
-
cleanup: "cleanup";
|
|
4529
5334
|
reporting: "reporting";
|
|
4530
5335
|
}>;
|
|
4531
5336
|
code: z.ZodString;
|
|
@@ -4567,13 +5372,13 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4567
5372
|
repetition: z.ZodNumber;
|
|
4568
5373
|
phase: z.ZodEnum<{
|
|
4569
5374
|
params: "params";
|
|
5375
|
+
cleanup: "cleanup";
|
|
4570
5376
|
execute: "execute";
|
|
4571
5377
|
setup: "setup";
|
|
4572
5378
|
responder: "responder";
|
|
4573
5379
|
reference: "reference";
|
|
4574
5380
|
grading: "grading";
|
|
4575
5381
|
verification: "verification";
|
|
4576
|
-
cleanup: "cleanup";
|
|
4577
5382
|
reporting: "reporting";
|
|
4578
5383
|
}>;
|
|
4579
5384
|
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
@@ -4763,13 +5568,13 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4763
5568
|
}, z.core.$strict>>;
|
|
4764
5569
|
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
4765
5570
|
params: "params";
|
|
5571
|
+
cleanup: "cleanup";
|
|
4766
5572
|
execute: "execute";
|
|
4767
5573
|
setup: "setup";
|
|
4768
5574
|
responder: "responder";
|
|
4769
5575
|
reference: "reference";
|
|
4770
5576
|
grading: "grading";
|
|
4771
5577
|
verification: "verification";
|
|
4772
|
-
cleanup: "cleanup";
|
|
4773
5578
|
reporting: "reporting";
|
|
4774
5579
|
}> & z.core.$partial, z.ZodNumber>>;
|
|
4775
5580
|
caseId: z.ZodString;
|
|
@@ -4955,17 +5760,44 @@ declare const StudioEvaluationResultsSchema: z.ZodObject<{
|
|
|
4955
5760
|
}, z.core.$strict>>;
|
|
4956
5761
|
}, z.core.$strict>>;
|
|
4957
5762
|
errors: z.ZodArray<z.ZodObject<{
|
|
5763
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
5764
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
5765
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
5766
|
+
capture: z.ZodObject<{
|
|
5767
|
+
status: z.ZodEnum<{
|
|
5768
|
+
complete: "complete";
|
|
5769
|
+
failed: "failed";
|
|
5770
|
+
partial: "partial";
|
|
5771
|
+
}>;
|
|
5772
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
5773
|
+
path: z.ZodString;
|
|
5774
|
+
reason: z.ZodString;
|
|
5775
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
5776
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
5777
|
+
}, z.core.$strict>>;
|
|
5778
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
5779
|
+
path: z.ZodString;
|
|
5780
|
+
reason: z.ZodString;
|
|
5781
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
5782
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
5783
|
+
}, z.core.$strict>>;
|
|
5784
|
+
}, z.core.$strict>;
|
|
5785
|
+
}, z.core.$strict>>;
|
|
5786
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
5787
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5788
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
5789
|
+
}, z.core.$strict>>;
|
|
4958
5790
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
4959
5791
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
4960
5792
|
phase: z.ZodEnum<{
|
|
4961
5793
|
params: "params";
|
|
5794
|
+
cleanup: "cleanup";
|
|
4962
5795
|
execute: "execute";
|
|
4963
5796
|
setup: "setup";
|
|
4964
5797
|
responder: "responder";
|
|
4965
5798
|
reference: "reference";
|
|
4966
5799
|
grading: "grading";
|
|
4967
5800
|
verification: "verification";
|
|
4968
|
-
cleanup: "cleanup";
|
|
4969
5801
|
reporting: "reporting";
|
|
4970
5802
|
}>;
|
|
4971
5803
|
code: z.ZodString;
|
|
@@ -4986,7 +5818,105 @@ declare function getEvalGradeEvidence(step: EvalStepResult, previous: readonly E
|
|
|
4986
5818
|
conversation: EvalStepResult[];
|
|
4987
5819
|
};
|
|
4988
5820
|
|
|
5821
|
+
declare const EvalSettingsSchema: z.ZodObject<{
|
|
5822
|
+
repetitions: z.ZodOptional<z.ZodNumber>;
|
|
5823
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
5824
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5825
|
+
runTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5826
|
+
judge: z.ZodOptional<z.ZodEnum<{
|
|
5827
|
+
studio: "studio";
|
|
5828
|
+
app: "app";
|
|
5829
|
+
}>>;
|
|
5830
|
+
}, z.core.$strict>;
|
|
5831
|
+
type EvalSettings = z.infer<typeof EvalSettingsSchema>;
|
|
5832
|
+
declare const EvalExecutionSettingsSchema: z.ZodObject<{
|
|
5833
|
+
defaults: z.ZodObject<{
|
|
5834
|
+
repetitions: z.ZodOptional<z.ZodNumber>;
|
|
5835
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
5836
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5837
|
+
}, z.core.$strict>;
|
|
5838
|
+
limits: z.ZodObject<{
|
|
5839
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
5840
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5841
|
+
}, z.core.$strict>;
|
|
5842
|
+
overrides: z.ZodArray<z.ZodEnum<{
|
|
5843
|
+
repetitions: "repetitions";
|
|
5844
|
+
concurrency: "concurrency";
|
|
5845
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
5846
|
+
}>>;
|
|
5847
|
+
}, z.core.$strict>;
|
|
5848
|
+
declare const ResolvedEvalSettingsSchema: z.ZodObject<{
|
|
5849
|
+
values: z.ZodObject<{
|
|
5850
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
5851
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
5852
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
5853
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
5854
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
5855
|
+
studio: "studio";
|
|
5856
|
+
app: "app";
|
|
5857
|
+
}>>>;
|
|
5858
|
+
}, z.core.$strict>;
|
|
5859
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
5860
|
+
repetitions: "repetitions";
|
|
5861
|
+
concurrency: "concurrency";
|
|
5862
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
5863
|
+
runTimeoutMs: "runTimeoutMs";
|
|
5864
|
+
judge: "judge";
|
|
5865
|
+
}>, z.ZodEnum<{
|
|
5866
|
+
studio: "studio";
|
|
5867
|
+
run: "run";
|
|
5868
|
+
sdk: "sdk";
|
|
5869
|
+
"sdk-default": "sdk-default";
|
|
5870
|
+
"studio-default": "studio-default";
|
|
5871
|
+
}>>;
|
|
5872
|
+
}, z.core.$strict>;
|
|
5873
|
+
type ResolvedEvalSettings = z.infer<typeof ResolvedEvalSettingsSchema>;
|
|
5874
|
+
declare const EvalSettingsResponseSchema: z.ZodObject<{
|
|
5875
|
+
defaults: z.ZodObject<{
|
|
5876
|
+
repetitions: z.ZodOptional<z.ZodNumber>;
|
|
5877
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
5878
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5879
|
+
runTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5880
|
+
judge: z.ZodOptional<z.ZodEnum<{
|
|
5881
|
+
studio: "studio";
|
|
5882
|
+
app: "app";
|
|
5883
|
+
}>>;
|
|
5884
|
+
}, z.core.$strict>;
|
|
5885
|
+
installation: z.ZodObject<{
|
|
5886
|
+
repetitions: z.ZodOptional<z.ZodNumber>;
|
|
5887
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
5888
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5889
|
+
runTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5890
|
+
judge: z.ZodOptional<z.ZodEnum<{
|
|
5891
|
+
studio: "studio";
|
|
5892
|
+
app: "app";
|
|
5893
|
+
}>>;
|
|
5894
|
+
}, z.core.$strict>;
|
|
5895
|
+
maxRunTimeoutMs: z.ZodNumber;
|
|
5896
|
+
canEdit: z.ZodBoolean;
|
|
5897
|
+
}, z.core.$strip>;
|
|
5898
|
+
declare const EvalTerminationSchema: z.ZodObject<{
|
|
5899
|
+
code: z.ZodEnum<{
|
|
5900
|
+
EVAL_RUN_TIMEOUT: "EVAL_RUN_TIMEOUT";
|
|
5901
|
+
EVAL_CANCELLED: "EVAL_CANCELLED";
|
|
5902
|
+
EVAL_LEASE_LOST: "EVAL_LEASE_LOST";
|
|
5903
|
+
EVAL_EXECUTION_UNKNOWN: "EVAL_EXECUTION_UNKNOWN";
|
|
5904
|
+
EVAL_EXECUTOR_FAILED: "EVAL_EXECUTOR_FAILED";
|
|
5905
|
+
}>;
|
|
5906
|
+
message: z.ZodString;
|
|
5907
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
5908
|
+
phase: z.ZodOptional<z.ZodString>;
|
|
5909
|
+
cleanup: z.ZodOptional<z.ZodEnum<{
|
|
5910
|
+
unknown: "unknown";
|
|
5911
|
+
signalled: "signalled";
|
|
5912
|
+
}>>;
|
|
5913
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
5914
|
+
}, z.core.$strict>;
|
|
5915
|
+
type EvalTermination = z.infer<typeof EvalTerminationSchema>;
|
|
5916
|
+
/** Resolve only defined fields. Ceilings never participate in precedence. */
|
|
5917
|
+
declare function resolveEvalSettings(overrides?: EvalSettings, studio?: EvalSettings, execution?: z.infer<typeof EvalExecutionSettingsSchema>, installation?: EvalSettings, maxRunTimeoutMs?: number): ResolvedEvalSettings;
|
|
5918
|
+
|
|
4989
5919
|
/** Discover and pin the Studio backend's judge; grading never runs in the browser. */
|
|
4990
5920
|
declare function createStudioEvalJudge(options: StudioEvalJudgeOptions): Promise<EvalJudge>;
|
|
4991
5921
|
|
|
4992
|
-
export { type
|
|
5922
|
+
export { type EvalStatus as $, EvalJudgeIdentitySchema as A, type EvalJudgeOptions as B, type CreateEvalsOptions as C, type EvalJudgeUsage as D, type EvalCase as E, EvalJudgeUsageSchema as F, type EvalManifest as G, EvalManifestSchema as H, type EvalObservation as I, EvalObservationSchema as J, type EvalOutputExpectation as K, type EvalOutputSelector as L, type EvalPhase as M, type EvalProgress as N, EvalProgressSchema as O, type EvalRemoteRunRequest as P, EvalRemoteRunRequestSchema as Q, type EvalResponder as R, EvalResumeResponseSchema as S, type EvalRunOptions as T, type EvalRunResult as U, EvalRunResultSchema as V, type EvalRuntimeExecution as W, type EvalSettings as X, EvalSettingsResponseSchema as Y, EvalSettingsSchema as Z, type EvalSetupContext as _, type EvalCaseResult as a, type EvalStep as a0, type EvalStepResult as a1, EvalStepResultSchema as a2, type EvalSuite as a3, EvalSuiteSchema as a4, type EvalTermination as a5, EvalTerminationSchema as a6, type EvalVerdict as a7, EvalVerdictSchema as a8, type EvalWireEvent as a9, type AgentProcessOptions as aA, type AgentProjectTopologyOptions as aB, type ChatResponseStatus as aC, type CreateAgentArgs as aD, type ExecutableWorkflow as aE, type ExecuteOptions as aF, type ExecutionInfo as aG, type ExecutionInterrupt as aH, ExecutionRequestError as aI, type ExecutionResult as aJ, type InterruptScope as aK, type ListInterruptsOptions as aL, type PendingInterrupt as aM, type ResumableInterrupt as aN, type ResumeHandle as aO, type ResumeOptions as aP, type ResumeResponse as aQ, createAgent as aR, projectWorkflowTopology as aS, EvalWireEventSchema as aa, EvaluationMetadataSchema as ab, type ResolvedEvalSettings as ac, ResolvedEvalSettingsSchema as ad, type StudioEvalJudgeOptions as ae, StudioEvalJudgeRequestSchema as af, StudioEvalJudgeResponseSchema as ag, StudioEvalStartRequestSchema as ah, type StudioEvaluationDetail as ai, StudioEvaluationDetailSchema as aj, StudioEvaluationHistorySchema as ak, StudioEvaluationResultsSchema as al, type StudioEvaluationStartRequest as am, StudioEvaluationStartRequestSchema as an, type StudioEvaluationSummary as ao, StudioEvaluationSummarySchema as ap, createEvalIssue as aq, createStudioEvalJudge as ar, defineSuite as as, getEvalGradeEvidence as at, parseEvalSuite as au, resolveEvalSettings as av, type ChatLifecyclePhase as aw, ChatLifecycleHookError as ax, type Agent as ay, type ChatResponseFinalized as az, EvalCaseResultSchema as b, type EvalCommand as c, EvalConfigurationError as d, type EvalContext as e, type EvalCriterion as f, type EvalCriterionResult as g, type EvalDefaults as h, type EvalEvidenceEvent as i, type EvalEvidenceEventType as j, type EvalEvidenceFilters as k, type EvalEvidencePolicy as l, EvalEvidencePolicySchema as m, type EvalExecution as n, EvalExecutionSettingsSchema as o, type EvalExecutionStatus as p, type EvalExpectation as q, type EvalGradeInput as r, type EvalHandlerRef as s, type EvalInterrupt as t, type EvalIssue as u, EvalIssueSchema as v, type EvalJson as w, type EvalJudge as x, type EvalJudgeEvidence as y, type EvalJudgeIdentity as z };
|