@kortyx/agent 0.38.0 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +44 -0
- package/README.md +11 -0
- package/dist/evals.d.ts +414 -7
- package/dist/evals.js +495 -330
- package/dist/evals.js.map +1 -1
- package/dist/index.d.ts +20 -3
- package/dist/index.js +1263 -2997
- package/dist/index.js.map +1 -1
- package/dist/{studio-judge-DCAw3v60.d.ts → studio-judge-caDvI8By.d.ts} +973 -43
- package/package.json +11 -10
package/CHANGELOG.md
CHANGED
|
@@ -2,10 +2,54 @@
|
|
|
2
2
|
|
|
3
3
|
## Unreleased
|
|
4
4
|
|
|
5
|
+
- Preserve completed tool results and addressed human answers across root and child execution-budget continuations. Continue renews only the saved allowance and does not answer a later review.
|
|
6
|
+
|
|
5
7
|
- Add bounded, idempotent queued eval execution with shared consumer capacity,
|
|
6
8
|
event replay, explicit cancellation and optional PostgreSQL recovery.
|
|
7
9
|
|
|
8
10
|
|
|
11
|
+
## [0.39.0](https://github.com/kortyx-io/kortyx/compare/agent-v0.38.1...agent-v0.39.0) (2026-10-16)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
### Features
|
|
15
|
+
|
|
16
|
+
* **evals:** configurable deadlines and original error visibility ([#354](https://github.com/kortyx-io/kortyx/issues/354)) ([7dde173](https://github.com/kortyx-io/kortyx/commit/7dde173b54eb37a2d28ff28ac342a3333e5ed902))
|
|
17
|
+
* **telemetry:** improve failure classification and add CLI usage ([#355](https://github.com/kortyx-io/kortyx/issues/355)) ([c5fbbed](https://github.com/kortyx-io/kortyx/commit/c5fbbed3064ea7b2b3020617db7ce09d1b175db3))
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
### Dependencies
|
|
21
|
+
|
|
22
|
+
* The following workspace dependencies were updated
|
|
23
|
+
* dependencies
|
|
24
|
+
* @kortyx/core bumped to 0.15.0
|
|
25
|
+
* @kortyx/providers bumped to 0.9.4
|
|
26
|
+
* @kortyx/runtime bumped to 0.25.0
|
|
27
|
+
* @kortyx/stream bumped to 0.15.6
|
|
28
|
+
* @kortyx/utils bumped to 0.3.9
|
|
29
|
+
* @kortyx/prompts bumped to 0.2.2
|
|
30
|
+
* @kortyx/usage-telemetry bumped to 0.4.0
|
|
31
|
+
* @kortyx/telemetry-contracts bumped to 0.18.0
|
|
32
|
+
* devDependencies
|
|
33
|
+
* @kortyx/hooks bumped to 0.35.0
|
|
34
|
+
* @kortyx/openai bumped to 0.5.10
|
|
35
|
+
|
|
36
|
+
## [0.38.1](https://github.com/kortyx-io/kortyx/compare/agent-v0.38.0...agent-v0.38.1) (2026-10-14)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
### Bug Fixes
|
|
40
|
+
|
|
41
|
+
* **runtime:** preserve interrupt state across pauses and resume ([#348](https://github.com/kortyx-io/kortyx/issues/348)) ([03b49f2](https://github.com/kortyx-io/kortyx/commit/03b49f2b8b6b44a2852d16fd4fe1b3e199316072))
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
### Dependencies
|
|
45
|
+
|
|
46
|
+
* The following workspace dependencies were updated
|
|
47
|
+
* dependencies
|
|
48
|
+
* @kortyx/runtime bumped to 0.24.1
|
|
49
|
+
* @kortyx/utils bumped to 0.3.8
|
|
50
|
+
* devDependencies
|
|
51
|
+
* @kortyx/hooks bumped to 0.34.3
|
|
52
|
+
|
|
9
53
|
## [0.38.0](https://github.com/kortyx-io/kortyx/compare/agent-v0.37.1...agent-v0.38.0) (2026-10-13)
|
|
10
54
|
|
|
11
55
|
|
package/README.md
CHANGED
|
@@ -111,3 +111,14 @@ observation and explicit cancellation. Use `createPostgresEvalExecutionStore` fr
|
|
|
111
111
|
The default memory store only survives within its process. See
|
|
112
112
|
[execution resilience](../../docs/evals/execution-resilience.md) for configuration,
|
|
113
113
|
retention, legacy compatibility and upgrade order.
|
|
114
|
+
|
|
115
|
+
### Evaluation settings and errors
|
|
116
|
+
|
|
117
|
+
`createEvals` supports `defaults: { repetitions, concurrency, caseTimeoutMs }`,
|
|
118
|
+
`limits: { concurrency, caseTimeoutMs }`, and per-run `caseTimeoutMs` overrides.
|
|
119
|
+
Studio resolves run overrides above project defaults, SDK defaults and built-ins;
|
|
120
|
+
its whole-evaluation timeout is independent of the case budget (30 minutes by
|
|
121
|
+
default). Eval failures retain original messages, stacks, causes and response
|
|
122
|
+
properties with credential redaction automatically. See the repository guides
|
|
123
|
+
[SDK evals](../../docs/evals/sdk-runner.md) and
|
|
124
|
+
[Studio settings and migration](../../docs/evals/studio-execution.md).
|
package/dist/evals.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
export { C as CreateEvalsOptions, E as EvalCase, a as EvalCaseResult, b as EvalCaseResultSchema, c as EvalCommand, d as EvalConfigurationError, e as EvalContext, f as EvalCriterion, g as EvalCriterionResult, h as EvalDefaults, i as EvalEvidenceEvent, j as EvalEvidenceEventType, k as EvalEvidenceFilters, l as EvalEvidencePolicy, m as EvalEvidencePolicySchema, n as EvalExecution, o as
|
|
1
|
+
export { C as CreateEvalsOptions, E as EvalCase, a as EvalCaseResult, b as EvalCaseResultSchema, c as EvalCommand, d as EvalConfigurationError, e as EvalContext, f as EvalCriterion, g as EvalCriterionResult, h as EvalDefaults, i as EvalEvidenceEvent, j as EvalEvidenceEventType, k as EvalEvidenceFilters, l as EvalEvidencePolicy, m as EvalEvidencePolicySchema, n as EvalExecution, o as EvalExecutionSettingsSchema, p as EvalExecutionStatus, q as EvalExpectation, r as EvalGradeInput, s as EvalHandlerRef, t as EvalInterrupt, u as EvalIssue, v as EvalIssueSchema, w as EvalJson, x as EvalJudge, y as EvalJudgeEvidence, z as EvalJudgeIdentity, A as EvalJudgeIdentitySchema, B as EvalJudgeOptions, D as EvalJudgeUsage, F as EvalJudgeUsageSchema, G as EvalManifest, H as EvalManifestSchema, I as EvalObservation, J as EvalObservationSchema, K as EvalOutputExpectation, L as EvalOutputSelector, M as EvalPhase, N as EvalProgress, O as EvalProgressSchema, P as EvalRemoteRunRequest, Q as EvalRemoteRunRequestSchema, R as EvalResponder, S as EvalResumeResponseSchema, T as EvalRunOptions, U as EvalRunResult, V as EvalRunResultSchema, W as EvalRuntimeExecution, X as EvalSettings, Y as EvalSettingsResponseSchema, Z as EvalSettingsSchema, _ as EvalSetupContext, $ as EvalStatus, a0 as EvalStep, a1 as EvalStepResult, a2 as EvalStepResultSchema, a3 as EvalSuite, a4 as EvalSuiteSchema, a5 as EvalTermination, a6 as EvalTerminationSchema, a7 as EvalVerdict, a8 as EvalVerdictSchema, a9 as EvalWireEvent, aa as EvalWireEventSchema, ab as EvaluationMetadataSchema, ac as ResolvedEvalSettings, ad as ResolvedEvalSettingsSchema, ae as StudioEvalJudgeOptions, af as StudioEvalJudgeRequestSchema, ag as StudioEvalJudgeResponseSchema, ah as StudioEvalStartRequestSchema, ai as StudioEvaluationDetail, aj as StudioEvaluationDetailSchema, ak as StudioEvaluationHistorySchema, al as StudioEvaluationResultsSchema, am as StudioEvaluationStartRequest, an as StudioEvaluationStartRequestSchema, ao as StudioEvaluationSummary, ap as StudioEvaluationSummarySchema, aq as createEvalIssue, ar as createStudioEvalJudge, as as defineSuite, at as getEvalGradeEvidence, au as parseEvalSuite, av as resolveEvalSettings } from './studio-judge-caDvI8By.js';
|
|
2
2
|
import * as _kortyx_core from '@kortyx/core';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
+
import '@kortyx/telemetry-contracts';
|
|
4
5
|
import '@kortyx/prompts';
|
|
5
6
|
import '@kortyx/core/errors';
|
|
6
7
|
import '@kortyx/providers';
|
|
@@ -11,6 +12,30 @@ import './chat-message-D4EsFcr1.js';
|
|
|
11
12
|
|
|
12
13
|
declare const StudioEvalTargetsResponseSchema: z.ZodObject<{
|
|
13
14
|
canRun: z.ZodBoolean;
|
|
15
|
+
settings: z.ZodOptional<z.ZodObject<{
|
|
16
|
+
defaults: z.ZodObject<{
|
|
17
|
+
repetitions: z.ZodOptional<z.ZodNumber>;
|
|
18
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
19
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
20
|
+
runTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
21
|
+
judge: z.ZodOptional<z.ZodEnum<{
|
|
22
|
+
studio: "studio";
|
|
23
|
+
app: "app";
|
|
24
|
+
}>>;
|
|
25
|
+
}, z.core.$strict>;
|
|
26
|
+
installation: z.ZodObject<{
|
|
27
|
+
repetitions: z.ZodOptional<z.ZodNumber>;
|
|
28
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
29
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
30
|
+
runTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
31
|
+
judge: z.ZodOptional<z.ZodEnum<{
|
|
32
|
+
studio: "studio";
|
|
33
|
+
app: "app";
|
|
34
|
+
}>>;
|
|
35
|
+
}, z.core.$strict>;
|
|
36
|
+
maxRunTimeoutMs: z.ZodNumber;
|
|
37
|
+
canEdit: z.ZodBoolean;
|
|
38
|
+
}, z.core.$strip>>;
|
|
14
39
|
studioJudge: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
15
40
|
id: z.ZodString;
|
|
16
41
|
version: z.ZodString;
|
|
@@ -37,6 +62,22 @@ declare const StudioEvalTargetsResponseSchema: z.ZodObject<{
|
|
|
37
62
|
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
38
63
|
}, z.core.$strip>>>;
|
|
39
64
|
manifest: z.ZodNullable<z.ZodObject<{
|
|
65
|
+
execution: z.ZodOptional<z.ZodObject<{
|
|
66
|
+
defaults: z.ZodObject<{
|
|
67
|
+
repetitions: z.ZodOptional<z.ZodNumber>;
|
|
68
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
69
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
70
|
+
}, z.core.$strict>;
|
|
71
|
+
limits: z.ZodObject<{
|
|
72
|
+
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
73
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
74
|
+
}, z.core.$strict>;
|
|
75
|
+
overrides: z.ZodArray<z.ZodEnum<{
|
|
76
|
+
repetitions: "repetitions";
|
|
77
|
+
concurrency: "concurrency";
|
|
78
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
79
|
+
}>>;
|
|
80
|
+
}, z.core.$strict>>;
|
|
40
81
|
schemaVersion: z.ZodLiteral<1>;
|
|
41
82
|
studioJudging: z.ZodOptional<z.ZodLiteral<true>>;
|
|
42
83
|
promptContracts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
@@ -238,6 +279,92 @@ declare const EvalCostsSchema: z.ZodObject<{
|
|
|
238
279
|
type EvalCosts = z.infer<typeof EvalCostsSchema>;
|
|
239
280
|
type EvalCostAmount = z.infer<typeof EvalCostAmountSchema>;
|
|
240
281
|
declare const StudioEvalRunSummarySchema: z.ZodObject<{
|
|
282
|
+
issue: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
283
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
284
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
285
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
286
|
+
capture: z.ZodObject<{
|
|
287
|
+
status: z.ZodEnum<{
|
|
288
|
+
complete: "complete";
|
|
289
|
+
failed: "failed";
|
|
290
|
+
partial: "partial";
|
|
291
|
+
}>;
|
|
292
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
293
|
+
path: z.ZodString;
|
|
294
|
+
reason: z.ZodString;
|
|
295
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
296
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
297
|
+
}, z.core.$strict>>;
|
|
298
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
299
|
+
path: z.ZodString;
|
|
300
|
+
reason: z.ZodString;
|
|
301
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
302
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
303
|
+
}, z.core.$strict>>;
|
|
304
|
+
}, z.core.$strict>;
|
|
305
|
+
}, z.core.$strict>>;
|
|
306
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
307
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
308
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
309
|
+
}, z.core.$strict>>;
|
|
310
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
311
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
312
|
+
phase: z.ZodEnum<{
|
|
313
|
+
params: "params";
|
|
314
|
+
cleanup: "cleanup";
|
|
315
|
+
execute: "execute";
|
|
316
|
+
setup: "setup";
|
|
317
|
+
responder: "responder";
|
|
318
|
+
reference: "reference";
|
|
319
|
+
grading: "grading";
|
|
320
|
+
verification: "verification";
|
|
321
|
+
reporting: "reporting";
|
|
322
|
+
}>;
|
|
323
|
+
code: z.ZodString;
|
|
324
|
+
message: z.ZodString;
|
|
325
|
+
}, z.core.$strict>>>;
|
|
326
|
+
termination: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
327
|
+
code: z.ZodEnum<{
|
|
328
|
+
EVAL_RUN_TIMEOUT: "EVAL_RUN_TIMEOUT";
|
|
329
|
+
EVAL_CANCELLED: "EVAL_CANCELLED";
|
|
330
|
+
EVAL_LEASE_LOST: "EVAL_LEASE_LOST";
|
|
331
|
+
EVAL_EXECUTION_UNKNOWN: "EVAL_EXECUTION_UNKNOWN";
|
|
332
|
+
EVAL_EXECUTOR_FAILED: "EVAL_EXECUTOR_FAILED";
|
|
333
|
+
}>;
|
|
334
|
+
message: z.ZodString;
|
|
335
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
336
|
+
phase: z.ZodOptional<z.ZodString>;
|
|
337
|
+
cleanup: z.ZodOptional<z.ZodEnum<{
|
|
338
|
+
unknown: "unknown";
|
|
339
|
+
signalled: "signalled";
|
|
340
|
+
}>>;
|
|
341
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
342
|
+
}, z.core.$strict>>>;
|
|
343
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
344
|
+
values: z.ZodObject<{
|
|
345
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
346
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
347
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
348
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
349
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
350
|
+
studio: "studio";
|
|
351
|
+
app: "app";
|
|
352
|
+
}>>>;
|
|
353
|
+
}, z.core.$strict>;
|
|
354
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
355
|
+
repetitions: "repetitions";
|
|
356
|
+
concurrency: "concurrency";
|
|
357
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
358
|
+
runTimeoutMs: "runTimeoutMs";
|
|
359
|
+
judge: "judge";
|
|
360
|
+
}>, z.ZodEnum<{
|
|
361
|
+
studio: "studio";
|
|
362
|
+
run: "run";
|
|
363
|
+
sdk: "sdk";
|
|
364
|
+
"sdk-default": "sdk-default";
|
|
365
|
+
"studio-default": "studio-default";
|
|
366
|
+
}>>;
|
|
367
|
+
}, z.core.$strict>>;
|
|
241
368
|
id: z.ZodUUID;
|
|
242
369
|
evaluationId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
|
|
243
370
|
targetId: z.ZodString;
|
|
@@ -330,6 +457,92 @@ declare const StudioEvalRunSummarySchema: z.ZodObject<{
|
|
|
330
457
|
}, z.core.$strip>;
|
|
331
458
|
declare const StudioEvalHistorySchema: z.ZodObject<{
|
|
332
459
|
runs: z.ZodArray<z.ZodObject<{
|
|
460
|
+
issue: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
461
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
462
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
463
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
464
|
+
capture: z.ZodObject<{
|
|
465
|
+
status: z.ZodEnum<{
|
|
466
|
+
complete: "complete";
|
|
467
|
+
failed: "failed";
|
|
468
|
+
partial: "partial";
|
|
469
|
+
}>;
|
|
470
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
471
|
+
path: z.ZodString;
|
|
472
|
+
reason: z.ZodString;
|
|
473
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
474
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
475
|
+
}, z.core.$strict>>;
|
|
476
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
477
|
+
path: z.ZodString;
|
|
478
|
+
reason: z.ZodString;
|
|
479
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
480
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
481
|
+
}, z.core.$strict>>;
|
|
482
|
+
}, z.core.$strict>;
|
|
483
|
+
}, z.core.$strict>>;
|
|
484
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
485
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
486
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
487
|
+
}, z.core.$strict>>;
|
|
488
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
489
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
490
|
+
phase: z.ZodEnum<{
|
|
491
|
+
params: "params";
|
|
492
|
+
cleanup: "cleanup";
|
|
493
|
+
execute: "execute";
|
|
494
|
+
setup: "setup";
|
|
495
|
+
responder: "responder";
|
|
496
|
+
reference: "reference";
|
|
497
|
+
grading: "grading";
|
|
498
|
+
verification: "verification";
|
|
499
|
+
reporting: "reporting";
|
|
500
|
+
}>;
|
|
501
|
+
code: z.ZodString;
|
|
502
|
+
message: z.ZodString;
|
|
503
|
+
}, z.core.$strict>>>;
|
|
504
|
+
termination: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
505
|
+
code: z.ZodEnum<{
|
|
506
|
+
EVAL_RUN_TIMEOUT: "EVAL_RUN_TIMEOUT";
|
|
507
|
+
EVAL_CANCELLED: "EVAL_CANCELLED";
|
|
508
|
+
EVAL_LEASE_LOST: "EVAL_LEASE_LOST";
|
|
509
|
+
EVAL_EXECUTION_UNKNOWN: "EVAL_EXECUTION_UNKNOWN";
|
|
510
|
+
EVAL_EXECUTOR_FAILED: "EVAL_EXECUTOR_FAILED";
|
|
511
|
+
}>;
|
|
512
|
+
message: z.ZodString;
|
|
513
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
514
|
+
phase: z.ZodOptional<z.ZodString>;
|
|
515
|
+
cleanup: z.ZodOptional<z.ZodEnum<{
|
|
516
|
+
unknown: "unknown";
|
|
517
|
+
signalled: "signalled";
|
|
518
|
+
}>>;
|
|
519
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
520
|
+
}, z.core.$strict>>>;
|
|
521
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
522
|
+
values: z.ZodObject<{
|
|
523
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
524
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
525
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
526
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
527
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
528
|
+
studio: "studio";
|
|
529
|
+
app: "app";
|
|
530
|
+
}>>>;
|
|
531
|
+
}, z.core.$strict>;
|
|
532
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
533
|
+
repetitions: "repetitions";
|
|
534
|
+
concurrency: "concurrency";
|
|
535
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
536
|
+
runTimeoutMs: "runTimeoutMs";
|
|
537
|
+
judge: "judge";
|
|
538
|
+
}>, z.ZodEnum<{
|
|
539
|
+
studio: "studio";
|
|
540
|
+
run: "run";
|
|
541
|
+
sdk: "sdk";
|
|
542
|
+
"sdk-default": "sdk-default";
|
|
543
|
+
"studio-default": "studio-default";
|
|
544
|
+
}>>;
|
|
545
|
+
}, z.core.$strict>>;
|
|
333
546
|
id: z.ZodUUID;
|
|
334
547
|
evaluationId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
|
|
335
548
|
targetId: z.ZodString;
|
|
@@ -423,6 +636,92 @@ declare const StudioEvalHistorySchema: z.ZodObject<{
|
|
|
423
636
|
}, z.core.$strip>;
|
|
424
637
|
declare const StudioEvalDetailSchema: z.ZodObject<{
|
|
425
638
|
run: z.ZodObject<{
|
|
639
|
+
issue: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
640
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
641
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
642
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
643
|
+
capture: z.ZodObject<{
|
|
644
|
+
status: z.ZodEnum<{
|
|
645
|
+
complete: "complete";
|
|
646
|
+
failed: "failed";
|
|
647
|
+
partial: "partial";
|
|
648
|
+
}>;
|
|
649
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
650
|
+
path: z.ZodString;
|
|
651
|
+
reason: z.ZodString;
|
|
652
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
653
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
654
|
+
}, z.core.$strict>>;
|
|
655
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
656
|
+
path: z.ZodString;
|
|
657
|
+
reason: z.ZodString;
|
|
658
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
659
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
660
|
+
}, z.core.$strict>>;
|
|
661
|
+
}, z.core.$strict>;
|
|
662
|
+
}, z.core.$strict>>;
|
|
663
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
664
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
665
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
666
|
+
}, z.core.$strict>>;
|
|
667
|
+
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
668
|
+
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
669
|
+
phase: z.ZodEnum<{
|
|
670
|
+
params: "params";
|
|
671
|
+
cleanup: "cleanup";
|
|
672
|
+
execute: "execute";
|
|
673
|
+
setup: "setup";
|
|
674
|
+
responder: "responder";
|
|
675
|
+
reference: "reference";
|
|
676
|
+
grading: "grading";
|
|
677
|
+
verification: "verification";
|
|
678
|
+
reporting: "reporting";
|
|
679
|
+
}>;
|
|
680
|
+
code: z.ZodString;
|
|
681
|
+
message: z.ZodString;
|
|
682
|
+
}, z.core.$strict>>>;
|
|
683
|
+
termination: z.ZodOptional<z.ZodNullable<z.ZodObject<{
|
|
684
|
+
code: z.ZodEnum<{
|
|
685
|
+
EVAL_RUN_TIMEOUT: "EVAL_RUN_TIMEOUT";
|
|
686
|
+
EVAL_CANCELLED: "EVAL_CANCELLED";
|
|
687
|
+
EVAL_LEASE_LOST: "EVAL_LEASE_LOST";
|
|
688
|
+
EVAL_EXECUTION_UNKNOWN: "EVAL_EXECUTION_UNKNOWN";
|
|
689
|
+
EVAL_EXECUTOR_FAILED: "EVAL_EXECUTOR_FAILED";
|
|
690
|
+
}>;
|
|
691
|
+
message: z.ZodString;
|
|
692
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
693
|
+
phase: z.ZodOptional<z.ZodString>;
|
|
694
|
+
cleanup: z.ZodOptional<z.ZodEnum<{
|
|
695
|
+
unknown: "unknown";
|
|
696
|
+
signalled: "signalled";
|
|
697
|
+
}>>;
|
|
698
|
+
diagnosticId: z.ZodOptional<z.ZodString>;
|
|
699
|
+
}, z.core.$strict>>>;
|
|
700
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
701
|
+
values: z.ZodObject<{
|
|
702
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
703
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
704
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
705
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
706
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
707
|
+
studio: "studio";
|
|
708
|
+
app: "app";
|
|
709
|
+
}>>>;
|
|
710
|
+
}, z.core.$strict>;
|
|
711
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
712
|
+
repetitions: "repetitions";
|
|
713
|
+
concurrency: "concurrency";
|
|
714
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
715
|
+
runTimeoutMs: "runTimeoutMs";
|
|
716
|
+
judge: "judge";
|
|
717
|
+
}>, z.ZodEnum<{
|
|
718
|
+
studio: "studio";
|
|
719
|
+
run: "run";
|
|
720
|
+
sdk: "sdk";
|
|
721
|
+
"sdk-default": "sdk-default";
|
|
722
|
+
"studio-default": "studio-default";
|
|
723
|
+
}>>;
|
|
724
|
+
}, z.core.$strict>>;
|
|
426
725
|
id: z.ZodUUID;
|
|
427
726
|
evaluationId: z.ZodOptional<z.ZodNullable<z.ZodUUID>>;
|
|
428
727
|
targetId: z.ZodString;
|
|
@@ -587,6 +886,33 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
|
|
|
587
886
|
}, z.core.$strict>>;
|
|
588
887
|
repetitions: z.ZodNumber;
|
|
589
888
|
concurrency: z.ZodOptional<z.ZodNumber>;
|
|
889
|
+
caseTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
890
|
+
runTimeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
891
|
+
resolvedSettings: z.ZodOptional<z.ZodObject<{
|
|
892
|
+
values: z.ZodObject<{
|
|
893
|
+
repetitions: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
894
|
+
concurrency: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
895
|
+
caseTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
896
|
+
runTimeoutMs: z.ZodNonOptional<z.ZodOptional<z.ZodNumber>>;
|
|
897
|
+
judge: z.ZodNonOptional<z.ZodOptional<z.ZodEnum<{
|
|
898
|
+
studio: "studio";
|
|
899
|
+
app: "app";
|
|
900
|
+
}>>>;
|
|
901
|
+
}, z.core.$strict>;
|
|
902
|
+
sources: z.ZodRecord<z.ZodEnum<{
|
|
903
|
+
repetitions: "repetitions";
|
|
904
|
+
concurrency: "concurrency";
|
|
905
|
+
caseTimeoutMs: "caseTimeoutMs";
|
|
906
|
+
runTimeoutMs: "runTimeoutMs";
|
|
907
|
+
judge: "judge";
|
|
908
|
+
}>, z.ZodEnum<{
|
|
909
|
+
studio: "studio";
|
|
910
|
+
run: "run";
|
|
911
|
+
sdk: "sdk";
|
|
912
|
+
"sdk-default": "sdk-default";
|
|
913
|
+
"studio-default": "studio-default";
|
|
914
|
+
}>>;
|
|
915
|
+
}, z.core.$strict>>;
|
|
590
916
|
promptSnapshot: z.ZodOptional<z.ZodObject<{
|
|
591
917
|
schemaVersion: z.ZodLiteral<1>;
|
|
592
918
|
environment: z.ZodString;
|
|
@@ -845,13 +1171,13 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
|
|
|
845
1171
|
}, z.core.$strict>>;
|
|
846
1172
|
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
847
1173
|
params: "params";
|
|
1174
|
+
cleanup: "cleanup";
|
|
848
1175
|
execute: "execute";
|
|
849
1176
|
setup: "setup";
|
|
850
1177
|
responder: "responder";
|
|
851
1178
|
reference: "reference";
|
|
852
1179
|
grading: "grading";
|
|
853
1180
|
verification: "verification";
|
|
854
|
-
cleanup: "cleanup";
|
|
855
1181
|
reporting: "reporting";
|
|
856
1182
|
}> & z.core.$partial, z.ZodNumber>>;
|
|
857
1183
|
caseId: z.ZodString;
|
|
@@ -1037,17 +1363,44 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
|
|
|
1037
1363
|
}, z.core.$strict>>;
|
|
1038
1364
|
}, z.core.$strict>>;
|
|
1039
1365
|
errors: z.ZodArray<z.ZodObject<{
|
|
1366
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
1367
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
1368
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
1369
|
+
capture: z.ZodObject<{
|
|
1370
|
+
status: z.ZodEnum<{
|
|
1371
|
+
complete: "complete";
|
|
1372
|
+
failed: "failed";
|
|
1373
|
+
partial: "partial";
|
|
1374
|
+
}>;
|
|
1375
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
1376
|
+
path: z.ZodString;
|
|
1377
|
+
reason: z.ZodString;
|
|
1378
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1379
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1380
|
+
}, z.core.$strict>>;
|
|
1381
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
1382
|
+
path: z.ZodString;
|
|
1383
|
+
reason: z.ZodString;
|
|
1384
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1385
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1386
|
+
}, z.core.$strict>>;
|
|
1387
|
+
}, z.core.$strict>;
|
|
1388
|
+
}, z.core.$strict>>;
|
|
1389
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
1390
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
1391
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
1392
|
+
}, z.core.$strict>>;
|
|
1040
1393
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
1041
1394
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1042
1395
|
phase: z.ZodEnum<{
|
|
1043
1396
|
params: "params";
|
|
1397
|
+
cleanup: "cleanup";
|
|
1044
1398
|
execute: "execute";
|
|
1045
1399
|
setup: "setup";
|
|
1046
1400
|
responder: "responder";
|
|
1047
1401
|
reference: "reference";
|
|
1048
1402
|
grading: "grading";
|
|
1049
1403
|
verification: "verification";
|
|
1050
|
-
cleanup: "cleanup";
|
|
1051
1404
|
reporting: "reporting";
|
|
1052
1405
|
}>;
|
|
1053
1406
|
code: z.ZodString;
|
|
@@ -1055,17 +1408,44 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
|
|
|
1055
1408
|
}, z.core.$strict>>;
|
|
1056
1409
|
}, z.core.$strict>>;
|
|
1057
1410
|
errors: z.ZodArray<z.ZodObject<{
|
|
1411
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
1412
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
1413
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
1414
|
+
capture: z.ZodObject<{
|
|
1415
|
+
status: z.ZodEnum<{
|
|
1416
|
+
complete: "complete";
|
|
1417
|
+
failed: "failed";
|
|
1418
|
+
partial: "partial";
|
|
1419
|
+
}>;
|
|
1420
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
1421
|
+
path: z.ZodString;
|
|
1422
|
+
reason: z.ZodString;
|
|
1423
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1424
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1425
|
+
}, z.core.$strict>>;
|
|
1426
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
1427
|
+
path: z.ZodString;
|
|
1428
|
+
reason: z.ZodString;
|
|
1429
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1430
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1431
|
+
}, z.core.$strict>>;
|
|
1432
|
+
}, z.core.$strict>;
|
|
1433
|
+
}, z.core.$strict>>;
|
|
1434
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
1435
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
1436
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
1437
|
+
}, z.core.$strict>>;
|
|
1058
1438
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
1059
1439
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1060
1440
|
phase: z.ZodEnum<{
|
|
1061
1441
|
params: "params";
|
|
1442
|
+
cleanup: "cleanup";
|
|
1062
1443
|
execute: "execute";
|
|
1063
1444
|
setup: "setup";
|
|
1064
1445
|
responder: "responder";
|
|
1065
1446
|
reference: "reference";
|
|
1066
1447
|
grading: "grading";
|
|
1067
1448
|
verification: "verification";
|
|
1068
|
-
cleanup: "cleanup";
|
|
1069
1449
|
reporting: "reporting";
|
|
1070
1450
|
}>;
|
|
1071
1451
|
code: z.ZodString;
|
|
@@ -1107,13 +1487,13 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
|
|
|
1107
1487
|
repetition: z.ZodNumber;
|
|
1108
1488
|
phase: z.ZodEnum<{
|
|
1109
1489
|
params: "params";
|
|
1490
|
+
cleanup: "cleanup";
|
|
1110
1491
|
execute: "execute";
|
|
1111
1492
|
setup: "setup";
|
|
1112
1493
|
responder: "responder";
|
|
1113
1494
|
reference: "reference";
|
|
1114
1495
|
grading: "grading";
|
|
1115
1496
|
verification: "verification";
|
|
1116
|
-
cleanup: "cleanup";
|
|
1117
1497
|
reporting: "reporting";
|
|
1118
1498
|
}>;
|
|
1119
1499
|
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
@@ -1303,13 +1683,13 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
|
|
|
1303
1683
|
}, z.core.$strict>>;
|
|
1304
1684
|
phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
|
|
1305
1685
|
params: "params";
|
|
1686
|
+
cleanup: "cleanup";
|
|
1306
1687
|
execute: "execute";
|
|
1307
1688
|
setup: "setup";
|
|
1308
1689
|
responder: "responder";
|
|
1309
1690
|
reference: "reference";
|
|
1310
1691
|
grading: "grading";
|
|
1311
1692
|
verification: "verification";
|
|
1312
|
-
cleanup: "cleanup";
|
|
1313
1693
|
reporting: "reporting";
|
|
1314
1694
|
}> & z.core.$partial, z.ZodNumber>>;
|
|
1315
1695
|
caseId: z.ZodString;
|
|
@@ -1495,17 +1875,44 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
|
|
|
1495
1875
|
}, z.core.$strict>>;
|
|
1496
1876
|
}, z.core.$strict>>;
|
|
1497
1877
|
errors: z.ZodArray<z.ZodObject<{
|
|
1878
|
+
exception: z.ZodOptional<z.ZodObject<{
|
|
1879
|
+
schemaVersion: z.ZodLiteral<1>;
|
|
1880
|
+
data: z.ZodRecord<z.ZodString, z.ZodUnknown>;
|
|
1881
|
+
capture: z.ZodObject<{
|
|
1882
|
+
status: z.ZodEnum<{
|
|
1883
|
+
complete: "complete";
|
|
1884
|
+
failed: "failed";
|
|
1885
|
+
partial: "partial";
|
|
1886
|
+
}>;
|
|
1887
|
+
omissions: z.ZodArray<z.ZodObject<{
|
|
1888
|
+
path: z.ZodString;
|
|
1889
|
+
reason: z.ZodString;
|
|
1890
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1891
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1892
|
+
}, z.core.$strict>>;
|
|
1893
|
+
redactions: z.ZodArray<z.ZodObject<{
|
|
1894
|
+
path: z.ZodString;
|
|
1895
|
+
reason: z.ZodString;
|
|
1896
|
+
originalBytes: z.ZodOptional<z.ZodNumber>;
|
|
1897
|
+
originalCount: z.ZodOptional<z.ZodNumber>;
|
|
1898
|
+
}, z.core.$strict>>;
|
|
1899
|
+
}, z.core.$strict>;
|
|
1900
|
+
}, z.core.$strict>>;
|
|
1901
|
+
details: z.ZodOptional<z.ZodObject<{
|
|
1902
|
+
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
1903
|
+
httpStatus: z.ZodOptional<z.ZodNumber>;
|
|
1904
|
+
}, z.core.$strict>>;
|
|
1498
1905
|
failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
|
|
1499
1906
|
diagnosticId: z.ZodOptional<z.ZodUUID>;
|
|
1500
1907
|
phase: z.ZodEnum<{
|
|
1501
1908
|
params: "params";
|
|
1909
|
+
cleanup: "cleanup";
|
|
1502
1910
|
execute: "execute";
|
|
1503
1911
|
setup: "setup";
|
|
1504
1912
|
responder: "responder";
|
|
1505
1913
|
reference: "reference";
|
|
1506
1914
|
grading: "grading";
|
|
1507
1915
|
verification: "verification";
|
|
1508
|
-
cleanup: "cleanup";
|
|
1509
1916
|
reporting: "reporting";
|
|
1510
1917
|
}>;
|
|
1511
1918
|
code: z.ZodString;
|