@kortyx/agent 0.38.1 → 0.39.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -29,1602 +29,6 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
29
29
  ));
30
30
  var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
31
31
 
32
- // ../telemetry-contracts/dist/index.js
33
- var require_dist = __commonJS({
34
- "../telemetry-contracts/dist/index.js"(exports2, module2) {
35
- var __defProp2 = Object.defineProperty;
36
- var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
37
- var __getOwnPropNames2 = Object.getOwnPropertyNames;
38
- var __hasOwnProp2 = Object.prototype.hasOwnProperty;
39
- var __export2 = (target, all) => {
40
- for (var name2 in all)
41
- __defProp2(target, name2, { get: all[name2], enumerable: true });
42
- };
43
- var __copyProps2 = (to, from, except, desc) => {
44
- if (from && typeof from === "object" || typeof from === "function") {
45
- for (let key of __getOwnPropNames2(from))
46
- if (!__hasOwnProp2.call(to, key) && key !== except)
47
- __defProp2(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc2(from, key)) || desc.enumerable });
48
- }
49
- return to;
50
- };
51
- var __toCommonJS2 = (mod) => __copyProps2(__defProp2({}, "__esModule", { value: true }), mod);
52
- var index_exports2 = {};
53
- __export2(index_exports2, {
54
- ClearScoreResponseSchema: () => ClearScoreResponseSchema,
55
- ClearUserFeedbackRequestSchema: () => ClearUserFeedbackRequestSchema,
56
- DIAGNOSTIC_LIMITS: () => DIAGNOSTIC_LIMITS,
57
- DiagnosticContentSchema: () => DiagnosticContentSchema,
58
- DiagnosticCorrelationSchema: () => DiagnosticCorrelationSchema,
59
- DiagnosticManifestSchema: () => DiagnosticManifestSchema,
60
- DiagnosticNoteSchema: () => DiagnosticNoteSchema,
61
- DiagnosticPartSchema: () => DiagnosticPartSchema,
62
- DiagnosticResponseSchema: () => DiagnosticResponseSchema,
63
- EnsureWorkflowTopologyRequestSchema: () => EnsureWorkflowTopologyRequestSchema,
64
- EnsureWorkflowTopologyResponseSchema: () => EnsureWorkflowTopologyResponseSchema,
65
- FEEDBACK_FILTERS: () => FEEDBACK_FILTERS,
66
- FeedbackReasonSchema: () => FeedbackReasonSchema,
67
- FeedbackSummarySchema: () => FeedbackSummarySchema,
68
- PromptAssetSchema: () => PromptAssetSchema,
69
- PromptAssignmentSchema: () => PromptAssignmentSchema,
70
- PromptBundleSchema: () => PromptBundleSchema,
71
- PromptCategorySchema: () => PromptCategorySchema,
72
- PromptDetailSchema: () => PromptDetailSchema,
73
- PromptEvidenceSchema: () => PromptEvidenceSchema,
74
- PromptGroupSchema: () => PromptGroupSchema,
75
- PromptLibrarySchema: () => PromptLibrarySchema,
76
- PromptMutationSchema: () => PromptMutationSchema,
77
- PromptPolicySchema: () => PromptPolicySchema,
78
- PromptSelectionSchema: () => PromptSelectionSchema,
79
- PromptStoredVersionSchema: () => PromptStoredVersionSchema,
80
- PromptTransferRequestSchema: () => PromptTransferRequestSchema,
81
- PromptUsageSchema: () => PromptUsageSchema,
82
- ReviewVerdictSchema: () => ReviewVerdictSchema,
83
- STUDIO_API_PROTOCOL_VERSION: () => STUDIO_API_PROTOCOL_VERSION,
84
- STUDIO_TIME_RANGES: () => STUDIO_TIME_RANGES,
85
- StudioCatalogsResponseSchema: () => StudioCatalogsResponseSchema,
86
- StudioChangeResourceSchema: () => StudioChangeResourceSchema,
87
- StudioChangeSchema: () => StudioChangeSchema,
88
- StudioContextResponseSchema: () => StudioContextResponseSchema,
89
- StudioCostSummarySchema: () => StudioCostSummarySchema,
90
- StudioDetailEventSchema: () => StudioDetailEventSchema,
91
- StudioInterruptDetailResponseSchema: () => StudioInterruptDetailResponseSchema,
92
- StudioInterruptInteractionModeSchema: () => StudioInterruptInteractionModeSchema,
93
- StudioInterruptOptionSchema: () => StudioInterruptOptionSchema,
94
- StudioInterruptSchema: () => StudioInterruptSchema,
95
- StudioInterruptStatusSchema: () => StudioInterruptStatusSchema,
96
- StudioInterruptTypeSchema: () => StudioInterruptTypeSchema,
97
- StudioInterruptValueSchema: () => StudioInterruptValueSchema,
98
- StudioInterruptsResponseSchema: () => StudioInterruptsResponseSchema,
99
- StudioMetricSchema: () => StudioMetricSchema,
100
- StudioPricingSourceSchema: () => StudioPricingSourceSchema,
101
- StudioPricingStatusSchema: () => StudioPricingStatusSchema,
102
- StudioReleaseDeploymentSchema: () => StudioReleaseDeploymentSchema,
103
- StudioReleaseSchema: () => StudioReleaseSchema,
104
- StudioReleaseVersionSchema: () => StudioReleaseVersionSchema,
105
- StudioResumeOutcomeSchema: () => StudioResumeOutcomeSchema,
106
- StudioReviewRequestSchema: () => StudioReviewRequestSchema,
107
- StudioRunDetailResponseSchema: () => StudioRunDetailResponseSchema,
108
- StudioRunSchema: () => StudioRunSchema,
109
- StudioRunStatusSchema: () => StudioRunStatusSchema,
110
- StudioRunsResponseSchema: () => StudioRunsResponseSchema,
111
- StudioScoreResponseSchema: () => StudioScoreResponseSchema,
112
- StudioScoreSchema: () => StudioScoreSchema,
113
- StudioSessionDetailResponseSchema: () => StudioSessionDetailResponseSchema,
114
- StudioSessionSchema: () => StudioSessionSchema,
115
- StudioSessionsResponseSchema: () => StudioSessionsResponseSchema,
116
- StudioTimeRangeContextSchema: () => StudioTimeRangeContextSchema,
117
- StudioTimeRangeQuerySchema: () => StudioTimeRangeQuerySchema,
118
- StudioTimeRangeSchema: () => StudioTimeRangeSchema,
119
- StudioToolSchema: () => StudioToolSchema,
120
- StudioUpdateOperationSchema: () => StudioUpdateOperationSchema,
121
- StudioUpdateSettingsSchema: () => StudioUpdateSettingsSchema,
122
- StudioUpdateStatusSchema: () => StudioUpdateStatusSchema,
123
- StudioWorkflowHealthSchema: () => StudioWorkflowHealthSchema,
124
- StudioWorkflowInternalEdgeSchema: () => StudioWorkflowInternalEdgeSchema,
125
- StudioWorkflowNodeSchema: () => StudioWorkflowNodeSchema,
126
- StudioWorkflowSchema: () => StudioWorkflowSchema,
127
- StudioWorkflowTransitionSchema: () => StudioWorkflowTransitionSchema,
128
- StudioWorkflowsResponseSchema: () => StudioWorkflowsResponseSchema,
129
- TELEMETRY_EVENT_TYPES: () => TELEMETRY_EVENT_TYPES,
130
- TelemetryEventBatchResponseSchema: () => TelemetryEventBatchResponseSchema,
131
- TelemetryEventBatchSchema: () => TelemetryEventBatchSchema,
132
- TelemetryEventSchema: () => TelemetryEventSchema,
133
- TelemetryEventTypeSchema: () => TelemetryEventTypeSchema,
134
- TelemetryPricingHintSchema: () => TelemetryPricingHintSchema,
135
- TelemetryPricingLineItemSchema: () => TelemetryPricingLineItemSchema,
136
- TelemetryPricingSourceSchema: () => TelemetryPricingSourceSchema,
137
- TelemetryPricingUnitSchema: () => TelemetryPricingUnitSchema,
138
- TelemetryPricingUsageTypeSchema: () => TelemetryPricingUsageTypeSchema,
139
- TelemetryServiceSchema: () => TelemetryServiceSchema,
140
- TelemetryUnitPriceSchema: () => TelemetryUnitPriceSchema,
141
- TelemetryUsageItemSchema: () => TelemetryUsageItemSchema,
142
- ToolDiscoverySchema: () => ToolDiscoverySchema,
143
- ToolObservationSchema: () => ToolObservationSchema,
144
- ToolOutcomeSchema: () => ToolOutcomeSchema,
145
- UserFeedbackRequestSchema: () => UserFeedbackRequestSchema,
146
- WorkflowToolSchema: () => WorkflowToolSchema,
147
- WorkflowTopologyCallSchema: () => WorkflowTopologyCallSchema,
148
- WorkflowTopologyEdgeSchema: () => WorkflowTopologyEdgeSchema,
149
- WorkflowTopologyNodeSchema: () => WorkflowTopologyNodeSchema,
150
- WorkflowTopologyTransitionSchema: () => WorkflowTopologyTransitionSchema,
151
- isPrivateDiagnosticField: () => isPrivateDiagnosticField,
152
- newerStudioVersion: () => newerStudioVersion,
153
- projectWorkflowCalls: () => projectWorkflowCalls,
154
- redactDiagnosticContent: () => redactDiagnosticContent,
155
- redactDiagnosticText: () => redactDiagnosticText,
156
- resolveStudioInterruptStatus: () => resolveStudioInterruptStatus,
157
- resolveStudioTimeRange: () => resolveStudioTimeRange,
158
- studioUpdateRunning: () => studioUpdateRunning,
159
- summarizeDiagnostic: () => summarizeDiagnostic2,
160
- summarizeUserFeedback: () => summarizeUserFeedback
161
- });
162
- module2.exports = __toCommonJS2(index_exports2);
163
- var import_zod62 = require("zod");
164
- var import_zod14 = require("zod");
165
- var FEEDBACK_FILTERS = ["positive", "negative", "unrated"];
166
- var FeedbackSummarySchema = import_zod14.z.object({
167
- positive: import_zod14.z.number().int().nonnegative(),
168
- negative: import_zod14.z.number().int().nonnegative()
169
- }).strict();
170
- var FeedbackReasonSchema = import_zod14.z.enum([
171
- "incorrect",
172
- "irrelevant",
173
- "incomplete",
174
- "unsafe",
175
- "other"
176
- ]);
177
- var ReviewVerdictSchema = import_zod14.z.enum([
178
- "correct",
179
- "partially-correct",
180
- "incorrect"
181
- ]);
182
- var runId = import_zod14.z.string().trim().min(1).max(512);
183
- var actorId = import_zod14.z.string().trim().min(1).max(256);
184
- var comment = import_zod14.z.string().trim().max(4e3).nullable().optional();
185
- var UserFeedbackRequestSchema = import_zod14.z.object({
186
- runId,
187
- actorId,
188
- value: import_zod14.z.union([import_zod14.z.literal(0), import_zod14.z.literal(1)]),
189
- reasons: import_zod14.z.array(FeedbackReasonSchema).max(5).optional(),
190
- comment
191
- }).strict();
192
- var ClearUserFeedbackRequestSchema = UserFeedbackRequestSchema.pick({
193
- runId: true,
194
- actorId: true
195
- });
196
- var StudioReviewRequestSchema = import_zod14.z.object({ value: ReviewVerdictSchema, comment }).strict();
197
- var BaseScoreSchema = import_zod14.z.object({
198
- id: import_zod14.z.string().uuid(),
199
- target: import_zod14.z.object({ type: import_zod14.z.literal("run"), runId }).strict(),
200
- environment: import_zod14.z.string().min(1),
201
- name: import_zod14.z.string().min(1),
202
- source: import_zod14.z.enum(["end-user", "human-review", "evaluator"]),
203
- actorId,
204
- reasons: import_zod14.z.array(FeedbackReasonSchema),
205
- comment: import_zod14.z.string().nullable(),
206
- createdAt: import_zod14.z.string().datetime({ offset: true }),
207
- updatedAt: import_zod14.z.string().datetime({ offset: true })
208
- }).strict();
209
- var StudioScoreSchema = import_zod14.z.discriminatedUnion("dataType", [
210
- BaseScoreSchema.extend({
211
- dataType: import_zod14.z.literal("BOOLEAN"),
212
- value: import_zod14.z.union([import_zod14.z.literal(0), import_zod14.z.literal(1)])
213
- }),
214
- BaseScoreSchema.extend({
215
- dataType: import_zod14.z.literal("CATEGORICAL"),
216
- value: import_zod14.z.string()
217
- }),
218
- BaseScoreSchema.extend({
219
- dataType: import_zod14.z.literal("NUMERIC"),
220
- value: import_zod14.z.number().finite()
221
- })
222
- ]);
223
- var StudioScoreResponseSchema = import_zod14.z.object({ score: StudioScoreSchema }).strict();
224
- var ClearScoreResponseSchema = import_zod14.z.object({ ok: import_zod14.z.literal(true) }).strict();
225
- function summarizeUserFeedback(scores) {
226
- const feedback = { positive: 0, negative: 0 };
227
- for (const score of scores) {
228
- if (score.source !== "end-user" || score.name !== "user-feedback" || score.dataType !== "BOOLEAN")
229
- continue;
230
- if (score.value === 1) feedback.positive += 1;
231
- if (score.value === 0) feedback.negative += 1;
232
- }
233
- return feedback;
234
- }
235
- var import_zod22 = require("zod");
236
- var DIAGNOSTIC_LIMITS = {
237
- bytes: 8 * 1024 * 1024,
238
- partBytes: 64 * 1024,
239
- parts: 128,
240
- nodes: 1e4,
241
- depth: 128,
242
- properties: 1e3
243
- };
244
- var DiagnosticNoteSchema = import_zod22.z.object({
245
- path: import_zod22.z.string().max(4096),
246
- reason: import_zod22.z.string().max(128),
247
- originalBytes: import_zod22.z.number().int().nonnegative().optional(),
248
- originalCount: import_zod22.z.number().int().nonnegative().optional()
249
- }).strict();
250
- var DiagnosticContentSchema = import_zod22.z.object({
251
- schemaVersion: import_zod22.z.literal(1),
252
- data: import_zod22.z.record(import_zod22.z.string(), import_zod22.z.unknown()),
253
- capture: import_zod22.z.object({
254
- status: import_zod22.z.enum(["complete", "partial", "failed"]),
255
- omissions: import_zod22.z.array(DiagnosticNoteSchema).max(1e4),
256
- redactions: import_zod22.z.array(DiagnosticNoteSchema).max(1e4)
257
- }).strict()
258
- }).strict();
259
- var DiagnosticCorrelationSchema = import_zod22.z.object({
260
- runId: import_zod22.z.string().optional(),
261
- workflowId: import_zod22.z.string().optional(),
262
- nodeId: import_zod22.z.string().optional(),
263
- sessionId: import_zod22.z.string().optional(),
264
- invocationId: import_zod22.z.string().optional(),
265
- parentInvocationId: import_zod22.z.string().optional(),
266
- branchId: import_zod22.z.string().optional(),
267
- toolCallId: import_zod22.z.string().optional(),
268
- attemptId: import_zod22.z.string().optional(),
269
- workflowRevisionId: import_zod22.z.string().optional(),
270
- traceId: import_zod22.z.string().optional(),
271
- spanId: import_zod22.z.string().optional(),
272
- parentSpanId: import_zod22.z.string().optional()
273
- }).strict();
274
- var DiagnosticManifestSchema = import_zod22.z.object({
275
- schemaVersion: import_zod22.z.literal(1),
276
- diagnosticId: import_zod22.z.string().uuid(),
277
- occurrenceId: import_zod22.z.string().uuid(),
278
- occurredAt: import_zod22.z.string().datetime({ offset: true }),
279
- environment: import_zod22.z.string().min(1).max(256),
280
- service: import_zod22.z.object({
281
- name: import_zod22.z.string().min(1).max(256),
282
- deploymentRef: import_zod22.z.string().max(1024).optional()
283
- }).strict(),
284
- correlation: DiagnosticCorrelationSchema,
285
- handled: import_zod22.z.boolean(),
286
- severity: import_zod22.z.enum(["warning", "error"]),
287
- mechanism: import_zod22.z.enum(["span", "report", "tool"]),
288
- summary: import_zod22.z.object({ type: import_zod22.z.string().max(256), message: import_zod22.z.string().max(8192) }).strict(),
289
- captureStatus: import_zod22.z.enum(["complete", "partial", "failed"]),
290
- checksum: import_zod22.z.string().regex(/^[a-f0-9]{64}$/),
291
- byteLength: import_zod22.z.number().int().min(1).max(DIAGNOSTIC_LIMITS.bytes),
292
- partCount: import_zod22.z.number().int().min(1).max(DIAGNOSTIC_LIMITS.parts)
293
- }).strict().superRefine((value, ctx) => {
294
- if (value.partCount !== Math.ceil(value.byteLength / DIAGNOSTIC_LIMITS.partBytes))
295
- ctx.addIssue({
296
- code: "custom",
297
- path: ["partCount"],
298
- message: "Part count does not match byte length."
299
- });
300
- });
301
- var DiagnosticPartSchema = import_zod22.z.object({
302
- index: import_zod22.z.number().int().min(0).max(DIAGNOSTIC_LIMITS.parts - 1),
303
- data: import_zod22.z.string().max(Math.ceil(DIAGNOSTIC_LIMITS.partBytes / 3) * 4).regex(
304
- /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/
305
- )
306
- }).strict();
307
- var DiagnosticResponseSchema = import_zod22.z.object({
308
- manifest: DiagnosticManifestSchema,
309
- state: import_zod22.z.enum(["pending", "available", "incomplete", "expired"]),
310
- receivedParts: import_zod22.z.number().int().nonnegative(),
311
- expiresAt: import_zod22.z.string().datetime({ offset: true }),
312
- content: DiagnosticContentSchema.nullable(),
313
- contentChecksum: import_zod22.z.string().regex(/^[a-f0-9]{64}$/).nullable(),
314
- contentByteLength: import_zod22.z.number().int().nonnegative().nullable()
315
- }).strict();
316
- var isPrivateDiagnosticField = (key) => /^(?:password|passwd|secret|clientsecret|privatekey|credential|credentials|authorization|proxyauthorization|bearer|bearertoken|jwt|token|accesstoken|refreshtoken|authtoken|apikey|xapikey|apitoken|xauthtoken|xaccesstoken|awsaccesskeyid|awssecretaccesskey|awssessiontoken|xamzsecuritytoken|apisecret|cookie|setcookie|resumehandle|resumetoken)$/.test(
317
- key.toLowerCase().replace(/[^a-z0-9]/g, "")
318
- );
319
- function redactDiagnosticText(value) {
320
- return value.replace(/\bBearer\s+[A-Za-z0-9._~+/-]+=*/gi, "Bearer [REDACTED]").replace(
321
- /\b(?:ktyx_(?:test|live)_[^\s"'<>]+|sk-(?:proj-|ant-)?[A-Za-z0-9_-]{12,}|AIza[A-Za-z0-9_-]{20,}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)/g,
322
- "[REDACTED]"
323
- ).replace(
324
- /(["']?(?:api[_-]?key|access[_-]?token|refresh[_-]?token|password|passwd|secret|token|client[_-]?secret|authorization|cookie)["']?\s*[:=]\s*)(["'])(.*?)\2/gi,
325
- "$1$2[REDACTED]$2"
326
- ).replace(
327
- /(\b(?:api[_-]?key|access[_-]?token|refresh[_-]?token|password|passwd|secret|token|client[_-]?secret)\s*[:=]\s*)(?!["'[])[^\s,;}&]+/gi,
328
- "$1[REDACTED]"
329
- ).replace(
330
- /(\b(?:authorization|proxy-authorization)\s*:\s*Basic\s+)[A-Za-z0-9+/=]+/gi,
331
- "$1[REDACTED]"
332
- ).replace(
333
- /([?&](?:api[_-]?key|token|access[_-]?token|password)=)[^&#\s]+/gi,
334
- "$1[REDACTED]"
335
- ).replace(/(https?:\/\/)[^\s/@:]+:[^\s/@]+@/gi, "$1[REDACTED]@");
336
- }
337
- function summarizeDiagnostic2(data, depth = 0) {
338
- return {
339
- type: typeof data.type === "string" ? data.type.slice(0, 256) : "Error",
340
- message: typeof data.message === "string" ? data.message.slice(0, 8192) : "Execution reported a fault.",
341
- ...typeof data.stack === "string" ? { stack: data.stack.slice(0, 32768) } : {},
342
- ...depth < 4 && data.cause && typeof data.cause === "object" && !Array.isArray(data.cause) ? {
343
- cause: summarizeDiagnostic2(
344
- data.cause,
345
- depth + 1
346
- )
347
- } : {}
348
- };
349
- }
350
- function redactDiagnosticContent(content) {
351
- const redactions = [...content.capture.redactions];
352
- const omissions = [...content.capture.omissions];
353
- let nodes = 0;
354
- const visit = (value, path, depth = 0) => {
355
- if (++nodes > DIAGNOSTIC_LIMITS.nodes || depth > DIAGNOSTIC_LIMITS.depth) {
356
- const reason = depth > DIAGNOSTIC_LIMITS.depth ? "server_depth_limit" : "server_node_limit";
357
- if (omissions.length < 1e4)
358
- omissions.push({ path: path.slice(0, 4096), reason });
359
- return { $omitted: reason };
360
- }
361
- if (typeof value === "string") {
362
- const clean = redactDiagnosticText(value);
363
- if (clean !== value && redactions.length < 1e4)
364
- redactions.push({
365
- path: path.slice(0, 4096),
366
- reason: "server_credential_text"
367
- });
368
- return clean;
369
- }
370
- if (Array.isArray(value)) {
371
- if (value.length > DIAGNOSTIC_LIMITS.properties && omissions.length < 1e4)
372
- omissions.push({
373
- path: path.slice(0, 4096),
374
- reason: "server_collection_limit",
375
- originalCount: value.length
376
- });
377
- return value.slice(0, DIAGNOSTIC_LIMITS.properties).map((item, index) => visit(item, `${path}/${index}`, depth + 1));
378
- }
379
- if (value && typeof value === "object") {
380
- const keys = Object.keys(value);
381
- if (keys.length > DIAGNOSTIC_LIMITS.properties && omissions.length < 1e4)
382
- omissions.push({
383
- path: path.slice(0, 4096),
384
- reason: "server_property_limit",
385
- originalCount: keys.length
386
- });
387
- return Object.fromEntries(
388
- keys.slice(0, DIAGNOSTIC_LIMITS.properties).map((key) => {
389
- const nested = value[key];
390
- if (isPrivateDiagnosticField(key)) {
391
- if (redactions.length < 1e4 && nested !== "[REDACTED]")
392
- redactions.push({
393
- path: `${path}/${key}`.slice(0, 4096),
394
- reason: "server_credential_field"
395
- });
396
- return [key, "[REDACTED]"];
397
- }
398
- return [key, visit(nested, `${path}/${key}`, depth + 1)];
399
- })
400
- );
401
- }
402
- return value;
403
- };
404
- return {
405
- ...content,
406
- data: visit(content.data, "#/data"),
407
- capture: {
408
- ...content.capture,
409
- status: content.capture.status === "failed" ? "failed" : omissions.length ? "partial" : "complete",
410
- omissions: omissions.map((note) => ({
411
- ...note,
412
- path: redactDiagnosticText(note.path),
413
- reason: redactDiagnosticText(note.reason)
414
- })),
415
- redactions: redactions.map((note) => ({
416
- ...note,
417
- path: redactDiagnosticText(note.path),
418
- reason: redactDiagnosticText(note.reason)
419
- }))
420
- }
421
- };
422
- }
423
- var import_zod32 = require("zod");
424
- var ToolOutcomeSchema = import_zod32.z.enum([
425
- "success",
426
- "denied",
427
- "fault",
428
- "cancelled"
429
- ]);
430
- var ToolObservationSchema = import_zod32.z.object({
431
- version: import_zod32.z.literal(1),
432
- name: import_zod32.z.string().min(1).max(256),
433
- toolCallId: import_zod32.z.string().min(1),
434
- attemptId: import_zod32.z.string().min(1),
435
- callingMode: import_zod32.z.enum(["direct", "model"]),
436
- executed: import_zod32.z.boolean(),
437
- outcome: ToolOutcomeSchema.optional(),
438
- denialCode: import_zod32.z.string().regex(/^[A-Z][A-Z0-9_]{0,63}$/).optional(),
439
- errorType: import_zod32.z.string().max(256).optional(),
440
- errorMessage: import_zod32.z.string().max(8192).optional(),
441
- diagnosticId: import_zod32.z.string().uuid().optional(),
442
- durationMs: import_zod32.z.number().finite().nonnegative().optional(),
443
- workflowId: import_zod32.z.string().optional(),
444
- nodeId: import_zod32.z.string().optional(),
445
- workflowRevisionId: import_zod32.z.string().optional(),
446
- runId: import_zod32.z.string().optional(),
447
- invocationId: import_zod32.z.string().optional(),
448
- parentInvocationId: import_zod32.z.string().optional(),
449
- branchId: import_zod32.z.string().optional(),
450
- observationKind: import_zod32.z.enum(["reused", "waiting"]).optional(),
451
- suspended: import_zod32.z.boolean().optional(),
452
- source: import_zod32.z.object({
453
- runId: import_zod32.z.string().optional(),
454
- invocationId: import_zod32.z.string().optional(),
455
- branchId: import_zod32.z.string().optional(),
456
- toolCallId: import_zod32.z.string(),
457
- attemptId: import_zod32.z.string()
458
- }).strict().optional()
459
- });
460
- var WorkflowToolSchema = import_zod32.z.object({
461
- name: import_zod32.z.string().min(1).max(256),
462
- description: import_zod32.z.string().max(4096).optional(),
463
- callingMode: import_zod32.z.enum(["direct", "model"]),
464
- provenance: import_zod32.z.enum(["source", "observed"]),
465
- inputFields: import_zod32.z.array(
466
- import_zod32.z.object({ name: import_zod32.z.string(), type: import_zod32.z.string(), required: import_zod32.z.boolean() }).strict()
467
- ).optional()
468
- }).strict();
469
- var ToolDiscoverySchema = import_zod32.z.object({
470
- status: import_zod32.z.enum(["complete", "unresolved"]),
471
- publishedAt: import_zod32.z.string().datetime().optional(),
472
- warnings: import_zod32.z.array(import_zod32.z.string()).optional()
473
- }).strict();
474
- var StudioToolSchema = WorkflowToolSchema.extend({
475
- callingMode: import_zod32.z.enum(["direct", "model", "unknown"]),
476
- calls: import_zod32.z.number().nonnegative(),
477
- replays: import_zod32.z.number().nonnegative(),
478
- successes: import_zod32.z.number().nonnegative(),
479
- denials: import_zod32.z.number().nonnegative(),
480
- faults: import_zod32.z.number().nonnegative(),
481
- cancellations: import_zod32.z.number().nonnegative(),
482
- p50DurationMs: import_zod32.z.number().nonnegative().nullable(),
483
- p95DurationMs: import_zod32.z.number().nonnegative().nullable()
484
- }).strict();
485
- var import_browser4 = require("@kortyx/prompts/browser");
486
- var import_zod42 = require("zod");
487
- var name = import_zod42.z.string().trim().min(1).max(200);
488
- var version = import_zod42.z.number().int().positive();
489
- var revision = import_zod42.z.number().int().nonnegative();
490
- var PromptCategorySchema = import_zod42.z.object({
491
- id: import_zod42.z.uuid(),
492
- name,
493
- parentId: import_zod42.z.uuid().nullable(),
494
- revision
495
- });
496
- var PromptGroupSchema = import_zod42.z.object({
497
- id: import_zod42.z.uuid(),
498
- name,
499
- revision,
500
- members: import_zod42.z.array(
501
- import_zod42.z.object({
502
- promptId: import_zod42.z.uuid(),
503
- version,
504
- key: import_browser4.PromptKeySchema.optional(),
505
- name: name.optional(),
506
- archived: import_zod42.z.boolean().optional()
507
- })
508
- ).max(100),
509
- updatedAt: import_zod42.z.string()
510
- });
511
- var PromptAssignmentSchema = import_zod42.z.object({
512
- tag: import_browser4.PromptTagSchema,
513
- version,
514
- revision,
515
- updatedAt: import_zod42.z.string()
516
- });
517
- var PromptAssetSchema = import_zod42.z.object({
518
- id: import_zod42.z.uuid(),
519
- key: import_browser4.PromptKeySchema,
520
- name,
521
- categoryId: import_zod42.z.uuid().nullable(),
522
- latestVersion: version,
523
- revision,
524
- archived: import_zod42.z.boolean(),
525
- updatedAt: import_zod42.z.string(),
526
- assignments: import_zod42.z.array(PromptAssignmentSchema)
527
- });
528
- var PromptLibrarySchema = import_zod42.z.object({
529
- schemaVersion: import_zod42.z.literal(1),
530
- assets: import_zod42.z.array(PromptAssetSchema),
531
- categories: import_zod42.z.array(PromptCategorySchema),
532
- groups: import_zod42.z.array(PromptGroupSchema),
533
- totalCount: import_zod42.z.number().int().nonnegative(),
534
- nextCursor: import_zod42.z.string().nullable(),
535
- permissions: import_zod42.z.object({
536
- edit: import_zod42.z.boolean(),
537
- promote: import_zod42.z.boolean(),
538
- review: import_zod42.z.boolean(),
539
- settings: import_zod42.z.boolean()
540
- })
541
- });
542
- var PromptStoredVersionSchema = import_browser4.PromptVersionSchema.extend({
543
- promptId: import_zod42.z.uuid(),
544
- note: import_zod42.z.string(),
545
- author: import_zod42.z.string(),
546
- createdAt: import_zod42.z.string(),
547
- origin: import_zod42.z.object({
548
- apiUrl: import_zod42.z.string(),
549
- projectId: import_zod42.z.string(),
550
- key: import_zod42.z.string(),
551
- version,
552
- hash: import_zod42.z.string()
553
- }).nullable()
554
- });
555
- var PromptPolicySchema = import_zod42.z.object({
556
- revision,
557
- requireTest: import_zod42.z.boolean(),
558
- requireChangeNote: import_zod42.z.boolean().default(false),
559
- requiredSuites: import_zod42.z.array(import_zod42.z.object({ targetId: name, suiteId: name })).max(100),
560
- requiredReviews: import_zod42.z.number().int().min(0).max(10),
561
- allowException: import_zod42.z.boolean()
562
- });
563
- var PromptEvidenceSchema = import_zod42.z.object({
564
- environment: import_zod42.z.string(),
565
- runId: import_zod42.z.string(),
566
- evaluationId: import_zod42.z.string().nullable().optional(),
567
- targetId: import_zod42.z.string(),
568
- suiteId: import_zod42.z.string(),
569
- suiteRevision: import_zod42.z.string().optional(),
570
- status: import_zod42.z.string(),
571
- version,
572
- fullSuite: import_zod42.z.boolean(),
573
- usage: import_zod42.z.enum([
574
- "verified",
575
- "pending",
576
- "not-used",
577
- "mismatch",
578
- "context-differs"
579
- ]),
580
- groupName: import_zod42.z.string().nullable(),
581
- createdAt: import_zod42.z.string(),
582
- companions: import_zod42.z.array(import_zod42.z.object({ key: import_browser4.PromptKeySchema, version }))
583
- });
584
- var PromptUsageSchema = import_zod42.z.object({
585
- eventId: import_zod42.z.string(),
586
- runId: import_zod42.z.string(),
587
- sessionId: import_zod42.z.string().nullable(),
588
- nodeId: import_zod42.z.string().nullable(),
589
- version: import_zod42.z.number().int(),
590
- hash: import_zod42.z.string().nullable(),
591
- source: import_zod42.z.string().nullable(),
592
- model: import_zod42.z.string().nullable(),
593
- environment: import_zod42.z.string(),
594
- occurredAt: import_zod42.z.string(),
595
- captured: import_zod42.z.boolean()
596
- });
597
- var PromptDetailSchema = import_zod42.z.object({
598
- asset: PromptAssetSchema,
599
- versions: import_zod42.z.array(PromptStoredVersionSchema),
600
- versionsNextCursor: import_zod42.z.string().nullable().optional(),
601
- draft: import_browser4.PromptContentSchema.nullable(),
602
- draftBase: import_zod42.z.number().int().nullable(),
603
- draftRevision: revision,
604
- evidence: import_zod42.z.array(PromptEvidenceSchema),
605
- usage: import_zod42.z.array(PromptUsageSchema),
606
- activity: import_zod42.z.array(
607
- import_zod42.z.object({
608
- id: import_zod42.z.string(),
609
- action: import_zod42.z.string(),
610
- actor: import_zod42.z.string(),
611
- details: import_zod42.z.record(import_zod42.z.string(), import_zod42.z.unknown()),
612
- createdAt: import_zod42.z.string()
613
- })
614
- ),
615
- policies: import_zod42.z.array(PromptPolicySchema),
616
- reviews: import_zod42.z.array(
617
- import_zod42.z.object({
618
- version,
619
- hash: import_zod42.z.string(),
620
- independent: import_zod42.z.boolean(),
621
- reviewer: import_zod42.z.string(),
622
- note: import_zod42.z.string(),
623
- createdAt: import_zod42.z.string()
624
- })
625
- ).optional()
626
- });
627
- var PromptMutationSchema = import_zod42.z.discriminatedUnion("action", [
628
- import_zod42.z.object({
629
- action: import_zod42.z.literal("tag-set"),
630
- id: import_zod42.z.uuid(),
631
- tag: import_browser4.PromptTagSchema.refine(
632
- (tag) => tag !== "live",
633
- "Use promotion to update live."
634
- ),
635
- version,
636
- expectedRevision: revision
637
- }).strict(),
638
- import_zod42.z.object({
639
- action: import_zod42.z.literal("tag-remove"),
640
- id: import_zod42.z.uuid(),
641
- tag: import_browser4.PromptTagSchema.refine(
642
- (tag) => tag !== "live",
643
- "The live tag cannot be removed manually."
644
- ),
645
- expectedRevision: revision
646
- }).strict(),
647
- import_zod42.z.object({
648
- action: import_zod42.z.literal("bulk-update"),
649
- assets: import_zod42.z.array(import_zod42.z.object({ id: import_zod42.z.uuid(), expectedRevision: revision })).min(1).max(100),
650
- categoryId: import_zod42.z.uuid().nullable().optional(),
651
- archived: import_zod42.z.boolean().optional()
652
- }).strict().refine(
653
- (value) => value.categoryId !== void 0 || value.archived !== void 0,
654
- "Choose a category or archive state."
655
- ),
656
- import_zod42.z.object({
657
- action: import_zod42.z.literal("create"),
658
- key: import_browser4.PromptKeySchema,
659
- name,
660
- categoryId: import_zod42.z.uuid().nullable().default(null),
661
- content: import_browser4.PromptContentSchema,
662
- note: import_zod42.z.string().trim().max(2e3).default(""),
663
- idempotencyKey: import_zod42.z.uuid().optional()
664
- }).strict(),
665
- import_zod42.z.object({
666
- action: import_zod42.z.literal("draft"),
667
- id: import_zod42.z.uuid(),
668
- content: import_browser4.PromptContentSchema,
669
- baseVersion: version,
670
- expectedRevision: revision
671
- }).strict(),
672
- import_zod42.z.object({
673
- action: import_zod42.z.literal("discard-draft"),
674
- id: import_zod42.z.uuid(),
675
- expectedRevision: revision
676
- }).strict(),
677
- import_zod42.z.object({
678
- action: import_zod42.z.literal("save"),
679
- id: import_zod42.z.uuid(),
680
- content: import_browser4.PromptContentSchema,
681
- baseVersion: version,
682
- expectedHash: import_zod42.z.string().regex(/^[a-f0-9]{64}$/),
683
- expectedDraftRevision: revision.optional(),
684
- note: import_zod42.z.string().trim().max(2e3).default(""),
685
- idempotencyKey: import_zod42.z.uuid()
686
- }).strict(),
687
- import_zod42.z.object({
688
- action: import_zod42.z.literal("update"),
689
- id: import_zod42.z.uuid(),
690
- expectedRevision: revision,
691
- name: name.optional(),
692
- categoryId: import_zod42.z.uuid().nullable().optional(),
693
- archived: import_zod42.z.boolean().optional()
694
- }).strict(),
695
- import_zod42.z.object({
696
- action: import_zod42.z.literal("category-create"),
697
- path: import_zod42.z.string().trim().min(1).max(1e3)
698
- }).strict(),
699
- import_zod42.z.object({
700
- action: import_zod42.z.literal("category-update"),
701
- id: import_zod42.z.uuid(),
702
- expectedRevision: revision,
703
- name: name.optional(),
704
- parentId: import_zod42.z.uuid().nullable().optional()
705
- }).strict(),
706
- import_zod42.z.object({
707
- action: import_zod42.z.literal("category-delete"),
708
- id: import_zod42.z.uuid(),
709
- expectedRevision: revision,
710
- destinationId: import_zod42.z.uuid().nullable()
711
- }).strict(),
712
- import_zod42.z.object({
713
- action: import_zod42.z.literal("group-create"),
714
- name,
715
- members: import_zod42.z.array(import_zod42.z.object({ promptId: import_zod42.z.uuid(), version })).max(100).optional()
716
- }).strict(),
717
- import_zod42.z.object({
718
- action: import_zod42.z.literal("group-update"),
719
- id: import_zod42.z.uuid(),
720
- expectedRevision: revision,
721
- name: name.optional(),
722
- members: import_zod42.z.array(import_zod42.z.object({ promptId: import_zod42.z.uuid(), version })).max(100).optional()
723
- }).strict(),
724
- import_zod42.z.object({
725
- action: import_zod42.z.literal("group-delete"),
726
- id: import_zod42.z.uuid(),
727
- expectedRevision: revision
728
- }).strict(),
729
- import_zod42.z.object({
730
- action: import_zod42.z.literal("promote"),
731
- id: import_zod42.z.uuid(),
732
- version,
733
- expectedRevision: revision,
734
- exceptionReason: import_zod42.z.string().trim().min(1).max(2e3).optional(),
735
- rollback: import_zod42.z.boolean().default(false)
736
- }).strict(),
737
- import_zod42.z.object({
738
- action: import_zod42.z.literal("review"),
739
- id: import_zod42.z.uuid(),
740
- version,
741
- note: import_zod42.z.string().trim().min(1).max(2e3)
742
- }).strict(),
743
- import_zod42.z.object({
744
- action: import_zod42.z.literal("policy"),
745
- id: import_zod42.z.uuid(),
746
- policy: PromptPolicySchema
747
- }).strict()
748
- ]);
749
- var PromptSelectionSchema = import_zod42.z.discriminatedUnion("type", [
750
- import_zod42.z.object({ type: import_zod42.z.literal("live") }).strict(),
751
- import_zod42.z.object({
752
- type: import_zod42.z.literal("single"),
753
- id: import_zod42.z.string().min(1),
754
- version: import_zod42.z.number().int().positive()
755
- }).strict(),
756
- import_zod42.z.object({ type: import_zod42.z.literal("group"), groupId: import_zod42.z.uuid() }).strict()
757
- ]);
758
- var PromptBundleSchema = import_zod42.z.object({
759
- schemaVersion: import_zod42.z.literal(1),
760
- origin: import_zod42.z.object({
761
- apiUrl: import_zod42.z.url().refine((value) => {
762
- const url = new URL(value);
763
- return /^https?:$/.test(url.protocol) && !url.username && !url.password && !url.search;
764
- }),
765
- projectId: import_zod42.z.string().min(1)
766
- }).strict(),
767
- prompts: import_zod42.z.array(
768
- import_zod42.z.object({
769
- key: import_browser4.PromptKeySchema,
770
- name,
771
- categoryPath: import_zod42.z.string().optional(),
772
- versions: import_zod42.z.array(import_browser4.PromptVersionSchema.extend({ note: import_zod42.z.string().max(2e3) })).min(1).max(200)
773
- }).strict()
774
- ).min(1).max(200),
775
- groups: import_zod42.z.array(
776
- import_zod42.z.object({
777
- name,
778
- members: import_zod42.z.array(import_zod42.z.object({ key: import_browser4.PromptKeySchema, version })).max(100)
779
- }).strict()
780
- ).max(100).default([])
781
- }).strict();
782
- var PromptTransferRequestSchema = import_zod42.z.object({
783
- bundle: PromptBundleSchema,
784
- conflicts: import_zod42.z.enum(["error", "append"]).default("error"),
785
- rename: import_zod42.z.record(import_browser4.PromptKeySchema, import_browser4.PromptKeySchema).default({}),
786
- categories: import_zod42.z.boolean().default(true),
787
- groups: import_zod42.z.boolean().default(false)
788
- }).strict();
789
- var import_zod52 = require("zod");
790
- var StudioReleaseVersionSchema = import_zod52.z.string().regex(/^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/);
791
- var StudioReleaseDeploymentSchema = import_zod52.z.object({
792
- strategy: import_zod52.z.enum(["rolling", "recreate"])
793
- });
794
- var StudioReleaseSchema = import_zod52.z.object({
795
- format: import_zod52.z.literal(1),
796
- installer: import_zod52.z.literal(1),
797
- version: StudioReleaseVersionSchema,
798
- api: import_zod52.z.string().regex(/^ghcr\.io\/kortyx-io\/kortyx-api@sha256:[a-f0-9]{64}$/),
799
- studio: import_zod52.z.string().regex(/^ghcr\.io\/kortyx-io\/kortyx-studio@sha256:[a-f0-9]{64}$/),
800
- deployment: StudioReleaseDeploymentSchema.default({
801
- strategy: "recreate"
802
- })
803
- });
804
- var StudioUpdateSettingsSchema = import_zod52.z.object({
805
- automatic: import_zod52.z.boolean(),
806
- hourUtc: import_zod52.z.number().int().min(0).max(23)
807
- });
808
- var StudioUpdateOperationSchema = import_zod52.z.object({
809
- id: import_zod52.z.uuid(),
810
- from: import_zod52.z.string(),
811
- release: StudioReleaseSchema,
812
- phase: import_zod52.z.enum([
813
- "starting",
814
- "pulling",
815
- "backup",
816
- "installing",
817
- "verifying",
818
- "succeeded",
819
- "failed"
820
- ]),
821
- startedAt: import_zod52.z.iso.datetime(),
822
- updatedAt: import_zod52.z.iso.datetime(),
823
- backup: import_zod52.z.string().nullable(),
824
- messages: import_zod52.z.array(import_zod52.z.string()).max(100)
825
- });
826
- var StudioUpdateStatusSchema = import_zod52.z.object({
827
- current: import_zod52.z.string().nullable(),
828
- available: StudioReleaseSchema.nullable(),
829
- checkedAt: import_zod52.z.iso.datetime().nullable(),
830
- checkError: import_zod52.z.string().nullable(),
831
- settings: StudioUpdateSettingsSchema,
832
- operation: StudioUpdateOperationSchema.nullable()
833
- });
834
- function newerStudioVersion(candidate, current) {
835
- const next = StudioReleaseVersionSchema.parse(candidate).split(".").map(BigInt);
836
- const previous = StudioReleaseVersionSchema.parse(current).split(".").map(BigInt);
837
- for (let i = 0; i < 3; i++) {
838
- if (next[i] !== previous[i]) return (next[i] ?? 0n) > (previous[i] ?? 0n);
839
- }
840
- return false;
841
- }
842
- function studioUpdateRunning(operation) {
843
- return !!operation && operation.phase !== "succeeded" && operation.phase !== "failed";
844
- }
845
- var str = (v, fallback = "") => typeof v === "string" ? v : fallback;
846
- function projectWorkflowCalls(events) {
847
- const groups = /* @__PURE__ */ new Map();
848
- for (const event of events) {
849
- if (!event.type.startsWith("workflow.call.") || !str(event.payload.invocationId) || !str(event.payload.targetWorkflowId))
850
- continue;
851
- const key = `${event.runId}:${str(event.payload.branchId, event.runId)}:${event.payload.invocationId}`;
852
- const group = groups.get(key) ?? [];
853
- if (!group.some((existing) => existing.id === event.id)) group.push(event);
854
- groups.set(key, group);
855
- }
856
- return [...groups.entries()].map(([id3, group]) => {
857
- group.sort(
858
- (a, b) => Number(a.payload.sequence ?? 0) - Number(b.payload.sequence ?? 0) || a.occurredAt.localeCompare(b.occurredAt) || a.id.localeCompare(b.id)
859
- );
860
- const first = group[0];
861
- const p = first.payload;
862
- const call = {
863
- id: id3,
864
- invocationId: str(p.invocationId),
865
- branchId: str(p.branchId, first.runId),
866
- parentInvocationId: str(p.parentInvocationId) || null,
867
- callId: str(p.callId),
868
- callerNodeId: str(p.callerNodeId),
869
- callerNodeExecutionId: str(p.callerNodeExecutionId),
870
- sourceWorkflowId: str(p.sourceWorkflowId, first.workflowId),
871
- targetWorkflowId: str(p.targetWorkflowId),
872
- targetVersion: str(p.targetVersion) || null,
873
- status: "incomplete",
874
- startedAt: first.occurredAt,
875
- endedAt: null,
876
- activeMs: 0,
877
- waitMs: 0,
878
- attempts: 0,
879
- reused: 0,
880
- inherited: false,
881
- sourceRunId: null,
882
- leaf: null,
883
- events: group
884
- };
885
- let active = null;
886
- let waiting = null;
887
- for (const event of group) {
888
- const at = Date.parse(event.occurredAt);
889
- const status2 = event.type.slice("workflow.call.".length);
890
- for (const side of [
891
- "input",
892
- "output",
893
- "inputOmitted",
894
- "outputOmitted"
895
- ])
896
- if (side in event.payload) call[side] = event.payload[side];
897
- if (status2 === "restored") {
898
- call.inherited = true;
899
- call.sourceRunId = str(event.payload.sourceRunId) || null;
900
- call.status = event.payload.status === "completed" ? "completed" : event.payload.status === "failed" ? "failed" : "interrupted";
901
- if (call.status === "interrupted") waiting = at;
902
- } else if (status2 === "started" || status2 === "resumed") {
903
- if (waiting !== null) call.waitMs += Math.max(0, at - waiting);
904
- waiting = null;
905
- active = at;
906
- call.attempts++;
907
- call.status = "running";
908
- call.endedAt = null;
909
- } else if (status2 === "reused") {
910
- call.reused++;
911
- call.status = "completed";
912
- } else if (status2 === "suspended" || status2 === "completed" || status2 === "failed") {
913
- if (active !== null) call.activeMs += Math.max(0, at - active);
914
- active = null;
915
- call.status = status2 === "suspended" ? "interrupted" : status2;
916
- if (status2 === "suspended") {
917
- waiting = at;
918
- const leaf = event.payload.leaf;
919
- if (leaf && typeof leaf === "object") {
920
- const value = leaf;
921
- call.leaf = {
922
- workflowId: str(value.workflowId),
923
- nodeId: str(value.nodeId),
924
- invocationId: str(value.invocationId)
925
- };
926
- }
927
- } else {
928
- call.endedAt = event.occurredAt;
929
- call.leaf = null;
930
- }
931
- }
932
- }
933
- const cancellation = events.find(
934
- (event) => event.type === "run.cancelled" && event.runId === first.runId && (event.payload.branchId ?? event.runId) === call.branchId && event.occurredAt >= (group.at(-1)?.occurredAt ?? first.occurredAt)
935
- );
936
- if (cancellation && (call.status === "interrupted" || call.status === "running")) {
937
- call.status = "cancelled";
938
- call.endedAt = cancellation.occurredAt;
939
- }
940
- return call;
941
- }).sort((a, b) => a.startedAt.localeCompare(b.startedAt));
942
- }
943
- var STUDIO_API_PROTOCOL_VERSION = "1";
944
- var TELEMETRY_EVENT_TYPES = [
945
- "span.started",
946
- "span.ended",
947
- "span.failed",
948
- "error.reported",
949
- "generation.completed",
950
- "tool.started",
951
- "tool.completed",
952
- "tool.failed",
953
- "tool.denied",
954
- "tool.cancelled",
955
- "tool.reused",
956
- "tool.waiting",
957
- "tool.suspended",
958
- "response.completed",
959
- "output.emitted",
960
- "output.invalidated",
961
- "interrupt.created",
962
- "interrupt.resolved",
963
- "interrupt.expired",
964
- "interrupt.cancelled",
965
- "run.cancelled",
966
- "run.limit_reached",
967
- "workflow.suspended",
968
- "workflow.resumed",
969
- "workflow.transitioned",
970
- "workflow.call.started",
971
- "workflow.call.suspended",
972
- "workflow.call.resumed",
973
- "workflow.call.completed",
974
- "workflow.call.failed",
975
- "workflow.call.reused",
976
- "workflow.call.restored",
977
- "session.checkpointed",
978
- "session.forked",
979
- "session.rolled_back"
980
- ];
981
- var TelemetryServiceSchema = import_zod62.z.object({
982
- name: import_zod62.z.string().min(1),
983
- deploymentRef: import_zod62.z.string().min(1).optional()
984
- }).strict();
985
- var WorkflowTopologyNodeSchema = import_zod62.z.object({
986
- id: import_zod62.z.string().min(1),
987
- label: import_zod62.z.string().optional(),
988
- type: import_zod62.z.string().optional(),
989
- provider: import_zod62.z.string().optional(),
990
- model: import_zod62.z.string().optional(),
991
- metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).optional(),
992
- tools: import_zod62.z.array(WorkflowToolSchema).optional(),
993
- toolDiscovery: ToolDiscoverySchema.optional()
994
- }).strict();
995
- var WorkflowTopologyEdgeSchema = import_zod62.z.object({
996
- sourceNodeId: import_zod62.z.string().min(1),
997
- targetNodeId: import_zod62.z.string().min(1),
998
- condition: import_zod62.z.string().optional()
999
- }).strict();
1000
- var WorkflowTopologyCallSchema = import_zod62.z.object({
1001
- sourceNodeId: import_zod62.z.string().min(1),
1002
- targetWorkflowId: import_zod62.z.string().min(1)
1003
- }).strict();
1004
- var WorkflowTopologyTransitionSchema = import_zod62.z.object({
1005
- sourceNodeId: import_zod62.z.string().min(1).optional(),
1006
- targetWorkflowId: import_zod62.z.string().min(1),
1007
- condition: import_zod62.z.string().optional(),
1008
- intent: import_zod62.z.string().optional(),
1009
- kind: import_zod62.z.enum(["call", "handoff"]).optional()
1010
- }).strict();
1011
- var EnsureWorkflowTopologyRequestSchema = import_zod62.z.object({
1012
- schemaVersion: import_zod62.z.literal(1),
1013
- environment: import_zod62.z.string().min(1),
1014
- service: TelemetryServiceSchema,
1015
- workflow: import_zod62.z.object({
1016
- id: import_zod62.z.string().min(1),
1017
- declaredVersion: import_zod62.z.string().min(1),
1018
- description: import_zod62.z.string().optional(),
1019
- tags: import_zod62.z.array(import_zod62.z.string()).optional(),
1020
- topologyHash: import_zod62.z.string().regex(/^[a-f0-9]{64}$/),
1021
- nodes: import_zod62.z.array(WorkflowTopologyNodeSchema),
1022
- edges: import_zod62.z.array(WorkflowTopologyEdgeSchema),
1023
- transitions: import_zod62.z.array(WorkflowTopologyTransitionSchema).optional(),
1024
- calls: import_zod62.z.array(WorkflowTopologyCallSchema).optional()
1025
- }).strict()
1026
- }).strict();
1027
- var EnsureWorkflowTopologyResponseSchema = import_zod62.z.object({ workflowRevisionId: import_zod62.z.string().min(1), created: import_zod62.z.boolean() }).strict();
1028
- var TelemetryEventTypeSchema = import_zod62.z.enum(TELEMETRY_EVENT_TYPES);
1029
- var TelemetryEventSchema = import_zod62.z.object({
1030
- schemaVersion: import_zod62.z.literal(1),
1031
- eventId: import_zod62.z.string().min(1),
1032
- occurredAt: import_zod62.z.string().datetime({ offset: true }),
1033
- environment: import_zod62.z.string().min(1),
1034
- service: TelemetryServiceSchema,
1035
- correlation: import_zod62.z.object({
1036
- invocationId: import_zod62.z.string().optional(),
1037
- parentInvocationId: import_zod62.z.string().optional(),
1038
- branchId: import_zod62.z.string().optional(),
1039
- traceId: import_zod62.z.string().optional(),
1040
- spanId: import_zod62.z.string().optional(),
1041
- parentSpanId: import_zod62.z.string().optional(),
1042
- runId: import_zod62.z.string().min(1),
1043
- sessionId: import_zod62.z.string().optional(),
1044
- workflowId: import_zod62.z.string().min(1),
1045
- workflowRevisionId: import_zod62.z.string().optional(),
1046
- topologyHash: import_zod62.z.string().regex(/^[a-f0-9]{64}$/).optional(),
1047
- nodeId: import_zod62.z.string().optional()
1048
- }).strict(),
1049
- context: import_zod62.z.object({
1050
- userId: import_zod62.z.string().optional(),
1051
- tenantId: import_zod62.z.string().optional(),
1052
- tags: import_zod62.z.array(import_zod62.z.string()).optional(),
1053
- metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).optional()
1054
- }).strict().optional(),
1055
- type: TelemetryEventTypeSchema,
1056
- payload: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown())
1057
- }).strict().superRefine((event, ctx) => {
1058
- if (!event.type.startsWith("tool.")) return;
1059
- const canonical3 = event.payload.version === 1 || [
1060
- "tool.denied",
1061
- "tool.cancelled",
1062
- "tool.reused",
1063
- "tool.waiting",
1064
- "tool.suspended"
1065
- ].includes(event.type);
1066
- if (!canonical3) return;
1067
- const parsed = ToolObservationSchema.strict().safeParse(event.payload);
1068
- if (!parsed.success)
1069
- for (const issue2 of parsed.error.issues)
1070
- ctx.addIssue({ ...issue2, path: ["payload", ...issue2.path] });
1071
- else {
1072
- if ((parsed.data.errorType !== void 0 || parsed.data.errorMessage !== void 0) && parsed.data.outcome !== "fault")
1073
- ctx.addIssue({
1074
- code: "custom",
1075
- path: ["payload"],
1076
- message: "Error diagnostics require a fault outcome."
1077
- });
1078
- const terminalOutcomes = {
1079
- "tool.completed": "success",
1080
- "tool.failed": "fault",
1081
- "tool.denied": "denied",
1082
- "tool.cancelled": "cancelled"
1083
- };
1084
- const expected = terminalOutcomes[event.type];
1085
- if (expected && parsed.data.outcome !== expected)
1086
- ctx.addIssue({
1087
- code: "custom",
1088
- path: ["payload", "outcome"],
1089
- message: "Outcome must match the tool event type."
1090
- });
1091
- if (event.type === "tool.reused" && (parsed.data.executed || !parsed.data.source))
1092
- ctx.addIssue({
1093
- code: "custom",
1094
- path: ["payload"],
1095
- message: "Reuse must be unexecuted and reference its source."
1096
- });
1097
- }
1098
- });
1099
- var TelemetryEventBatchSchema = import_zod62.z.object({ events: import_zod62.z.array(TelemetryEventSchema).min(1) }).strict();
1100
- var TelemetryEventBatchResponseSchema = import_zod62.z.object({
1101
- accepted: import_zod62.z.number().int().nonnegative(),
1102
- inserted: import_zod62.z.number().int().nonnegative(),
1103
- duplicates: import_zod62.z.number().int().nonnegative()
1104
- }).strict();
1105
- var TelemetryPricingSourceSchema = import_zod62.z.enum([
1106
- "sdk",
1107
- "provider",
1108
- "custom",
1109
- "project-rate-card",
1110
- "default-rate-card"
1111
- ]);
1112
- var TelemetryPricingUnitSchema = import_zod62.z.enum([
1113
- "token",
1114
- "character",
1115
- "request",
1116
- "image",
1117
- "audio_second",
1118
- "audio_minute",
1119
- "video_second",
1120
- "video_minute",
1121
- "second",
1122
- "minute",
1123
- "custom"
1124
- ]);
1125
- var TelemetryPricingUsageTypeSchema = import_zod62.z.enum([
1126
- "input",
1127
- "output",
1128
- "reasoning",
1129
- "cache_read",
1130
- "cache_write",
1131
- "embedding",
1132
- "image_input",
1133
- "image_output",
1134
- "audio_input",
1135
- "audio_output",
1136
- "video_input",
1137
- "video_output",
1138
- "request",
1139
- "custom"
1140
- ]);
1141
- var TelemetryUsageItemSchema = import_zod62.z.object({
1142
- usageType: TelemetryPricingUsageTypeSchema,
1143
- quantity: import_zod62.z.number().nonnegative(),
1144
- unit: TelemetryPricingUnitSchema,
1145
- label: import_zod62.z.string().optional()
1146
- }).strict();
1147
- var TelemetryUnitPriceSchema = import_zod62.z.object({
1148
- usageType: TelemetryPricingUsageTypeSchema,
1149
- unit: TelemetryPricingUnitSchema,
1150
- unitQuantity: import_zod62.z.number().positive().default(1),
1151
- priceMicros: import_zod62.z.number().int().nonnegative(),
1152
- label: import_zod62.z.string().optional()
1153
- }).strict();
1154
- var TelemetryPricingLineItemSchema = import_zod62.z.object({
1155
- usageType: TelemetryPricingUsageTypeSchema,
1156
- quantity: import_zod62.z.number().nonnegative(),
1157
- unit: TelemetryPricingUnitSchema,
1158
- unitQuantity: import_zod62.z.number().positive().default(1),
1159
- unitPriceMicros: import_zod62.z.number().int().nonnegative().optional(),
1160
- totalCostMicros: import_zod62.z.number().int().nonnegative().optional(),
1161
- label: import_zod62.z.string().optional()
1162
- }).strict().refine(
1163
- (value) => value.totalCostMicros !== void 0 || value.unitPriceMicros !== void 0,
1164
- "line item requires totalCostMicros or unitPriceMicros"
1165
- );
1166
- var TelemetryPricingHintSchema = import_zod62.z.object({
1167
- source: TelemetryPricingSourceSchema.extract(["sdk", "provider", "custom"]),
1168
- currency: import_zod62.z.string().min(3).max(12).default("USD"),
1169
- actualCostMicros: import_zod62.z.number().int().nonnegative().optional(),
1170
- lineItems: import_zod62.z.array(TelemetryPricingLineItemSchema).optional(),
1171
- unitPrices: import_zod62.z.array(TelemetryUnitPriceSchema).optional(),
1172
- usageItems: import_zod62.z.array(TelemetryUsageItemSchema).optional(),
1173
- pricingRef: import_zod62.z.string().min(1).optional(),
1174
- metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).optional()
1175
- }).strict();
1176
- var StudioRunStatusSchema = import_zod62.z.enum([
1177
- "running",
1178
- "completed",
1179
- "interrupted",
1180
- "incomplete",
1181
- "failed",
1182
- "cancelled"
1183
- ]);
1184
- var STUDIO_TIME_RANGES = [
1185
- "Last hour",
1186
- "24 hours",
1187
- "7 days",
1188
- "30 days",
1189
- "All time",
1190
- "Custom range"
1191
- ];
1192
- var StudioTimeRangeSchema = import_zod62.z.enum(STUDIO_TIME_RANGES);
1193
- var StudioTimeRangeQuerySchema = import_zod62.z.object({
1194
- range: StudioTimeRangeSchema.optional(),
1195
- startedAfter: import_zod62.z.string().optional(),
1196
- startedBefore: import_zod62.z.string().optional()
1197
- }).strict();
1198
- var StudioTimeRangeContextSchema = import_zod62.z.object({
1199
- range: StudioTimeRangeSchema,
1200
- startedAfter: import_zod62.z.string().datetime({ offset: true }).nullable(),
1201
- startedBefore: import_zod62.z.string().datetime({ offset: true }).nullable()
1202
- }).strict();
1203
- var STUDIO_TIME_RANGE_MILLISECONDS = {
1204
- "Last hour": 60 * 60 * 1e3,
1205
- "24 hours": 24 * 60 * 60 * 1e3,
1206
- "7 days": 7 * 24 * 60 * 60 * 1e3,
1207
- "30 days": 30 * 24 * 60 * 60 * 1e3
1208
- };
1209
- var parseIsoBoundary = (value, label) => {
1210
- if (!value) {
1211
- return { ok: false, error: `${label} is required for a custom range.` };
1212
- }
1213
- if (!/(?:Z|[+-]\d{2}:\d{2})$/.test(value)) {
1214
- return {
1215
- ok: false,
1216
- error: `${label} must include an explicit UTC offset.`
1217
- };
1218
- }
1219
- const date = new Date(value);
1220
- return Number.isFinite(date.getTime()) ? { ok: true, date } : { ok: false, error: `${label} must be a valid ISO date and time.` };
1221
- };
1222
- var resolveStudioTimeRange = (input, now = /* @__PURE__ */ new Date()) => {
1223
- const parsedRange = StudioTimeRangeSchema.safeParse(
1224
- input.range || "24 hours"
1225
- );
1226
- if (!parsedRange.success) {
1227
- return {
1228
- ok: false,
1229
- error: `Unknown time range "${input.range}".`
1230
- };
1231
- }
1232
- if (!Number.isFinite(now.getTime())) {
1233
- return { ok: false, error: "The time range reference date is invalid." };
1234
- }
1235
- const range = parsedRange.data;
1236
- if (range === "All time") {
1237
- return {
1238
- ok: true,
1239
- value: { range, startedAfter: null, startedBefore: null }
1240
- };
1241
- }
1242
- if (range === "Custom range") {
1243
- const after = parseIsoBoundary(input.startedAfter, "Start time");
1244
- if ("error" in after) return { ok: false, error: after.error };
1245
- const before = parseIsoBoundary(input.startedBefore, "End time");
1246
- if ("error" in before) return { ok: false, error: before.error };
1247
- if (after.date.getTime() >= before.date.getTime()) {
1248
- return {
1249
- ok: false,
1250
- error: "Start time must be before end time."
1251
- };
1252
- }
1253
- return {
1254
- ok: true,
1255
- value: {
1256
- range,
1257
- startedAfter: after.date.toISOString(),
1258
- startedBefore: before.date.toISOString()
1259
- }
1260
- };
1261
- }
1262
- const duration = STUDIO_TIME_RANGE_MILLISECONDS[range];
1263
- if (!duration) {
1264
- return { ok: false, error: `Unknown time range "${range}".` };
1265
- }
1266
- return {
1267
- ok: true,
1268
- value: {
1269
- range,
1270
- startedAfter: new Date(now.getTime() - duration).toISOString(),
1271
- startedBefore: now.toISOString()
1272
- }
1273
- };
1274
- };
1275
- var StudioInterruptStatusSchema = import_zod62.z.enum([
1276
- "pending",
1277
- "resolved",
1278
- "expired",
1279
- "failed",
1280
- "cancelled"
1281
- ]);
1282
- var resolveStudioInterruptStatus = (input, now = Date.now()) => {
1283
- if (input.status !== "pending" || !input.expiresAt) return input.status;
1284
- const expiresAt = Date.parse(input.expiresAt);
1285
- const nowMs = typeof now === "number" ? now : now.getTime();
1286
- return Number.isFinite(expiresAt) && Number.isFinite(nowMs) && expiresAt <= nowMs ? "expired" : input.status;
1287
- };
1288
- var StudioInterruptTypeSchema = import_zod62.z.enum([
1289
- "choice",
1290
- "multi-choice",
1291
- "text",
1292
- "structured",
1293
- "unknown"
1294
- ]);
1295
- var StudioInterruptInteractionModeSchema = import_zod62.z.enum([
1296
- "static-options",
1297
- "dynamic-picker",
1298
- "freeform",
1299
- "unknown"
1300
- ]);
1301
- var StudioInterruptOptionSchema = import_zod62.z.object({
1302
- id: import_zod62.z.string().min(1),
1303
- label: import_zod62.z.string().min(1),
1304
- description: import_zod62.z.string().nullable()
1305
- }).strip();
1306
- var StudioInterruptValueSchema = import_zod62.z.union([
1307
- import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()),
1308
- import_zod62.z.array(import_zod62.z.unknown()),
1309
- import_zod62.z.string(),
1310
- import_zod62.z.number(),
1311
- import_zod62.z.boolean()
1312
- ]);
1313
- var StudioResumeOutcomeSchema = import_zod62.z.enum([
1314
- "resumed",
1315
- "resume failed",
1316
- "expired before resume",
1317
- "cancelled"
1318
- ]);
1319
- var StudioWorkflowHealthSchema = import_zod62.z.enum([
1320
- "unknown",
1321
- "healthy",
1322
- "degraded",
1323
- "failing",
1324
- "idle"
1325
- ]);
1326
- var StudioPricingStatusSchema = import_zod62.z.enum([
1327
- "priced",
1328
- "unpriced",
1329
- "unknown"
1330
- ]);
1331
- var StudioPricingSourceSchema = TelemetryPricingSourceSchema.nullable();
1332
- var StudioChangeResourceSchema = import_zod62.z.enum([
1333
- "evals",
1334
- "workflows",
1335
- "runs",
1336
- "sessions",
1337
- "interrupts"
1338
- ]);
1339
- var StudioChangeSchema = import_zod62.z.object({
1340
- schemaVersion: import_zod62.z.literal(1),
1341
- changeId: import_zod62.z.string().min(1),
1342
- emittedAt: import_zod62.z.string().datetime({ offset: true }),
1343
- organizationId: import_zod62.z.string().min(1),
1344
- projectId: import_zod62.z.string().min(1),
1345
- environment: import_zod62.z.string().min(1).optional(),
1346
- resources: import_zod62.z.array(StudioChangeResourceSchema).min(1)
1347
- }).strict();
1348
- var StudioCostSummarySchema = import_zod62.z.object({
1349
- amounts: import_zod62.z.array(
1350
- import_zod62.z.object({
1351
- amount: import_zod62.z.number().nonnegative(),
1352
- currency: import_zod62.z.string().nullable()
1353
- }).strip()
1354
- ),
1355
- complete: import_zod62.z.boolean()
1356
- }).strip();
1357
- var StudioMetricSchema = import_zod62.z.object({
1358
- runCount: import_zod62.z.number().int().nonnegative(),
1359
- successRate: import_zod62.z.number().min(0).max(100).nullable(),
1360
- errorRate: import_zod62.z.number().min(0).max(100).nullable(),
1361
- retryCount: import_zod62.z.number().int().nonnegative().nullable(),
1362
- interruptRate: import_zod62.z.number().min(0).max(100).nullable(),
1363
- p50DurationMs: import_zod62.z.number().nonnegative().nullable(),
1364
- p95DurationMs: import_zod62.z.number().nonnegative().nullable(),
1365
- averageTokens: import_zod62.z.number().nonnegative().nullable(),
1366
- averageCost: import_zod62.z.number().nonnegative().nullable(),
1367
- averageCostSummary: StudioCostSummarySchema.optional(),
1368
- currency: import_zod62.z.string().nullable()
1369
- }).strip();
1370
- var StudioRunSchema = import_zod62.z.object({
1371
- id: import_zod62.z.string().min(1),
1372
- feedback: FeedbackSummarySchema.optional(),
1373
- callerNodeId: import_zod62.z.string().optional(),
1374
- parentRunId: import_zod62.z.string().optional(),
1375
- parentWorkflowId: import_zod62.z.string().optional(),
1376
- invocationId: import_zod62.z.string().optional(),
1377
- branchId: import_zod62.z.string().optional(),
1378
- callId: import_zod62.z.string().optional(),
1379
- status: StudioRunStatusSchema,
1380
- startedAt: import_zod62.z.string().datetime({ offset: true }),
1381
- endedAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
1382
- workflowId: import_zod62.z.string().min(1),
1383
- workflowIds: import_zod62.z.array(import_zod62.z.string().min(1)),
1384
- workflowRefs: import_zod62.z.array(
1385
- import_zod62.z.object({
1386
- workflowId: import_zod62.z.string().min(1),
1387
- workflowRevisionId: import_zod62.z.string().nullable(),
1388
- declaredVersion: import_zod62.z.string().nullable()
1389
- }).strip()
1390
- ),
1391
- workflowRevisionId: import_zod62.z.string().nullable(),
1392
- declaredVersion: import_zod62.z.string().nullable(),
1393
- transitionIds: import_zod62.z.array(import_zod62.z.string().min(1)),
1394
- path: import_zod62.z.array(import_zod62.z.string()),
1395
- sessionId: import_zod62.z.string().nullable(),
1396
- provider: import_zod62.z.string().nullable(),
1397
- model: import_zod62.z.string().nullable(),
1398
- models: import_zod62.z.array(import_zod62.z.string()),
1399
- durationMs: import_zod62.z.number().nonnegative().nullable(),
1400
- tokens: import_zod62.z.number().int().nonnegative().nullable(),
1401
- cost: import_zod62.z.number().nonnegative().nullable(),
1402
- costSummary: StudioCostSummarySchema.optional(),
1403
- pricingStatus: StudioPricingStatusSchema,
1404
- pricingSource: StudioPricingSourceSchema,
1405
- currency: import_zod62.z.string().nullable(),
1406
- result: import_zod62.z.string().nullable(),
1407
- environment: import_zod62.z.string().min(1),
1408
- userId: import_zod62.z.string().nullable(),
1409
- tenantId: import_zod62.z.string().nullable(),
1410
- hasTool: import_zod62.z.boolean(),
1411
- hasRetry: import_zod62.z.boolean(),
1412
- interruptNodeId: import_zod62.z.string().nullable(),
1413
- interruptId: import_zod62.z.string().nullable().optional(),
1414
- interruptStatus: StudioInterruptStatusSchema.nullable().optional(),
1415
- interruptExpiresAt: import_zod62.z.string().datetime({ offset: true }).nullable().optional()
1416
- }).strip();
1417
- var StudioSessionSchema = import_zod62.z.object({
1418
- id: import_zod62.z.string().min(1),
1419
- status: StudioRunStatusSchema,
1420
- lastActivityAt: import_zod62.z.string().datetime({ offset: true }),
1421
- workflowIds: import_zod62.z.array(import_zod62.z.string()),
1422
- workflowCount: import_zod62.z.number().int().nonnegative(),
1423
- activeWorkflowId: import_zod62.z.string().nullable(),
1424
- activeVersion: import_zod62.z.string().nullable(),
1425
- userId: import_zod62.z.string().nullable(),
1426
- tenantId: import_zod62.z.string().nullable(),
1427
- runs: import_zod62.z.number().int().nonnegative(),
1428
- succeeded: import_zod62.z.number().int().nonnegative(),
1429
- failed: import_zod62.z.number().int().nonnegative(),
1430
- interrupted: import_zod62.z.number().int().nonnegative(),
1431
- checkpoints: import_zod62.z.number().int().nonnegative(),
1432
- hasFork: import_zod62.z.boolean(),
1433
- durationMs: import_zod62.z.number().nonnegative().nullable(),
1434
- tokens: import_zod62.z.number().int().nonnegative().nullable(),
1435
- cost: import_zod62.z.number().nonnegative().nullable(),
1436
- costSummary: StudioCostSummarySchema.optional(),
1437
- pricingStatus: StudioPricingStatusSchema,
1438
- pricingSource: StudioPricingSourceSchema,
1439
- currency: import_zod62.z.string().nullable(),
1440
- latestResult: import_zod62.z.string().nullable(),
1441
- latestError: import_zod62.z.string().nullable(),
1442
- pendingInterruptId: import_zod62.z.string().nullable(),
1443
- interruptStatus: StudioInterruptStatusSchema.nullable().optional(),
1444
- interruptExpiresAt: import_zod62.z.string().datetime({ offset: true }).nullable().optional(),
1445
- providers: import_zod62.z.array(import_zod62.z.string()),
1446
- models: import_zod62.z.array(import_zod62.z.string()),
1447
- tags: import_zod62.z.array(import_zod62.z.string()),
1448
- environment: import_zod62.z.string().min(1)
1449
- }).strip();
1450
- var StudioInterruptSchema = import_zod62.z.object({
1451
- afterResponseCompleted: import_zod62.z.boolean().optional(),
1452
- id: import_zod62.z.string().min(1),
1453
- status: StudioInterruptStatusSchema,
1454
- type: StudioInterruptTypeSchema,
1455
- interactionMode: StudioInterruptInteractionModeSchema,
1456
- contract: import_zod62.z.string().nullable().default(null),
1457
- schemaId: import_zod62.z.string().nullable(),
1458
- schemaVersion: import_zod62.z.string().nullable(),
1459
- createdAt: import_zod62.z.string().datetime({ offset: true }),
1460
- resolvedAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
1461
- expiresAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
1462
- question: import_zod62.z.string().nullable(),
1463
- contentCaptured: import_zod62.z.boolean(),
1464
- request: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).nullable().default(null),
1465
- requestCaptured: import_zod62.z.boolean().default(false),
1466
- optionCount: import_zod62.z.number().int().nonnegative().nullable(),
1467
- options: import_zod62.z.array(StudioInterruptOptionSchema).nullable(),
1468
- workflowId: import_zod62.z.string().min(1),
1469
- nodeId: import_zod62.z.string().nullable(),
1470
- sessionId: import_zod62.z.string().nullable(),
1471
- userId: import_zod62.z.string().nullable(),
1472
- tenantId: import_zod62.z.string().nullable(),
1473
- response: import_zod62.z.string().nullable(),
1474
- responseValue: StudioInterruptValueSchema.nullable().default(null),
1475
- responseCaptured: import_zod62.z.boolean(),
1476
- resumeOutcome: StudioResumeOutcomeSchema.nullable(),
1477
- resumeError: import_zod62.z.string().nullable(),
1478
- runId: import_zod62.z.string().min(1),
1479
- resumeToken: import_zod62.z.string().nullable(),
1480
- resolvedBy: import_zod62.z.string().nullable(),
1481
- environment: import_zod62.z.string().min(1)
1482
- }).strip();
1483
- var StudioDetailEventSchema = import_zod62.z.object({
1484
- id: import_zod62.z.string().min(1),
1485
- // Studio readers must preserve future event kinds within the same API major.
1486
- type: import_zod62.z.string().min(1),
1487
- occurredAt: import_zod62.z.string().datetime({ offset: true }),
1488
- receivedAt: import_zod62.z.string().datetime({ offset: true }),
1489
- environment: import_zod62.z.string().min(1),
1490
- serviceName: import_zod62.z.string().min(1),
1491
- deploymentRef: import_zod62.z.string().nullable(),
1492
- traceId: import_zod62.z.string().nullable(),
1493
- spanId: import_zod62.z.string().nullable(),
1494
- parentSpanId: import_zod62.z.string().nullable(),
1495
- runId: import_zod62.z.string().min(1),
1496
- sessionId: import_zod62.z.string().nullable(),
1497
- workflowId: import_zod62.z.string().min(1),
1498
- workflowRevisionId: import_zod62.z.string().nullable(),
1499
- nodeId: import_zod62.z.string().nullable(),
1500
- userId: import_zod62.z.string().nullable(),
1501
- tenantId: import_zod62.z.string().nullable(),
1502
- tags: import_zod62.z.array(import_zod62.z.string()),
1503
- metadata: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown()).nullable(),
1504
- payload: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown())
1505
- }).strip();
1506
- var StudioWorkflowNodeSchema = import_zod62.z.object({
1507
- tools: import_zod62.z.array(StudioToolSchema).optional(),
1508
- toolDiscovery: ToolDiscoverySchema.optional(),
1509
- id: import_zod62.z.string().min(1),
1510
- label: import_zod62.z.string(),
1511
- type: import_zod62.z.string().nullable(),
1512
- state: import_zod62.z.enum(["healthy", "warning", "failed", "interrupted", "retried"]).nullable(),
1513
- provider: import_zod62.z.string().nullable(),
1514
- model: import_zod62.z.string().nullable(),
1515
- metrics: StudioMetricSchema
1516
- }).strip();
1517
- var StudioWorkflowInternalEdgeSchema = import_zod62.z.object({
1518
- id: import_zod62.z.string().min(1),
1519
- source: import_zod62.z.string().min(1),
1520
- target: import_zod62.z.string().min(1),
1521
- condition: import_zod62.z.string().nullable()
1522
- }).strip();
1523
- var StudioWorkflowTransitionSchema = import_zod62.z.object({
1524
- kind: import_zod62.z.enum(["call", "handoff"]).optional(),
1525
- id: import_zod62.z.string().min(1),
1526
- sourceWorkflowId: import_zod62.z.string().min(1),
1527
- sourceNodeId: import_zod62.z.string().nullable(),
1528
- targetWorkflowId: import_zod62.z.string().min(1),
1529
- condition: import_zod62.z.string().nullable(),
1530
- volume: import_zod62.z.number().int().nonnegative(),
1531
- successRate: import_zod62.z.number().min(0).max(100).nullable(),
1532
- medianDurationMs: import_zod62.z.number().nonnegative().nullable(),
1533
- errorRate: import_zod62.z.number().min(0).max(100).nullable()
1534
- }).strip();
1535
- var StudioWorkflowSchema = import_zod62.z.object({
1536
- id: import_zod62.z.string().min(1),
1537
- name: import_zod62.z.string().min(1),
1538
- description: import_zod62.z.string().nullable(),
1539
- versions: import_zod62.z.array(import_zod62.z.string()),
1540
- activeVersion: import_zod62.z.string().nullable(),
1541
- activeRevisionId: import_zod62.z.string().nullable(),
1542
- health: StudioWorkflowHealthSchema,
1543
- tags: import_zod62.z.array(import_zod62.z.string()),
1544
- lastActivityAt: import_zod62.z.string().datetime({ offset: true }).nullable(),
1545
- metrics: StudioMetricSchema,
1546
- nodes: import_zod62.z.array(StudioWorkflowNodeSchema),
1547
- internalEdges: import_zod62.z.array(StudioWorkflowInternalEdgeSchema)
1548
- }).strip();
1549
- var StudioRunsResponseSchema = import_zod62.z.object({
1550
- runs: import_zod62.z.array(StudioRunSchema),
1551
- totalCount: import_zod62.z.number().int().nonnegative()
1552
- }).strip();
1553
- var StudioSessionsResponseSchema = import_zod62.z.object({
1554
- sessions: import_zod62.z.array(StudioSessionSchema),
1555
- totalCount: import_zod62.z.number().int().nonnegative()
1556
- }).strip();
1557
- var StudioInterruptsResponseSchema = import_zod62.z.object({
1558
- interrupts: import_zod62.z.array(StudioInterruptSchema),
1559
- totalCount: import_zod62.z.number().int().nonnegative()
1560
- }).strip();
1561
- var StudioRunDetailResponseSchema = import_zod62.z.object({
1562
- run: StudioRunSchema,
1563
- scores: import_zod62.z.array(StudioScoreSchema).optional(),
1564
- canReview: import_zod62.z.boolean().optional(),
1565
- reviewActorId: import_zod62.z.string().optional(),
1566
- events: import_zod62.z.array(StudioDetailEventSchema),
1567
- session: StudioSessionSchema.nullable(),
1568
- interrupts: import_zod62.z.array(StudioInterruptSchema),
1569
- updatedAt: import_zod62.z.string().datetime({ offset: true })
1570
- }).strip();
1571
- var StudioSessionDetailResponseSchema = import_zod62.z.object({
1572
- session: StudioSessionSchema,
1573
- runs: import_zod62.z.array(StudioRunSchema),
1574
- events: import_zod62.z.array(StudioDetailEventSchema),
1575
- interrupts: import_zod62.z.array(StudioInterruptSchema),
1576
- updatedAt: import_zod62.z.string().datetime({ offset: true })
1577
- }).strip();
1578
- var StudioInterruptDetailResponseSchema = import_zod62.z.object({
1579
- interrupt: StudioInterruptSchema,
1580
- events: import_zod62.z.array(StudioDetailEventSchema),
1581
- run: StudioRunSchema.nullable(),
1582
- session: StudioSessionSchema.nullable(),
1583
- updatedAt: import_zod62.z.string().datetime({ offset: true })
1584
- }).strip();
1585
- var StudioWorkflowsResponseSchema = import_zod62.z.object({
1586
- workflows: import_zod62.z.array(StudioWorkflowSchema),
1587
- observedCalls: import_zod62.z.array(
1588
- StudioWorkflowTransitionSchema.extend({
1589
- runId: import_zod62.z.string(),
1590
- invocationId: import_zod62.z.string(),
1591
- branchId: import_zod62.z.string()
1592
- })
1593
- ).optional(),
1594
- transitions: import_zod62.z.array(StudioWorkflowTransitionSchema),
1595
- cohort: StudioTimeRangeContextSchema.extend({
1596
- workflowId: import_zod62.z.string().nullable(),
1597
- version: import_zod62.z.string().nullable()
1598
- }).strip()
1599
- }).strip();
1600
- var StudioCatalogsResponseSchema = import_zod62.z.object({
1601
- environments: import_zod62.z.array(import_zod62.z.string()),
1602
- providers: import_zod62.z.array(import_zod62.z.string()),
1603
- models: import_zod62.z.array(import_zod62.z.string()),
1604
- workflows: import_zod62.z.array(import_zod62.z.string()),
1605
- tags: import_zod62.z.array(import_zod62.z.string())
1606
- }).strip();
1607
- var StudioContextResponseSchema = import_zod62.z.object({
1608
- organization: import_zod62.z.object({
1609
- name: import_zod62.z.string().min(1)
1610
- }).strip(),
1611
- project: import_zod62.z.object({
1612
- name: import_zod62.z.string().min(1)
1613
- }).strip(),
1614
- environments: import_zod62.z.array(import_zod62.z.string()),
1615
- apiKey: import_zod62.z.object({
1616
- mode: import_zod62.z.enum(["test", "live"]),
1617
- scopes: import_zod62.z.array(import_zod62.z.string())
1618
- }).strip().nullable(),
1619
- api: import_zod62.z.object({
1620
- status: import_zod62.z.literal("ok"),
1621
- service: import_zod62.z.literal("kortyx-api"),
1622
- version: import_zod62.z.string().min(1)
1623
- }).strip()
1624
- }).strip();
1625
- }
1626
- });
1627
-
1628
32
  // ../hooks/dist/internal.js
1629
33
  var require_internal = __commonJS({
1630
34
  "../hooks/dist/internal.js"(exports2, module2) {
@@ -1647,7 +51,7 @@ var require_internal = __commonJS({
1647
51
  var __toCommonJS2 = (mod) => __copyProps2(__defProp2({}, "__esModule", { value: true }), mod);
1648
52
  var internal_exports = {};
1649
53
  __export2(internal_exports, {
1650
- captureDiagnosticContent: () => captureDiagnosticContent,
54
+ captureDiagnosticContent: () => import_telemetry_contracts4.captureDiagnosticContent,
1651
55
  errorDiagnostics: () => errorDiagnostics,
1652
56
  exceptionDiagnostics: () => exceptionDiagnostics,
1653
57
  isModelTraceSpan: () => isModelTraceSpan,
@@ -1655,255 +59,20 @@ var require_internal = __commonJS({
1655
59
  safeSpan: () => safeSpan,
1656
60
  safeStartSpan: () => safeStartSpan2,
1657
61
  safeTelemetryMetadata: () => safeTelemetryMetadata2,
1658
- summarizeDiagnostic: () => import_telemetry_contracts3.summarizeDiagnostic,
62
+ summarizeDiagnostic: () => import_telemetry_contracts32.summarizeDiagnostic,
1659
63
  withSafeTraceSpan: () => withSafeTraceSpan2
1660
64
  });
1661
65
  module2.exports = __toCommonJS2(internal_exports);
1662
- var import_telemetry_contracts3 = require_dist();
1663
- var import_errors16 = require("@kortyx/core/errors");
1664
- var import_telemetry_contracts = require_dist();
1665
- var encoder = new TextEncoder();
1666
- var nativeStackGetter = Object.getOwnPropertyDescriptor(
1667
- new Error(),
1668
- "stack"
1669
- )?.get;
1670
- function nativeStack(object2, getter) {
1671
- if (!getter || getter !== nativeStackGetter)
1672
- throw new Error("Custom stack accessor");
1673
- for (const key of ["name", "message"]) {
1674
- let cursor = object2;
1675
- while (cursor) {
1676
- const descriptor = Object.getOwnPropertyDescriptor(cursor, key);
1677
- if (descriptor) {
1678
- if (!("value" in descriptor) || typeof descriptor.value !== "string")
1679
- throw new Error("Custom error formatting");
1680
- break;
1681
- }
1682
- cursor = Object.getPrototypeOf(cursor);
1683
- }
1684
- }
1685
- const formatter = Object.getOwnPropertyDescriptor(Error, "prepareStackTrace");
1686
- if (formatter && (!("value" in formatter) || !formatter.writable))
1687
- throw new Error("Custom stack formatter");
1688
- Object.defineProperty(Error, "prepareStackTrace", {
1689
- ...formatter ?? { configurable: true, writable: true },
1690
- value: void 0
1691
- });
1692
- try {
1693
- return getter.call(object2);
1694
- } finally {
1695
- if (formatter) Object.defineProperty(Error, "prepareStackTrace", formatter);
1696
- else Reflect.deleteProperty(Error, "prepareStackTrace");
1697
- }
1698
- }
1699
- var pointer = (key) => key.replace(/~/g, "~0").replace(/\//g, "~1");
1700
- function captureDiagnosticContent(error, project) {
1701
- const omissions = [];
1702
- const redactions = [];
1703
- const seen = /* @__PURE__ */ new WeakMap();
1704
- let nodes = 0;
1705
- let remaining = import_telemetry_contracts.DIAGNOSTIC_LIMITS.bytes - 64 * 1024;
1706
- const omit = (path, reason, extra = {}) => {
1707
- if (omissions.length < 1e3)
1708
- omissions.push({ path: path.slice(0, 4096), reason, ...extra });
1709
- return { $omitted: reason };
1710
- };
1711
- const visit = (value, path, depth) => {
1712
- if (++nodes > import_telemetry_contracts.DIAGNOSTIC_LIMITS.nodes) return omit(path, "node_limit");
1713
- if (depth > import_telemetry_contracts.DIAGNOSTIC_LIMITS.depth) return omit(path, "depth_limit");
1714
- if (typeof value === "string") {
1715
- const clean = (0, import_telemetry_contracts.redactDiagnosticText)(value);
1716
- if (clean !== value && redactions.length < 1e3)
1717
- redactions.push({
1718
- path: path.slice(0, 4096),
1719
- reason: "credential_text"
1720
- });
1721
- const bytes = encoder.encode(JSON.stringify(clean)).length;
1722
- if (bytes > remaining)
1723
- return omit(path, "byte_limit", {
1724
- originalBytes: encoder.encode(value).length
1725
- });
1726
- remaining -= bytes;
1727
- return clean;
1728
- }
1729
- if (value === null || typeof value === "boolean") return value;
1730
- if (typeof value === "number")
1731
- return Number.isFinite(value) ? value : { $type: "number", value: String(value) };
1732
- if (typeof value === "bigint")
1733
- return { $type: "bigint", value: value.toString() };
1734
- if (typeof value === "undefined") return { $type: "undefined" };
1735
- if (typeof value === "function" || typeof value === "symbol")
1736
- return omit(path, `unsupported_${typeof value}`);
1737
- const object2 = value;
1738
- const reference = seen.get(object2);
1739
- if (reference) return { $ref: reference };
1740
- seen.set(object2, path);
1741
- remaining -= 32;
1742
- if (remaining < 0) return omit(path, "byte_limit");
1743
- try {
1744
- if (value instanceof Date)
1745
- return { $type: "Date", value: Date.prototype.toISOString.call(value) };
1746
- if (value instanceof Map || value instanceof Set) {
1747
- const isMap = value instanceof Map;
1748
- const iterator = isMap ? Map.prototype.entries.call(value) : Set.prototype.values.call(value);
1749
- const count = Object.getOwnPropertyDescriptor(
1750
- isMap ? Map.prototype : Set.prototype,
1751
- "size"
1752
- )?.get?.call(value);
1753
- const entries = [];
1754
- for (let index = 0; index < Math.min(count, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties); index++)
1755
- entries.push(iterator.next().value);
1756
- if (count > entries.length)
1757
- omissions.push({
1758
- path,
1759
- reason: "collection_limit",
1760
- originalCount: count
1761
- });
1762
- return isMap ? {
1763
- $type: "Map",
1764
- entries: visit(entries, `${path}/entries`, depth + 1)
1765
- } : {
1766
- $type: "Set",
1767
- values: visit(entries, `${path}/values`, depth + 1)
1768
- };
1769
- }
1770
- if (Array.isArray(value)) {
1771
- const count = Math.min(value.length, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties);
1772
- const items = Array.from(
1773
- { length: count },
1774
- (_, index) => read(value, String(index), `${path}/${index}`, depth)
1775
- );
1776
- if (count < value.length)
1777
- items.push(
1778
- omit(path, "collection_limit", { originalCount: value.length })
1779
- );
1780
- return items;
1781
- }
1782
- const keys = Object.getOwnPropertyNames(object2);
1783
- const result = /* @__PURE__ */ Object.create(null);
1784
- if (value instanceof Error) {
1785
- const ownName = Object.getOwnPropertyDescriptor(object2, "name");
1786
- let type = ownName && "value" in ownName ? ownName.value : void 0;
1787
- if (typeof type !== "string") {
1788
- const name = Object.getOwnPropertyDescriptor(
1789
- Object.getPrototypeOf(value),
1790
- "name"
1791
- );
1792
- type = name && "value" in name ? name.value : "Error";
1793
- }
1794
- result.type = visit(
1795
- typeof type === "string" ? type : "Error",
1796
- `${path}/type`,
1797
- depth + 1
1798
- );
1799
- }
1800
- for (const key of keys.slice(0, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties)) {
1801
- if (["__proto__", "constructor", "prototype"].includes(key)) {
1802
- result[key] = omit(`${path}/${pointer(key)}`, "unsafe_key");
1803
- continue;
1804
- }
1805
- remaining -= encoder.encode(key).length + 4;
1806
- result[key] = read(object2, key, `${path}/${pointer(key)}`, depth);
1807
- }
1808
- if (keys.length > import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties)
1809
- result.$omittedProperties = omit(path, "property_limit", {
1810
- originalCount: keys.length
1811
- });
1812
- return result;
1813
- } catch {
1814
- return omit(path, "unreadable_object");
1815
- }
1816
- };
1817
- const read = (object2, key, path, depth) => {
1818
- if ((0, import_telemetry_contracts.isPrivateDiagnosticField)(key)) {
1819
- if (redactions.length < 1e3)
1820
- redactions.push({
1821
- path: path.slice(0, 4096),
1822
- reason: "credential_field"
1823
- });
1824
- return "[REDACTED]";
1825
- }
1826
- try {
1827
- const property = Object.getOwnPropertyDescriptor(object2, key);
1828
- if (property && "value" in property)
1829
- return visit(property.value, path, depth + 1);
1830
- if (key === "stack" && object2 instanceof Error && property?.get && property.get === nativeStackGetter)
1831
- return visit(nativeStack(object2, property.get), path, depth + 1);
1832
- return omit(
1833
- path,
1834
- property ? "accessor_not_evaluated" : "unavailable_property"
1835
- );
1836
- } catch {
1837
- return omit(path, "unreadable_property");
1838
- }
1839
- };
1840
- try {
1841
- const original = project ? project(error) : error;
1842
- if (project && original === null) return null;
1843
- const captured = visit(original, "#/data", 0);
1844
- const upstreamOmissions = (0, import_errors16.errorProperty)(original, "diagnosticOmissions");
1845
- if (Array.isArray(upstreamOmissions))
1846
- for (const note of upstreamOmissions.slice(0, 1e3)) {
1847
- const path = (0, import_errors16.errorProperty)(note, "path");
1848
- const reason = (0, import_errors16.errorProperty)(note, "reason");
1849
- if (typeof path === "string" && typeof reason === "string")
1850
- omissions.push({
1851
- path: (0, import_telemetry_contracts.redactDiagnosticText)(path).slice(0, 4096),
1852
- reason: (0, import_telemetry_contracts.redactDiagnosticText)(reason).slice(0, 128)
1853
- });
1854
- }
1855
- const data = captured && typeof captured === "object" && !Array.isArray(captured) ? captured : {
1856
- type: "Error",
1857
- message: typeof captured === "string" ? captured : "Execution reported a fault.",
1858
- value: captured
1859
- };
1860
- if (typeof data.message !== "string" && !("message" in data))
1861
- data.message = "Execution reported a fault.";
1862
- if (!("type" in data))
1863
- data.type = typeof data.name === "string" ? data.name : "Error";
1864
- const content = {
1865
- schemaVersion: 1,
1866
- data,
1867
- capture: {
1868
- status: omissions.length ? "partial" : "complete",
1869
- omissions,
1870
- redactions
1871
- }
1872
- };
1873
- if (encoder.encode(JSON.stringify(content)).length > import_telemetry_contracts.DIAGNOSTIC_LIMITS.bytes)
1874
- return {
1875
- schemaVersion: 1,
1876
- data: {
1877
- type: "Error",
1878
- message: "Diagnostic exceeded the capture budget."
1879
- },
1880
- capture: {
1881
- status: "partial",
1882
- redactions: [],
1883
- omissions: [{ path: "#/data", reason: "serialized_byte_limit" }]
1884
- }
1885
- };
1886
- return content;
1887
- } catch {
1888
- return {
1889
- schemaVersion: 1,
1890
- data: { type: "Error", message: "Diagnostic capture failed." },
1891
- capture: {
1892
- status: "failed",
1893
- redactions: [],
1894
- omissions: [{ path: "#/data", reason: "projection_or_capture_failed" }]
1895
- }
1896
- };
1897
- }
1898
- }
1899
- var import_telemetry_contracts2 = require_dist();
66
+ var import_telemetry_contracts32 = require("@kortyx/telemetry-contracts");
67
+ var import_telemetry_contracts4 = require("@kortyx/telemetry-contracts");
68
+ var import_telemetry_contracts22 = require("@kortyx/telemetry-contracts");
1900
69
  function exceptionDiagnostics(error, project) {
1901
- const content = captureDiagnosticContent(error, project);
70
+ const content = (0, import_telemetry_contracts4.captureDiagnosticContent)(error, project);
1902
71
  return !content || content.capture.status === "failed" ? null : content.data;
1903
72
  }
1904
73
  function errorDiagnostics(error, project) {
1905
74
  const captured = exceptionDiagnostics(error, project);
1906
- const details = captured ? (0, import_telemetry_contracts2.summarizeDiagnostic)(captured) : null;
75
+ const details = captured ? (0, import_telemetry_contracts22.summarizeDiagnostic)(captured) : null;
1907
76
  return details ? {
1908
77
  errorMessage: details.message.slice(0, 8192),
1909
78
  ...details.type ? { errorType: details.type.slice(0, 256) } : {}
@@ -1914,7 +83,7 @@ var require_internal = __commonJS({
1914
83
  }
1915
84
  var import_node_crypto8 = require("crypto");
1916
85
  var import_core6 = require("@kortyx/core");
1917
- var import_errors22 = require("@kortyx/core/errors");
86
+ var import_errors16 = require("@kortyx/core/errors");
1918
87
  function safeSpan(span) {
1919
88
  const guard = (fn) => (...args) => {
1920
89
  try {
@@ -1996,7 +165,7 @@ var require_internal = __commonJS({
1996
165
  var import_node2 = require("@kortyx/usage-telemetry/node");
1997
166
  var package_default2 = {
1998
167
  name: "@kortyx/hooks",
1999
- version: "0.34.3",
168
+ version: "0.35.0",
2000
169
  private: false,
2001
170
  description: "Lifecycle hooks and extension points for Kortyx.",
2002
171
  keywords: [
@@ -2076,7 +245,7 @@ var require_internal = __commonJS({
2076
245
  function assertModelFinish(result) {
2077
246
  if (result.finishReason?.unified !== "content-filter") return;
2078
247
  throw Object.assign(
2079
- new import_errors22.KortyxError("PROVIDER_REFUSAL", "The provider refused the request.", {
248
+ new import_errors16.KortyxError("PROVIDER_REFUSAL", "The provider refused the request.", {
2080
249
  category: "provider",
2081
250
  retryable: false,
2082
251
  safeMessage: "The provider refused the request."
@@ -2393,7 +562,7 @@ var require_internal = __commonJS({
2393
562
  });
2394
563
 
2395
564
  // ../hooks/dist/index.js
2396
- var require_dist2 = __commonJS({
565
+ var require_dist = __commonJS({
2397
566
  "../hooks/dist/index.js"(exports2, module2) {
2398
567
  var __defProp2 = Object.defineProperty;
2399
568
  var __getOwnPropDesc2 = Object.getOwnPropertyDescriptor;
@@ -2437,7 +606,7 @@ var require_dist2 = __commonJS({
2437
606
  workflowCallFingerprint: () => workflowCallFingerprint
2438
607
  });
2439
608
  module2.exports = __toCommonJS2(index_exports2);
2440
- var import_zod14 = require("zod");
609
+ var import_zod15 = require("zod");
2441
610
  var import_node_async_hooks2 = require("async_hooks");
2442
611
  var storage = new import_node_async_hooks2.AsyncLocalStorage();
2443
612
  var isRecord5 = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
@@ -2632,7 +801,7 @@ var require_dist2 = __commonJS({
2632
801
  var import_node2 = require("@kortyx/usage-telemetry/node");
2633
802
  var package_default2 = {
2634
803
  name: "@kortyx/hooks",
2635
- version: "0.34.3",
804
+ version: "0.35.0",
2636
805
  private: false,
2637
806
  description: "Lifecycle hooks and extension points for Kortyx.",
2638
807
  keywords: [
@@ -2709,9 +878,9 @@ var require_dist2 = __commonJS({
2709
878
  measure: (...args) => client2().measure(...args),
2710
879
  flush: () => client2().flush()
2711
880
  };
2712
- var optionsSchema = import_zod14.z.object({
2713
- message: import_zod14.z.string().optional(),
2714
- data: import_zod14.z.json().optional()
881
+ var optionsSchema = import_zod15.z.object({
882
+ message: import_zod15.z.string().optional(),
883
+ data: import_zod15.z.json().optional()
2715
884
  }).strict();
2716
885
  async function completeResponseInternal(options = {}) {
2717
886
  const ctx = getHookContext();
@@ -3181,261 +1350,26 @@ var require_dist2 = __commonJS({
3181
1350
  var import_node_async_hooks22 = require("async_hooks");
3182
1351
  var import_node_crypto32 = require("crypto");
3183
1352
  var import_core22 = require("@kortyx/core");
3184
- var import_errors32 = require("@kortyx/core/errors");
3185
- var import_telemetry_contracts2 = require_dist();
3186
1353
  var import_errors22 = require("@kortyx/core/errors");
3187
- var import_telemetry_contracts = require_dist();
3188
- var encoder = new TextEncoder();
3189
- var nativeStackGetter = Object.getOwnPropertyDescriptor(
3190
- new Error(),
3191
- "stack"
3192
- )?.get;
3193
- function nativeStack(object2, getter) {
3194
- if (!getter || getter !== nativeStackGetter)
3195
- throw new Error("Custom stack accessor");
3196
- for (const key of ["name", "message"]) {
3197
- let cursor = object2;
3198
- while (cursor) {
3199
- const descriptor = Object.getOwnPropertyDescriptor(cursor, key);
3200
- if (descriptor) {
3201
- if (!("value" in descriptor) || typeof descriptor.value !== "string")
3202
- throw new Error("Custom error formatting");
3203
- break;
3204
- }
3205
- cursor = Object.getPrototypeOf(cursor);
3206
- }
3207
- }
3208
- const formatter = Object.getOwnPropertyDescriptor(Error, "prepareStackTrace");
3209
- if (formatter && (!("value" in formatter) || !formatter.writable))
3210
- throw new Error("Custom stack formatter");
3211
- Object.defineProperty(Error, "prepareStackTrace", {
3212
- ...formatter ?? { configurable: true, writable: true },
3213
- value: void 0
3214
- });
3215
- try {
3216
- return getter.call(object2);
3217
- } finally {
3218
- if (formatter) Object.defineProperty(Error, "prepareStackTrace", formatter);
3219
- else Reflect.deleteProperty(Error, "prepareStackTrace");
3220
- }
3221
- }
3222
- var pointer = (key) => key.replace(/~/g, "~0").replace(/\//g, "~1");
3223
- function captureDiagnosticContent(error, project) {
3224
- const omissions = [];
3225
- const redactions = [];
3226
- const seen = /* @__PURE__ */ new WeakMap();
3227
- let nodes = 0;
3228
- let remaining = import_telemetry_contracts.DIAGNOSTIC_LIMITS.bytes - 64 * 1024;
3229
- const omit = (path, reason, extra = {}) => {
3230
- if (omissions.length < 1e3)
3231
- omissions.push({ path: path.slice(0, 4096), reason, ...extra });
3232
- return { $omitted: reason };
3233
- };
3234
- const visit = (value, path, depth) => {
3235
- if (++nodes > import_telemetry_contracts.DIAGNOSTIC_LIMITS.nodes) return omit(path, "node_limit");
3236
- if (depth > import_telemetry_contracts.DIAGNOSTIC_LIMITS.depth) return omit(path, "depth_limit");
3237
- if (typeof value === "string") {
3238
- const clean = (0, import_telemetry_contracts.redactDiagnosticText)(value);
3239
- if (clean !== value && redactions.length < 1e3)
3240
- redactions.push({
3241
- path: path.slice(0, 4096),
3242
- reason: "credential_text"
3243
- });
3244
- const bytes = encoder.encode(JSON.stringify(clean)).length;
3245
- if (bytes > remaining)
3246
- return omit(path, "byte_limit", {
3247
- originalBytes: encoder.encode(value).length
3248
- });
3249
- remaining -= bytes;
3250
- return clean;
3251
- }
3252
- if (value === null || typeof value === "boolean") return value;
3253
- if (typeof value === "number")
3254
- return Number.isFinite(value) ? value : { $type: "number", value: String(value) };
3255
- if (typeof value === "bigint")
3256
- return { $type: "bigint", value: value.toString() };
3257
- if (typeof value === "undefined") return { $type: "undefined" };
3258
- if (typeof value === "function" || typeof value === "symbol")
3259
- return omit(path, `unsupported_${typeof value}`);
3260
- const object2 = value;
3261
- const reference = seen.get(object2);
3262
- if (reference) return { $ref: reference };
3263
- seen.set(object2, path);
3264
- remaining -= 32;
3265
- if (remaining < 0) return omit(path, "byte_limit");
3266
- try {
3267
- if (value instanceof Date)
3268
- return { $type: "Date", value: Date.prototype.toISOString.call(value) };
3269
- if (value instanceof Map || value instanceof Set) {
3270
- const isMap = value instanceof Map;
3271
- const iterator = isMap ? Map.prototype.entries.call(value) : Set.prototype.values.call(value);
3272
- const count = Object.getOwnPropertyDescriptor(
3273
- isMap ? Map.prototype : Set.prototype,
3274
- "size"
3275
- )?.get?.call(value);
3276
- const entries = [];
3277
- for (let index = 0; index < Math.min(count, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties); index++)
3278
- entries.push(iterator.next().value);
3279
- if (count > entries.length)
3280
- omissions.push({
3281
- path,
3282
- reason: "collection_limit",
3283
- originalCount: count
3284
- });
3285
- return isMap ? {
3286
- $type: "Map",
3287
- entries: visit(entries, `${path}/entries`, depth + 1)
3288
- } : {
3289
- $type: "Set",
3290
- values: visit(entries, `${path}/values`, depth + 1)
3291
- };
3292
- }
3293
- if (Array.isArray(value)) {
3294
- const count = Math.min(value.length, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties);
3295
- const items = Array.from(
3296
- { length: count },
3297
- (_, index) => read(value, String(index), `${path}/${index}`, depth)
3298
- );
3299
- if (count < value.length)
3300
- items.push(
3301
- omit(path, "collection_limit", { originalCount: value.length })
3302
- );
3303
- return items;
3304
- }
3305
- const keys = Object.getOwnPropertyNames(object2);
3306
- const result = /* @__PURE__ */ Object.create(null);
3307
- if (value instanceof Error) {
3308
- const ownName = Object.getOwnPropertyDescriptor(object2, "name");
3309
- let type = ownName && "value" in ownName ? ownName.value : void 0;
3310
- if (typeof type !== "string") {
3311
- const name = Object.getOwnPropertyDescriptor(
3312
- Object.getPrototypeOf(value),
3313
- "name"
3314
- );
3315
- type = name && "value" in name ? name.value : "Error";
3316
- }
3317
- result.type = visit(
3318
- typeof type === "string" ? type : "Error",
3319
- `${path}/type`,
3320
- depth + 1
3321
- );
3322
- }
3323
- for (const key of keys.slice(0, import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties)) {
3324
- if (["__proto__", "constructor", "prototype"].includes(key)) {
3325
- result[key] = omit(`${path}/${pointer(key)}`, "unsafe_key");
3326
- continue;
3327
- }
3328
- remaining -= encoder.encode(key).length + 4;
3329
- result[key] = read(object2, key, `${path}/${pointer(key)}`, depth);
3330
- }
3331
- if (keys.length > import_telemetry_contracts.DIAGNOSTIC_LIMITS.properties)
3332
- result.$omittedProperties = omit(path, "property_limit", {
3333
- originalCount: keys.length
3334
- });
3335
- return result;
3336
- } catch {
3337
- return omit(path, "unreadable_object");
3338
- }
3339
- };
3340
- const read = (object2, key, path, depth) => {
3341
- if ((0, import_telemetry_contracts.isPrivateDiagnosticField)(key)) {
3342
- if (redactions.length < 1e3)
3343
- redactions.push({
3344
- path: path.slice(0, 4096),
3345
- reason: "credential_field"
3346
- });
3347
- return "[REDACTED]";
3348
- }
3349
- try {
3350
- const property = Object.getOwnPropertyDescriptor(object2, key);
3351
- if (property && "value" in property)
3352
- return visit(property.value, path, depth + 1);
3353
- if (key === "stack" && object2 instanceof Error && property?.get && property.get === nativeStackGetter)
3354
- return visit(nativeStack(object2, property.get), path, depth + 1);
3355
- return omit(
3356
- path,
3357
- property ? "accessor_not_evaluated" : "unavailable_property"
3358
- );
3359
- } catch {
3360
- return omit(path, "unreadable_property");
3361
- }
3362
- };
3363
- try {
3364
- const original = project ? project(error) : error;
3365
- if (project && original === null) return null;
3366
- const captured = visit(original, "#/data", 0);
3367
- const upstreamOmissions = (0, import_errors22.errorProperty)(original, "diagnosticOmissions");
3368
- if (Array.isArray(upstreamOmissions))
3369
- for (const note of upstreamOmissions.slice(0, 1e3)) {
3370
- const path = (0, import_errors22.errorProperty)(note, "path");
3371
- const reason = (0, import_errors22.errorProperty)(note, "reason");
3372
- if (typeof path === "string" && typeof reason === "string")
3373
- omissions.push({
3374
- path: (0, import_telemetry_contracts.redactDiagnosticText)(path).slice(0, 4096),
3375
- reason: (0, import_telemetry_contracts.redactDiagnosticText)(reason).slice(0, 128)
3376
- });
3377
- }
3378
- const data = captured && typeof captured === "object" && !Array.isArray(captured) ? captured : {
3379
- type: "Error",
3380
- message: typeof captured === "string" ? captured : "Execution reported a fault.",
3381
- value: captured
3382
- };
3383
- if (typeof data.message !== "string" && !("message" in data))
3384
- data.message = "Execution reported a fault.";
3385
- if (!("type" in data))
3386
- data.type = typeof data.name === "string" ? data.name : "Error";
3387
- const content = {
3388
- schemaVersion: 1,
3389
- data,
3390
- capture: {
3391
- status: omissions.length ? "partial" : "complete",
3392
- omissions,
3393
- redactions
3394
- }
3395
- };
3396
- if (encoder.encode(JSON.stringify(content)).length > import_telemetry_contracts.DIAGNOSTIC_LIMITS.bytes)
3397
- return {
3398
- schemaVersion: 1,
3399
- data: {
3400
- type: "Error",
3401
- message: "Diagnostic exceeded the capture budget."
3402
- },
3403
- capture: {
3404
- status: "partial",
3405
- redactions: [],
3406
- omissions: [{ path: "#/data", reason: "serialized_byte_limit" }]
3407
- }
3408
- };
3409
- return content;
3410
- } catch {
3411
- return {
3412
- schemaVersion: 1,
3413
- data: { type: "Error", message: "Diagnostic capture failed." },
3414
- capture: {
3415
- status: "failed",
3416
- redactions: [],
3417
- omissions: [{ path: "#/data", reason: "projection_or_capture_failed" }]
3418
- }
3419
- };
3420
- }
3421
- }
3422
- function exceptionDiagnostics(error, project) {
3423
- const content = captureDiagnosticContent(error, project);
3424
- return !content || content.capture.status === "failed" ? null : content.data;
3425
- }
3426
- function errorDiagnostics(error, project) {
3427
- const captured = exceptionDiagnostics(error, project);
3428
- const details = captured ? (0, import_telemetry_contracts2.summarizeDiagnostic)(captured) : null;
3429
- return details ? {
3430
- errorMessage: details.message.slice(0, 8192),
3431
- ...details.type ? { errorType: details.type.slice(0, 256) } : {}
3432
- } : {};
3433
- }
3434
- var import_node_crypto22 = require("crypto");
3435
- function emitWorkflowCall2(telemetry, type, payload) {
3436
- const base = telemetry?.correlation;
3437
- if (!telemetry?.reporter || !telemetry.environment || !telemetry.service || !base?.runId || !base.workflowId)
3438
- return;
1354
+ var import_telemetry_contracts22 = require("@kortyx/telemetry-contracts");
1355
+ var import_telemetry_contracts4 = require("@kortyx/telemetry-contracts");
1356
+ function exceptionDiagnostics(error, project) {
1357
+ const content = (0, import_telemetry_contracts4.captureDiagnosticContent)(error, project);
1358
+ return !content || content.capture.status === "failed" ? null : content.data;
1359
+ }
1360
+ function errorDiagnostics(error, project) {
1361
+ const captured = exceptionDiagnostics(error, project);
1362
+ const details = captured ? (0, import_telemetry_contracts22.summarizeDiagnostic)(captured) : null;
1363
+ return details ? {
1364
+ errorMessage: details.message.slice(0, 8192),
1365
+ ...details.type ? { errorType: details.type.slice(0, 256) } : {}
1366
+ } : {};
1367
+ }
1368
+ var import_node_crypto22 = require("crypto");
1369
+ function emitWorkflowCall2(telemetry, type, payload) {
1370
+ const base = telemetry?.correlation;
1371
+ if (!telemetry?.reporter || !telemetry.environment || !telemetry.service || !base?.runId || !base.workflowId)
1372
+ return;
3439
1373
  try {
3440
1374
  const active = telemetry.trace?.getActiveContext?.();
3441
1375
  void Promise.resolve(
@@ -3681,7 +1615,7 @@ var require_dist2 = __commonJS({
3681
1615
  return result;
3682
1616
  } catch (error) {
3683
1617
  if (args.abortSignal?.aborted) observation.outcome = "cancelled";
3684
- else if (!(0, import_errors32.isControlFlowError)(error)) {
1618
+ else if (!(0, import_errors22.isControlFlowError)(error)) {
3685
1619
  const outcome = classification(
3686
1620
  args.tool,
3687
1621
  error,
@@ -3816,11 +1750,11 @@ var require_dist2 = __commonJS({
3816
1750
  var import_prompts3 = require("@kortyx/prompts");
3817
1751
  var import_node_crypto42 = require("crypto");
3818
1752
  var import_core32 = require("@kortyx/core");
3819
- var import_errors42 = require("@kortyx/core/errors");
1753
+ var import_errors32 = require("@kortyx/core/errors");
3820
1754
  function assertModelFinish(result) {
3821
1755
  if (result.finishReason?.unified !== "content-filter") return;
3822
1756
  throw Object.assign(
3823
- new import_errors42.KortyxError("PROVIDER_REFUSAL", "The provider refused the request.", {
1757
+ new import_errors32.KortyxError("PROVIDER_REFUSAL", "The provider refused the request.", {
3824
1758
  category: "provider",
3825
1759
  retryable: false,
3826
1760
  safeMessage: "The provider refused the request."
@@ -4884,7 +2818,7 @@ var require_dist2 = __commonJS({
4884
2818
  }
4885
2819
  };
4886
2820
  }
4887
- var import_errors52 = require("@kortyx/core/errors");
2821
+ var import_errors42 = require("@kortyx/core/errors");
4888
2822
  var tryJsonParse = (value) => {
4889
2823
  try {
4890
2824
  return { ok: true, value: JSON.parse(value) };
@@ -4922,12 +2856,12 @@ var require_dist2 = __commonJS({
4922
2856
  }
4923
2857
  return text;
4924
2858
  };
4925
- var createTruncatedStructuredOutputError = (label, cause) => new import_errors52.ValidationError(
2859
+ var createTruncatedStructuredOutputError = (label, cause) => new import_errors42.ValidationError(
4926
2860
  "OUTPUT_TRUNCATED",
4927
2861
  `${label} was truncated before producing valid structured output. The model stopped due to output length. Increase maxOutputTokens or simplify the requested output.`,
4928
2862
  cause
4929
2863
  );
4930
- var createInvalidStructuredOutputError = (label, cause) => new import_errors52.ValidationError(
2864
+ var createInvalidStructuredOutputError = (label, cause) => new import_errors42.ValidationError(
4931
2865
  "INVALID_MODEL_JSON",
4932
2866
  `${label} did not produce valid structured output. The model returned text instead of the expected JSON payload. Check the output contract or outputSchema and provider structured-output settings.`,
4933
2867
  cause
@@ -4939,7 +2873,7 @@ var require_dist2 = __commonJS({
4939
2873
  try {
4940
2874
  return parseWithSchema(args.schema, candidate, args.label);
4941
2875
  } catch (error) {
4942
- if (!(error instanceof import_errors52.ValidationError)) throw error;
2876
+ if (!(error instanceof import_errors42.ValidationError)) throw error;
4943
2877
  const failure = args.finishReason?.unified === "length" ? createTruncatedStructuredOutputError(args.label, error) : typeof candidate === "string" && !validJson ? createInvalidStructuredOutputError(args.label, error) : error;
4944
2878
  if (failure && typeof failure === "object")
4945
2879
  Object.assign(failure, {
@@ -4959,7 +2893,7 @@ var require_dist2 = __commonJS({
4959
2893
  if (typeof candidate === "string" && candidate.trim().length > 0) {
4960
2894
  throw createInvalidStructuredOutputError("useReason output");
4961
2895
  }
4962
- throw new import_errors52.ValidationError(
2896
+ throw new import_errors42.ValidationError(
4963
2897
  "MODEL_OUTPUT_SCHEMA",
4964
2898
  "useReason first pass with interrupt must return a JSON object."
4965
2899
  );
@@ -4968,7 +2902,7 @@ var require_dist2 = __commonJS({
4968
2902
  let interruptRequired = true;
4969
2903
  if (mode === "optional") {
4970
2904
  if (candidate.decision !== "continue" && candidate.decision !== "interrupt") {
4971
- throw new import_errors52.ValidationError(
2905
+ throw new import_errors42.ValidationError(
4972
2906
  "MODEL_OUTPUT_SCHEMA",
4973
2907
  'useReason optional interrupt first pass must include decision "continue" or "interrupt".'
4974
2908
  );
@@ -4981,7 +2915,7 @@ var require_dist2 = __commonJS({
4981
2915
  candidate.interrupt
4982
2916
  ].some((value) => value !== null && value !== void 0);
4983
2917
  if (hasUnexpectedRequest) {
4984
- throw new import_errors52.ValidationError(
2918
+ throw new import_errors42.ValidationError(
4985
2919
  "MODEL_OUTPUT_SCHEMA",
4986
2920
  'useReason optional interrupt first pass with decision "continue" must not include an interrupt request.'
4987
2921
  );
@@ -5015,7 +2949,7 @@ var require_dist2 = __commonJS({
5015
2949
  "useReason output"
5016
2950
  );
5017
2951
  } catch (error) {
5018
- if (!(error instanceof import_errors52.ValidationError)) throw error;
2952
+ if (!(error instanceof import_errors42.ValidationError)) throw error;
5019
2953
  if (args.finishReason?.unified === "length") {
5020
2954
  throw createTruncatedStructuredOutputError("useReason output", error);
5021
2955
  }
@@ -5024,7 +2958,7 @@ var require_dist2 = __commonJS({
5024
2958
  }
5025
2959
  const explicitDraft = typeof candidate.draftText === "string" && candidate.draftText.length > 0 ? candidate.draftText : typeof candidate.text === "string" && candidate.text.length > 0 ? candidate.text : void 0;
5026
2960
  if (!args.outputSchema && !explicitDraft) {
5027
- throw new import_errors52.ValidationError(
2961
+ throw new import_errors42.ValidationError(
5028
2962
  "MODEL_OUTPUT_SCHEMA",
5029
2963
  "useReason first pass with interrupt requires `draftText` when outputSchema is not provided."
5030
2964
  );
@@ -5290,7 +3224,7 @@ ${tryStringify2(args.response)}`
5290
3224
  });
5291
3225
  };
5292
3226
  var import_core52 = require("@kortyx/core");
5293
- var import_errors62 = require("@kortyx/core/errors");
3227
+ var import_errors52 = require("@kortyx/core/errors");
5294
3228
  var import_zod52 = require("zod");
5295
3229
  var record = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
5296
3230
  function schemaPath(schema, path) {
@@ -5526,7 +3460,7 @@ ${tryStringify2(args.response)}`
5526
3460
  const maxSteps = Math.max(1, useReasonArgs.toolExecution?.maxSteps ?? 3);
5527
3461
  const missingReturnCorrections = useReasonArgs.toolExecution?.missingReturnCorrections ?? 0;
5528
3462
  if (!Number.isSafeInteger(missingReturnCorrections) || missingReturnCorrections < 0)
5529
- throw new import_errors62.KortyxError(
3463
+ throw new import_errors52.KortyxError(
5530
3464
  "INVALID_REASON_CORRECTION_POLICY",
5531
3465
  "Invalid missing return correction budget.",
5532
3466
  {
@@ -5587,7 +3521,7 @@ ${tryStringify2(args.response)}`
5587
3521
  let outputContractCorrections = checkpoint?.outputContractCorrections ?? 0;
5588
3522
  const correctionExhausted = () => {
5589
3523
  const message = `useReason could not complete output-contract correction within toolExecution.maxSteps (${maxSteps}). Select exactly one terminal return contract; multiple emit contracts are allowed. Increase maxSteps to allow correction and output generation.`;
5590
- return new import_errors62.KortyxError(
3524
+ return new import_errors52.KortyxError(
5591
3525
  "REASON_OUTPUT_CONTRACT_CORRECTION_EXHAUSTED",
5592
3526
  message,
5593
3527
  {
@@ -5684,7 +3618,7 @@ ${tryStringify2(args.response)}`
5684
3618
  aggregatedWarnings = mergeWarnings(aggregatedWarnings, step.warnings);
5685
3619
  const toolCalls = step.toolCalls ?? [];
5686
3620
  if (missingReturnAttempts > 0 && toolCalls.some((call) => !outputByToolName.has(call.name)))
5687
- throw new import_errors62.KortyxError(
3621
+ throw new import_errors52.KortyxError(
5688
3622
  "REASON_OUTPUT_CORRECTION_TOOL_DISALLOWED",
5689
3623
  "A return correction requested a non-output tool.",
5690
3624
  {
@@ -5727,7 +3661,7 @@ ${tryStringify2(args.response)}`
5727
3661
  save();
5728
3662
  continue;
5729
3663
  }
5730
- throw new import_errors62.KortyxError(
3664
+ throw new import_errors52.KortyxError(
5731
3665
  "REASON_TERMINAL_OUTPUT_MISSING",
5732
3666
  "useReason outputs.return requires one terminal output contract call.",
5733
3667
  {
@@ -5861,7 +3795,7 @@ ${tryStringify2(args.response)}`
5861
3795
  );
5862
3796
  if (steps.length >= maxSteps) {
5863
3797
  const message = `useReason output generation requires another model pass within toolExecution.maxSteps (${maxSteps}). Increase maxSteps to allow outputs to use completed tool and interrupt results.`;
5864
- throw new import_errors62.ProviderRequestError(message, {
3798
+ throw new import_errors52.ProviderRequestError(message, {
5865
3799
  code: "REASON_OUTPUT_BUDGET_EXHAUSTED",
5866
3800
  operation: "useReason",
5867
3801
  safeMessage: message,
@@ -6056,7 +3990,7 @@ ${tryStringify2(args.response)}`
6056
3990
  `useReason interrupt contract "${resolved.name}" request`
6057
3991
  );
6058
3992
  } catch (error) {
6059
- if (!(error instanceof import_errors62.ValidationError)) throw error;
3993
+ if (!(error instanceof import_errors52.ValidationError)) throw error;
6060
3994
  const result3 = {
6061
3995
  toolCallId: toolCall.id,
6062
3996
  name: toolCall.name,
@@ -6266,12 +4200,12 @@ ${tryStringify2(args.response)}`
6266
4200
  }
6267
4201
  } catch (error) {
6268
4202
  (0, import_core52.throwIfExecutionAborted)(abortSignal);
6269
- if ((0, import_errors62.isControlFlowError)(error)) throw error;
4203
+ if ((0, import_errors52.isControlFlowError)(error)) throw error;
6270
4204
  const result2 = {
6271
4205
  toolCallId: toolCall.id,
6272
4206
  name: tool.name,
6273
- content: (0, import_errors62.serializeFailure)(error).message,
6274
- failure: (0, import_errors62.serializeFailure)(error),
4207
+ content: (0, import_errors52.serializeFailure)(error).message,
4208
+ failure: (0, import_errors52.serializeFailure)(error),
6275
4209
  isError: true
6276
4210
  };
6277
4211
  toolResults.push(result2);
@@ -6868,7 +4802,7 @@ ${tryStringify2(args.response)}`
6868
4802
  return [getValue(), setValue];
6869
4803
  }
6870
4804
  var import_core7 = require("@kortyx/core");
6871
- var import_errors72 = require("@kortyx/core/errors");
4805
+ var import_errors62 = require("@kortyx/core/errors");
6872
4806
  var ParallelChildWaiting = class extends Error {
6873
4807
  name = "ParallelChildWaiting";
6874
4808
  };
@@ -6882,14 +4816,14 @@ ${tryStringify2(args.response)}`
6882
4816
  );
6883
4817
  this.results = results;
6884
4818
  this.name = "ParallelError";
6885
- this.failure = (0, import_errors72.serializeFailure)({
4819
+ this.failure = (0, import_errors62.serializeFailure)({
6886
4820
  version: 1,
6887
4821
  code: this.code,
6888
4822
  category: "workflow",
6889
4823
  message: this.message,
6890
4824
  retryable: null,
6891
4825
  results: results.map(
6892
- (result) => result.status === "fulfilled" ? { status: "fulfilled" } : { status: "rejected", failure: (0, import_errors72.serializeFailure)(result.reason) }
4826
+ (result) => result.status === "fulfilled" ? { status: "fulfilled" } : { status: "rejected", failure: (0, import_errors62.serializeFailure)(result.reason) }
6893
4827
  )
6894
4828
  });
6895
4829
  }
@@ -6897,7 +4831,7 @@ ${tryStringify2(args.response)}`
6897
4831
  code = "PARALLEL_FAILED";
6898
4832
  failure;
6899
4833
  toJSON() {
6900
- return (0, import_errors72.serializeFailure)(this);
4834
+ return (0, import_errors62.serializeFailure)(this);
6901
4835
  }
6902
4836
  };
6903
4837
  function registerWorkflowTask(id3) {
@@ -7072,11 +5006,11 @@ ${tryStringify2(args.response)}`
7072
5006
  }
7073
5007
  var import_node_crypto62 = require("crypto");
7074
5008
  var import_core8 = require("@kortyx/core");
7075
- var import_errors82 = require("@kortyx/core/errors");
7076
- var WorkflowCallError = class extends import_errors82.KortyxError {
5009
+ var import_errors72 = require("@kortyx/core/errors");
5010
+ var WorkflowCallError = class extends import_errors72.KortyxError {
7077
5011
  constructor(workflow, message, cause, failure) {
7078
- const detail = (0, import_errors82.serializeFailure)(
7079
- failure ?? cause ?? new import_errors82.WorkflowContractError(message)
5012
+ const detail = (0, import_errors72.serializeFailure)(
5013
+ failure ?? cause ?? new import_errors72.WorkflowContractError(message)
7080
5014
  );
7081
5015
  super(detail.code, `Workflow '${workflow}': ${message}`, {
7082
5016
  ...detail,
@@ -7097,12 +5031,12 @@ ${tryStringify2(args.response)}`
7097
5031
  if (v === null || typeof v === "string" || typeof v === "boolean") return v;
7098
5032
  if (typeof v === "number" && Number.isFinite(v)) return v;
7099
5033
  if (typeof v !== "object" || !v || seen.has(v)) {
7100
- throw new import_errors82.WorkflowContractError(
5034
+ throw new import_errors72.WorkflowContractError(
7101
5035
  "Workflow input and output must be JSON serializable."
7102
5036
  );
7103
5037
  }
7104
5038
  if (!Array.isArray(v) && Object.getPrototypeOf(v) !== Object.prototype) {
7105
- throw new import_errors82.WorkflowContractError(
5039
+ throw new import_errors72.WorkflowContractError(
7106
5040
  "Workflow input and output must contain plain JSON objects."
7107
5041
  );
7108
5042
  }
@@ -7123,7 +5057,7 @@ ${tryStringify2(args.response)}`
7123
5057
  useWorkflow: (args) => {
7124
5058
  const workflow = workflows[args.workflow];
7125
5059
  if (!workflow || workflow.id !== args.workflow)
7126
- throw new import_errors82.WorkflowContractError(
5060
+ throw new import_errors72.WorkflowContractError(
7127
5061
  "Workflow registry keys must match definition ids."
7128
5062
  );
7129
5063
  return useWorkflow({ ...args, workflow });
@@ -7149,23 +5083,23 @@ ${tryStringify2(args.response)}`
7149
5083
  (0, import_core8.throwIfExecutionAborted)(ctx.node.abortSignal);
7150
5084
  const service = ctx.node.callWorkflow;
7151
5085
  if (!service)
7152
- throw new import_errors82.WorkflowContractError(
5086
+ throw new import_errors72.WorkflowContractError(
7153
5087
  "useWorkflow requires an agent workflow registry."
7154
5088
  );
7155
5089
  if (!args.id.trim())
7156
- throw new import_errors82.WorkflowContractError(
5090
+ throw new import_errors72.WorkflowContractError(
7157
5091
  "useWorkflow requires a stable, nonempty id."
7158
5092
  );
7159
5093
  if (ctx.workflowCallActive && !task.group)
7160
- throw new import_errors82.WorkflowContractError(
5094
+ throw new import_errors72.WorkflowContractError(
7161
5095
  "Concurrent workflow calls require parallel([...]); otherwise await each call."
7162
5096
  );
7163
5097
  if (ctx.workflowCallIds.has(args.id))
7164
- throw new import_errors82.WorkflowContractError(
5098
+ throw new import_errors72.WorkflowContractError(
7165
5099
  `Duplicate workflow call id '${args.id}' in one node activation.`
7166
5100
  );
7167
5101
  if (ctx.workflowCallIds.size >= 64)
7168
- throw new import_errors82.WorkflowContractError(
5102
+ throw new import_errors72.WorkflowContractError(
7169
5103
  "A node may call at most 64 child workflows per activation."
7170
5104
  );
7171
5105
  ctx.workflowCallIds.add(args.id);
@@ -7246,7 +5180,7 @@ ${tryStringify2(args.response)}`
7246
5180
  workflow,
7247
5181
  record2.error ?? "child failed",
7248
5182
  void 0,
7249
- record2.failure ?? (0, import_errors82.serializeFailure)(void 0)
5183
+ record2.failure ?? (0, import_errors72.serializeFailure)(void 0)
7250
5184
  );
7251
5185
  if (task.group && record2.status === "interrupted" && record2.interrupts.at(-1)?.response === void 0)
7252
5186
  throw new ParallelChildWaiting();
@@ -7254,7 +5188,7 @@ ${tryStringify2(args.response)}`
7254
5188
  let outcome;
7255
5189
  try {
7256
5190
  if (Array.isArray(record2.snapshot) && record2.snapshot.length !== 1)
7257
- throw new import_errors82.WorkflowContractError("Invalid child workflow snapshot.");
5191
+ throw new import_errors72.WorkflowContractError("Invalid child workflow snapshot.");
7258
5192
  const snapshot = Array.isArray(record2.snapshot) ? record2.snapshot[0] : record2.snapshot;
7259
5193
  report(
7260
5194
  snapshot ? "resumed" : "started",
@@ -7285,7 +5219,7 @@ ${tryStringify2(args.response)}`
7285
5219
  throw error;
7286
5220
  }
7287
5221
  record2.status = "failed";
7288
- record2.failure = (0, import_errors82.serializeFailure)(error);
5222
+ record2.failure = (0, import_errors72.serializeFailure)(error);
7289
5223
  record2.error = record2.failure.message;
7290
5224
  report("failed", { error: "Child workflow failed" });
7291
5225
  throw new WorkflowCallError(
@@ -7349,6 +5283,7 @@ __export(index_exports, {
7349
5283
  ChatLifecycleHookError: () => ChatLifecycleHookError,
7350
5284
  EvalConfigurationError: () => EvalConfigurationError,
7351
5285
  EvalEvidencePolicySchema: () => EvalEvidencePolicySchema,
5286
+ EvalExecutionSettingsSchema: () => EvalExecutionSettingsSchema,
7352
5287
  EvalIssueSchema: () => EvalIssueSchema,
7353
5288
  EvalJudgeIdentitySchema: () => EvalJudgeIdentitySchema,
7354
5289
  EvalJudgeUsageSchema: () => EvalJudgeUsageSchema,
@@ -7356,10 +5291,14 @@ __export(index_exports, {
7356
5291
  EvalProgressSchema: () => EvalProgressSchema,
7357
5292
  EvalRemoteRunRequestSchema: () => EvalRemoteRunRequestSchema,
7358
5293
  EvalRunResultSchema: () => EvalRunResultSchema,
5294
+ EvalSettingsResponseSchema: () => EvalSettingsResponseSchema,
5295
+ EvalSettingsSchema: () => EvalSettingsSchema,
7359
5296
  EvalSuiteSchema: () => EvalSuiteSchema,
5297
+ EvalTerminationSchema: () => EvalTerminationSchema,
7360
5298
  EvalWireEventSchema: () => EvalWireEventSchema,
7361
5299
  EvaluationMetadataSchema: () => EvaluationMetadataSchema,
7362
5300
  ExecutionRequestError: () => ExecutionRequestError,
5301
+ ResolvedEvalSettingsSchema: () => ResolvedEvalSettingsSchema,
7363
5302
  StudioEvalJudgeRequestSchema: () => StudioEvalJudgeRequestSchema,
7364
5303
  StudioEvalJudgeResponseSchema: () => StudioEvalJudgeResponseSchema,
7365
5304
  StudioEvalStartRequestSchema: () => StudioEvalStartRequestSchema,
@@ -7390,6 +5329,7 @@ __export(index_exports, {
7390
5329
  parseResumeMeta: () => parseResumeMeta,
7391
5330
  projectWorkflowTopology: () => projectWorkflowTopology,
7392
5331
  readRequestJson: () => readRequestJson,
5332
+ resolveEvalSettings: () => resolveEvalSettings,
7393
5333
  streamChat: () => streamChat,
7394
5334
  streamChatFromRoute: () => streamChatFromRoute,
7395
5335
  transformGraphStreamForUI: () => transformGraphStreamForUI,
@@ -7445,6 +5385,7 @@ var ExecutionRequestError = class extends import_errors2.KortyxError {
7445
5385
  var import_node_stream = require("stream");
7446
5386
  var import_core2 = require("@kortyx/core");
7447
5387
  var import_errors4 = require("@kortyx/core/errors");
5388
+ var import_eval_attribution2 = require("@kortyx/core/eval-attribution");
7448
5389
  var import_internal = __toESM(require_internal());
7449
5390
  var import_runtime = require("@kortyx/runtime");
7450
5391
  var import_stream = require("@kortyx/stream");
@@ -7906,7 +5847,7 @@ var import_node = require("@kortyx/usage-telemetry/node");
7906
5847
  // package.json
7907
5848
  var package_default = {
7908
5849
  name: "@kortyx/agent",
7909
- version: "0.38.1",
5850
+ version: "0.39.0",
7910
5851
  private: false,
7911
5852
  description: "Agent composition utilities for Kortyx.",
7912
5853
  keywords: [
@@ -7964,7 +5905,8 @@ var package_default = {
7964
5905
  "@langchain/langgraph": "catalog:",
7965
5906
  zod: "catalog:",
7966
5907
  "@kortyx/prompts": "workspace:*",
7967
- "@kortyx/usage-telemetry": "workspace:*"
5908
+ "@kortyx/usage-telemetry": "workspace:*",
5909
+ "@kortyx/telemetry-contracts": "workspace:*"
7968
5910
  },
7969
5911
  devDependencies: {
7970
5912
  "@kortyx/hooks": "workspace:*",
@@ -8170,7 +6112,8 @@ async function orchestrateGraphStream({
8170
6112
  isRecord3(config.context) ? { userId: config.context.userId, tenantId: config.context.tenantId } : {}
8171
6113
  );
8172
6114
  const telemetryConfig = isRecord3(config.telemetry) ? config.telemetry : {};
8173
- const errorChunk = (error, existingDiagnosticId) => {
6115
+ const errorChunk = (error, existingDiagnosticId, original = true) => {
6116
+ if (original) (0, import_eval_attribution2.observeEvalError)(error);
8174
6117
  const failure = (0, import_errors4.serializeFailure)(error);
8175
6118
  let diagnosticId = existingDiagnosticId;
8176
6119
  if (!diagnosticId) {
@@ -8457,7 +6400,8 @@ async function orchestrateGraphStream({
8457
6400
  write(
8458
6401
  errorChunk(
8459
6402
  outcomeError,
8460
- payload?.diagnosticId
6403
+ payload?.diagnosticId,
6404
+ false
8461
6405
  )
8462
6406
  );
8463
6407
  write({ type: "done" });
@@ -10195,7 +8139,7 @@ function createInterruptDiscovery(store) {
10195
8139
 
10196
8140
  // src/telemetry/workflow-call-branch.ts
10197
8141
  var import_node_crypto3 = require("crypto");
10198
- var import_hooks = __toESM(require_dist2());
8142
+ var import_hooks = __toESM(require_dist());
10199
8143
  function restoreWorkflowCallBranch(requests, telemetry, sourceRunId) {
10200
8144
  const branchId = (0, import_node_crypto3.randomUUID)();
10201
8145
  const emitted = /* @__PURE__ */ new Set();
@@ -10888,7 +8832,8 @@ function createAgent(args) {
10888
8832
  // src/evals/contracts.ts
10889
8833
  var import_errors8 = require("@kortyx/core/errors");
10890
8834
  var import_browser2 = require("@kortyx/prompts/browser");
10891
- var import_zod8 = require("zod");
8835
+ var import_telemetry_contracts = require("@kortyx/telemetry-contracts");
8836
+ var import_zod9 = require("zod");
10892
8837
 
10893
8838
  // src/evals/evidence-policy.ts
10894
8839
  var import_zod7 = require("zod");
@@ -10928,30 +8873,126 @@ var EvalEvidencePolicySchema = import_zod7.z.object({
10928
8873
  ]).optional()
10929
8874
  }).strict();
10930
8875
 
8876
+ // src/evals/settings.ts
8877
+ var import_zod8 = require("zod");
8878
+ var timeout = import_zod8.z.number().int().min(1).max(2147483647);
8879
+ var EvalSettingsSchema = import_zod8.z.object({
8880
+ repetitions: import_zod8.z.number().int().min(1).max(20).optional(),
8881
+ concurrency: import_zod8.z.number().int().min(1).max(20).optional(),
8882
+ caseTimeoutMs: timeout.optional(),
8883
+ runTimeoutMs: timeout.optional(),
8884
+ judge: import_zod8.z.enum(["studio", "app"]).optional()
8885
+ }).strict();
8886
+ var EvalExecutionSettingsSchema = import_zod8.z.object({
8887
+ defaults: EvalSettingsSchema.omit({ runTimeoutMs: true, judge: true }),
8888
+ limits: import_zod8.z.object({
8889
+ concurrency: import_zod8.z.number().int().positive().max(20).optional(),
8890
+ caseTimeoutMs: timeout.optional()
8891
+ }).strict(),
8892
+ overrides: import_zod8.z.array(import_zod8.z.enum(["repetitions", "concurrency", "caseTimeoutMs"]))
8893
+ }).strict();
8894
+ var ResolvedEvalSettingsSchema = import_zod8.z.object({
8895
+ values: EvalSettingsSchema.required(),
8896
+ sources: import_zod8.z.record(
8897
+ import_zod8.z.enum([
8898
+ "repetitions",
8899
+ "concurrency",
8900
+ "caseTimeoutMs",
8901
+ "runTimeoutMs",
8902
+ "judge"
8903
+ ]),
8904
+ import_zod8.z.enum(["run", "studio", "sdk", "sdk-default", "studio-default"])
8905
+ )
8906
+ }).strict();
8907
+ var EvalSettingsResponseSchema = import_zod8.z.object({
8908
+ defaults: EvalSettingsSchema,
8909
+ installation: EvalSettingsSchema,
8910
+ maxRunTimeoutMs: timeout,
8911
+ canEdit: import_zod8.z.boolean()
8912
+ });
8913
+ var EvalTerminationSchema = import_zod8.z.object({
8914
+ code: import_zod8.z.enum([
8915
+ "EVAL_RUN_TIMEOUT",
8916
+ "EVAL_CANCELLED",
8917
+ "EVAL_LEASE_LOST",
8918
+ "EVAL_EXECUTION_UNKNOWN",
8919
+ "EVAL_EXECUTOR_FAILED"
8920
+ ]),
8921
+ message: import_zod8.z.string(),
8922
+ timeoutMs: timeout.optional(),
8923
+ phase: import_zod8.z.string().optional(),
8924
+ cleanup: import_zod8.z.enum(["signalled", "unknown"]).optional(),
8925
+ diagnosticId: import_zod8.z.string().uuid().optional()
8926
+ }).strict();
8927
+ function resolveEvalSettings(overrides = {}, studio = {}, execution, installation = {}, maxRunTimeoutMs = 2147483647) {
8928
+ const values = {
8929
+ repetitions: 1,
8930
+ concurrency: 1,
8931
+ caseTimeoutMs: 6e4,
8932
+ runTimeoutMs: 30 * 6e4,
8933
+ judge: "studio"
8934
+ };
8935
+ const sources = {
8936
+ repetitions: "sdk-default",
8937
+ concurrency: "sdk-default",
8938
+ caseTimeoutMs: "sdk-default",
8939
+ runTimeoutMs: "studio-default",
8940
+ judge: "studio-default"
8941
+ };
8942
+ for (const [input, source] of [
8943
+ [execution?.defaults ?? {}, "sdk"],
8944
+ [installation, "studio"],
8945
+ [studio, "studio"],
8946
+ [overrides, "run"]
8947
+ ]) {
8948
+ const parsed = EvalSettingsSchema.parse(input);
8949
+ for (const key of Object.keys(parsed)) {
8950
+ if (parsed[key] !== void 0) {
8951
+ Object.assign(values, { [key]: parsed[key] });
8952
+ sources[key] = source;
8953
+ }
8954
+ }
8955
+ }
8956
+ if (values.runTimeoutMs > maxRunTimeoutMs)
8957
+ throw new Error(
8958
+ `Evaluation timeout exceeds the administrator maximum of ${maxRunTimeoutMs} ms.`
8959
+ );
8960
+ for (const key of ["concurrency", "caseTimeoutMs"]) {
8961
+ const max = execution?.limits[key];
8962
+ if (max !== void 0 && values[key] > max)
8963
+ throw new Error(`${key} exceeds the application maximum of ${max}.`);
8964
+ }
8965
+ if (!execution?.overrides.includes("caseTimeoutMs") && (overrides.caseTimeoutMs !== void 0 || studio.caseTimeoutMs !== void 0 || installation.caseTimeoutMs !== void 0))
8966
+ throw new Error(
8967
+ "Update the consumer SDK to support case timeout overrides."
8968
+ );
8969
+ return { values, sources };
8970
+ }
8971
+
10931
8972
  // src/evals/contracts.ts
10932
- var id2 = import_zod8.z.string().trim().min(1).max(256);
10933
- var failureDescriptor = import_zod8.z.custom(import_errors8.isFailureDescriptor).transform(
8973
+ var id2 = import_zod9.z.string().trim().min(1).max(256);
8974
+ var failureDescriptor = import_zod9.z.custom(import_errors8.isFailureDescriptor).transform(
10934
8975
  (value) => JSON.parse(JSON.stringify((0, import_errors8.serializeFailure)(value)))
10935
8976
  );
10936
- var handler = import_zod8.z.object({ using: id2, params: import_zod8.z.json().optional() }).strict();
10937
- var EvalResumeResponseSchema = import_zod8.z.discriminatedUnion("type", [
10938
- import_zod8.z.object({ type: import_zod8.z.literal("text"), text: import_zod8.z.string() }).strict(),
10939
- import_zod8.z.object({ type: import_zod8.z.literal("select"), ids: import_zod8.z.array(import_zod8.z.string()) }).strict(),
10940
- import_zod8.z.object({ type: import_zod8.z.literal("value"), value: import_zod8.z.json() }).strict(),
10941
- import_zod8.z.object({ type: import_zod8.z.literal("cancel") }).strict()
8977
+ var handler = import_zod9.z.object({ using: id2, params: import_zod9.z.json().optional() }).strict();
8978
+ var EvalResumeResponseSchema = import_zod9.z.discriminatedUnion("type", [
8979
+ import_zod9.z.object({ type: import_zod9.z.literal("text"), text: import_zod9.z.string() }).strict(),
8980
+ import_zod9.z.object({ type: import_zod9.z.literal("select"), ids: import_zod9.z.array(import_zod9.z.string()) }).strict(),
8981
+ import_zod9.z.object({ type: import_zod9.z.literal("value"), value: import_zod9.z.json() }).strict(),
8982
+ import_zod9.z.object({ type: import_zod9.z.literal("cancel") }).strict()
10942
8983
  ]);
10943
- var expectation = import_zod8.z.object({
10944
- type: import_zod8.z.enum(["answer", "interrupt"]),
8984
+ var expectation = import_zod9.z.object({
8985
+ type: import_zod9.z.enum(["answer", "interrupt"]),
10945
8986
  schemaId: id2.optional(),
10946
8987
  schemaVersion: id2.optional(),
10947
- outputs: import_zod8.z.array(import_zod8.z.object({ schemaId: id2, schemaVersion: id2.optional() }).strict()).optional(),
10948
- criteria: import_zod8.z.array(
10949
- import_zod8.z.union([
10950
- import_zod8.z.string().trim().min(1),
10951
- import_zod8.z.object({ id: id2, text: import_zod8.z.string().trim().min(1) }).strict()
8988
+ outputs: import_zod9.z.array(import_zod9.z.object({ schemaId: id2, schemaVersion: id2.optional() }).strict()).optional(),
8989
+ criteria: import_zod9.z.array(
8990
+ import_zod9.z.union([
8991
+ import_zod9.z.string().trim().min(1),
8992
+ import_zod9.z.object({ id: id2, text: import_zod9.z.string().trim().min(1) }).strict()
10952
8993
  ])
10953
8994
  ).optional(),
10954
- reference: import_zod8.z.json().optional()
8995
+ reference: import_zod9.z.json().optional()
10955
8996
  }).strict().superRefine((value, ctx) => {
10956
8997
  if (value.type === "answer" && (value.schemaId || value.schemaVersion))
10957
8998
  ctx.addIssue({
@@ -10967,21 +9008,21 @@ var expectation = import_zod8.z.object({
10967
9008
  message: "Criterion IDs must be unique within a step."
10968
9009
  });
10969
9010
  });
10970
- var EvalSuiteSchema = import_zod8.z.object({
9011
+ var EvalSuiteSchema = import_zod9.z.object({
10971
9012
  evidence: EvalEvidencePolicySchema.optional(),
10972
9013
  id: id2,
10973
- name: import_zod8.z.string().optional(),
10974
- cases: import_zod8.z.array(
10975
- import_zod8.z.object({
9014
+ name: import_zod9.z.string().optional(),
9015
+ cases: import_zod9.z.array(
9016
+ import_zod9.z.object({
10976
9017
  id: id2,
10977
- name: import_zod8.z.string().optional(),
10978
- params: import_zod8.z.json().optional(),
9018
+ name: import_zod9.z.string().optional(),
9019
+ params: import_zod9.z.json().optional(),
10979
9020
  workflowId: id2.optional(),
10980
- steps: import_zod8.z.array(
10981
- import_zod8.z.union([
10982
- import_zod8.z.object({ message: import_zod8.z.string().min(1), expect: expectation }).strict(),
10983
- import_zod8.z.object({
10984
- resume: import_zod8.z.union([EvalResumeResponseSchema, handler]),
9021
+ steps: import_zod9.z.array(
9022
+ import_zod9.z.union([
9023
+ import_zod9.z.object({ message: import_zod9.z.string().min(1), expect: expectation }).strict(),
9024
+ import_zod9.z.object({
9025
+ resume: import_zod9.z.union([EvalResumeResponseSchema, handler]),
10985
9026
  expect: expectation
10986
9027
  }).strict()
10987
9028
  ])
@@ -10989,44 +9030,44 @@ var EvalSuiteSchema = import_zod8.z.object({
10989
9030
  }).strict()
10990
9031
  ).min(1)
10991
9032
  }).strict();
10992
- var EvalVerdictSchema = import_zod8.z.object({
10993
- passed: import_zod8.z.boolean(),
10994
- reason: import_zod8.z.string().trim().min(1),
10995
- evidence: import_zod8.z.array(import_zod8.z.string().min(1))
9033
+ var EvalVerdictSchema = import_zod9.z.object({
9034
+ passed: import_zod9.z.boolean(),
9035
+ reason: import_zod9.z.string().trim().min(1),
9036
+ evidence: import_zod9.z.array(import_zod9.z.string().min(1))
10996
9037
  }).strict();
10997
- var EvalObservationSchema = import_zod8.z.object({
9038
+ var EvalObservationSchema = import_zod9.z.object({
10998
9039
  failure: failureDescriptor.optional(),
10999
- diagnosticId: import_zod8.z.uuid().optional(),
11000
- promptUsage: import_zod8.z.array(
11001
- import_zod8.z.object({
11002
- id: import_zod8.z.string(),
11003
- version: import_zod8.z.number().int().positive(),
11004
- hash: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
11005
- environment: import_zod8.z.string(),
11006
- snapshotRevision: import_zod8.z.string(),
11007
- model: import_zod8.z.string().optional()
9040
+ diagnosticId: import_zod9.z.uuid().optional(),
9041
+ promptUsage: import_zod9.z.array(
9042
+ import_zod9.z.object({
9043
+ id: import_zod9.z.string(),
9044
+ version: import_zod9.z.number().int().positive(),
9045
+ hash: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
9046
+ environment: import_zod9.z.string(),
9047
+ snapshotRevision: import_zod9.z.string(),
9048
+ model: import_zod9.z.string().optional()
11008
9049
  }).strict()
11009
9050
  ).optional(),
11010
- type: import_zod8.z.enum(["answer", "interrupt", "error", "cancelled"]),
11011
- text: import_zod8.z.string(),
11012
- structured: import_zod8.z.array(import_zod8.z.json()),
11013
- events: import_zod8.z.array(import_zod8.z.json()).optional(),
11014
- runId: import_zod8.z.string().optional(),
11015
- checkpointId: import_zod8.z.string().optional(),
11016
- interrupt: import_zod8.z.object({
11017
- requestId: import_zod8.z.string(),
11018
- kind: import_zod8.z.enum(["text", "choice", "multi-choice", "custom"]),
11019
- question: import_zod8.z.string().optional(),
11020
- schemaId: import_zod8.z.string().optional(),
11021
- schemaVersion: import_zod8.z.string().optional(),
11022
- options: import_zod8.z.array(
11023
- import_zod8.z.object({
11024
- id: import_zod8.z.string(),
11025
- label: import_zod8.z.string(),
11026
- description: import_zod8.z.string().optional()
9051
+ type: import_zod9.z.enum(["answer", "interrupt", "error", "cancelled"]),
9052
+ text: import_zod9.z.string(),
9053
+ structured: import_zod9.z.array(import_zod9.z.json()),
9054
+ events: import_zod9.z.array(import_zod9.z.json()).optional(),
9055
+ runId: import_zod9.z.string().optional(),
9056
+ checkpointId: import_zod9.z.string().optional(),
9057
+ interrupt: import_zod9.z.object({
9058
+ requestId: import_zod9.z.string(),
9059
+ kind: import_zod9.z.enum(["text", "choice", "multi-choice", "custom"]),
9060
+ question: import_zod9.z.string().optional(),
9061
+ schemaId: import_zod9.z.string().optional(),
9062
+ schemaVersion: import_zod9.z.string().optional(),
9063
+ options: import_zod9.z.array(
9064
+ import_zod9.z.object({
9065
+ id: import_zod9.z.string(),
9066
+ label: import_zod9.z.string(),
9067
+ description: import_zod9.z.string().optional()
11027
9068
  }).strict()
11028
9069
  ),
11029
- request: import_zod8.z.json().optional()
9070
+ request: import_zod9.z.json().optional()
11030
9071
  }).strict().optional()
11031
9072
  }).strict().superRefine((value, ctx) => {
11032
9073
  if (value.type === "interrupt" !== Boolean(value.interrupt))
@@ -11035,11 +9076,16 @@ var EvalObservationSchema = import_zod8.z.object({
11035
9076
  message: "Interrupt observations require exactly one interrupt."
11036
9077
  });
11037
9078
  });
11038
- var status = import_zod8.z.enum(["passed", "failed", "error", "cancelled", "ungraded"]);
11039
- var EvalIssueSchema = import_zod8.z.object({
9079
+ var status = import_zod9.z.enum(["passed", "failed", "error", "cancelled", "ungraded"]);
9080
+ var EvalIssueSchema = import_zod9.z.object({
9081
+ exception: import_telemetry_contracts.DiagnosticContentSchema.optional(),
9082
+ details: import_zod9.z.object({
9083
+ timeoutMs: import_zod9.z.number().positive().optional(),
9084
+ httpStatus: import_zod9.z.number().int().min(100).max(599).optional()
9085
+ }).strict().optional(),
11040
9086
  failure: failureDescriptor.optional(),
11041
- diagnosticId: import_zod8.z.uuid().optional(),
11042
- phase: import_zod8.z.enum([
9087
+ diagnosticId: import_zod9.z.uuid().optional(),
9088
+ phase: import_zod9.z.enum([
11043
9089
  "params",
11044
9090
  "setup",
11045
9091
  "execute",
@@ -11050,196 +9096,200 @@ var EvalIssueSchema = import_zod8.z.object({
11050
9096
  "cleanup",
11051
9097
  "reporting"
11052
9098
  ]),
11053
- code: import_zod8.z.string(),
11054
- message: import_zod8.z.string()
9099
+ code: import_zod9.z.string(),
9100
+ message: import_zod9.z.string()
11055
9101
  }).strict();
11056
9102
  var issue = EvalIssueSchema;
11057
- var EvalJudgeUsageSchema = import_zod8.z.object({
11058
- provider: import_zod8.z.string().min(1),
11059
- model: import_zod8.z.string().min(1),
11060
- occurredAt: import_zod8.z.iso.datetime(),
11061
- usage: import_zod8.z.object({
11062
- input: import_zod8.z.number().finite().nonnegative().optional(),
11063
- output: import_zod8.z.number().finite().nonnegative().optional(),
11064
- total: import_zod8.z.number().finite().nonnegative().optional(),
11065
- reasoning: import_zod8.z.number().finite().nonnegative().optional(),
11066
- cacheRead: import_zod8.z.number().finite().nonnegative().optional(),
11067
- cacheWrite: import_zod8.z.number().finite().nonnegative().optional(),
11068
- cacheWrite1h: import_zod8.z.number().finite().nonnegative().optional(),
11069
- outputIncludesReasoning: import_zod8.z.boolean().optional(),
11070
- inputIncludesCacheRead: import_zod8.z.boolean().optional(),
11071
- inputIncludesCacheWrite: import_zod8.z.boolean().optional()
9103
+ var EvalJudgeUsageSchema = import_zod9.z.object({
9104
+ provider: import_zod9.z.string().min(1),
9105
+ model: import_zod9.z.string().min(1),
9106
+ occurredAt: import_zod9.z.iso.datetime(),
9107
+ usage: import_zod9.z.object({
9108
+ input: import_zod9.z.number().finite().nonnegative().optional(),
9109
+ output: import_zod9.z.number().finite().nonnegative().optional(),
9110
+ total: import_zod9.z.number().finite().nonnegative().optional(),
9111
+ reasoning: import_zod9.z.number().finite().nonnegative().optional(),
9112
+ cacheRead: import_zod9.z.number().finite().nonnegative().optional(),
9113
+ cacheWrite: import_zod9.z.number().finite().nonnegative().optional(),
9114
+ cacheWrite1h: import_zod9.z.number().finite().nonnegative().optional(),
9115
+ outputIncludesReasoning: import_zod9.z.boolean().optional(),
9116
+ inputIncludesCacheRead: import_zod9.z.boolean().optional(),
9117
+ inputIncludesCacheWrite: import_zod9.z.boolean().optional()
11072
9118
  }).strict().optional(),
11073
- pricingContext: import_zod8.z.object({
11074
- serviceTier: import_zod8.z.string().min(1).max(100).optional(),
11075
- inferenceGeo: import_zod8.z.string().min(1).max(100).optional(),
11076
- speed: import_zod8.z.string().min(1).max(100).optional()
9119
+ pricingContext: import_zod9.z.object({
9120
+ serviceTier: import_zod9.z.string().min(1).max(100).optional(),
9121
+ inferenceGeo: import_zod9.z.string().min(1).max(100).optional(),
9122
+ speed: import_zod9.z.string().min(1).max(100).optional()
11077
9123
  }).strict().optional(),
11078
- pricing: import_zod8.z.object({
11079
- source: import_zod8.z.literal("provider"),
11080
- currency: import_zod8.z.literal("USD"),
11081
- actualCostMicros: import_zod8.z.number().int().nonnegative().safe()
9124
+ pricing: import_zod9.z.object({
9125
+ source: import_zod9.z.literal("provider"),
9126
+ currency: import_zod9.z.literal("USD"),
9127
+ actualCostMicros: import_zod9.z.number().int().nonnegative().safe()
11082
9128
  }).strict().optional()
11083
9129
  }).strict();
11084
- var EvalStepResultSchema = import_zod8.z.object({
11085
- evidence: import_zod8.z.object({
11086
- version: import_zod8.z.literal("compact-v1"),
11087
- history: import_zod8.z.boolean(),
9130
+ var EvalStepResultSchema = import_zod9.z.object({
9131
+ evidence: import_zod9.z.object({
9132
+ version: import_zod9.z.literal("compact-v1"),
9133
+ history: import_zod9.z.boolean(),
11088
9134
  observation: EvalObservationSchema
11089
9135
  }).strict().optional(),
11090
- index: import_zod8.z.number().int().nonnegative(),
11091
- input: import_zod8.z.union([
11092
- import_zod8.z.object({ message: import_zod8.z.string() }).strict(),
11093
- import_zod8.z.object({ resume: EvalResumeResponseSchema }).strict()
9136
+ index: import_zod9.z.number().int().nonnegative(),
9137
+ input: import_zod9.z.union([
9138
+ import_zod9.z.object({ message: import_zod9.z.string() }).strict(),
9139
+ import_zod9.z.object({ resume: EvalResumeResponseSchema }).strict()
11094
9140
  ]),
11095
9141
  expectation,
11096
9142
  observation: EvalObservationSchema,
11097
- reference: import_zod8.z.json().optional(),
9143
+ reference: import_zod9.z.json().optional(),
11098
9144
  status,
11099
- reason: import_zod8.z.string().optional(),
11100
- judgeCalls: import_zod8.z.number().int().nonnegative().optional(),
11101
- judgeUsage: import_zod8.z.array(EvalJudgeUsageSchema).optional(),
11102
- criteria: import_zod8.z.array(
11103
- EvalVerdictSchema.extend({ id: import_zod8.z.string(), text: import_zod8.z.string() })
9145
+ reason: import_zod9.z.string().optional(),
9146
+ judgeCalls: import_zod9.z.number().int().nonnegative().optional(),
9147
+ judgeUsage: import_zod9.z.array(EvalJudgeUsageSchema).optional(),
9148
+ criteria: import_zod9.z.array(
9149
+ EvalVerdictSchema.extend({ id: import_zod9.z.string(), text: import_zod9.z.string() })
11104
9150
  )
11105
9151
  }).strict();
11106
- var runtimeExecution = import_zod8.z.object({ sessionId: import_zod8.z.string().min(1), runId: import_zod8.z.string().min(1) }).strict();
11107
- var EvalCaseResultSchema = import_zod8.z.object({
9152
+ var runtimeExecution = import_zod9.z.object({ sessionId: import_zod9.z.string().min(1), runId: import_zod9.z.string().min(1) }).strict();
9153
+ var EvalCaseResultSchema = import_zod9.z.object({
11108
9154
  verification: EvalVerdictSchema.optional(),
11109
- phaseDurationsMs: import_zod8.z.partialRecord(issue.shape.phase, import_zod8.z.number().finite().nonnegative()).optional(),
9155
+ phaseDurationsMs: import_zod9.z.partialRecord(issue.shape.phase, import_zod9.z.number().finite().nonnegative()).optional(),
11110
9156
  caseId: id2,
11111
- repetition: import_zod8.z.number().int().positive(),
11112
- attemptId: import_zod8.z.uuid().optional(),
11113
- runtimeExecutions: import_zod8.z.array(runtimeExecution).optional(),
11114
- sessionId: import_zod8.z.string(),
9157
+ repetition: import_zod9.z.number().int().positive(),
9158
+ attemptId: import_zod9.z.uuid().optional(),
9159
+ runtimeExecutions: import_zod9.z.array(runtimeExecution).optional(),
9160
+ sessionId: import_zod9.z.string(),
11115
9161
  status,
11116
- durationMs: import_zod8.z.number().nonnegative(),
11117
- steps: import_zod8.z.array(EvalStepResultSchema),
11118
- errors: import_zod8.z.array(issue)
9162
+ durationMs: import_zod9.z.number().nonnegative(),
9163
+ steps: import_zod9.z.array(EvalStepResultSchema),
9164
+ errors: import_zod9.z.array(issue)
11119
9165
  }).strict();
11120
- var EvalJudgeIdentitySchema = import_zod8.z.object({
9166
+ var EvalJudgeIdentitySchema = import_zod9.z.object({
11121
9167
  id: id2,
11122
9168
  version: id2,
11123
- location: import_zod8.z.enum(["app", "studio"]).optional()
9169
+ location: import_zod9.z.enum(["app", "studio"]).optional()
11124
9170
  }).strict();
11125
9171
  var judge = EvalJudgeIdentitySchema;
11126
- var StudioEvalJudgeRequestSchema = import_zod8.z.object({
9172
+ var StudioEvalJudgeRequestSchema = import_zod9.z.object({
11127
9173
  environment: id2,
11128
9174
  judge: EvalJudgeIdentitySchema,
11129
- criterion: import_zod8.z.object({ id: id2, text: import_zod8.z.string().trim().min(1).max(16384) }).strict(),
9175
+ criterion: import_zod9.z.object({ id: id2, text: import_zod9.z.string().trim().min(1).max(16384) }).strict(),
11130
9176
  input: EvalStepResultSchema.shape.input,
11131
9177
  observation: EvalObservationSchema,
11132
- reference: import_zod8.z.json().optional(),
11133
- conversation: import_zod8.z.array(EvalStepResultSchema).max(100)
9178
+ reference: import_zod9.z.json().optional(),
9179
+ conversation: import_zod9.z.array(EvalStepResultSchema).max(100)
11134
9180
  }).strict();
11135
- var StudioEvalJudgeResponseSchema = import_zod8.z.object({
9181
+ var StudioEvalJudgeResponseSchema = import_zod9.z.object({
11136
9182
  judge: EvalJudgeIdentitySchema,
11137
9183
  verdict: EvalVerdictSchema,
11138
- usage: import_zod8.z.array(EvalJudgeUsageSchema).optional()
9184
+ usage: import_zod9.z.array(EvalJudgeUsageSchema).optional()
11139
9185
  }).strict();
11140
- var EvalRunResultSchema = import_zod8.z.object({
11141
- id: import_zod8.z.uuid(),
9186
+ var EvalRunResultSchema = import_zod9.z.object({
9187
+ id: import_zod9.z.uuid(),
11142
9188
  suiteId: id2,
11143
- suiteRevision: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
9189
+ suiteRevision: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
11144
9190
  suite: EvalSuiteSchema,
11145
9191
  judge: judge.optional(),
11146
- startedAt: import_zod8.z.iso.datetime(),
11147
- durationMs: import_zod8.z.number().nonnegative(),
9192
+ startedAt: import_zod9.z.iso.datetime(),
9193
+ durationMs: import_zod9.z.number().nonnegative(),
11148
9194
  status,
11149
- counts: import_zod8.z.object({
11150
- passed: import_zod8.z.number().int().nonnegative(),
11151
- failed: import_zod8.z.number().int().nonnegative(),
11152
- error: import_zod8.z.number().int().nonnegative(),
11153
- cancelled: import_zod8.z.number().int().nonnegative(),
11154
- ungraded: import_zod8.z.number().int().nonnegative().optional()
9195
+ counts: import_zod9.z.object({
9196
+ passed: import_zod9.z.number().int().nonnegative(),
9197
+ failed: import_zod9.z.number().int().nonnegative(),
9198
+ error: import_zod9.z.number().int().nonnegative(),
9199
+ cancelled: import_zod9.z.number().int().nonnegative(),
9200
+ ungraded: import_zod9.z.number().int().nonnegative().optional()
11155
9201
  }).strict(),
11156
- cases: import_zod8.z.array(EvalCaseResultSchema),
11157
- errors: import_zod8.z.array(issue)
9202
+ cases: import_zod9.z.array(EvalCaseResultSchema),
9203
+ errors: import_zod9.z.array(issue)
11158
9204
  }).strict();
11159
- var EvalManifestSchema = import_zod8.z.object({
11160
- schemaVersion: import_zod8.z.literal(1),
11161
- studioJudging: import_zod8.z.literal(true).optional(),
11162
- promptContracts: import_zod8.z.array(
11163
- import_zod8.z.object({
11164
- id: import_zod8.z.string(),
11165
- format: import_zod8.z.enum(["system-user", "chat"]),
11166
- variablesSchema: import_zod8.z.record(import_zod8.z.string(), import_zod8.z.unknown()),
11167
- configSchema: import_zod8.z.record(import_zod8.z.string(), import_zod8.z.unknown())
9205
+ var EvalManifestSchema = import_zod9.z.object({
9206
+ execution: EvalExecutionSettingsSchema.optional(),
9207
+ schemaVersion: import_zod9.z.literal(1),
9208
+ studioJudging: import_zod9.z.literal(true).optional(),
9209
+ promptContracts: import_zod9.z.array(
9210
+ import_zod9.z.object({
9211
+ id: import_zod9.z.string(),
9212
+ format: import_zod9.z.enum(["system-user", "chat"]),
9213
+ variablesSchema: import_zod9.z.record(import_zod9.z.string(), import_zod9.z.unknown()),
9214
+ configSchema: import_zod9.z.record(import_zod9.z.string(), import_zod9.z.unknown())
11168
9215
  }).strict()
11169
9216
  ).optional(),
11170
- attemptScheduling: import_zod8.z.literal(true).optional(),
11171
- suites: import_zod8.z.array(EvalSuiteSchema),
11172
- responders: import_zod8.z.array(
11173
- import_zod8.z.object({
9217
+ attemptScheduling: import_zod9.z.literal(true).optional(),
9218
+ suites: import_zod9.z.array(EvalSuiteSchema),
9219
+ responders: import_zod9.z.array(
9220
+ import_zod9.z.object({
11174
9221
  name: id2,
11175
9222
  schemaId: id2.optional(),
11176
9223
  schemaVersion: id2.optional()
11177
9224
  }).strict()
11178
9225
  ),
11179
- references: import_zod8.z.array(id2),
11180
- paramsSchema: import_zod8.z.record(import_zod8.z.string(), import_zod8.z.unknown()).optional(),
9226
+ references: import_zod9.z.array(id2),
9227
+ paramsSchema: import_zod9.z.record(import_zod9.z.string(), import_zod9.z.unknown()).optional(),
11181
9228
  judge: judge.optional()
11182
9229
  }).strict();
11183
- var EvalProgressSchema = import_zod8.z.discriminatedUnion("type", [
11184
- import_zod8.z.object({
11185
- type: import_zod8.z.literal("case-dispatch"),
9230
+ var EvalProgressSchema = import_zod9.z.discriminatedUnion("type", [
9231
+ import_zod9.z.object({
9232
+ type: import_zod9.z.literal("case-dispatch"),
11186
9233
  caseId: id2,
11187
- repetition: import_zod8.z.number().int().positive(),
11188
- status: import_zod8.z.enum(["queued", "retrying", "reconnecting"]),
11189
- retryAt: import_zod8.z.iso.datetime().optional(),
11190
- httpStatus: import_zod8.z.number().int().min(100).max(599).optional(),
11191
- executionId: import_zod8.z.uuid().optional(),
11192
- generation: import_zod8.z.uuid().optional()
9234
+ repetition: import_zod9.z.number().int().positive(),
9235
+ status: import_zod9.z.enum(["queued", "retrying", "reconnecting"]),
9236
+ retryAt: import_zod9.z.iso.datetime().optional(),
9237
+ httpStatus: import_zod9.z.number().int().min(100).max(599).optional(),
9238
+ executionId: import_zod9.z.uuid().optional(),
9239
+ generation: import_zod9.z.uuid().optional()
11193
9240
  }).strict(),
11194
- import_zod8.z.object({
11195
- type: import_zod8.z.literal("case-runtime-associated"),
9241
+ import_zod9.z.object({
9242
+ type: import_zod9.z.literal("case-runtime-associated"),
11196
9243
  caseId: id2,
11197
- repetition: import_zod8.z.number().int().positive(),
11198
- attemptId: import_zod8.z.uuid(),
9244
+ repetition: import_zod9.z.number().int().positive(),
9245
+ attemptId: import_zod9.z.uuid(),
11199
9246
  execution: runtimeExecution
11200
9247
  }).strict(),
11201
- import_zod8.z.object({
11202
- type: import_zod8.z.literal("run-started"),
11203
- caseIds: import_zod8.z.array(id2).min(1),
11204
- repetitions: import_zod8.z.number().int().positive(),
11205
- concurrency: import_zod8.z.number().int().positive()
9248
+ import_zod9.z.object({
9249
+ type: import_zod9.z.literal("run-started"),
9250
+ caseIds: import_zod9.z.array(id2).min(1),
9251
+ repetitions: import_zod9.z.number().int().positive(),
9252
+ concurrency: import_zod9.z.number().int().positive()
11206
9253
  }).strict(),
11207
- import_zod8.z.object({
11208
- type: import_zod8.z.literal("case-progress"),
9254
+ import_zod9.z.object({
9255
+ type: import_zod9.z.literal("case-progress"),
11209
9256
  caseId: id2,
11210
- repetition: import_zod8.z.number().int().positive(),
9257
+ repetition: import_zod9.z.number().int().positive(),
11211
9258
  phase: issue.shape.phase,
11212
- stepIndex: import_zod8.z.number().int().nonnegative().optional()
9259
+ stepIndex: import_zod9.z.number().int().nonnegative().optional()
11213
9260
  }).strict(),
11214
- import_zod8.z.object({
11215
- type: import_zod8.z.literal("case-started"),
11216
- attemptId: import_zod8.z.uuid().optional(),
9261
+ import_zod9.z.object({
9262
+ type: import_zod9.z.literal("case-started"),
9263
+ attemptId: import_zod9.z.uuid().optional(),
11217
9264
  caseId: id2,
11218
- repetition: import_zod8.z.number().int().positive(),
11219
- sessionId: import_zod8.z.string()
9265
+ repetition: import_zod9.z.number().int().positive(),
9266
+ sessionId: import_zod9.z.string()
11220
9267
  }).strict(),
11221
- import_zod8.z.object({
11222
- type: import_zod8.z.literal("step-completed"),
9268
+ import_zod9.z.object({
9269
+ type: import_zod9.z.literal("step-completed"),
11223
9270
  caseId: id2,
11224
- repetition: import_zod8.z.number().int().positive(),
9271
+ repetition: import_zod9.z.number().int().positive(),
11225
9272
  step: EvalStepResultSchema
11226
9273
  }).strict(),
11227
- import_zod8.z.object({ type: import_zod8.z.literal("case-completed"), result: EvalCaseResultSchema }).strict()
9274
+ import_zod9.z.object({ type: import_zod9.z.literal("case-completed"), result: EvalCaseResultSchema }).strict()
11228
9275
  ]);
11229
- var EvalRemoteRunRequestSchema = import_zod8.z.object({
9276
+ var EvalRemoteRunRequestSchema = import_zod9.z.object({
11230
9277
  promptSnapshot: import_browser2.PromptSnapshotSchema.optional(),
11231
- promptGroupName: import_zod8.z.string().optional(),
9278
+ promptGroupName: import_zod9.z.string().optional(),
9279
+ caseTimeoutMs: EvalSettingsSchema.shape.caseTimeoutMs,
9280
+ runTimeoutMs: EvalSettingsSchema.shape.runTimeoutMs,
9281
+ resolvedSettings: ResolvedEvalSettingsSchema.optional(),
11232
9282
  suiteId: id2,
11233
- suiteRevision: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
11234
- grading: import_zod8.z.enum(["app", "studio"]).optional(),
9283
+ suiteRevision: import_zod9.z.string().regex(/^[a-f0-9]{64}$/),
9284
+ grading: import_zod9.z.enum(["app", "studio"]).optional(),
11235
9285
  judge: EvalJudgeIdentitySchema.optional(),
11236
- caseIds: import_zod8.z.array(id2).min(1).max(100).optional(),
11237
- repetitions: import_zod8.z.number().int().min(1).max(20).default(1),
11238
- concurrency: import_zod8.z.number().int().min(1).max(20).default(1),
11239
- attempt: import_zod8.z.object({
9286
+ caseIds: import_zod9.z.array(id2).min(1).max(100).optional(),
9287
+ repetitions: import_zod9.z.number().int().min(1).max(20).default(1),
9288
+ concurrency: import_zod9.z.number().int().min(1).max(20).default(1),
9289
+ attempt: import_zod9.z.object({
11240
9290
  caseId: id2,
11241
- repetition: import_zod8.z.number().int().min(1).max(20),
11242
- evaluationId: import_zod8.z.uuid()
9291
+ repetition: import_zod9.z.number().int().min(1).max(20),
9292
+ evaluationId: import_zod9.z.uuid()
11243
9293
  }).strict().optional()
11244
9294
  }).strict();
11245
9295
  var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
@@ -11249,39 +9299,42 @@ var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
11249
9299
  promptGroupName: true,
11250
9300
  attempt: true
11251
9301
  }).extend({
11252
- promptSelection: import_zod8.z.discriminatedUnion("type", [
11253
- import_zod8.z.object({ type: import_zod8.z.literal("live") }).strict(),
11254
- import_zod8.z.object({
11255
- type: import_zod8.z.literal("single"),
11256
- id: import_zod8.z.string(),
11257
- version: import_zod8.z.number().int().positive()
9302
+ promptSelection: import_zod9.z.discriminatedUnion("type", [
9303
+ import_zod9.z.object({ type: import_zod9.z.literal("live") }).strict(),
9304
+ import_zod9.z.object({
9305
+ type: import_zod9.z.literal("single"),
9306
+ id: import_zod9.z.string(),
9307
+ version: import_zod9.z.number().int().positive()
11258
9308
  }).strict(),
11259
- import_zod8.z.object({ type: import_zod8.z.literal("group"), groupId: import_zod8.z.uuid() }).strict()
9309
+ import_zod9.z.object({ type: import_zod9.z.literal("group"), groupId: import_zod9.z.uuid() }).strict()
11260
9310
  ]).optional(),
11261
- targetId: import_zod8.z.string().min(1).max(128),
11262
- judge: import_zod8.z.enum(["studio", "app"]).default("studio")
9311
+ targetId: import_zod9.z.string().min(1).max(128),
9312
+ judge: import_zod9.z.enum(["studio", "app"]).optional(),
9313
+ repetitions: EvalSettingsSchema.shape.repetitions,
9314
+ concurrency: EvalSettingsSchema.shape.concurrency
11263
9315
  });
11264
- var EvalWireEventSchema = import_zod8.z.discriminatedUnion("type", [
11265
- import_zod8.z.object({
11266
- type: import_zod8.z.literal("execution-status"),
11267
- state: import_zod8.z.enum(["queued", "running", "completed", "cancelled", "unknown"])
9316
+ var EvalWireEventSchema = import_zod9.z.discriminatedUnion("type", [
9317
+ import_zod9.z.object({
9318
+ type: import_zod9.z.literal("execution-status"),
9319
+ state: import_zod9.z.enum(["queued", "running", "completed", "cancelled", "unknown"])
11268
9320
  }).strict(),
11269
- import_zod8.z.object({
11270
- type: import_zod8.z.literal("progress"),
9321
+ import_zod9.z.object({
9322
+ type: import_zod9.z.literal("progress"),
11271
9323
  event: EvalProgressSchema,
11272
- sequence: import_zod8.z.number().int().positive().optional()
9324
+ sequence: import_zod9.z.number().int().positive().optional()
11273
9325
  }).strict(),
11274
- import_zod8.z.object({
11275
- type: import_zod8.z.literal("result"),
9326
+ import_zod9.z.object({
9327
+ type: import_zod9.z.literal("result"),
11276
9328
  result: EvalRunResultSchema,
11277
- sequence: import_zod8.z.number().int().positive().optional()
9329
+ sequence: import_zod9.z.number().int().positive().optional()
11278
9330
  }).strict(),
11279
- import_zod8.z.object({
11280
- type: import_zod8.z.literal("error"),
11281
- message: import_zod8.z.string(),
9331
+ import_zod9.z.object({
9332
+ type: import_zod9.z.literal("error"),
9333
+ exception: import_telemetry_contracts.DiagnosticContentSchema.optional(),
9334
+ message: import_zod9.z.string(),
11282
9335
  failure: failureDescriptor.optional(),
11283
- diagnosticId: import_zod8.z.uuid().optional(),
11284
- sequence: import_zod8.z.number().int().positive().optional()
9336
+ diagnosticId: import_zod9.z.uuid().optional(),
9337
+ sequence: import_zod9.z.number().int().positive().optional()
11285
9338
  }).strict()
11286
9339
  ]);
11287
9340
  var EvalConfigurationError = class extends Error {
@@ -11311,13 +9364,14 @@ function parseEvalSuite(value) {
11311
9364
 
11312
9365
  // src/evals/create-evals.ts
11313
9366
  var import_node_crypto5 = require("crypto");
11314
- var import_eval_attribution2 = require("@kortyx/core/eval-attribution");
9367
+ var import_eval_attribution3 = require("@kortyx/core/eval-attribution");
11315
9368
  var import_usage_telemetry = require("@kortyx/usage-telemetry");
11316
- var import_zod10 = require("zod");
9369
+ var import_zod11 = require("zod");
11317
9370
 
11318
9371
  // src/evals/errors.ts
11319
9372
  var import_errors9 = require("@kortyx/core/errors");
11320
- var import_zod9 = require("zod");
9373
+ var import_telemetry_contracts2 = require("@kortyx/telemetry-contracts");
9374
+ var import_zod10 = require("zod");
11321
9375
  var messages = {
11322
9376
  params: "Case parameters failed validation.",
11323
9377
  setup: "Case setup failed.",
@@ -11333,6 +9387,9 @@ function createEvalIssue(phase, error, options = {}) {
11333
9387
  const control = (0, import_errors9.isControlFlowError)(error);
11334
9388
  const declared = !control && ((0, import_errors9.isFailureDescriptor)(error) || (0, import_errors9.isFailureDescriptor)((0, import_errors9.errorProperty)(error, "failure")));
11335
9389
  const failure = (0, import_errors9.serializeFailure)(control ? void 0 : error);
9390
+ const exception = !control && error !== void 0 ? (0, import_telemetry_contracts2.captureDiagnosticContent)(error) : null;
9391
+ const rawMessage = typeof error === "string" ? error : (0, import_errors9.errorProperty)(error, "message");
9392
+ const originalMessage = typeof rawMessage === "string" ? (0, import_telemetry_contracts2.redactDiagnosticText)(rawMessage) : void 0;
11336
9393
  let diagnosticId = options.diagnosticId ?? (0, import_errors9.errorProperty)(error, "diagnosticId");
11337
9394
  if (!control && !diagnosticId && options.reportError) {
11338
9395
  try {
@@ -11340,12 +9397,16 @@ function createEvalIssue(phase, error, options = {}) {
11340
9397
  } catch {
11341
9398
  }
11342
9399
  }
9400
+ const candidate = (0, import_errors9.errorProperty)(error, "statusCode") ?? (0, import_errors9.errorProperty)(error, "status");
9401
+ const httpStatus = typeof candidate === "number" && Number.isInteger(candidate) && candidate >= 100 && candidate <= 599 ? candidate : void 0;
11343
9402
  return {
9403
+ ...exception ? { exception } : {},
9404
+ ...httpStatus ? { details: { httpStatus } } : {},
11344
9405
  phase,
11345
9406
  code: declared ? failure.code : options.code ?? "EVAL_HOOK_FAILED",
11346
- message: declared ? failure.message : messages[phase],
9407
+ message: originalMessage ?? (declared ? failure.message : `${messages[phase]}${httpStatus ? ` HTTP ${httpStatus}.` : ""}`),
11347
9408
  failure,
11348
- ...typeof diagnosticId === "string" && import_zod9.z.uuid().safeParse(diagnosticId).success ? { diagnosticId } : {}
9409
+ ...typeof diagnosticId === "string" && import_zod10.z.uuid().safeParse(diagnosticId).success ? { diagnosticId } : {}
11349
9410
  };
11350
9411
  }
11351
9412
 
@@ -11852,12 +9913,28 @@ function createEvals(options) {
11852
9913
  correlation,
11853
9914
  ...code ? { code } : {}
11854
9915
  });
9916
+ for (const key of ["concurrency", "caseTimeoutMs"]) {
9917
+ const max = options.limits?.[key];
9918
+ if (max !== void 0 && defaults[key] > positive(max, `limits.${key}`))
9919
+ throw new EvalConfigurationError(
9920
+ `${key} default exceeds its application limit.`
9921
+ );
9922
+ }
11855
9923
  return {
11856
9924
  listSuites() {
11857
9925
  return clone(suites);
11858
9926
  },
11859
9927
  describe() {
11860
9928
  return {
9929
+ execution: {
9930
+ defaults: {
9931
+ ...options.defaults?.repetitions !== void 0 ? { repetitions: options.defaults.repetitions } : {},
9932
+ ...options.defaults?.concurrency !== void 0 ? { concurrency: options.defaults.concurrency } : {},
9933
+ ...options.defaults?.caseTimeoutMs !== void 0 ? { caseTimeoutMs: options.defaults.caseTimeoutMs } : {}
9934
+ },
9935
+ limits: options.limits ?? {},
9936
+ overrides: ["repetitions", "concurrency", "caseTimeoutMs"]
9937
+ },
11861
9938
  schemaVersion: 1,
11862
9939
  studioJudging: true,
11863
9940
  ...options.agent.describePromptContracts ? { promptContracts: options.agent.describePromptContracts() } : {},
@@ -11872,7 +9949,7 @@ function createEvals(options) {
11872
9949
  })),
11873
9950
  references: Object.keys(references),
11874
9951
  ...options.paramsSchema ? {
11875
- paramsSchema: import_zod10.z.toJSONSchema(options.paramsSchema, {
9952
+ paramsSchema: import_zod11.z.toJSONSchema(options.paramsSchema, {
11876
9953
  io: "input"
11877
9954
  })
11878
9955
  } : {},
@@ -11886,252 +9963,368 @@ function createEvals(options) {
11886
9963
  };
11887
9964
  },
11888
9965
  async run(args) {
11889
- const suite = suites.find((value) => value.id === args.suiteId);
11890
- if (!suite)
11891
- throw new EvalConfigurationError(`Unknown suite: ${args.suiteId}`);
11892
- if (args.caseIds && (!args.caseIds.length || new Set(args.caseIds).size !== args.caseIds.length || args.caseIds.some(
11893
- (id3) => !suite.cases.some((item) => item.id === id3)
11894
- )))
11895
- throw new EvalConfigurationError("Select unique existing case IDs.");
11896
- const repetitions = positive(
11897
- args.repetitions ?? defaults.repetitions,
11898
- "repetitions"
11899
- );
11900
- const concurrency = positive(
11901
- args.concurrency ?? defaults.concurrency,
11902
- "concurrency"
11903
- );
11904
- const selected = suite.cases.filter(
11905
- (item) => !args.caseIds || args.caseIds.includes(item.id)
11906
- );
11907
- if (!Number.isSafeInteger(selected.length * repetitions))
11908
- throw new EvalConfigurationError("Too many case repetitions.");
11909
- if (args.attempt && (!selected.some((item) => item.id === args.attempt?.caseId) || !Number.isSafeInteger(args.attempt.repetition) || args.attempt.repetition < 1 || args.attempt.repetition > repetitions))
11910
- throw new EvalConfigurationError("Invalid scheduled attempt.");
11911
- const studioGrading = args.grading === "studio";
11912
- if (!studioGrading && !options.judge && selected.some(
11913
- (item) => item.steps.some((step) => step.expect.criteria?.length)
11914
- ))
11915
- throw new EvalConfigurationError(
11916
- "Local runs with criteria require a judge. Run from Studio or provide a code judge."
11917
- );
11918
- const selectedJudge = studioGrading ? args.judgeIdentity : options.judge;
11919
- if (!studioGrading && args.judgeIdentity && (!options.judge || args.judgeIdentity.id !== options.judge.id || args.judgeIdentity.version !== options.judge.version || args.judgeIdentity.location !== options.judge.location))
11920
- throw new EvalConfigurationError(
11921
- "App judge changed. Refresh before running."
11922
- );
9966
+ let usagePhase = "configuration";
11923
9967
  const evalUsage = usage.start("sdk.eval", {
11924
- scale: (0, import_usage_telemetry.scale)(selected.length * repetitions)
9968
+ case_timeout_override: args.caseTimeoutMs !== void 0,
9969
+ settings_override: args.caseTimeoutMs !== void 0 || args.concurrency !== void 0 || args.repetitions !== void 0
11925
9970
  });
11926
- const start = Date.now();
11927
- const result = {
11928
- id: (0, import_node_crypto5.randomUUID)(),
11929
- suiteId: suite.id,
11930
- suiteRevision: getEvalSuiteRevision(suite),
11931
- suite: clone(suite),
11932
- ...selectedJudge ? {
11933
- judge: {
11934
- id: selectedJudge.id,
11935
- version: selectedJudge.version,
11936
- ...selectedJudge.location ? { location: selectedJudge.location } : {}
11937
- }
11938
- } : {},
11939
- startedAt: new Date(start).toISOString(),
11940
- durationMs: 0,
11941
- status: "passed",
11942
- counts: { passed: 0, failed: 0, error: 0, cancelled: 0 },
11943
- cases: [],
11944
- errors: []
11945
- };
11946
- let reporting = true;
11947
- const emit = async (event) => {
11948
- if (!args.onProgress || !reporting) return;
11949
- try {
11950
- await (0, import_eval_attribution2.withEvalAttribution)(
11951
- void 0,
11952
- () => args.onProgress(clone(event))
9971
+ try {
9972
+ const suite = suites.find((value) => value.id === args.suiteId);
9973
+ if (!suite)
9974
+ throw new EvalConfigurationError(`Unknown suite: ${args.suiteId}`);
9975
+ if (args.caseIds && (!args.caseIds.length || new Set(args.caseIds).size !== args.caseIds.length || args.caseIds.some(
9976
+ (id3) => !suite.cases.some((item) => item.id === id3)
9977
+ )))
9978
+ throw new EvalConfigurationError("Select unique existing case IDs.");
9979
+ const repetitions = positive(
9980
+ args.repetitions ?? defaults.repetitions,
9981
+ "repetitions"
9982
+ );
9983
+ const concurrency = positive(
9984
+ args.concurrency ?? defaults.concurrency,
9985
+ "concurrency"
9986
+ );
9987
+ const caseTimeoutMs = positive(
9988
+ args.caseTimeoutMs ?? defaults.caseTimeoutMs,
9989
+ "caseTimeoutMs"
9990
+ );
9991
+ if (caseTimeoutMs > 2147483647)
9992
+ throw new EvalConfigurationError(
9993
+ "caseTimeoutMs exceeds the supported timer range."
11953
9994
  );
11954
- } catch (error) {
11955
- reporting = false;
11956
- result.errors.push(issueFor("reporting", error));
9995
+ for (const [key, value] of [
9996
+ ["concurrency", concurrency],
9997
+ ["caseTimeoutMs", caseTimeoutMs]
9998
+ ]) {
9999
+ const max = options.limits?.[key];
10000
+ if (max !== void 0 && value > max)
10001
+ throw new EvalConfigurationError(
10002
+ `${key} exceeds the application maximum of ${max}.`
10003
+ );
11957
10004
  }
11958
- };
11959
- const attempt = async (item, repetition) => {
11960
- const started = Date.now();
11961
- const controller = new AbortController();
11962
- let timedOut = false;
11963
- let timer;
11964
- const deadline = (milliseconds) => {
11965
- clearTimeout(timer);
11966
- timer = setTimeout(() => {
11967
- timedOut = true;
11968
- controller.abort();
11969
- }, milliseconds);
11970
- };
11971
- deadline(defaults.setupTimeoutMs);
11972
- const signal = args.signal ? AbortSignal.any([args.signal, controller.signal]) : controller.signal;
11973
- const attemptId = (0, import_node_crypto5.randomUUID)();
11974
- const runtimeExecutions = [];
11975
- const attemptResult = {
11976
- caseId: item.id,
11977
- repetition,
11978
- attemptId,
11979
- runtimeExecutions,
11980
- sessionId: `eval-${(0, import_node_crypto5.randomUUID)()}`,
11981
- status: "passed",
10005
+ const selected = suite.cases.filter(
10006
+ (item) => !args.caseIds || args.caseIds.includes(item.id)
10007
+ );
10008
+ if (!Number.isSafeInteger(selected.length * repetitions))
10009
+ throw new EvalConfigurationError("Too many case repetitions.");
10010
+ if (args.attempt && (!selected.some((item) => item.id === args.attempt?.caseId) || !Number.isSafeInteger(args.attempt.repetition) || args.attempt.repetition < 1 || args.attempt.repetition > repetitions))
10011
+ throw new EvalConfigurationError("Invalid scheduled attempt.");
10012
+ const studioGrading = args.grading === "studio";
10013
+ if (!studioGrading && !options.judge && selected.some(
10014
+ (item) => item.steps.some((step) => step.expect.criteria?.length)
10015
+ ))
10016
+ throw new EvalConfigurationError(
10017
+ "Local runs with criteria require a judge. Run from Studio or provide a code judge."
10018
+ );
10019
+ const selectedJudge = studioGrading ? args.judgeIdentity : options.judge;
10020
+ if (!studioGrading && args.judgeIdentity && (!options.judge || args.judgeIdentity.id !== options.judge.id || args.judgeIdentity.version !== options.judge.version || args.judgeIdentity.location !== options.judge.location))
10021
+ throw new EvalConfigurationError(
10022
+ "App judge changed. Refresh before running."
10023
+ );
10024
+ usagePhase = "execution";
10025
+ const start = Date.now();
10026
+ const result = {
10027
+ id: (0, import_node_crypto5.randomUUID)(),
10028
+ suiteId: suite.id,
10029
+ suiteRevision: getEvalSuiteRevision(suite),
10030
+ suite: clone(suite),
10031
+ ...selectedJudge ? {
10032
+ judge: {
10033
+ id: selectedJudge.id,
10034
+ version: selectedJudge.version,
10035
+ ...selectedJudge.location ? { location: selectedJudge.location } : {}
10036
+ }
10037
+ } : {},
10038
+ startedAt: new Date(start).toISOString(),
11982
10039
  durationMs: 0,
11983
- steps: [],
11984
- errors: [],
11985
- phaseDurationsMs: {}
10040
+ status: "passed",
10041
+ counts: { passed: 0, failed: 0, error: 0, cancelled: 0 },
10042
+ cases: [],
10043
+ errors: []
11986
10044
  };
11987
- let associationProgress = Promise.resolve();
11988
- let acceptingExecutions = true;
11989
- const executeAttempt = async () => {
11990
- let phase = "params";
11991
- let phaseStarted = Date.now();
11992
- const recordPhase = () => {
11993
- const now = Date.now();
11994
- const durations = attemptResult.phaseDurationsMs;
11995
- durations[phase] = (durations[phase] ?? 0) + now - phaseStarted;
11996
- phaseStarted = now;
10045
+ let reporting = true;
10046
+ const emit = async (event) => {
10047
+ if (!args.onProgress || !reporting) return;
10048
+ try {
10049
+ await (0, import_eval_attribution3.withEvalAttribution)(
10050
+ void 0,
10051
+ () => args.onProgress?.(clone(event))
10052
+ );
10053
+ } catch (error) {
10054
+ reporting = false;
10055
+ result.errors.push(issueFor("reporting", error));
10056
+ }
10057
+ };
10058
+ const attempt = async (item, repetition) => {
10059
+ const started = Date.now();
10060
+ const controller = new AbortController();
10061
+ let timedOut = false;
10062
+ let timer;
10063
+ const deadline = (milliseconds) => {
10064
+ clearTimeout(timer);
10065
+ timer = setTimeout(() => {
10066
+ timedOut = true;
10067
+ controller.abort();
10068
+ }, milliseconds);
11997
10069
  };
11998
- const phaseIssue = (error, code) => issueFor(
11999
- phase,
12000
- error,
12001
- {
12002
- correlation: { sessionId: attemptResult.sessionId, attemptId }
12003
- },
12004
- code
12005
- );
12006
- let stepIndex;
12007
- const reportPhase = async (next) => {
12008
- recordPhase();
12009
- phase = next;
12010
- if (args.includeActivity)
12011
- await emit({
12012
- type: "case-progress",
12013
- caseId: item.id,
12014
- repetition,
12015
- phase,
12016
- ...stepIndex !== void 0 ? { stepIndex } : {}
12017
- });
10070
+ deadline(defaults.setupTimeoutMs);
10071
+ const signal = args.signal ? AbortSignal.any([args.signal, controller.signal]) : controller.signal;
10072
+ let originalExecutionError;
10073
+ const attemptId = (0, import_node_crypto5.randomUUID)();
10074
+ const runtimeExecutions = [];
10075
+ const attemptResult = {
10076
+ caseId: item.id,
10077
+ repetition,
10078
+ attemptId,
10079
+ runtimeExecutions,
10080
+ sessionId: `eval-${(0, import_node_crypto5.randomUUID)()}`,
10081
+ status: "passed",
10082
+ durationMs: 0,
10083
+ steps: [],
10084
+ errors: [],
10085
+ phaseDurationsMs: {}
12018
10086
  };
12019
- let context;
12020
- let continuation;
12021
- let activeStep;
12022
- let activeStepReported = false;
12023
- const history = [];
12024
- const call = async (readyContext, command, activeSignal) => {
12025
- const run = (overrides) => executeEvalChat({
12026
- agent: options.agent,
12027
- ...args.promptSnapshot ? { promptSnapshot: args.promptSnapshot } : {},
12028
- command,
12029
- continuation,
12030
- history: clone(history),
12031
- sessionId: attemptResult.sessionId,
12032
- clientTurnId: (0, import_node_crypto5.randomUUID)(),
12033
- signal: activeSignal,
12034
- ...item.workflowId ? { workflowId: item.workflowId } : {},
12035
- ...overrides
12036
- });
12037
- const execution = options.execute ? await options.execute({
12038
- ...readyContext,
12039
- signal: activeSignal,
12040
- command,
12041
- continuation,
12042
- history: clone(history),
12043
- run
12044
- }) : await run();
12045
- continuation = execution.continuation;
12046
- const observation = EvalObservationSchema.parse(
12047
- execution.observation
10087
+ let associationProgress = Promise.resolve();
10088
+ let acceptingExecutions = true;
10089
+ const executeAttempt = async () => {
10090
+ let phase = "params";
10091
+ let phaseStarted = Date.now();
10092
+ const recordPhase = () => {
10093
+ const now = Date.now();
10094
+ const durations = attemptResult.phaseDurationsMs;
10095
+ durations[phase] = (durations[phase] ?? 0) + now - phaseStarted;
10096
+ phaseStarted = now;
10097
+ };
10098
+ const phaseIssue = (error, code) => issueFor(
10099
+ phase,
10100
+ error,
10101
+ {
10102
+ correlation: {
10103
+ sessionId: attemptResult.sessionId,
10104
+ attemptId
10105
+ }
10106
+ },
10107
+ code
12048
10108
  );
12049
- if (observation.type === "interrupt" && execution.continuation === void 0)
12050
- throw new Error(
12051
- "A waiting interrupt requires a private continuation."
12052
- );
12053
- return {
12054
- observation,
12055
- ...continuation !== void 0 ? { continuation } : {}
10109
+ let stepIndex;
10110
+ const reportPhase = async (next) => {
10111
+ recordPhase();
10112
+ phase = next;
10113
+ if (args.includeActivity)
10114
+ await emit({
10115
+ type: "case-progress",
10116
+ caseId: item.id,
10117
+ repetition,
10118
+ phase,
10119
+ ...stepIndex !== void 0 ? { stepIndex } : {}
10120
+ });
12056
10121
  };
12057
- };
12058
- try {
12059
- await emit({
12060
- type: "case-started",
12061
- attemptId,
12062
- caseId: item.id,
12063
- repetition,
12064
- sessionId: attemptResult.sessionId
12065
- });
12066
- signal.throwIfAborted();
12067
- await reportPhase("params");
12068
- const params = options.paramsSchema ? await options.paramsSchema.parseAsync(item.params) : item.params;
12069
- const base = {
12070
- suiteId: suite.id,
12071
- case: item,
12072
- repetition,
12073
- sessionId: attemptResult.sessionId,
12074
- params,
12075
- signal
10122
+ let context;
10123
+ let continuation;
10124
+ let activeStep;
10125
+ let activeStepReported = false;
10126
+ const history = [];
10127
+ const call = async (readyContext, command, activeSignal) => {
10128
+ const run = (overrides) => executeEvalChat({
10129
+ agent: options.agent,
10130
+ ...args.promptSnapshot ? { promptSnapshot: args.promptSnapshot } : {},
10131
+ command,
10132
+ continuation,
10133
+ history: clone(history),
10134
+ sessionId: attemptResult.sessionId,
10135
+ clientTurnId: (0, import_node_crypto5.randomUUID)(),
10136
+ signal: activeSignal,
10137
+ ...item.workflowId ? { workflowId: item.workflowId } : {},
10138
+ ...overrides
10139
+ });
10140
+ const execution = options.execute ? await options.execute({
10141
+ ...readyContext,
10142
+ signal: activeSignal,
10143
+ command,
10144
+ continuation,
10145
+ history: clone(history),
10146
+ run
10147
+ }) : await run();
10148
+ continuation = execution.continuation;
10149
+ const observation = EvalObservationSchema.parse(
10150
+ execution.observation
10151
+ );
10152
+ if (observation.type === "interrupt" && execution.continuation === void 0)
10153
+ throw new Error(
10154
+ "A waiting interrupt requires a private continuation."
10155
+ );
10156
+ return {
10157
+ observation,
10158
+ ...continuation !== void 0 ? { continuation } : {}
10159
+ };
12076
10160
  };
12077
- await reportPhase("setup");
12078
- const prepared = options.setup ? await options.setup(base) : void 0;
12079
- context = { ...base, prepared };
12080
- signal.throwIfAborted();
12081
- deadline(defaults.caseTimeoutMs);
12082
- for (const [index, step] of item.steps.entries()) {
12083
- stepIndex = index;
12084
- activeStep = void 0;
12085
- activeStepReported = false;
12086
- signal.throwIfAborted();
12087
- let command;
12088
- if (step.message !== void 0)
12089
- command = { type: "message", message: step.message };
12090
- else {
12091
- await reportPhase("responder");
12092
- const interrupt = attemptResult.steps.at(-1)?.observation.interrupt;
12093
- let response = step.resume;
12094
- if (handlerRef(response)) {
12095
- const responder = responders[response.using];
12096
- if (typeof responder !== "function" && (responder.schemaId !== interrupt.schemaId || responder.schemaVersion && responder.schemaVersion !== interrupt.schemaVersion))
12097
- throw new Error("Incompatible responder.");
12098
- response = await (typeof responder === "function" ? responder : responder.respond)({
12099
- ...context,
12100
- interrupt: clone(interrupt),
12101
- ...response.params !== void 0 ? { args: response.params } : {}
12102
- });
12103
- }
12104
- command = {
12105
- type: "resume",
12106
- // z.json() is inferred as optional under the package's relaxed
12107
- // declaration build. Runtime validation still requires value.
12108
- response: EvalResumeResponseSchema.parse(
12109
- response
12110
- )
12111
- };
12112
- }
10161
+ try {
10162
+ await emit({
10163
+ type: "case-started",
10164
+ attemptId,
10165
+ caseId: item.id,
10166
+ repetition,
10167
+ sessionId: attemptResult.sessionId
10168
+ });
12113
10169
  signal.throwIfAborted();
12114
- await reportPhase("execute");
12115
- const { observation } = await call(context, command, signal);
12116
- const input = command.type === "message" ? { message: command.message } : { resume: command.response };
12117
- const evaluated = {
12118
- index,
12119
- input,
12120
- expectation: clone(step.expect),
12121
- observation,
12122
- status: "passed",
12123
- criteria: []
10170
+ await reportPhase("params");
10171
+ const params = options.paramsSchema ? await options.paramsSchema.parseAsync(item.params) : item.params;
10172
+ const base = {
10173
+ suiteId: suite.id,
10174
+ case: item,
10175
+ repetition,
10176
+ sessionId: attemptResult.sessionId,
10177
+ params,
10178
+ signal
12124
10179
  };
12125
- attemptResult.steps.push(evaluated);
12126
- activeStep = evaluated;
10180
+ await reportPhase("setup");
10181
+ const prepared = options.setup ? await options.setup(base) : void 0;
10182
+ context = { ...base, prepared };
12127
10183
  signal.throwIfAborted();
12128
- if (observation.type === "error") {
12129
- const issue2 = createEvalIssue("execute", observation.failure, {
12130
- ...observation.diagnosticId ? { diagnosticId: observation.diagnosticId } : {}
12131
- });
12132
- attemptResult.errors.push(issue2);
12133
- attemptResult.status = evaluated.status = "error";
12134
- evaluated.reason = issue2.message;
10184
+ deadline(caseTimeoutMs);
10185
+ for (const [index, step] of item.steps.entries()) {
10186
+ stepIndex = index;
10187
+ activeStep = void 0;
10188
+ activeStepReported = false;
10189
+ signal.throwIfAborted();
10190
+ let command;
10191
+ if (step.message !== void 0)
10192
+ command = { type: "message", message: step.message };
10193
+ else {
10194
+ await reportPhase("responder");
10195
+ const interrupt = attemptResult.steps.at(-1)?.observation.interrupt;
10196
+ let response = step.resume;
10197
+ if (handlerRef(response)) {
10198
+ const responder = responders[response.using];
10199
+ if (typeof responder !== "function" && (responder.schemaId !== interrupt.schemaId || responder.schemaVersion && responder.schemaVersion !== interrupt.schemaVersion))
10200
+ throw new Error("Incompatible responder.");
10201
+ response = await (typeof responder === "function" ? responder : responder.respond)({
10202
+ ...context,
10203
+ interrupt: clone(interrupt),
10204
+ ...response.params !== void 0 ? { args: response.params } : {}
10205
+ });
10206
+ }
10207
+ command = {
10208
+ type: "resume",
10209
+ // z.json() is inferred as optional under the package's relaxed
10210
+ // declaration build. Runtime validation still requires value.
10211
+ response: EvalResumeResponseSchema.parse(
10212
+ response
10213
+ )
10214
+ };
10215
+ }
10216
+ signal.throwIfAborted();
10217
+ await reportPhase("execute");
10218
+ originalExecutionError = void 0;
10219
+ const { observation } = await call(context, command, signal);
10220
+ const input = command.type === "message" ? { message: command.message } : { resume: command.response };
10221
+ const evaluated = {
10222
+ index,
10223
+ input,
10224
+ expectation: clone(step.expect),
10225
+ observation,
10226
+ status: "passed",
10227
+ criteria: []
10228
+ };
10229
+ attemptResult.steps.push(evaluated);
10230
+ activeStep = evaluated;
10231
+ signal.throwIfAborted();
10232
+ if (observation.type === "error") {
10233
+ const issue2 = createEvalIssue(
10234
+ "execute",
10235
+ originalExecutionError ?? observation.failure ?? observation.text,
10236
+ {
10237
+ ...observation.diagnosticId ? { diagnosticId: observation.diagnosticId } : {}
10238
+ }
10239
+ );
10240
+ attemptResult.errors.push(issue2);
10241
+ attemptResult.status = evaluated.status = "error";
10242
+ evaluated.reason = issue2.message;
10243
+ await emit({
10244
+ type: "step-completed",
10245
+ caseId: item.id,
10246
+ repetition,
10247
+ step: evaluated
10248
+ });
10249
+ activeStepReported = true;
10250
+ break;
10251
+ }
10252
+ if (observation.type === "cancelled") {
10253
+ evaluated.status = "cancelled";
10254
+ attemptResult.status = "cancelled";
10255
+ await emit({
10256
+ type: "step-completed",
10257
+ caseId: item.id,
10258
+ repetition,
10259
+ step: evaluated
10260
+ });
10261
+ activeStepReported = true;
10262
+ break;
10263
+ }
10264
+ if (observation.type !== step.expect.type || step.expect.schemaId && step.expect.schemaId !== observation.interrupt?.schemaId || step.expect.schemaVersion && step.expect.schemaVersion !== observation.interrupt?.schemaVersion) {
10265
+ evaluated.status = "failed";
10266
+ evaluated.reason = "The observed interaction did not match the expected answer or interrupt contract.";
10267
+ } else {
10268
+ const outputReason = missingOutputReason(
10269
+ step.expect.outputs,
10270
+ observation
10271
+ );
10272
+ if (outputReason) {
10273
+ evaluated.status = "failed";
10274
+ evaluated.reason = outputReason;
10275
+ } else {
10276
+ const reference = step.expect.reference;
10277
+ if (reference !== void 0) {
10278
+ await reportPhase("reference");
10279
+ evaluated.reference = handlerRef(reference) ? import_zod11.z.json().parse(
10280
+ await references[reference.using]?.({
10281
+ ...context,
10282
+ observation: clone(observation),
10283
+ ...reference.params !== void 0 ? { args: reference.params } : {}
10284
+ })
10285
+ ) : clone(reference);
10286
+ }
10287
+ await reportPhase("grading");
10288
+ evaluated.evidence = createEvalJudgeEvidence(
10289
+ observation,
10290
+ suite.evidence,
10291
+ evidenceFilters
10292
+ );
10293
+ if (studioGrading && step.expect.criteria?.length) {
10294
+ evaluated.status = "ungraded";
10295
+ attemptResult.status = "ungraded";
10296
+ }
10297
+ for (const [criterionIndex, value] of (studioGrading ? [] : step.expect.criteria ?? []).entries()) {
10298
+ signal.throwIfAborted();
10299
+ const criterion = typeof value === "string" ? { id: String(criterionIndex), text: value } : value;
10300
+ evaluated.judgeCalls = (evaluated.judgeCalls ?? 0) + 1;
10301
+ const verdict = EvalVerdictSchema.parse(
10302
+ await (0, import_eval_attribution3.withEvalAttribution)(
10303
+ void 0,
10304
+ () => options.judge.grade({
10305
+ criterion,
10306
+ onUsage: (usage2) => {
10307
+ if (!signal.aborted) {
10308
+ evaluated.judgeUsage ??= [];
10309
+ evaluated.judgeUsage.push(clone(usage2));
10310
+ }
10311
+ },
10312
+ input: clone(input),
10313
+ ...getEvalGradeEvidence(
10314
+ evaluated,
10315
+ attemptResult.steps.slice(0, -1)
10316
+ ),
10317
+ signal,
10318
+ ...evaluated.reference !== void 0 ? { reference: clone(evaluated.reference) } : {}
10319
+ })
10320
+ )
10321
+ );
10322
+ signal.throwIfAborted();
10323
+ evaluated.criteria.push({ ...criterion, ...verdict });
10324
+ if (!verdict.passed) evaluated.status = "failed";
10325
+ }
10326
+ }
10327
+ }
12135
10328
  await emit({
12136
10329
  type: "step-completed",
12137
10330
  caseId: item.id,
@@ -12139,255 +10332,191 @@ function createEvals(options) {
12139
10332
  step: evaluated
12140
10333
  });
12141
10334
  activeStepReported = true;
12142
- break;
10335
+ if (evaluated.status === "failed") {
10336
+ attemptResult.status = "failed";
10337
+ break;
10338
+ }
10339
+ history.push(
10340
+ {
10341
+ role: "user",
10342
+ content: command.type === "message" ? command.message : JSON.stringify(command.response)
10343
+ },
10344
+ { role: "assistant", content: observation.text }
10345
+ );
10346
+ }
10347
+ if (options.verify && attemptResult.status !== "cancelled" && attemptResult.status !== "error") {
10348
+ stepIndex = void 0;
10349
+ deadline(defaults.verificationTimeoutMs);
10350
+ await reportPhase("verification");
10351
+ const verification = EvalVerdictSchema.parse(
10352
+ await options.verify({
10353
+ ...context,
10354
+ result: clone(attemptResult)
10355
+ })
10356
+ );
10357
+ signal.throwIfAborted();
10358
+ attemptResult.verification = verification;
10359
+ if (!verification.passed) attemptResult.status = "failed";
12143
10360
  }
12144
- if (observation.type === "cancelled") {
12145
- evaluated.status = "cancelled";
12146
- attemptResult.status = "cancelled";
10361
+ } catch (error) {
10362
+ if (signal.aborted) {
10363
+ attemptResult.status = timedOut ? "error" : "cancelled";
10364
+ if (timedOut)
10365
+ attemptResult.errors.push({
10366
+ ...phaseIssue(error),
10367
+ details: {
10368
+ timeoutMs: phase === "params" || phase === "setup" ? defaults.setupTimeoutMs : phase === "verification" ? defaults.verificationTimeoutMs : caseTimeoutMs
10369
+ },
10370
+ code: phase === "params" || phase === "setup" ? "EVAL_SETUP_TIMEOUT" : phase === "verification" ? "EVAL_VERIFICATION_TIMEOUT" : "EVAL_TIMEOUT",
10371
+ message: phase === "params" || phase === "setup" ? "Case setup exceeded its configured time limit." : phase === "verification" ? "Case verification exceeded its configured time limit." : "Case execution exceeded its configured time limit."
10372
+ });
10373
+ } else {
10374
+ attemptResult.status = "error";
10375
+ attemptResult.errors.push(phaseIssue(error));
10376
+ }
10377
+ if (activeStep && !activeStepReported) {
10378
+ activeStep.status = attemptResult.status;
10379
+ activeStep.reason = attemptResult.errors.at(-1)?.message ?? "Case was cancelled.";
12147
10380
  await emit({
12148
10381
  type: "step-completed",
12149
10382
  caseId: item.id,
12150
10383
  repetition,
12151
- step: evaluated
10384
+ step: activeStep
12152
10385
  });
12153
- activeStepReported = true;
12154
- break;
12155
10386
  }
12156
- if (observation.type !== step.expect.type || step.expect.schemaId && step.expect.schemaId !== observation.interrupt?.schemaId || step.expect.schemaVersion && step.expect.schemaVersion !== observation.interrupt?.schemaVersion) {
12157
- evaluated.status = "failed";
12158
- evaluated.reason = "The observed interaction did not match the expected answer or interrupt contract.";
12159
- } else {
12160
- const outputReason = missingOutputReason(
12161
- step.expect.outputs,
12162
- observation
10387
+ } finally {
10388
+ clearTimeout(timer);
10389
+ if (context) {
10390
+ await reportPhase("cleanup");
10391
+ const cleanup = new AbortController();
10392
+ const cleanupTimer = setTimeout(
10393
+ () => cleanup.abort(),
10394
+ defaults.cleanupTimeoutMs
12163
10395
  );
12164
- if (outputReason) {
12165
- evaluated.status = "failed";
12166
- evaluated.reason = outputReason;
12167
- } else {
12168
- const reference = step.expect.reference;
12169
- if (reference !== void 0) {
12170
- await reportPhase("reference");
12171
- evaluated.reference = handlerRef(reference) ? import_zod10.z.json().parse(
12172
- await references[reference.using]({
12173
- ...context,
12174
- observation: clone(observation),
12175
- ...reference.params !== void 0 ? { args: reference.params } : {}
12176
- })
12177
- ) : clone(reference);
12178
- }
12179
- await reportPhase("grading");
12180
- evaluated.evidence = createEvalJudgeEvidence(
12181
- observation,
12182
- suite.evidence,
12183
- evidenceFilters
12184
- );
12185
- if (studioGrading && step.expect.criteria?.length) {
12186
- evaluated.status = "ungraded";
12187
- attemptResult.status = "ungraded";
12188
- }
12189
- for (const [criterionIndex, value] of (studioGrading ? [] : step.expect.criteria ?? []).entries()) {
12190
- signal.throwIfAborted();
12191
- const criterion = typeof value === "string" ? { id: String(criterionIndex), text: value } : value;
12192
- evaluated.judgeCalls = (evaluated.judgeCalls ?? 0) + 1;
12193
- const verdict = EvalVerdictSchema.parse(
12194
- await (0, import_eval_attribution2.withEvalAttribution)(
12195
- void 0,
12196
- () => options.judge.grade({
12197
- criterion,
12198
- onUsage: (usage2) => {
12199
- if (!signal.aborted) {
12200
- evaluated.judgeUsage ??= [];
12201
- evaluated.judgeUsage.push(clone(usage2));
10396
+ try {
10397
+ if (continuation !== void 0) {
10398
+ try {
10399
+ const cancelled = await call(
10400
+ context,
10401
+ { type: "cancel" },
10402
+ cleanup.signal
10403
+ );
10404
+ if (cancelled.observation.type === "error" || cancelled.continuation !== void 0)
10405
+ attemptResult.errors.push(
10406
+ createEvalIssue(
10407
+ "cleanup",
10408
+ cancelled.observation.failure,
10409
+ {
10410
+ code: "EVAL_INTERRUPT_CLEANUP_FAILED",
10411
+ ...cancelled.observation.diagnosticId ? {
10412
+ diagnosticId: cancelled.observation.diagnosticId
10413
+ } : {}
12202
10414
  }
12203
- },
12204
- input: clone(input),
12205
- ...getEvalGradeEvidence(
12206
- evaluated,
12207
- attemptResult.steps.slice(0, -1)
12208
- ),
12209
- signal,
12210
- ...evaluated.reference !== void 0 ? { reference: clone(evaluated.reference) } : {}
12211
- })
12212
- )
12213
- );
12214
- signal.throwIfAborted();
12215
- evaluated.criteria.push({ ...criterion, ...verdict });
12216
- if (!verdict.passed) evaluated.status = "failed";
12217
- }
12218
- }
12219
- }
12220
- await emit({
12221
- type: "step-completed",
12222
- caseId: item.id,
12223
- repetition,
12224
- step: evaluated
12225
- });
12226
- activeStepReported = true;
12227
- if (evaluated.status === "failed") {
12228
- attemptResult.status = "failed";
12229
- break;
12230
- }
12231
- history.push(
12232
- {
12233
- role: "user",
12234
- content: command.type === "message" ? command.message : JSON.stringify(command.response)
12235
- },
12236
- { role: "assistant", content: observation.text }
12237
- );
12238
- }
12239
- if (options.verify && attemptResult.status !== "cancelled" && attemptResult.status !== "error") {
12240
- stepIndex = void 0;
12241
- deadline(defaults.verificationTimeoutMs);
12242
- await reportPhase("verification");
12243
- const verification = EvalVerdictSchema.parse(
12244
- await options.verify({
12245
- ...context,
12246
- result: clone(attemptResult)
12247
- })
12248
- );
12249
- signal.throwIfAborted();
12250
- attemptResult.verification = verification;
12251
- if (!verification.passed) attemptResult.status = "failed";
12252
- }
12253
- } catch (error) {
12254
- if (signal.aborted) {
12255
- attemptResult.status = timedOut ? "error" : "cancelled";
12256
- if (timedOut)
12257
- attemptResult.errors.push({
12258
- ...phaseIssue(error),
12259
- code: phase === "params" || phase === "setup" ? "EVAL_SETUP_TIMEOUT" : phase === "verification" ? "EVAL_VERIFICATION_TIMEOUT" : "EVAL_TIMEOUT",
12260
- message: phase === "params" || phase === "setup" ? "Case setup exceeded its configured time limit." : phase === "verification" ? "Case verification exceeded its configured time limit." : "Case execution exceeded its configured time limit."
12261
- });
12262
- } else {
12263
- attemptResult.status = "error";
12264
- attemptResult.errors.push(phaseIssue(error));
12265
- }
12266
- if (activeStep && !activeStepReported) {
12267
- activeStep.status = attemptResult.status;
12268
- activeStep.reason = attemptResult.errors.at(-1)?.message ?? "Case was cancelled.";
12269
- await emit({
12270
- type: "step-completed",
12271
- caseId: item.id,
12272
- repetition,
12273
- step: activeStep
12274
- });
12275
- }
12276
- } finally {
12277
- clearTimeout(timer);
12278
- if (context) {
12279
- await reportPhase("cleanup");
12280
- const cleanup = new AbortController();
12281
- const cleanupTimer = setTimeout(
12282
- () => cleanup.abort(),
12283
- defaults.cleanupTimeoutMs
12284
- );
12285
- try {
12286
- if (continuation !== void 0) {
12287
- try {
12288
- const cancelled = await call(
12289
- context,
12290
- { type: "cancel" },
12291
- cleanup.signal
12292
- );
12293
- if (cancelled.observation.type === "error" || cancelled.continuation !== void 0)
10415
+ )
10416
+ );
10417
+ cleanup.signal.throwIfAborted();
10418
+ } catch (error) {
12294
10419
  attemptResult.errors.push(
12295
- createEvalIssue(
12296
- "cleanup",
12297
- cancelled.observation.failure,
12298
- {
12299
- code: "EVAL_INTERRUPT_CLEANUP_FAILED",
12300
- ...cancelled.observation.diagnosticId ? {
12301
- diagnosticId: cancelled.observation.diagnosticId
12302
- } : {}
12303
- }
12304
- )
10420
+ phaseIssue(error, "EVAL_INTERRUPT_CLEANUP_FAILED")
12305
10421
  );
10422
+ }
10423
+ }
10424
+ try {
10425
+ await options.teardown?.({
10426
+ ...context,
10427
+ signal: cleanup.signal,
10428
+ result: clone(attemptResult)
10429
+ });
12306
10430
  cleanup.signal.throwIfAborted();
12307
10431
  } catch (error) {
12308
- attemptResult.errors.push(
12309
- phaseIssue(error, "EVAL_INTERRUPT_CLEANUP_FAILED")
12310
- );
10432
+ attemptResult.errors.push(phaseIssue(error));
12311
10433
  }
10434
+ } finally {
10435
+ clearTimeout(cleanupTimer);
12312
10436
  }
12313
- try {
12314
- await options.teardown?.({
12315
- ...context,
12316
- signal: cleanup.signal,
12317
- result: clone(attemptResult)
12318
- });
12319
- cleanup.signal.throwIfAborted();
12320
- } catch (error) {
12321
- attemptResult.errors.push(phaseIssue(error));
12322
- }
12323
- } finally {
12324
- clearTimeout(cleanupTimer);
10437
+ if (attemptResult.errors.some(
10438
+ (error) => error.phase === "cleanup"
10439
+ ))
10440
+ attemptResult.status = "error";
12325
10441
  }
12326
- if (attemptResult.errors.some((error) => error.phase === "cleanup"))
12327
- attemptResult.status = "error";
10442
+ recordPhase();
10443
+ attemptResult.durationMs = Date.now() - started;
12328
10444
  }
12329
- recordPhase();
12330
- attemptResult.durationMs = Date.now() - started;
12331
- }
12332
- acceptingExecutions = false;
12333
- await associationProgress;
12334
- await emit({ type: "case-completed", result: attemptResult });
12335
- return attemptResult;
10445
+ acceptingExecutions = false;
10446
+ await associationProgress;
10447
+ await emit({ type: "case-completed", result: attemptResult });
10448
+ return attemptResult;
10449
+ };
10450
+ return (0, import_eval_attribution3.withEvalAttribution)(
10451
+ {
10452
+ attemptId,
10453
+ onError: (error) => {
10454
+ originalExecutionError = error;
10455
+ },
10456
+ onExecution: (execution) => {
10457
+ if (!acceptingExecutions || runtimeExecutions.some(
10458
+ (entry) => entry.runId === execution.runId && entry.sessionId === execution.sessionId
10459
+ ))
10460
+ return;
10461
+ runtimeExecutions.push(execution);
10462
+ associationProgress = associationProgress.then(
10463
+ () => emit({
10464
+ type: "case-runtime-associated",
10465
+ caseId: item.id,
10466
+ repetition,
10467
+ attemptId,
10468
+ execution
10469
+ })
10470
+ );
10471
+ }
10472
+ },
10473
+ executeAttempt
10474
+ );
12336
10475
  };
12337
- return (0, import_eval_attribution2.withEvalAttribution)(
12338
- {
12339
- attemptId,
12340
- onExecution: (execution) => {
12341
- if (!acceptingExecutions || runtimeExecutions.some(
12342
- (entry) => entry.runId === execution.runId && entry.sessionId === execution.sessionId
12343
- ))
12344
- return;
12345
- runtimeExecutions.push(execution);
12346
- associationProgress = associationProgress.then(
12347
- () => emit({
12348
- type: "case-runtime-associated",
12349
- caseId: item.id,
12350
- repetition,
12351
- attemptId,
12352
- execution
12353
- })
10476
+ const total = args.attempt ? 1 : selected.length * repetitions;
10477
+ if (args.includeActivity)
10478
+ await emit({
10479
+ type: "run-started",
10480
+ caseIds: selected.map((item) => item.id),
10481
+ repetitions,
10482
+ concurrency
10483
+ });
10484
+ const completed = new Array(total);
10485
+ let cursor = 0;
10486
+ await Promise.all(
10487
+ Array.from({ length: Math.min(concurrency, total) }, async () => {
10488
+ while (cursor < total) {
10489
+ const index = cursor++;
10490
+ const item = args.attempt ? selected.find((item2) => item2.id === args.attempt?.caseId) : selected[Math.floor(index / repetitions)];
10491
+ completed[index] = await attempt(
10492
+ item,
10493
+ args.attempt?.repetition ?? index % repetitions + 1
12354
10494
  );
12355
10495
  }
12356
- },
12357
- executeAttempt
10496
+ })
12358
10497
  );
12359
- };
12360
- const total = args.attempt ? 1 : selected.length * repetitions;
12361
- if (args.includeActivity)
12362
- await emit({
12363
- type: "run-started",
12364
- caseIds: selected.map((item) => item.id),
12365
- repetitions,
12366
- concurrency
12367
- });
12368
- const completed = new Array(total);
12369
- let cursor = 0;
12370
- await Promise.all(
12371
- Array.from({ length: Math.min(concurrency, total) }, async () => {
12372
- while (cursor < total) {
12373
- const index = cursor++;
12374
- const item = args.attempt ? selected.find((item2) => item2.id === args.attempt?.caseId) : selected[Math.floor(index / repetitions)];
12375
- completed[index] = await attempt(
12376
- item,
12377
- args.attempt?.repetition ?? index % repetitions + 1
12378
- );
10498
+ result.cases = completed;
10499
+ for (const item of completed)
10500
+ result.counts[item.status] = (result.counts[item.status] ?? 0) + 1;
10501
+ result.status = statusFor(result.counts, result.errors);
10502
+ result.durationMs = Date.now() - start;
10503
+ const failedIssue = result.errors[0] ?? result.cases.find((item) => item.errors.length)?.errors[0];
10504
+ evalUsage.end(
10505
+ result.status === "passed" ? "success" : result.status === "failed" ? "quality_failed" : result.status === "error" ? "failed" : result.status,
10506
+ {
10507
+ scale: (0, import_usage_telemetry.scale)(selected.length * repetitions),
10508
+ phase: usagePhase,
10509
+ ...result.status === "error" ? {
10510
+ ...(0, import_usage_telemetry.classifyUsageError)(failedIssue).properties,
10511
+ eval_phase: failedIssue?.phase
10512
+ } : {}
12379
10513
  }
12380
- })
12381
- );
12382
- result.cases = completed;
12383
- for (const item of completed)
12384
- result.counts[item.status] = (result.counts[item.status] ?? 0) + 1;
12385
- result.status = statusFor(result.counts, result.errors);
12386
- result.durationMs = Date.now() - start;
12387
- evalUsage.end(
12388
- result.status === "passed" ? "success" : result.status === "failed" ? "quality_failed" : result.status === "error" ? "failed" : result.status
12389
- );
12390
- return result;
10514
+ );
10515
+ return result;
10516
+ } catch (error) {
10517
+ evalUsage.fail(error, { phase: usagePhase });
10518
+ throw error;
10519
+ }
12391
10520
  }
12392
10521
  };
12393
10522
  }
@@ -12398,23 +10527,24 @@ function defineSuite(suite) {
12398
10527
  }
12399
10528
 
12400
10529
  // src/evals/evaluation-contracts.ts
12401
- var import_zod12 = require("zod");
10530
+ var import_zod13 = require("zod");
12402
10531
 
12403
10532
  // src/evals/studio-contracts.ts
12404
10533
  var import_browser3 = require("@kortyx/prompts/browser");
12405
- var import_zod11 = require("zod");
12406
- var StudioEvalTargetsResponseSchema = import_zod11.z.object({
12407
- canRun: import_zod11.z.boolean(),
10534
+ var import_zod12 = require("zod");
10535
+ var StudioEvalTargetsResponseSchema = import_zod12.z.object({
10536
+ canRun: import_zod12.z.boolean(),
10537
+ settings: EvalSettingsResponseSchema.optional(),
12408
10538
  studioJudge: EvalJudgeIdentitySchema.nullable().optional(),
12409
- targets: import_zod11.z.array(
12410
- import_zod11.z.object({
12411
- id: import_zod11.z.string(),
12412
- name: import_zod11.z.string(),
12413
- environment: import_zod11.z.string(),
12414
- error: import_zod11.z.string().nullable(),
10539
+ targets: import_zod12.z.array(
10540
+ import_zod12.z.object({
10541
+ id: import_zod12.z.string(),
10542
+ name: import_zod12.z.string(),
10543
+ environment: import_zod12.z.string(),
10544
+ error: import_zod12.z.string().nullable(),
12415
10545
  // Safe discovery diagnostics; optional for compatibility with older APIs.
12416
- diagnostic: import_zod11.z.object({
12417
- code: import_zod11.z.enum([
10546
+ diagnostic: import_zod12.z.object({
10547
+ code: import_zod12.z.enum([
12418
10548
  "environment_forbidden",
12419
10549
  "environment_unavailable",
12420
10550
  "endpoint_not_found",
@@ -12423,44 +10553,47 @@ var StudioEvalTargetsResponseSchema = import_zod11.z.object({
12423
10553
  "endpoint_unreachable",
12424
10554
  "manifest_invalid"
12425
10555
  ]),
12426
- httpStatus: import_zod11.z.number().int().min(100).max(599).optional()
10556
+ httpStatus: import_zod12.z.number().int().min(100).max(599).optional()
12427
10557
  }).nullable().optional(),
12428
10558
  manifest: EvalManifestSchema.nullable(),
12429
- revisions: import_zod11.z.record(import_zod11.z.string(), import_zod11.z.string())
10559
+ revisions: import_zod12.z.record(import_zod12.z.string(), import_zod12.z.string())
12430
10560
  })
12431
10561
  )
12432
10562
  });
12433
- var EvalCostAmountSchema = import_zod11.z.object({
12434
- costSummary: import_zod11.z.object({
12435
- amounts: import_zod11.z.array(
12436
- import_zod11.z.object({
12437
- amount: import_zod11.z.number().finite().nonnegative(),
12438
- currency: import_zod11.z.string().nullable()
10563
+ var EvalCostAmountSchema = import_zod12.z.object({
10564
+ costSummary: import_zod12.z.object({
10565
+ amounts: import_zod12.z.array(
10566
+ import_zod12.z.object({
10567
+ amount: import_zod12.z.number().finite().nonnegative(),
10568
+ currency: import_zod12.z.string().nullable()
12439
10569
  })
12440
10570
  ),
12441
- complete: import_zod11.z.boolean()
10571
+ complete: import_zod12.z.boolean()
12442
10572
  }).optional(),
12443
- amount: import_zod11.z.number().finite().nonnegative().nullable(),
12444
- currency: import_zod11.z.string().nullable(),
12445
- status: import_zod11.z.enum(["complete", "partial", "unavailable"]),
12446
- calls: import_zod11.z.number().int().nonnegative(),
12447
- unpricedCalls: import_zod11.z.number().int().nonnegative(),
12448
- estimated: import_zod11.z.boolean()
10573
+ amount: import_zod12.z.number().finite().nonnegative().nullable(),
10574
+ currency: import_zod12.z.string().nullable(),
10575
+ status: import_zod12.z.enum(["complete", "partial", "unavailable"]),
10576
+ calls: import_zod12.z.number().int().nonnegative(),
10577
+ unpricedCalls: import_zod12.z.number().int().nonnegative(),
10578
+ estimated: import_zod12.z.boolean()
12449
10579
  });
12450
- var EvalCostsSchema = import_zod11.z.object({
10580
+ var EvalCostsSchema = import_zod12.z.object({
12451
10581
  workflow: EvalCostAmountSchema,
12452
10582
  judge: EvalCostAmountSchema,
12453
10583
  total: EvalCostAmountSchema
12454
10584
  });
12455
- var StudioEvalRunSummarySchema = import_zod11.z.object({
12456
- id: import_zod11.z.uuid(),
12457
- evaluationId: import_zod11.z.uuid().nullable().optional(),
12458
- targetId: import_zod11.z.string(),
12459
- targetName: import_zod11.z.string(),
12460
- environment: import_zod11.z.string(),
12461
- suiteId: import_zod11.z.string(),
12462
- suiteRevision: import_zod11.z.string(),
12463
- status: import_zod11.z.enum([
10585
+ var StudioEvalRunSummarySchema = import_zod12.z.object({
10586
+ issue: EvalIssueSchema.nullable().optional(),
10587
+ termination: EvalTerminationSchema.nullable().optional(),
10588
+ resolvedSettings: ResolvedEvalSettingsSchema.optional(),
10589
+ id: import_zod12.z.uuid(),
10590
+ evaluationId: import_zod12.z.uuid().nullable().optional(),
10591
+ targetId: import_zod12.z.string(),
10592
+ targetName: import_zod12.z.string(),
10593
+ environment: import_zod12.z.string(),
10594
+ suiteId: import_zod12.z.string(),
10595
+ suiteRevision: import_zod12.z.string(),
10596
+ status: import_zod12.z.enum([
12464
10597
  "queued",
12465
10598
  "running",
12466
10599
  "passed",
@@ -12468,49 +10601,52 @@ var StudioEvalRunSummarySchema = import_zod11.z.object({
12468
10601
  "error",
12469
10602
  "cancelled"
12470
10603
  ]),
12471
- createdAt: import_zod11.z.string(),
12472
- startedAt: import_zod11.z.string().nullable(),
12473
- endedAt: import_zod11.z.string().nullable(),
12474
- error: import_zod11.z.string().nullable(),
12475
- cancelRequestedAt: import_zod11.z.string().nullable(),
10604
+ createdAt: import_zod12.z.string(),
10605
+ startedAt: import_zod12.z.string().nullable(),
10606
+ endedAt: import_zod12.z.string().nullable(),
10607
+ error: import_zod12.z.string().nullable(),
10608
+ cancelRequestedAt: import_zod12.z.string().nullable(),
12476
10609
  costs: EvalCostsSchema.optional(),
12477
- counts: import_zod11.z.object({
12478
- passed: import_zod11.z.number(),
12479
- failed: import_zod11.z.number(),
12480
- error: import_zod11.z.number(),
12481
- cancelled: import_zod11.z.number()
10610
+ counts: import_zod12.z.object({
10611
+ passed: import_zod12.z.number(),
10612
+ failed: import_zod12.z.number(),
10613
+ error: import_zod12.z.number(),
10614
+ cancelled: import_zod12.z.number()
12482
10615
  }).nullable().optional(),
12483
- suiteName: import_zod11.z.string().nullable().optional(),
12484
- totalAttempts: import_zod11.z.number().int().nonnegative().optional(),
12485
- completedAttempts: import_zod11.z.number().int().nonnegative().optional(),
12486
- phase: import_zod11.z.string().optional()
10616
+ suiteName: import_zod12.z.string().nullable().optional(),
10617
+ totalAttempts: import_zod12.z.number().int().nonnegative().optional(),
10618
+ completedAttempts: import_zod12.z.number().int().nonnegative().optional(),
10619
+ phase: import_zod12.z.string().optional()
12487
10620
  });
12488
- var StudioEvalHistorySchema = import_zod11.z.object({
12489
- runs: import_zod11.z.array(StudioEvalRunSummarySchema)
10621
+ var StudioEvalHistorySchema = import_zod12.z.object({
10622
+ runs: import_zod12.z.array(StudioEvalRunSummarySchema)
12490
10623
  });
12491
- var StudioEvalDetailSchema = import_zod11.z.object({
10624
+ var StudioEvalDetailSchema = import_zod12.z.object({
12492
10625
  run: StudioEvalRunSummarySchema.extend({
12493
- caseCosts: import_zod11.z.record(import_zod11.z.string(), EvalCostsSchema).optional(),
12494
- request: import_zod11.z.object({
12495
- caseIds: import_zod11.z.array(import_zod11.z.string()).optional(),
12496
- grading: import_zod11.z.enum(["app", "studio"]).optional(),
10626
+ caseCosts: import_zod12.z.record(import_zod12.z.string(), EvalCostsSchema).optional(),
10627
+ request: import_zod12.z.object({
10628
+ caseIds: import_zod12.z.array(import_zod12.z.string()).optional(),
10629
+ grading: import_zod12.z.enum(["app", "studio"]).optional(),
12497
10630
  judge: EvalJudgeIdentitySchema.optional(),
12498
- repetitions: import_zod11.z.number().int().positive(),
12499
- concurrency: import_zod11.z.number().optional(),
10631
+ repetitions: import_zod12.z.number().int().positive(),
10632
+ concurrency: import_zod12.z.number().optional(),
10633
+ caseTimeoutMs: import_zod12.z.number().optional(),
10634
+ runTimeoutMs: import_zod12.z.number().optional(),
10635
+ resolvedSettings: ResolvedEvalSettingsSchema.optional(),
12500
10636
  promptSnapshot: import_browser3.PromptSnapshotSchema.optional(),
12501
- promptGroupName: import_zod11.z.string().optional()
10637
+ promptGroupName: import_zod12.z.string().optional()
12502
10638
  }).optional(),
12503
10639
  suite: EvalSuiteSchema,
12504
10640
  result: EvalRunResultSchema.nullable(),
12505
- events: import_zod11.z.array(import_zod11.z.object({ id: import_zod11.z.number(), event: EvalProgressSchema }))
10641
+ events: import_zod12.z.array(import_zod12.z.object({ id: import_zod12.z.number(), event: EvalProgressSchema }))
12506
10642
  })
12507
10643
  });
12508
10644
 
12509
10645
  // src/evals/evaluation-contracts.ts
12510
- var EvaluationMetadataSchema = import_zod12.z.object({
12511
- source: import_zod12.z.enum(["manual", "deployment", "schedule", "ci"]).default("manual"),
12512
- commit: import_zod12.z.string().trim().min(1).max(100).optional(),
12513
- deploymentUrl: import_zod12.z.url().refine((value) => {
10646
+ var EvaluationMetadataSchema = import_zod13.z.object({
10647
+ source: import_zod13.z.enum(["manual", "deployment", "schedule", "ci"]).default("manual"),
10648
+ commit: import_zod13.z.string().trim().min(1).max(100).optional(),
10649
+ deploymentUrl: import_zod13.z.url().refine((value) => {
12514
10650
  try {
12515
10651
  const url = new URL(value);
12516
10652
  return ["https:", "http:"].includes(url.protocol) && !url.username && !url.password;
@@ -12519,67 +10655,84 @@ var EvaluationMetadataSchema = import_zod12.z.object({
12519
10655
  }
12520
10656
  }, "Expected an HTTP(S) URL without credentials").optional()
12521
10657
  }).strict();
12522
- var StudioEvaluationStartRequestSchema = import_zod12.z.object({
12523
- targetId: import_zod12.z.string().trim().min(1),
12524
- name: import_zod12.z.string().trim().min(1).max(200).optional(),
10658
+ var StudioEvaluationStartRequestSchema = import_zod13.z.object({
10659
+ targetId: import_zod13.z.string().trim().min(1),
10660
+ name: import_zod13.z.string().trim().min(1).max(200).optional(),
12525
10661
  promptSelection: StudioEvalStartRequestSchema.shape.promptSelection,
12526
- selection: import_zod12.z.enum(["all", "selected"]),
12527
- suites: import_zod12.z.array(
12528
- import_zod12.z.object({
12529
- suiteId: import_zod12.z.string().min(1),
12530
- suiteRevision: import_zod12.z.string().regex(/^[a-f0-9]{64}$/),
12531
- caseIds: import_zod12.z.array(import_zod12.z.string().min(1)).min(1).max(100).optional()
10662
+ selection: import_zod13.z.enum(["all", "selected"]),
10663
+ suites: import_zod13.z.array(
10664
+ import_zod13.z.object({
10665
+ suiteId: import_zod13.z.string().min(1),
10666
+ suiteRevision: import_zod13.z.string().regex(/^[a-f0-9]{64}$/),
10667
+ caseIds: import_zod13.z.array(import_zod13.z.string().min(1)).min(1).max(100).optional()
12532
10668
  }).strict()
12533
10669
  ).min(1).max(100),
12534
- judge: import_zod12.z.enum(["studio", "app"]).default("studio"),
12535
- repetitions: import_zod12.z.number().int().min(1).max(20).default(1),
12536
- concurrency: import_zod12.z.number().int().min(1).max(20).default(1),
10670
+ ...EvalSettingsSchema.shape,
12537
10671
  metadata: EvaluationMetadataSchema.default({ source: "manual" }),
12538
- idempotencyKey: import_zod12.z.string().trim().min(1).max(200).optional()
10672
+ idempotencyKey: import_zod13.z.string().trim().min(1).max(200).optional()
12539
10673
  }).strict();
12540
- var StudioEvaluationSummarySchema = import_zod12.z.object({
12541
- id: import_zod12.z.uuid(),
12542
- name: import_zod12.z.string(),
12543
- targetId: import_zod12.z.string(),
12544
- targetName: import_zod12.z.string(),
12545
- environment: import_zod12.z.string(),
12546
- selection: import_zod12.z.enum(["all", "selected"]),
10674
+ var StudioEvaluationSummarySchema = import_zod13.z.object({
10675
+ termination: EvalTerminationSchema.nullable().optional(),
10676
+ resolvedSettings: ResolvedEvalSettingsSchema.optional(),
10677
+ interruptedSuites: import_zod13.z.number().int().nonnegative().optional(),
10678
+ failures: import_zod13.z.array(
10679
+ import_zod13.z.object({
10680
+ suiteRunId: import_zod13.z.uuid(),
10681
+ caseId: import_zod13.z.string().optional(),
10682
+ repetition: import_zod13.z.number().int().positive().optional(),
10683
+ phase: import_zod13.z.string(),
10684
+ code: import_zod13.z.string(),
10685
+ message: import_zod13.z.string(),
10686
+ diagnosticId: import_zod13.z.string().uuid().optional()
10687
+ })
10688
+ ).optional(),
10689
+ pendingAttempts: import_zod13.z.object({
10690
+ notStarted: import_zod13.z.number().int().nonnegative(),
10691
+ executing: import_zod13.z.number().int().nonnegative(),
10692
+ awaitingGrading: import_zod13.z.number().int().nonnegative()
10693
+ }).optional(),
10694
+ id: import_zod13.z.uuid(),
10695
+ name: import_zod13.z.string(),
10696
+ targetId: import_zod13.z.string(),
10697
+ targetName: import_zod13.z.string(),
10698
+ environment: import_zod13.z.string(),
10699
+ selection: import_zod13.z.enum(["all", "selected"]),
12547
10700
  metadata: EvaluationMetadataSchema,
12548
10701
  judge: EvalJudgeIdentitySchema.optional(),
12549
10702
  status: StudioEvalRunSummarySchema.shape.status,
12550
- createdAt: import_zod12.z.string(),
12551
- startedAt: import_zod12.z.string().nullable(),
12552
- endedAt: import_zod12.z.string().nullable(),
12553
- cancelRequestedAt: import_zod12.z.string().nullable(),
12554
- suiteCount: import_zod12.z.number().int().nonnegative(),
12555
- completedSuites: import_zod12.z.number().int().nonnegative(),
12556
- totalAttempts: import_zod12.z.number().int().nonnegative(),
12557
- completedAttempts: import_zod12.z.number().int().nonnegative(),
12558
- counts: import_zod12.z.object({
12559
- passed: import_zod12.z.number(),
12560
- failed: import_zod12.z.number(),
12561
- error: import_zod12.z.number(),
12562
- cancelled: import_zod12.z.number()
10703
+ createdAt: import_zod13.z.string(),
10704
+ startedAt: import_zod13.z.string().nullable(),
10705
+ endedAt: import_zod13.z.string().nullable(),
10706
+ cancelRequestedAt: import_zod13.z.string().nullable(),
10707
+ suiteCount: import_zod13.z.number().int().nonnegative(),
10708
+ completedSuites: import_zod13.z.number().int().nonnegative(),
10709
+ totalAttempts: import_zod13.z.number().int().nonnegative(),
10710
+ completedAttempts: import_zod13.z.number().int().nonnegative(),
10711
+ counts: import_zod13.z.object({
10712
+ passed: import_zod13.z.number(),
10713
+ failed: import_zod13.z.number(),
10714
+ error: import_zod13.z.number(),
10715
+ cancelled: import_zod13.z.number()
12563
10716
  }),
12564
10717
  costs: EvalCostsSchema.optional(),
12565
- legacy: import_zod12.z.boolean().optional()
10718
+ legacy: import_zod13.z.boolean().optional()
12566
10719
  });
12567
- var StudioEvaluationHistorySchema = import_zod12.z.object({
12568
- runs: import_zod12.z.array(StudioEvaluationSummarySchema)
10720
+ var StudioEvaluationHistorySchema = import_zod13.z.object({
10721
+ runs: import_zod13.z.array(StudioEvaluationSummarySchema)
12569
10722
  });
12570
- var StudioEvaluationDetailSchema = import_zod12.z.object({
10723
+ var StudioEvaluationDetailSchema = import_zod13.z.object({
12571
10724
  run: StudioEvaluationSummarySchema.extend({
12572
- suites: import_zod12.z.array(StudioEvalRunSummarySchema)
10725
+ suites: import_zod13.z.array(StudioEvalRunSummarySchema)
12573
10726
  })
12574
10727
  });
12575
- var StudioEvaluationResultsSchema = import_zod12.z.object({
10728
+ var StudioEvaluationResultsSchema = import_zod13.z.object({
12576
10729
  run: StudioEvaluationSummarySchema.extend({
12577
- suites: import_zod12.z.array(StudioEvalDetailSchema.shape.run)
10730
+ suites: import_zod13.z.array(StudioEvalDetailSchema.shape.run)
12578
10731
  })
12579
10732
  });
12580
10733
 
12581
10734
  // src/evals/judge.ts
12582
- var import_zod13 = require("zod");
10735
+ var import_zod14 = require("zod");
12583
10736
  function createEvalJudge({
12584
10737
  model,
12585
10738
  id: id3 = `${model.provider.id}/${model.modelId}`,
@@ -12606,7 +10759,7 @@ function createEvalJudge({
12606
10759
  responseFormat: {
12607
10760
  type: "json",
12608
10761
  name: "eval_verdict",
12609
- schema: import_zod13.z.toJSONSchema(EvalVerdictSchema)
10762
+ schema: import_zod14.z.toJSONSchema(EvalVerdictSchema)
12610
10763
  }
12611
10764
  });
12612
10765
  const result = await judge2.invoke([
@@ -12730,6 +10883,7 @@ function createEvalExecutionTransport({
12730
10883
  await store.append(job.id, owner, {
12731
10884
  type: "error",
12732
10885
  message: issue2.message,
10886
+ exception: issue2.exception,
12733
10887
  failure: issue2.failure,
12734
10888
  ...issue2.diagnosticId ? { diagnosticId: issue2.diagnosticId } : {}
12735
10889
  });
@@ -12771,6 +10925,7 @@ function createEvalExecutionTransport({
12771
10925
  suiteId: body.suiteId,
12772
10926
  repetitions: body.repetitions,
12773
10927
  concurrency: body.concurrency,
10928
+ ...body.caseTimeoutMs !== void 0 ? { caseTimeoutMs: body.caseTimeoutMs } : {},
12774
10929
  signal: controller.signal,
12775
10930
  ...body.attempt ? { attempt: body.attempt } : {},
12776
10931
  ...body.caseIds ? { caseIds: body.caseIds } : {},
@@ -12846,10 +11001,16 @@ function createEvalExecutionTransport({
12846
11001
  "x-kortyx-eval-generation": await store.generation()
12847
11002
  });
12848
11003
  const problem = (error, negotiated) => {
11004
+ const issue2 = createEvalIssue("reporting", error);
12849
11005
  const failure = (0, import_errors13.serializeFailure)(error);
12850
11006
  const busy = failure.code === "EVAL_EXECUTOR_BUSY";
12851
11007
  return Response.json(
12852
- { error: failure.message, failure, ...busy ? { accepted: false } : {} },
11008
+ {
11009
+ error: issue2.message,
11010
+ issue: issue2,
11011
+ failure,
11012
+ ...busy ? { accepted: false } : {}
11013
+ },
12853
11014
  {
12854
11015
  status: busy ? negotiated ? 503 : 429 : !negotiated && failure.code === "EVAL_CONCURRENCY_CHANGED" ? 429 : failure.status ?? 503,
12855
11016
  headers: {
@@ -13186,6 +11347,15 @@ function createEvalRouteHandler({
13186
11347
  { error: "Unsupported or invalid scheduled attempt." },
13187
11348
  { status: 409 }
13188
11349
  );
11350
+ for (const key of ["concurrency", "caseTimeoutMs"]) {
11351
+ const maximum = manifest.execution?.limits[key];
11352
+ const value = body[key] ?? manifest.execution?.defaults[key];
11353
+ if (maximum !== void 0 && value !== void 0 && value > maximum)
11354
+ return Response.json(
11355
+ { error: `${key} exceeds the application maximum of ${maximum}.` },
11356
+ { status: 400 }
11357
+ );
11358
+ }
13189
11359
  if (request.signal.aborted)
13190
11360
  return Response.json({ error: "Request cancelled." }, { status: 409 });
13191
11361
  return execution.submit(request, body);
@@ -13194,11 +11364,12 @@ function createEvalRouteHandler({
13194
11364
  }
13195
11365
 
13196
11366
  // src/evals/studio-judge.ts
11367
+ var import_telemetry_contracts3 = require("@kortyx/telemetry-contracts");
13197
11368
  async function readJson(response, signal) {
13198
- if (!response.ok || !response.body) {
13199
- await response.body?.cancel().catch(() => {
13200
- });
13201
- throw new Error("Studio judge is unavailable or rejected the request.");
11369
+ if (!response.body) {
11370
+ throw new Error(
11371
+ `Studio judge returned HTTP ${response.status} with no response body.`
11372
+ );
13202
11373
  }
13203
11374
  const reader = response.body.getReader();
13204
11375
  const abort = () => {
@@ -13220,7 +11391,18 @@ async function readJson(response, signal) {
13220
11391
  text += decoder.decode(part.value, { stream: true });
13221
11392
  }
13222
11393
  signal.throwIfAborted();
13223
- return JSON.parse(text + decoder.decode());
11394
+ text += decoder.decode();
11395
+ if (!response.ok)
11396
+ throw Object.assign(
11397
+ new Error(
11398
+ `Studio judge returned HTTP ${response.status}: ${(0, import_telemetry_contracts3.redactDiagnosticText)(text)}`
11399
+ ),
11400
+ {
11401
+ statusCode: response.status,
11402
+ responseBody: (0, import_telemetry_contracts3.redactDiagnosticText)(text)
11403
+ }
11404
+ );
11405
+ return JSON.parse(text);
13224
11406
  } finally {
13225
11407
  signal.removeEventListener("abort", abort);
13226
11408
  await reader.cancel().catch(() => {
@@ -13304,6 +11486,7 @@ async function createStudioEvalJudge(options) {
13304
11486
  ChatLifecycleHookError,
13305
11487
  EvalConfigurationError,
13306
11488
  EvalEvidencePolicySchema,
11489
+ EvalExecutionSettingsSchema,
13307
11490
  EvalIssueSchema,
13308
11491
  EvalJudgeIdentitySchema,
13309
11492
  EvalJudgeUsageSchema,
@@ -13311,10 +11494,14 @@ async function createStudioEvalJudge(options) {
13311
11494
  EvalProgressSchema,
13312
11495
  EvalRemoteRunRequestSchema,
13313
11496
  EvalRunResultSchema,
11497
+ EvalSettingsResponseSchema,
11498
+ EvalSettingsSchema,
13314
11499
  EvalSuiteSchema,
11500
+ EvalTerminationSchema,
13315
11501
  EvalWireEventSchema,
13316
11502
  EvaluationMetadataSchema,
13317
11503
  ExecutionRequestError,
11504
+ ResolvedEvalSettingsSchema,
13318
11505
  StudioEvalJudgeRequestSchema,
13319
11506
  StudioEvalJudgeResponseSchema,
13320
11507
  StudioEvalStartRequestSchema,
@@ -13345,6 +11532,7 @@ async function createStudioEvalJudge(options) {
13345
11532
  parseResumeMeta,
13346
11533
  projectWorkflowTopology,
13347
11534
  readRequestJson,
11535
+ resolveEvalSettings,
13348
11536
  streamChat,
13349
11537
  streamChatFromRoute,
13350
11538
  transformGraphStreamForUI,