@kortyx/agent 0.37.1 → 0.38.0

package/dist/index.js

@@ -161,67 +161,67 @@ var require_dist = __commonJS({
     });
     module2.exports = __toCommonJS2(index_exports2);
     var import_zod62 = require("zod");
-    var import_zod12 = require("zod");
+    var import_zod14 = require("zod");
     var FEEDBACK_FILTERS = ["positive", "negative", "unrated"];
-    var FeedbackSummarySchema = import_zod12.z.object({
-      positive: import_zod12.z.number().int().nonnegative(),
-      negative: import_zod12.z.number().int().nonnegative()
+    var FeedbackSummarySchema = import_zod14.z.object({
+      positive: import_zod14.z.number().int().nonnegative(),
+      negative: import_zod14.z.number().int().nonnegative()
     }).strict();
-    var FeedbackReasonSchema = import_zod12.z.enum([
+    var FeedbackReasonSchema = import_zod14.z.enum([
       "incorrect",
       "irrelevant",
       "incomplete",
       "unsafe",
       "other"
     ]);
-    var ReviewVerdictSchema = import_zod12.z.enum([
+    var ReviewVerdictSchema = import_zod14.z.enum([
       "correct",
       "partially-correct",
       "incorrect"
     ]);
-    var runId = import_zod12.z.string().trim().min(1).max(512);
-    var actorId = import_zod12.z.string().trim().min(1).max(256);
-    var comment = import_zod12.z.string().trim().max(4e3).nullable().optional();
-    var UserFeedbackRequestSchema = import_zod12.z.object({
+    var runId = import_zod14.z.string().trim().min(1).max(512);
+    var actorId = import_zod14.z.string().trim().min(1).max(256);
+    var comment = import_zod14.z.string().trim().max(4e3).nullable().optional();
+    var UserFeedbackRequestSchema = import_zod14.z.object({
       runId,
       actorId,
-      value: import_zod12.z.union([import_zod12.z.literal(0), import_zod12.z.literal(1)]),
-      reasons: import_zod12.z.array(FeedbackReasonSchema).max(5).optional(),
+      value: import_zod14.z.union([import_zod14.z.literal(0), import_zod14.z.literal(1)]),
+      reasons: import_zod14.z.array(FeedbackReasonSchema).max(5).optional(),
       comment
     }).strict();
     var ClearUserFeedbackRequestSchema = UserFeedbackRequestSchema.pick({
       runId: true,
       actorId: true
     });
-    var StudioReviewRequestSchema = import_zod12.z.object({ value: ReviewVerdictSchema, comment }).strict();
-    var BaseScoreSchema = import_zod12.z.object({
-      id: import_zod12.z.string().uuid(),
-      target: import_zod12.z.object({ type: import_zod12.z.literal("run"), runId }).strict(),
-      environment: import_zod12.z.string().min(1),
-      name: import_zod12.z.string().min(1),
-      source: import_zod12.z.enum(["end-user", "human-review", "evaluator"]),
+    var StudioReviewRequestSchema = import_zod14.z.object({ value: ReviewVerdictSchema, comment }).strict();
+    var BaseScoreSchema = import_zod14.z.object({
+      id: import_zod14.z.string().uuid(),
+      target: import_zod14.z.object({ type: import_zod14.z.literal("run"), runId }).strict(),
+      environment: import_zod14.z.string().min(1),
+      name: import_zod14.z.string().min(1),
+      source: import_zod14.z.enum(["end-user", "human-review", "evaluator"]),
       actorId,
-      reasons: import_zod12.z.array(FeedbackReasonSchema),
-      comment: import_zod12.z.string().nullable(),
-      createdAt: import_zod12.z.string().datetime({ offset: true }),
-      updatedAt: import_zod12.z.string().datetime({ offset: true })
+      reasons: import_zod14.z.array(FeedbackReasonSchema),
+      comment: import_zod14.z.string().nullable(),
+      createdAt: import_zod14.z.string().datetime({ offset: true }),
+      updatedAt: import_zod14.z.string().datetime({ offset: true })
     }).strict();
-    var StudioScoreSchema = import_zod12.z.discriminatedUnion("dataType", [
+    var StudioScoreSchema = import_zod14.z.discriminatedUnion("dataType", [
       BaseScoreSchema.extend({
-        dataType: import_zod12.z.literal("BOOLEAN"),
-        value: import_zod12.z.union([import_zod12.z.literal(0), import_zod12.z.literal(1)])
+        dataType: import_zod14.z.literal("BOOLEAN"),
+        value: import_zod14.z.union([import_zod14.z.literal(0), import_zod14.z.literal(1)])
       }),
       BaseScoreSchema.extend({
-        dataType: import_zod12.z.literal("CATEGORICAL"),
-        value: import_zod12.z.string()
+        dataType: import_zod14.z.literal("CATEGORICAL"),
+        value: import_zod14.z.string()
       }),
       BaseScoreSchema.extend({
-        dataType: import_zod12.z.literal("NUMERIC"),
-        value: import_zod12.z.number().finite()
+        dataType: import_zod14.z.literal("NUMERIC"),
+        value: import_zod14.z.number().finite()
       })
     ]);
-    var StudioScoreResponseSchema = import_zod12.z.object({ score: StudioScoreSchema }).strict();
-    var ClearScoreResponseSchema = import_zod12.z.object({ ok: import_zod12.z.literal(true) }).strict();
+    var StudioScoreResponseSchema = import_zod14.z.object({ score: StudioScoreSchema }).strict();
+    var ClearScoreResponseSchema = import_zod14.z.object({ ok: import_zod14.z.literal(true) }).strict();
     function summarizeUserFeedback(scores) {
       const feedback = { positive: 0, negative: 0 };
       for (const score of scores) {
@@ -1056,14 +1056,14 @@ var require_dist = __commonJS({
       payload: import_zod62.z.record(import_zod62.z.string(), import_zod62.z.unknown())
     }).strict().superRefine((event, ctx) => {
       if (!event.type.startsWith("tool.")) return;
-      const canonical2 = event.payload.version === 1 || [
+      const canonical3 = event.payload.version === 1 || [
         "tool.denied",
         "tool.cancelled",
         "tool.reused",
         "tool.waiting",
         "tool.suspended"
       ].includes(event.type);
-      if (!canonical2) return;
+      if (!canonical3) return;
       const parsed = ToolObservationSchema.strict().safeParse(event.payload);
       if (!parsed.success)
         for (const issue2 of parsed.error.issues)
@@ -1660,7 +1660,7 @@ var require_internal = __commonJS({
     });
     module2.exports = __toCommonJS2(internal_exports);
     var import_telemetry_contracts3 = require_dist();
-    var import_errors8 = require("@kortyx/core/errors");
+    var import_errors16 = require("@kortyx/core/errors");
     var import_telemetry_contracts = require_dist();
     var encoder = new TextEncoder();
     var nativeStackGetter = Object.getOwnPropertyDescriptor(
@@ -1841,11 +1841,11 @@ var require_internal = __commonJS({
         const original = project ? project(error) : error;
         if (project && original === null) return null;
         const captured = visit(original, "#/data", 0);
-        const upstreamOmissions = (0, import_errors8.errorProperty)(original, "diagnosticOmissions");
+        const upstreamOmissions = (0, import_errors16.errorProperty)(original, "diagnosticOmissions");
         if (Array.isArray(upstreamOmissions))
           for (const note of upstreamOmissions.slice(0, 1e3)) {
-            const path = (0, import_errors8.errorProperty)(note, "path");
-            const reason = (0, import_errors8.errorProperty)(note, "reason");
+            const path = (0, import_errors16.errorProperty)(note, "path");
+            const reason = (0, import_errors16.errorProperty)(note, "reason");
             if (typeof path === "string" && typeof reason === "string")
               omissions.push({
                 path: (0, import_telemetry_contracts.redactDiagnosticText)(path).slice(0, 4096),
@@ -1912,7 +1912,7 @@ var require_internal = __commonJS({
     function isModelTraceSpan(name) {
       return name === "runReasonEngine" || name === "useReason";
     }
-    var import_node_crypto7 = require("crypto");
+    var import_node_crypto8 = require("crypto");
     var import_core6 = require("@kortyx/core");
     var import_errors22 = require("@kortyx/core/errors");
     function safeSpan(span) {
@@ -1996,7 +1996,7 @@ var require_internal = __commonJS({
     var import_node2 = require("@kortyx/usage-telemetry/node");
     var package_default2 = {
       name: "@kortyx/hooks",
-      version: "0.34.1",
+      version: "0.34.2",
       private: false,
       description: "Lifecycle hooks and extension points for Kortyx.",
       keywords: [
@@ -2090,7 +2090,7 @@ var require_internal = __commonJS({
     var toError = (value) => value instanceof Error ? value : new Error(String(value));
     var createRuntimeId = () => {
       try {
-        return (0, import_node_crypto7.randomUUID)();
+        return (0, import_node_crypto8.randomUUID)();
       } catch {
         return `${Date.now()}-${Math.random().toString(36).slice(2)}`;
       }
@@ -2144,7 +2144,7 @@ var require_internal = __commonJS({
         tools: Boolean(tools?.length),
         provider: args.model.provider.id
       });
-      const messages = args.messages !== void 0 ? args.messages : [
+      const messages2 = args.messages !== void 0 ? args.messages : [
         ...typeof args.system === "string" && args.system.length > 0 ? [{ role: "system", content: args.system }] : [],
         { role: "user", content: String(args.input ?? "") }
       ];
@@ -2171,7 +2171,7 @@ var require_internal = __commonJS({
         },
         telemetry: {
           ...args.telemetry ?? {},
-          input: args.telemetry?.input ?? messages.map(
+          input: args.telemetry?.input ?? messages2.map(
             ({ continuation: _continuation, raw: _raw, ...message }) => message
           )
         }
@@ -2233,7 +2233,7 @@ var require_internal = __commonJS({
               ...commonMeta
             });
           }
-          const response2 = await model.stream(messages);
+          const response2 = await model.stream(messages2);
           for await (const chunk of response2) {
             (0, import_core6.throwIfExecutionAborted)(abortSignal);
             switch (chunk.type) {
@@ -2322,7 +2322,7 @@ var require_internal = __commonJS({
           endTrace(traceSpan, result2);
           return result2;
         }
-        const response = await model.invoke(messages);
+        const response = await model.invoke(messages2);
         (0, import_core6.throwIfExecutionAborted)(abortSignal);
         assertModelFinish(response);
         if (response.content) args.onTextChunk?.(response.content);
@@ -2353,9 +2353,9 @@ var require_internal = __commonJS({
         endTrace(traceSpan, result);
         return result;
       } catch (error) {
-        const failure2 = abortSignal?.aborted ? (0, import_core6.createExecutionCancelledError)() : error;
-        failTrace(traceSpan, failure2);
-        throw failure2;
+        const failure = abortSignal?.aborted ? (0, import_core6.createExecutionCancelledError)() : error;
+        failTrace(traceSpan, failure);
+        throw failure;
       }
     }
     function runReasonEngine(args) {
@@ -2437,7 +2437,7 @@ var require_dist2 = __commonJS({
       workflowCallFingerprint: () => workflowCallFingerprint
     });
     module2.exports = __toCommonJS2(index_exports2);
-    var import_zod12 = require("zod");
+    var import_zod14 = require("zod");
     var import_node_async_hooks2 = require("async_hooks");
     var storage = new import_node_async_hooks2.AsyncLocalStorage();
     var isRecord5 = (value) => Boolean(value) && typeof value === "object" && !Array.isArray(value);
@@ -2632,7 +2632,7 @@ var require_dist2 = __commonJS({
     var import_node2 = require("@kortyx/usage-telemetry/node");
     var package_default2 = {
       name: "@kortyx/hooks",
-      version: "0.34.1",
+      version: "0.34.2",
       private: false,
       description: "Lifecycle hooks and extension points for Kortyx.",
       keywords: [
@@ -2709,9 +2709,9 @@ var require_dist2 = __commonJS({
       measure: (...args) => client2().measure(...args),
       flush: () => client2().flush()
     };
-    var optionsSchema = import_zod12.z.object({
-      message: import_zod12.z.string().optional(),
-      data: import_zod12.z.json().optional()
+    var optionsSchema = import_zod14.z.object({
+      message: import_zod14.z.string().optional(),
+      data: import_zod14.z.json().optional()
     }).strict();
     async function completeResponseInternal(options = {}) {
       const ctx = getHookContext();
@@ -2771,14 +2771,14 @@ var require_dist2 = __commonJS({
         workflowState: args.workflowState
       }
     });
-    var import_errors8 = require("@kortyx/core/errors");
+    var import_errors16 = require("@kortyx/core/errors");
     var parseWithSchema = (schema, value, label) => {
       if (!schema) return value;
       const parsed = schema.safeParse(value);
       if (parsed.success) return parsed.data;
       const parsedError = "error" in parsed && parsed.error ? parsed.error : void 0;
       const reason = typeof parsedError?.message === "string" && parsedError.message.length > 0 ? parsedError.message : "invalid payload";
-      throw new import_errors8.ValidationError(
+      throw new import_errors16.ValidationError(
         label === "useReason output" ? "MODEL_OUTPUT_SCHEMA" : "SCHEMA_VALIDATION",
         `${label} validation failed: ${reason}`,
         parsedError
@@ -3097,10 +3097,10 @@ var require_dist2 = __commonJS({
       }
       return execution ?? invoke(safeSpan(void 0));
     }
-    var import_node_crypto7 = require("crypto");
+    var import_node_crypto8 = require("crypto");
     var createStructuredStreamId = () => {
       try {
-        return (0, import_node_crypto7.randomUUID)();
+        return (0, import_node_crypto8.randomUUID)();
       } catch {
         return `${Date.now()}-${Math.random().toString(36).slice(2)}`;
       }
@@ -3888,7 +3888,7 @@ var require_dist2 = __commonJS({
         tools: Boolean(tools2?.length),
         provider: args.model.provider.id
       });
-      const messages = args.messages !== void 0 ? args.messages : [
+      const messages2 = args.messages !== void 0 ? args.messages : [
         ...typeof args.system === "string" && args.system.length > 0 ? [{ role: "system", content: args.system }] : [],
         { role: "user", content: String(args.input ?? "") }
       ];
@@ -3915,7 +3915,7 @@ var require_dist2 = __commonJS({
         },
         telemetry: {
           ...args.telemetry ?? {},
-          input: args.telemetry?.input ?? messages.map(
+          input: args.telemetry?.input ?? messages2.map(
             ({ continuation: _continuation, raw: _raw, ...message }) => message
           )
         }
@@ -3977,7 +3977,7 @@ var require_dist2 = __commonJS({
               ...commonMeta
             });
           }
-          const response2 = await model.stream(messages);
+          const response2 = await model.stream(messages2);
           for await (const chunk of response2) {
             (0, import_core32.throwIfExecutionAborted)(abortSignal);
             switch (chunk.type) {
@@ -4066,7 +4066,7 @@ var require_dist2 = __commonJS({
           endTrace(traceSpan, result2);
           return result2;
         }
-        const response = await model.invoke(messages);
+        const response = await model.invoke(messages2);
         (0, import_core32.throwIfExecutionAborted)(abortSignal);
         assertModelFinish(response);
         if (response.content) args.onTextChunk?.(response.content);
@@ -4097,9 +4097,9 @@ var require_dist2 = __commonJS({
         endTrace(traceSpan, result);
         return result;
       } catch (error) {
-        const failure2 = abortSignal?.aborted ? (0, import_core32.createExecutionCancelledError)() : error;
-        failTrace(traceSpan, failure2);
-        throw failure2;
+        const failure = abortSignal?.aborted ? (0, import_core32.createExecutionCancelledError)() : error;
+        failTrace(traceSpan, failure);
+        throw failure;
       }
     }
     function runReasonEngine(args) {
@@ -4940,13 +4940,13 @@ var require_dist2 = __commonJS({
         return parseWithSchema(args.schema, candidate, args.label);
       } catch (error) {
         if (!(error instanceof import_errors52.ValidationError)) throw error;
-        const failure2 = args.finishReason?.unified === "length" ? createTruncatedStructuredOutputError(args.label, error) : typeof candidate === "string" && !validJson ? createInvalidStructuredOutputError(args.label, error) : error;
-        if (failure2 && typeof failure2 === "object")
-          Object.assign(failure2, {
+        const failure = args.finishReason?.unified === "length" ? createTruncatedStructuredOutputError(args.label, error) : typeof candidate === "string" && !validJson ? createInvalidStructuredOutputError(args.label, error) : error;
+        if (failure && typeof failure === "object")
+          Object.assign(failure, {
             ...args.usage ? { usage: args.usage } : {},
             ...args.finishReason ? { finishReason: args.finishReason } : {}
           });
-        throw failure2;
+        throw failure;
       }
     };
     var parseInterruptFirstPassResult = (args) => {
@@ -5494,7 +5494,19 @@ ${tryStringify2(args.response)}`
       let finalizing = checkpoint?.finalizing ?? false;
       let completed = false;
       const maxSteps = Math.max(1, useReasonArgs.toolExecution?.maxSteps ?? 3);
-      const messages = checkpoint?.messages ?? createInitialMessages({
+      const missingReturnCorrections = useReasonArgs.toolExecution?.missingReturnCorrections ?? 0;
+      if (!Number.isSafeInteger(missingReturnCorrections) || missingReturnCorrections < 0)
+        throw new import_errors62.KortyxError(
+          "INVALID_REASON_CORRECTION_POLICY",
+          "Invalid missing return correction budget.",
+          {
+            category: "configuration",
+            retryable: false,
+            safeMessage: "toolExecution.missingReturnCorrections must be a nonnegative safe integer."
+          }
+        );
+      let missingReturnAttempts = checkpoint?.missingReturnAttempts ?? 0;
+      const messages2 = checkpoint?.messages ?? createInitialMessages({
         messages: useReasonArgs.messages,
         inputOverride: Boolean(useReasonArgs.outputSchema),
         system: useReasonArgs.system,
@@ -5545,13 +5557,17 @@ ${tryStringify2(args.response)}`
       let outputContractCorrections = checkpoint?.outputContractCorrections ?? 0;
       const correctionExhausted = () => {
         const message = `useReason could not complete output-contract correction within toolExecution.maxSteps (${maxSteps}). Select exactly one terminal return contract; multiple emit contracts are allowed. Increase maxSteps to allow correction and output generation.`;
-        return new import_errors62.ProviderRequestError(message, {
-          code: "REASON_OUTPUT_CONTRACT_CORRECTION_EXHAUSTED",
-          operation: "useReason",
-          safeMessage: message,
-          retryable: false,
-          details: { maxSteps, outputContractCorrections }
-        });
+        return new import_errors62.KortyxError(
+          "REASON_OUTPUT_CONTRACT_CORRECTION_EXHAUSTED",
+          message,
+          {
+            category: "validation",
+            operation: "useReason",
+            safeMessage: message,
+            retryable: false,
+            details: { maxSteps, outputContractCorrections }
+          }
+        );
       };
       const resolvedInterrupts = checkpoint?.resolvedInterrupts ?? [];
       for (const request of resolvedInterrupts) ctx.node.replayInterrupt?.(request);
@@ -5563,13 +5579,14 @@ ${tryStringify2(args.response)}`
         ctx.currentNodeState.byKey[checkpointKey] = {
           status: "tool_loop",
           opId,
-          messages,
+          messages: messages2,
           steps,
           approvedCalls,
           resolvedInterrupts,
           interruptHistory,
           emissions,
           outputContractCorrections,
+          missingReturnAttempts,
           ...returned ? { returned } : {},
           finalizing,
           ...pending ? { pending } : {},
@@ -5605,9 +5622,9 @@ ${tryStringify2(args.response)}`
           const step = pending ?? await reasonEngine(
             {
               ...useReasonArgs,
-              tools: finalizing ? [] : toolDefinitions,
+              tools: finalizing ? [] : missingReturnAttempts > 0 ? outputDefinitions : toolDefinitions,
               ...separateOutput && !finalizing ? { responseFormat: { type: "text" } } : {},
-              messages,
+              messages: messages2,
               emit: false,
               stream,
               onTextChunk: (delta) => {
@@ -5636,6 +5653,16 @@ ${tryStringify2(args.response)}`
           );
           aggregatedWarnings = mergeWarnings(aggregatedWarnings, step.warnings);
           const toolCalls = step.toolCalls ?? [];
+          if (missingReturnAttempts > 0 && toolCalls.some((call) => !outputByToolName.has(call.name)))
+            throw new import_errors62.KortyxError(
+              "REASON_OUTPUT_CORRECTION_TOOL_DISALLOWED",
+              "A return correction requested a non-output tool.",
+              {
+                category: "validation",
+                retryable: false,
+                safeMessage: "A terminal output correction cannot rerun tools or human-input requests."
+              }
+            );
           const ids = /* @__PURE__ */ new Set();
           for (const call of toolCalls) {
             if (!call.id || ids.has(call.id) || !reused && knownToolCallIds.has(call.id))
@@ -5656,10 +5683,32 @@ ${tryStringify2(args.response)}`
             });
           if (!reused) for (const call of toolCalls) knownToolCallIds.add(call.id);
           if (toolCalls.length === 0) {
-            if (Object.keys(useReasonArgs.outputs?.return ?? {}).length > 0 && !returned)
-              throw new Error(
-                "useReason outputs.return requires one terminal output contract call."
+            if (Object.keys(useReasonArgs.outputs?.return ?? {}).length > 0 && !returned) {
+              if (missingReturnAttempts < missingReturnCorrections) {
+                messages2.push({ role: "assistant", content: step.text });
+                messages2.push({
+                  role: "user",
+                  content: "MISSING_TERMINAL_OUTPUT: No terminal output was published. Select exactly one configured return contract using the completed tool and human-input results. Do not repeat completed calls."
+                });
+                missingReturnAttempts += 1;
+                outputContractCorrections += 1;
+                pending = void 0;
+                pendingStepIndex = void 0;
+                save();
+                continue;
+              }
+              throw new import_errors62.KortyxError(
+                "REASON_TERMINAL_OUTPUT_MISSING",
+                "useReason outputs.return requires one terminal output contract call.",
+                {
+                  category: "validation",
+                  operation: "useReason",
+                  retryable: false,
+                  safeMessage: "useReason outputs.return requires one terminal output contract call.",
+                  details: { missingReturnAttempts }
+                }
               );
+            }
             if (interruptConfig?.mode === "required" && interruptHistory.length === 0)
               throw new Error(
                 "useReason interrupts.mode is required, but the model returned a final answer without requesting human input."
@@ -5686,12 +5735,12 @@ ${tryStringify2(args.response)}`
                 } catch {
                 }
               }
-              messages.push({
+              messages2.push({
                 role: "assistant",
                 content: step.text,
                 ...step.continuation ? { continuation: step.continuation } : {}
               });
-              messages.push({
+              messages2.push({
                 role: "user",
                 content: "Return the final answer using the requested output schema and the tool results above."
               });
@@ -5714,7 +5763,7 @@ ${tryStringify2(args.response)}`
                 (call) => !interruptByToolName.has(call.name) && !outputByToolName.has(call.name)
               )
             );
-            messages.push({
+            messages2.push({
               role: "assistant",
               content: step.text,
               toolCalls,
@@ -5742,7 +5791,7 @@ ${tryStringify2(args.response)}`
                 content: "MULTIPLE_TERMINAL_OUTPUTS: No terminal output was published. Select exactly one return contract using the tool and human-input results. Do not repeat completed calls. Multiple emit contracts are allowed."
               };
               toolResults.push(result2);
-              messages.push({ role: "tool", ...result2 });
+              messages2.push({ role: "tool", ...result2 });
               corrected = true;
             }
             if (corrected) outputContractCorrections += 1;
@@ -5827,7 +5876,7 @@ ${tryStringify2(args.response)}`
                       tools: [],
                       ...inferred.responseFormat ? { responseFormat: inferred.responseFormat } : {},
                       messages: [
-                        ...messages,
+                        ...messages2,
                         ...toolCalls.filter(
                           (pendingCall) => !toolResults.some(
                             (result3) => result3.toolCallId === pendingCall.id
@@ -5946,7 +5995,7 @@ ${tryStringify2(args.response)}`
               });
               const result2 = normalizeToolResult(call, { emitted: true, data });
               toolResults.push(result2);
-              messages.push({
+              messages2.push({
                 role: "tool",
                 content: result2.content,
                 toolCallId: result2.toolCallId,
@@ -6003,7 +6052,7 @@ ${tryStringify2(args.response)}`
               });
               const result2 = normalizeToolResult(toolCall, response);
               toolResults.push(result2);
-              messages.push({
+              messages2.push({
                 role: "tool",
                 content: result2.content,
                 toolCallId: result2.toolCallId,
@@ -6117,7 +6166,7 @@ ${tryStringify2(args.response)}`
                     isError: true
                   });
                 }
-                messages.push({
+                messages2.push({
                   role: "tool",
                   content: denied.content,
                   toolCallId: denied.toolCallId,
@@ -6174,17 +6223,21 @@ ${tryStringify2(args.response)}`
               toolResults.push(result2);
               allToolResults.push(result2);
               if (shouldEmitTool) {
+                const diagnosticId = steps[stepIndex]?.toolObservations?.slice().reverse().find(
+                  (observation) => observation.toolCallId === `${opId}:${toolCall.id}`
+                )?.diagnosticId;
                 emitToolEvent("tool-call-error", {
                   tool: tool.name,
                   toolCallId: toolCall.id,
                   message: result2.content,
-                  failure: result2.failure
+                  failure: result2.failure,
+                  ...diagnosticId ? { diagnosticId } : {}
                 });
               }
             }
             const completedToolResult = toolResults.at(-1);
             if (completedToolResult)
-              messages.push({
+              messages2.push({
                 role: "tool",
                 content: completedToolResult.content,
                 toolCallId: completedToolResult.toolCallId,
@@ -6967,9 +7020,9 @@ ${tryStringify2(args.response)}`
     var import_core8 = require("@kortyx/core");
     var import_errors82 = require("@kortyx/core/errors");
     var WorkflowCallError = class extends import_errors82.KortyxError {
-      constructor(workflow, message, cause, failure2) {
+      constructor(workflow, message, cause, failure) {
         const detail = (0, import_errors82.serializeFailure)(
-          failure2 ?? cause ?? new import_errors82.WorkflowContractError(message)
+          failure ?? cause ?? new import_errors82.WorkflowContractError(message)
         );
         super(detail.code, `Workflow '${workflow}': ${message}`, {
           ...detail,
@@ -6986,7 +7039,7 @@ ${tryStringify2(args.response)}`
     };
     function workflowCallFingerprint(value) {
       const seen = /* @__PURE__ */ new Set();
-      const canonical2 = (v) => {
+      const canonical3 = (v) => {
         if (v === null || typeof v === "string" || typeof v === "boolean") return v;
         if (typeof v === "number" && Number.isFinite(v)) return v;
         if (typeof v !== "object" || !v || seen.has(v)) {
@@ -7000,16 +7053,16 @@ ${tryStringify2(args.response)}`
           );
         }
         seen.add(v);
-        const result = Array.isArray(v) ? v.map(canonical2) : Object.fromEntries(
+        const result = Array.isArray(v) ? v.map(canonical3) : Object.fromEntries(
           Object.keys(v).sort().map((key) => [
             key,
-            canonical2(v[key])
+            canonical3(v[key])
           ])
         );
         seen.delete(v);
         return result;
       };
-      return JSON.stringify(canonical2(value));
+      return JSON.stringify(canonical3(value));
     }
     function createWorkflowHooks(workflows) {
       return {
@@ -7239,6 +7292,7 @@ __export(index_exports, {
   ChatLifecycleHookError: () => ChatLifecycleHookError,
   EvalConfigurationError: () => EvalConfigurationError,
   EvalEvidencePolicySchema: () => EvalEvidencePolicySchema,
+  EvalIssueSchema: () => EvalIssueSchema,
   EvalJudgeIdentitySchema: () => EvalJudgeIdentitySchema,
   EvalJudgeUsageSchema: () => EvalJudgeUsageSchema,
   EvalManifestSchema: () => EvalManifestSchema,
@@ -7260,6 +7314,7 @@ __export(index_exports, {
   createAgent: () => createAgent,
   createChatRouteHandler: () => createChatRouteHandler,
   createCheckpointRouteHandler: () => createCheckpointRouteHandler,
+  createEvalIssue: () => createEvalIssue,
   createEvalJudge: () => createEvalJudge,
   createEvalRouteHandler: () => createEvalRouteHandler,
   createEvals: () => createEvals,
@@ -7288,7 +7343,7 @@ module.exports = __toCommonJS(index_exports);
 // src/adapters/http.ts
 var import_errors5 = require("@kortyx/core/errors");
 var import_stream2 = require("@kortyx/stream");
-var import_zod2 = require("zod");
+var import_zod3 = require("zod");
 
 // src/chat/lifecycle.ts
 var import_errors = require("@kortyx/core/errors");
@@ -7336,6 +7391,7 @@ var import_internal = __toESM(require_internal());
 var import_runtime = require("@kortyx/runtime");
 var import_stream = require("@kortyx/stream");
 var import_langgraph = require("@langchain/langgraph");
+var import_zod2 = require("zod");
 
 // src/execution/contracts.ts
 var import_errors3 = require("@kortyx/core/errors");
@@ -7397,12 +7453,12 @@ async function validateExecutionOutput(state, select) {
     if (terminal.outputSchema || root.outputSchema) import_zod.z.json().parse(data);
   } catch (error) {
     if (!(error instanceof import_zod.z.ZodError)) throw error;
-    const failure2 = new import_errors3.ValidationError(
+    const failure = new import_errors3.ValidationError(
       "INVALID_OUTPUT",
       `Workflow output validation failed: ${error.message}`,
       error
     );
-    throw failure2;
+    throw failure;
   }
   return { ...state, data };
 }
@@ -7792,7 +7848,7 @@ var import_node = require("@kortyx/usage-telemetry/node");
 // package.json
 var package_default = {
   name: "@kortyx/agent",
-  version: "0.37.1",
+  version: "0.38.0",
   private: false,
   description: "Agent composition utilities for Kortyx.",
   keywords: [
@@ -8056,6 +8112,24 @@ async function orchestrateGraphStream({
     isRecord3(config.context) ? { userId: config.context.userId, tenantId: config.context.tenantId } : {}
   );
   const telemetryConfig = isRecord3(config.telemetry) ? config.telemetry : {};
+  const errorChunk = (error, existingDiagnosticId) => {
+    const failure = (0, import_errors4.serializeFailure)(error);
+    let diagnosticId = existingDiagnosticId;
+    if (!diagnosticId) {
+      try {
+        diagnosticId = traceAdapter?.reportError?.(error, {
+          correlation: { runId, ...sessionId ? { sessionId } : {} }
+        });
+      } catch {
+      }
+    }
+    return {
+      type: "error",
+      message: failure.message,
+      failure,
+      ...typeof diagnosticId === "string" && import_zod2.z.uuid().safeParse(diagnosticId).success ? { diagnosticId } : {}
+    };
+  };
   const emitResumeFailure = (resumeError) => {
     const interruptId = typeof config.telemetryInterruptId === "string" ? config.telemetryInterruptId : void 0;
     if (!interruptId) return;
@@ -8322,11 +8396,12 @@ async function orchestrateGraphStream({
       );
       const supplied = payload?.failure;
       outcomeError = (0, import_errors4.isFailureDescriptor)(supplied) ? (0, import_errors4.errorFromFailure)(supplied) : new Error(msg);
-      write({
-        type: "error",
-        message: (0, import_errors4.serializeFailure)(outcomeError).message,
-        failure: (0, import_errors4.serializeFailure)(outcomeError)
-      });
+      write(
+        errorChunk(
+          outcomeError,
+          payload?.diagnosticId
+        )
+      );
       write({ type: "done" });
       finished = true;
       out.end();
@@ -8695,11 +8770,7 @@ async function orchestrateGraphStream({
           runTraceSpan.addEvent?.("kortyx.transition.error", {
             transitionTo
           });
-          write({
-            type: "error",
-            message: (0, import_errors4.serializeFailure)(err).message,
-            failure: (0, import_errors4.serializeFailure)(err)
-          });
+          write(errorChunk(err));
           write({ type: "done" });
           out.end();
           return;
@@ -9083,11 +9154,7 @@ async function orchestrateGraphStream({
     fallbackRunSpan?.fail?.(err);
     emitResumeFailure(err);
     console.error("[error:orchestrateGraphStream]", (0, import_errors4.serializeFailure)(err));
-    write({
-      type: "error",
-      message: (0, import_errors4.serializeFailure)(err).message,
-      failure: (0, import_errors4.serializeFailure)(err)
-    });
+    write(errorChunk(err));
     write({ type: "done" });
     finished = true;
     out.end();
@@ -9432,38 +9499,38 @@ var reportLifecycleError = async (callback, event) => {
     console.error("[chat:onLifecycleError]", (0, import_errors5.serializeFailure)(error));
   }
 };
-var chatMessageSchema = import_zod2.z.object({
-  role: import_zod2.z.enum(["user", "assistant", "system"]),
-  content: import_zod2.z.string(),
-  metadata: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.unknown()).optional(),
-  id: import_zod2.z.string().optional(),
-  timestamp: import_zod2.z.number().refine(Number.isFinite).optional()
+var chatMessageSchema = import_zod3.z.object({
+  role: import_zod3.z.enum(["user", "assistant", "system"]),
+  content: import_zod3.z.string(),
+  metadata: import_zod3.z.record(import_zod3.z.string(), import_zod3.z.unknown()).optional(),
+  id: import_zod3.z.string().optional(),
+  timestamp: import_zod3.z.number().refine(Number.isFinite).optional()
 }).strict();
-var chatRequestBodySchema = import_zod2.z.looseObject({
-  sessionId: import_zod2.z.string().optional(),
-  clientTurnId: import_zod2.z.string().max(128).optional(),
-  workflowId: import_zod2.z.string().optional(),
-  stream: import_zod2.z.boolean().optional(),
-  context: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.unknown()).optional(),
-  messages: import_zod2.z.array(chatMessageSchema)
+var chatRequestBodySchema = import_zod3.z.looseObject({
+  sessionId: import_zod3.z.string().optional(),
+  clientTurnId: import_zod3.z.string().max(128).optional(),
+  workflowId: import_zod3.z.string().optional(),
+  stream: import_zod3.z.boolean().optional(),
+  context: import_zod3.z.record(import_zod3.z.string(), import_zod3.z.unknown()).optional(),
+  messages: import_zod3.z.array(chatMessageSchema)
 });
-var checkpointRequestBodySchema = import_zod2.z.discriminatedUnion("action", [
-  import_zod2.z.object({
-    action: import_zod2.z.literal("list"),
-    sessionId: import_zod2.z.string().min(1)
+var checkpointRequestBodySchema = import_zod3.z.discriminatedUnion("action", [
+  import_zod3.z.object({
+    action: import_zod3.z.literal("list"),
+    sessionId: import_zod3.z.string().min(1)
   }).strict(),
-  import_zod2.z.object({
-    action: import_zod2.z.literal("get"),
-    checkpointId: import_zod2.z.string().min(1)
+  import_zod3.z.object({
+    action: import_zod3.z.literal("get"),
+    checkpointId: import_zod3.z.string().min(1)
   }).strict(),
-  import_zod2.z.object({
-    action: import_zod2.z.literal("rollback"),
-    checkpointId: import_zod2.z.string().min(1)
+  import_zod3.z.object({
+    action: import_zod3.z.literal("rollback"),
+    checkpointId: import_zod3.z.string().min(1)
   }).strict(),
-  import_zod2.z.object({
-    action: import_zod2.z.literal("fork"),
-    checkpointId: import_zod2.z.string().min(1),
-    newSessionId: import_zod2.z.string().min(1).optional()
+  import_zod3.z.object({
+    action: import_zod3.z.literal("fork"),
+    checkpointId: import_zod3.z.string().min(1),
+    newSessionId: import_zod3.z.string().min(1).optional()
   }).strict()
 ]);
 async function readRequestJson(request) {
@@ -9486,10 +9553,10 @@ async function readRequestJson(request) {
   }
 }
 function createFailureResponse(error, status2) {
-  const failure2 = (0, import_errors5.serializeFailure)(error);
+  const failure = (0, import_errors5.serializeFailure)(error);
   return Response.json(
-    { error: failure2.message, failure: failure2 },
-    { status: status2 ?? (0, import_errors5.failureHttpStatus)(failure2) }
+    { error: failure.message, failure },
+    { status: status2 ?? (0, import_errors5.failureHttpStatus)(failure) }
   );
 }
 function parseChatRequestBody(value) {
@@ -9723,7 +9790,7 @@ var import_node_path = require("path");
 var import_core5 = require("@kortyx/core");
 var import_providers = require("@kortyx/providers");
 var import_runtime5 = require("@kortyx/runtime");
-var import_zod5 = require("zod");
+var import_zod6 = require("zod");
 
 // src/evals/prompt-context.ts
 var import_node_async_hooks = require("async_hooks");
@@ -9733,21 +9800,21 @@ var promptEvaluationContext = new import_node_async_hooks.AsyncLocalStorage();
 var import_core4 = require("@kortyx/core");
 var import_errors6 = require("@kortyx/core/errors");
 var import_runtime3 = require("@kortyx/runtime");
-var import_zod3 = require("zod");
-var handleSchema = import_zod3.z.object({
-  token: import_zod3.z.string().min(1),
-  requestId: import_zod3.z.string().min(1),
-  sessionId: import_zod3.z.string().min(1),
-  runId: import_zod3.z.string().min(1)
+var import_zod4 = require("zod");
+var handleSchema = import_zod4.z.object({
+  token: import_zod4.z.string().min(1),
+  requestId: import_zod4.z.string().min(1),
+  sessionId: import_zod4.z.string().min(1),
+  runId: import_zod4.z.string().min(1)
 }).strict();
-var responseSchema = import_zod3.z.discriminatedUnion("type", [
-  import_zod3.z.object({ type: import_zod3.z.literal("text"), text: import_zod3.z.string().min(1) }).strict(),
-  import_zod3.z.object({
-    type: import_zod3.z.literal("select"),
-    ids: import_zod3.z.array(import_zod3.z.string().min(1)).min(1)
+var responseSchema = import_zod4.z.discriminatedUnion("type", [
+  import_zod4.z.object({ type: import_zod4.z.literal("text"), text: import_zod4.z.string().min(1) }).strict(),
+  import_zod4.z.object({
+    type: import_zod4.z.literal("select"),
+    ids: import_zod4.z.array(import_zod4.z.string().min(1)).min(1)
   }).strict(),
-  import_zod3.z.object({ type: import_zod3.z.literal("value"), value: import_zod3.z.unknown() }).strict(),
-  import_zod3.z.object({ type: import_zod3.z.literal("cancel") }).strict()
+  import_zod4.z.object({ type: import_zod4.z.literal("value"), value: import_zod4.z.unknown() }).strict(),
+  import_zod4.z.object({ type: import_zod4.z.literal("cancel") }).strict()
 ]);
 function resultFromOutcome(outcome, runId, sessionId) {
   const usage2 = outcome.pending?.schema.meta?.__kortyxResumeStatePatch?.tokenUsage ?? outcome.error?.__kortyxHookStatePatch?.tokenUsage ?? outcome.state.runtime?.tokenUsage;
@@ -9872,7 +9939,7 @@ async function executeWorkflow(services, args) {
   });
 }
 async function resumeWorkflow(services, args) {
-  const parsed = import_zod3.z.object({ resume: handleSchema, response: responseSchema }).safeParse(args);
+  const parsed = import_zod4.z.object({ resume: handleSchema, response: responseSchema }).safeParse(args);
   if (!parsed.success)
     throw new ExecutionRequestError("INVALID_RESUME", parsed.error.message);
   const { resume: handle, response } = parsed.data;
@@ -9957,13 +10024,13 @@ async function resumeWorkflow(services, args) {
 }
 
 // src/interrupt/discovery.ts
-var import_zod4 = require("zod");
-var scopeSchema = import_zod4.z.object({
-  sessionId: import_zod4.z.string().min(1).optional(),
-  runId: import_zod4.z.string().min(1).optional(),
-  context: import_zod4.z.record(
-    import_zod4.z.string(),
-    import_zod4.z.union([import_zod4.z.string(), import_zod4.z.number(), import_zod4.z.boolean(), import_zod4.z.null()])
+var import_zod5 = require("zod");
+var scopeSchema = import_zod5.z.object({
+  sessionId: import_zod5.z.string().min(1).optional(),
+  runId: import_zod5.z.string().min(1).optional(),
+  context: import_zod5.z.record(
+    import_zod5.z.string(),
+    import_zod5.z.union([import_zod5.z.string(), import_zod5.z.number(), import_zod5.z.boolean(), import_zod5.z.null()])
   ).optional()
 }).strict().refine(
   (scope) => Boolean(
@@ -10011,15 +10078,15 @@ function createInterruptDiscovery(store) {
         status: _status,
         afterResponseCompleted,
         ...scope
-      } = import_zod4.z.object({
-        sessionId: import_zod4.z.string().optional(),
-        runId: import_zod4.z.string().optional(),
-        context: import_zod4.z.record(
-          import_zod4.z.string(),
-          import_zod4.z.union([import_zod4.z.string(), import_zod4.z.number(), import_zod4.z.boolean(), import_zod4.z.null()])
+      } = import_zod5.z.object({
+        sessionId: import_zod5.z.string().optional(),
+        runId: import_zod5.z.string().optional(),
+        context: import_zod5.z.record(
+          import_zod5.z.string(),
+          import_zod5.z.union([import_zod5.z.string(), import_zod5.z.number(), import_zod5.z.boolean(), import_zod5.z.null()])
         ).optional(),
-        status: import_zod4.z.literal("pending").optional(),
-        afterResponseCompleted: import_zod4.z.boolean().optional()
+        status: import_zod5.z.literal("pending").optional(),
+        afterResponseCompleted: import_zod5.z.boolean().optional()
       }).strict().parse(options);
       scopeSchema.parse(scope);
       return (await records()).filter(
@@ -10029,7 +10096,7 @@ function createInterruptDiscovery(store) {
       ).map(summary);
     },
     async getInterrupt(id3, scope) {
-      import_zod4.z.string().min(1).parse(id3);
+      import_zod5.z.string().min(1).parse(id3);
       scopeSchema.parse(scope);
       const record = (await records()).find(
         (record2) => record2.requestId === id3 && matches(record2, scope)
@@ -10153,10 +10220,10 @@ var import_errors7 = require("@kortyx/core/errors");
 var import_runtime4 = require("@kortyx/runtime");
 
 // src/utils/extract-latest-message.ts
-function extractLatestUserMessage(messages) {
-  if (!messages || messages.length === 0) return "";
-  for (let i = messages.length - 1; i >= 0; i -= 1) {
-    const msg = messages[i];
+function extractLatestUserMessage(messages2) {
+  if (!messages2 || messages2.length === 0) return "";
+  for (let i = messages2.length - 1; i >= 0; i -= 1) {
+    const msg = messages2[i];
     if (msg?.role === "user" && msg?.content?.trim()) {
       return msg.content.trim();
     }
@@ -10166,7 +10233,7 @@ function extractLatestUserMessage(messages) {
 
 // src/chat/process-chat.ts
 async function streamChat({
-  messages,
+  messages: messages2,
   abortSignal,
   executionSignal,
   onExecution,
@@ -10211,9 +10278,9 @@ async function streamChat({
   runtimeConfig2.selectWorkflow = workflowSelector;
   const fallbackSessionId = options?.sessionId;
   const resolvedSessionId = sessionId || fallbackSessionId || "anonymous-session";
-  const last = messages[messages.length - 1];
-  const input = extractLatestUserMessage(messages);
-  const previousMessages = messages.slice(0, -1);
+  const last = messages2[messages2.length - 1];
+  const input = extractLatestUserMessage(messages2);
+  const previousMessages = messages2.slice(0, -1);
   const runtime = {
     ...previousMessages.length > 0 ? { priorMessages: previousMessages } : {}
   };
@@ -10326,30 +10393,30 @@ async function streamChat({
 }
 
 // src/chat/create-agent.ts
-var agentProcessOptionsSchema = import_zod5.z.object({
-  executionSignal: import_zod5.z.custom((value) => value instanceof AbortSignal).optional(),
-  onExecution: import_zod5.z.custom(
+var agentProcessOptionsSchema = import_zod6.z.object({
+  executionSignal: import_zod6.z.custom((value) => value instanceof AbortSignal).optional(),
+  onExecution: import_zod6.z.custom(
     (value) => typeof value === "function"
   ).optional(),
-  clientTurnId: import_zod5.z.string().min(1).max(128).optional(),
-  continueOnDisconnect: import_zod5.z.boolean().optional(),
-  onResponseFinalized: import_zod5.z.custom((value) => typeof value === "function").optional(),
+  clientTurnId: import_zod6.z.string().min(1).max(128).optional(),
+  continueOnDisconnect: import_zod6.z.boolean().optional(),
+  onResponseFinalized: import_zod6.z.custom((value) => typeof value === "function").optional(),
   limits: import_core5.ExecutionLimitsSchema.optional(),
-  abortSignal: import_zod5.z.custom((value) => value instanceof AbortSignal).optional(),
-  sessionId: import_zod5.z.string().optional(),
-  workflowId: import_zod5.z.string().optional(),
-  context: import_zod5.z.record(import_zod5.z.string(), import_zod5.z.unknown()).optional()
+  abortSignal: import_zod6.z.custom((value) => value instanceof AbortSignal).optional(),
+  sessionId: import_zod6.z.string().optional(),
+  workflowId: import_zod6.z.string().optional(),
+  context: import_zod6.z.record(import_zod6.z.string(), import_zod6.z.unknown()).optional()
 }).strict();
-var createAgentArgsBaseSchema = import_zod5.z.object({
+var createAgentArgsBaseSchema = import_zod6.z.object({
   limits: import_core5.ExecutionLimitsSchema.optional(),
-  getProvider: import_zod5.z.unknown().optional(),
-  workflows: import_zod5.z.array(import_zod5.z.unknown()).optional(),
-  workflowsDir: import_zod5.z.string().optional(),
-  workflowRegistry: import_zod5.z.unknown().optional(),
-  defaultWorkflowId: import_zod5.z.string().optional(),
-  frameworkAdapter: import_zod5.z.unknown().optional(),
-  telemetry: import_zod5.z.unknown().optional(),
-  prompts: import_zod5.z.unknown().optional()
+  getProvider: import_zod6.z.unknown().optional(),
+  workflows: import_zod6.z.array(import_zod6.z.unknown()).optional(),
+  workflowsDir: import_zod6.z.string().optional(),
+  workflowRegistry: import_zod6.z.unknown().optional(),
+  defaultWorkflowId: import_zod6.z.string().optional(),
+  frameworkAdapter: import_zod6.z.unknown().optional(),
+  telemetry: import_zod6.z.unknown().optional(),
+  prompts: import_zod6.z.unknown().optional()
 }).strict();
 var createAgentArgsSchema = createAgentArgsBaseSchema.superRefine(
   (value, ctx) => {
@@ -10451,7 +10518,22 @@ function createAgentInternal(args) {
     (registry) => registry?.list?.()
   );
   void registeredWorkflowsPromise.then(
-    () => registryUsage.end(),
+    (registered) => {
+      try {
+        registryUsage.end(
+          "success",
+          registered ? {
+            data_reducers: registered.some(
+              (workflow) => Object.values(workflow.reduce ?? {}).some(
+                (callback) => typeof callback === "function"
+              )
+            )
+          } : {}
+        );
+      } catch {
+        registryUsage.end();
+      }
+    },
     (error) => registryUsage.fail(error)
   );
   const projectTopology = async (options) => {
@@ -10467,7 +10549,7 @@ function createAgentInternal(args) {
       })
     );
   };
-  const streamChatInternal = async (messages, options) => {
+  const streamChatInternal = async (messages2, options) => {
     const parsedOptions = parseAgentProcessOptions(options);
     const promptEvaluation = promptEvaluationContext.getStore();
     if (promptEvaluation && !prompts)
@@ -10486,7 +10568,7 @@ function createAgentInternal(args) {
     );
     return streamChat({
       ...resolvedDefaultWorkflowId ? { defaultWorkflowId: resolvedDefaultWorkflowId } : {},
-      messages,
+      messages: messages2,
       abortSignal: parsedOptions?.abortSignal,
       executionSignal: parsedOptions?.executionSignal,
       onExecution: parsedOptions?.onExecution,
@@ -10524,9 +10606,9 @@ function createAgentInternal(args) {
       })
     });
   };
-  const streamChat2 = (messages, options) => usage.measure(
+  const streamChat2 = (messages2, options) => usage.measure(
     "sdk.chat.admission",
-    () => streamChatInternal(messages, options),
+    () => streamChatInternal(messages2, options),
     {
       phase: "admission",
       continue_on_disconnect: options?.continueOnDisconnect === true
@@ -10682,6 +10764,12 @@ function createAgentInternal(args) {
   ));
   return {
     ...createInterruptDiscovery(resolvedFrameworkAdapter.pendingRequests),
+    reportError: (error, options) => {
+      try {
+        return telemetry?.trace?.reportError?.(error, options);
+      } catch {
+      }
+    },
     execute,
     resume,
     projectTopology,
@@ -10689,8 +10777,8 @@ function createAgentInternal(args) {
     describePromptContracts: () => (prompts?.definitions ?? []).map((ref) => ({
       id: ref.id,
       format: ref.format,
-      variablesSchema: import_zod5.z.toJSONSchema(ref.variables, { io: "input" }),
-      configSchema: import_zod5.z.toJSONSchema(ref.config, { io: "input" })
+      variablesSchema: import_zod6.z.toJSONSchema(ref.variables, { io: "input" }),
+      configSchema: import_zod6.z.toJSONSchema(ref.config, { io: "input" })
     })),
     listCheckpoints,
     getCheckpoint,
@@ -10720,11 +10808,12 @@ function createAgent(args) {
 }
 
 // src/evals/contracts.ts
+var import_errors8 = require("@kortyx/core/errors");
 var import_browser2 = require("@kortyx/prompts/browser");
-var import_zod7 = require("zod");
+var import_zod8 = require("zod");
 
 // src/evals/evidence-policy.ts
-var import_zod6 = require("zod");
+var import_zod7 = require("zod");
 var EVAL_EVIDENCE_EVENT_TYPES = [
   "tool-call-start",
   "tool-call-result",
@@ -10737,19 +10826,19 @@ var EVAL_EVIDENCE_EVENT_TYPES = [
   "cancelled",
   "limit-reached"
 ];
-var id = import_zod6.z.string().trim().min(1).max(256);
-var filter = import_zod6.z.object({ using: id, params: import_zod6.z.json().optional() }).strict();
-var EvalEvidencePolicySchema = import_zod6.z.object({
-  history: import_zod6.z.boolean().optional(),
-  events: import_zod6.z.union([
-    import_zod6.z.literal(false),
-    import_zod6.z.array(import_zod6.z.enum(EVAL_EVIDENCE_EVENT_TYPES)),
+var id = import_zod7.z.string().trim().min(1).max(256);
+var filter = import_zod7.z.object({ using: id, params: import_zod7.z.json().optional() }).strict();
+var EvalEvidencePolicySchema = import_zod7.z.object({
+  history: import_zod7.z.boolean().optional(),
+  events: import_zod7.z.union([
+    import_zod7.z.literal(false),
+    import_zod7.z.array(import_zod7.z.enum(EVAL_EVIDENCE_EVENT_TYPES)),
     filter
   ]).optional(),
-  outputs: import_zod6.z.union([
-    import_zod6.z.literal(false),
-    import_zod6.z.array(
-      import_zod6.z.object({
+  outputs: import_zod7.z.union([
+    import_zod7.z.literal(false),
+    import_zod7.z.array(
+      import_zod7.z.object({
         dataType: id.optional(),
         schemaId: id.optional(),
         schemaVersion: id.optional()
@@ -10762,26 +10851,29 @@ var EvalEvidencePolicySchema = import_zod6.z.object({
 }).strict();
 
 // src/evals/contracts.ts
-var id2 = import_zod7.z.string().trim().min(1).max(256);
-var handler = import_zod7.z.object({ using: id2, params: import_zod7.z.json().optional() }).strict();
-var EvalResumeResponseSchema = import_zod7.z.discriminatedUnion("type", [
-  import_zod7.z.object({ type: import_zod7.z.literal("text"), text: import_zod7.z.string() }).strict(),
-  import_zod7.z.object({ type: import_zod7.z.literal("select"), ids: import_zod7.z.array(import_zod7.z.string()) }).strict(),
-  import_zod7.z.object({ type: import_zod7.z.literal("value"), value: import_zod7.z.json() }).strict(),
-  import_zod7.z.object({ type: import_zod7.z.literal("cancel") }).strict()
+var id2 = import_zod8.z.string().trim().min(1).max(256);
+var failureDescriptor = import_zod8.z.custom(import_errors8.isFailureDescriptor).transform(
+  (value) => JSON.parse(JSON.stringify((0, import_errors8.serializeFailure)(value)))
+);
+var handler = import_zod8.z.object({ using: id2, params: import_zod8.z.json().optional() }).strict();
+var EvalResumeResponseSchema = import_zod8.z.discriminatedUnion("type", [
+  import_zod8.z.object({ type: import_zod8.z.literal("text"), text: import_zod8.z.string() }).strict(),
+  import_zod8.z.object({ type: import_zod8.z.literal("select"), ids: import_zod8.z.array(import_zod8.z.string()) }).strict(),
+  import_zod8.z.object({ type: import_zod8.z.literal("value"), value: import_zod8.z.json() }).strict(),
+  import_zod8.z.object({ type: import_zod8.z.literal("cancel") }).strict()
 ]);
-var expectation = import_zod7.z.object({
-  type: import_zod7.z.enum(["answer", "interrupt"]),
+var expectation = import_zod8.z.object({
+  type: import_zod8.z.enum(["answer", "interrupt"]),
   schemaId: id2.optional(),
   schemaVersion: id2.optional(),
-  outputs: import_zod7.z.array(import_zod7.z.object({ schemaId: id2, schemaVersion: id2.optional() }).strict()).optional(),
-  criteria: import_zod7.z.array(
-    import_zod7.z.union([
-      import_zod7.z.string().trim().min(1),
-      import_zod7.z.object({ id: id2, text: import_zod7.z.string().trim().min(1) }).strict()
+  outputs: import_zod8.z.array(import_zod8.z.object({ schemaId: id2, schemaVersion: id2.optional() }).strict()).optional(),
+  criteria: import_zod8.z.array(
+    import_zod8.z.union([
+      import_zod8.z.string().trim().min(1),
+      import_zod8.z.object({ id: id2, text: import_zod8.z.string().trim().min(1) }).strict()
     ])
   ).optional(),
-  reference: import_zod7.z.json().optional()
+  reference: import_zod8.z.json().optional()
 }).strict().superRefine((value, ctx) => {
   if (value.type === "answer" && (value.schemaId || value.schemaVersion))
     ctx.addIssue({
@@ -10797,21 +10889,21 @@ var expectation = import_zod7.z.object({
       message: "Criterion IDs must be unique within a step."
     });
 });
-var EvalSuiteSchema = import_zod7.z.object({
+var EvalSuiteSchema = import_zod8.z.object({
   evidence: EvalEvidencePolicySchema.optional(),
   id: id2,
-  name: import_zod7.z.string().optional(),
-  cases: import_zod7.z.array(
-    import_zod7.z.object({
+  name: import_zod8.z.string().optional(),
+  cases: import_zod8.z.array(
+    import_zod8.z.object({
       id: id2,
-      name: import_zod7.z.string().optional(),
-      params: import_zod7.z.json().optional(),
+      name: import_zod8.z.string().optional(),
+      params: import_zod8.z.json().optional(),
       workflowId: id2.optional(),
-      steps: import_zod7.z.array(
-        import_zod7.z.union([
-          import_zod7.z.object({ message: import_zod7.z.string().min(1), expect: expectation }).strict(),
-          import_zod7.z.object({
-            resume: import_zod7.z.union([EvalResumeResponseSchema, handler]),
+      steps: import_zod8.z.array(
+        import_zod8.z.union([
+          import_zod8.z.object({ message: import_zod8.z.string().min(1), expect: expectation }).strict(),
+          import_zod8.z.object({
+            resume: import_zod8.z.union([EvalResumeResponseSchema, handler]),
             expect: expectation
           }).strict()
         ])
@@ -10819,42 +10911,44 @@ var EvalSuiteSchema = import_zod7.z.object({
     }).strict()
   ).min(1)
 }).strict();
-var EvalVerdictSchema = import_zod7.z.object({
-  passed: import_zod7.z.boolean(),
-  reason: import_zod7.z.string().trim().min(1),
-  evidence: import_zod7.z.array(import_zod7.z.string().min(1))
+var EvalVerdictSchema = import_zod8.z.object({
+  passed: import_zod8.z.boolean(),
+  reason: import_zod8.z.string().trim().min(1),
+  evidence: import_zod8.z.array(import_zod8.z.string().min(1))
 }).strict();
-var EvalObservationSchema = import_zod7.z.object({
-  promptUsage: import_zod7.z.array(
-    import_zod7.z.object({
-      id: import_zod7.z.string(),
-      version: import_zod7.z.number().int().positive(),
-      hash: import_zod7.z.string().regex(/^[a-f0-9]{64}$/),
-      environment: import_zod7.z.string(),
-      snapshotRevision: import_zod7.z.string(),
-      model: import_zod7.z.string().optional()
+var EvalObservationSchema = import_zod8.z.object({
+  failure: failureDescriptor.optional(),
+  diagnosticId: import_zod8.z.uuid().optional(),
+  promptUsage: import_zod8.z.array(
+    import_zod8.z.object({
+      id: import_zod8.z.string(),
+      version: import_zod8.z.number().int().positive(),
+      hash: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
+      environment: import_zod8.z.string(),
+      snapshotRevision: import_zod8.z.string(),
+      model: import_zod8.z.string().optional()
     }).strict()
   ).optional(),
-  type: import_zod7.z.enum(["answer", "interrupt", "error", "cancelled"]),
-  text: import_zod7.z.string(),
-  structured: import_zod7.z.array(import_zod7.z.json()),
-  events: import_zod7.z.array(import_zod7.z.json()).optional(),
-  runId: import_zod7.z.string().optional(),
-  checkpointId: import_zod7.z.string().optional(),
-  interrupt: import_zod7.z.object({
-    requestId: import_zod7.z.string(),
-    kind: import_zod7.z.enum(["text", "choice", "multi-choice", "custom"]),
-    question: import_zod7.z.string().optional(),
-    schemaId: import_zod7.z.string().optional(),
-    schemaVersion: import_zod7.z.string().optional(),
-    options: import_zod7.z.array(
-      import_zod7.z.object({
-        id: import_zod7.z.string(),
-        label: import_zod7.z.string(),
-        description: import_zod7.z.string().optional()
+  type: import_zod8.z.enum(["answer", "interrupt", "error", "cancelled"]),
+  text: import_zod8.z.string(),
+  structured: import_zod8.z.array(import_zod8.z.json()),
+  events: import_zod8.z.array(import_zod8.z.json()).optional(),
+  runId: import_zod8.z.string().optional(),
+  checkpointId: import_zod8.z.string().optional(),
+  interrupt: import_zod8.z.object({
+    requestId: import_zod8.z.string(),
+    kind: import_zod8.z.enum(["text", "choice", "multi-choice", "custom"]),
+    question: import_zod8.z.string().optional(),
+    schemaId: import_zod8.z.string().optional(),
+    schemaVersion: import_zod8.z.string().optional(),
+    options: import_zod8.z.array(
+      import_zod8.z.object({
+        id: import_zod8.z.string(),
+        label: import_zod8.z.string(),
+        description: import_zod8.z.string().optional()
       }).strict()
     ),
-    request: import_zod7.z.json().optional()
+    request: import_zod8.z.json().optional()
   }).strict().optional()
 }).strict().superRefine((value, ctx) => {
   if (value.type === "interrupt" !== Boolean(value.interrupt))
@@ -10863,195 +10957,211 @@ var EvalObservationSchema = import_zod7.z.object({
       message: "Interrupt observations require exactly one interrupt."
     });
 });
-var status = import_zod7.z.enum(["passed", "failed", "error", "cancelled", "ungraded"]);
-var issue = import_zod7.z.object({
-  phase: import_zod7.z.enum([
+var status = import_zod8.z.enum(["passed", "failed", "error", "cancelled", "ungraded"]);
+var EvalIssueSchema = import_zod8.z.object({
+  failure: failureDescriptor.optional(),
+  diagnosticId: import_zod8.z.uuid().optional(),
+  phase: import_zod8.z.enum([
     "params",
     "setup",
     "execute",
     "responder",
     "reference",
     "grading",
+    "verification",
     "cleanup",
     "reporting"
   ]),
-  code: import_zod7.z.string(),
-  message: import_zod7.z.string()
+  code: import_zod8.z.string(),
+  message: import_zod8.z.string()
 }).strict();
-var EvalJudgeUsageSchema = import_zod7.z.object({
-  provider: import_zod7.z.string().min(1),
-  model: import_zod7.z.string().min(1),
-  occurredAt: import_zod7.z.iso.datetime(),
-  usage: import_zod7.z.object({
-    input: import_zod7.z.number().finite().nonnegative().optional(),
-    output: import_zod7.z.number().finite().nonnegative().optional(),
-    total: import_zod7.z.number().finite().nonnegative().optional(),
-    reasoning: import_zod7.z.number().finite().nonnegative().optional(),
-    cacheRead: import_zod7.z.number().finite().nonnegative().optional(),
-    cacheWrite: import_zod7.z.number().finite().nonnegative().optional(),
-    cacheWrite1h: import_zod7.z.number().finite().nonnegative().optional(),
-    outputIncludesReasoning: import_zod7.z.boolean().optional(),
-    inputIncludesCacheRead: import_zod7.z.boolean().optional(),
-    inputIncludesCacheWrite: import_zod7.z.boolean().optional()
+var issue = EvalIssueSchema;
+var EvalJudgeUsageSchema = import_zod8.z.object({
+  provider: import_zod8.z.string().min(1),
+  model: import_zod8.z.string().min(1),
+  occurredAt: import_zod8.z.iso.datetime(),
+  usage: import_zod8.z.object({
+    input: import_zod8.z.number().finite().nonnegative().optional(),
+    output: import_zod8.z.number().finite().nonnegative().optional(),
+    total: import_zod8.z.number().finite().nonnegative().optional(),
+    reasoning: import_zod8.z.number().finite().nonnegative().optional(),
+    cacheRead: import_zod8.z.number().finite().nonnegative().optional(),
+    cacheWrite: import_zod8.z.number().finite().nonnegative().optional(),
+    cacheWrite1h: import_zod8.z.number().finite().nonnegative().optional(),
+    outputIncludesReasoning: import_zod8.z.boolean().optional(),
+    inputIncludesCacheRead: import_zod8.z.boolean().optional(),
+    inputIncludesCacheWrite: import_zod8.z.boolean().optional()
   }).strict().optional(),
-  pricingContext: import_zod7.z.object({
-    serviceTier: import_zod7.z.string().min(1).max(100).optional(),
-    inferenceGeo: import_zod7.z.string().min(1).max(100).optional(),
-    speed: import_zod7.z.string().min(1).max(100).optional()
+  pricingContext: import_zod8.z.object({
+    serviceTier: import_zod8.z.string().min(1).max(100).optional(),
+    inferenceGeo: import_zod8.z.string().min(1).max(100).optional(),
+    speed: import_zod8.z.string().min(1).max(100).optional()
   }).strict().optional(),
-  pricing: import_zod7.z.object({
-    source: import_zod7.z.literal("provider"),
-    currency: import_zod7.z.literal("USD"),
-    actualCostMicros: import_zod7.z.number().int().nonnegative().safe()
+  pricing: import_zod8.z.object({
+    source: import_zod8.z.literal("provider"),
+    currency: import_zod8.z.literal("USD"),
+    actualCostMicros: import_zod8.z.number().int().nonnegative().safe()
   }).strict().optional()
 }).strict();
-var EvalStepResultSchema = import_zod7.z.object({
-  evidence: import_zod7.z.object({
-    version: import_zod7.z.literal("compact-v1"),
-    history: import_zod7.z.boolean(),
+var EvalStepResultSchema = import_zod8.z.object({
+  evidence: import_zod8.z.object({
+    version: import_zod8.z.literal("compact-v1"),
+    history: import_zod8.z.boolean(),
     observation: EvalObservationSchema
   }).strict().optional(),
-  index: import_zod7.z.number().int().nonnegative(),
-  input: import_zod7.z.union([
-    import_zod7.z.object({ message: import_zod7.z.string() }).strict(),
-    import_zod7.z.object({ resume: EvalResumeResponseSchema }).strict()
+  index: import_zod8.z.number().int().nonnegative(),
+  input: import_zod8.z.union([
+    import_zod8.z.object({ message: import_zod8.z.string() }).strict(),
+    import_zod8.z.object({ resume: EvalResumeResponseSchema }).strict()
   ]),
   expectation,
   observation: EvalObservationSchema,
-  reference: import_zod7.z.json().optional(),
+  reference: import_zod8.z.json().optional(),
   status,
-  reason: import_zod7.z.string().optional(),
-  judgeCalls: import_zod7.z.number().int().nonnegative().optional(),
-  judgeUsage: import_zod7.z.array(EvalJudgeUsageSchema).optional(),
-  criteria: import_zod7.z.array(
-    EvalVerdictSchema.extend({ id: import_zod7.z.string(), text: import_zod7.z.string() })
+  reason: import_zod8.z.string().optional(),
+  judgeCalls: import_zod8.z.number().int().nonnegative().optional(),
+  judgeUsage: import_zod8.z.array(EvalJudgeUsageSchema).optional(),
+  criteria: import_zod8.z.array(
+    EvalVerdictSchema.extend({ id: import_zod8.z.string(), text: import_zod8.z.string() })
   )
 }).strict();
-var runtimeExecution = import_zod7.z.object({ sessionId: import_zod7.z.string().min(1), runId: import_zod7.z.string().min(1) }).strict();
-var EvalCaseResultSchema = import_zod7.z.object({
+var runtimeExecution = import_zod8.z.object({ sessionId: import_zod8.z.string().min(1), runId: import_zod8.z.string().min(1) }).strict();
+var EvalCaseResultSchema = import_zod8.z.object({
+  verification: EvalVerdictSchema.optional(),
+  phaseDurationsMs: import_zod8.z.partialRecord(issue.shape.phase, import_zod8.z.number().finite().nonnegative()).optional(),
   caseId: id2,
-  repetition: import_zod7.z.number().int().positive(),
-  attemptId: import_zod7.z.uuid().optional(),
-  runtimeExecutions: import_zod7.z.array(runtimeExecution).optional(),
-  sessionId: import_zod7.z.string(),
+  repetition: import_zod8.z.number().int().positive(),
+  attemptId: import_zod8.z.uuid().optional(),
+  runtimeExecutions: import_zod8.z.array(runtimeExecution).optional(),
+  sessionId: import_zod8.z.string(),
   status,
-  durationMs: import_zod7.z.number().nonnegative(),
-  steps: import_zod7.z.array(EvalStepResultSchema),
-  errors: import_zod7.z.array(issue)
+  durationMs: import_zod8.z.number().nonnegative(),
+  steps: import_zod8.z.array(EvalStepResultSchema),
+  errors: import_zod8.z.array(issue)
 }).strict();
-var EvalJudgeIdentitySchema = import_zod7.z.object({
+var EvalJudgeIdentitySchema = import_zod8.z.object({
   id: id2,
   version: id2,
-  location: import_zod7.z.enum(["app", "studio"]).optional()
+  location: import_zod8.z.enum(["app", "studio"]).optional()
 }).strict();
 var judge = EvalJudgeIdentitySchema;
-var StudioEvalJudgeRequestSchema = import_zod7.z.object({
+var StudioEvalJudgeRequestSchema = import_zod8.z.object({
   environment: id2,
   judge: EvalJudgeIdentitySchema,
-  criterion: import_zod7.z.object({ id: id2, text: import_zod7.z.string().trim().min(1).max(16384) }).strict(),
+  criterion: import_zod8.z.object({ id: id2, text: import_zod8.z.string().trim().min(1).max(16384) }).strict(),
   input: EvalStepResultSchema.shape.input,
   observation: EvalObservationSchema,
-  reference: import_zod7.z.json().optional(),
-  conversation: import_zod7.z.array(EvalStepResultSchema).max(100)
+  reference: import_zod8.z.json().optional(),
+  conversation: import_zod8.z.array(EvalStepResultSchema).max(100)
 }).strict();
-var StudioEvalJudgeResponseSchema = import_zod7.z.object({
+var StudioEvalJudgeResponseSchema = import_zod8.z.object({
   judge: EvalJudgeIdentitySchema,
   verdict: EvalVerdictSchema,
-  usage: import_zod7.z.array(EvalJudgeUsageSchema).optional()
+  usage: import_zod8.z.array(EvalJudgeUsageSchema).optional()
 }).strict();
-var EvalRunResultSchema = import_zod7.z.object({
-  id: import_zod7.z.uuid(),
+var EvalRunResultSchema = import_zod8.z.object({
+  id: import_zod8.z.uuid(),
   suiteId: id2,
-  suiteRevision: import_zod7.z.string().regex(/^[a-f0-9]{64}$/),
+  suiteRevision: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
   suite: EvalSuiteSchema,
   judge: judge.optional(),
-  startedAt: import_zod7.z.iso.datetime(),
-  durationMs: import_zod7.z.number().nonnegative(),
+  startedAt: import_zod8.z.iso.datetime(),
+  durationMs: import_zod8.z.number().nonnegative(),
   status,
-  counts: import_zod7.z.object({
-    passed: import_zod7.z.number().int().nonnegative(),
-    failed: import_zod7.z.number().int().nonnegative(),
-    error: import_zod7.z.number().int().nonnegative(),
-    cancelled: import_zod7.z.number().int().nonnegative(),
-    ungraded: import_zod7.z.number().int().nonnegative().optional()
+  counts: import_zod8.z.object({
+    passed: import_zod8.z.number().int().nonnegative(),
+    failed: import_zod8.z.number().int().nonnegative(),
+    error: import_zod8.z.number().int().nonnegative(),
+    cancelled: import_zod8.z.number().int().nonnegative(),
+    ungraded: import_zod8.z.number().int().nonnegative().optional()
   }).strict(),
-  cases: import_zod7.z.array(EvalCaseResultSchema),
-  errors: import_zod7.z.array(issue)
+  cases: import_zod8.z.array(EvalCaseResultSchema),
+  errors: import_zod8.z.array(issue)
 }).strict();
-var EvalManifestSchema = import_zod7.z.object({
-  schemaVersion: import_zod7.z.literal(1),
-  studioJudging: import_zod7.z.literal(true).optional(),
-  promptContracts: import_zod7.z.array(
-    import_zod7.z.object({
-      id: import_zod7.z.string(),
-      format: import_zod7.z.enum(["system-user", "chat"]),
-      variablesSchema: import_zod7.z.record(import_zod7.z.string(), import_zod7.z.unknown()),
-      configSchema: import_zod7.z.record(import_zod7.z.string(), import_zod7.z.unknown())
+var EvalManifestSchema = import_zod8.z.object({
+  schemaVersion: import_zod8.z.literal(1),
+  studioJudging: import_zod8.z.literal(true).optional(),
+  promptContracts: import_zod8.z.array(
+    import_zod8.z.object({
+      id: import_zod8.z.string(),
+      format: import_zod8.z.enum(["system-user", "chat"]),
+      variablesSchema: import_zod8.z.record(import_zod8.z.string(), import_zod8.z.unknown()),
+      configSchema: import_zod8.z.record(import_zod8.z.string(), import_zod8.z.unknown())
     }).strict()
   ).optional(),
-  attemptScheduling: import_zod7.z.literal(true).optional(),
-  suites: import_zod7.z.array(EvalSuiteSchema),
-  responders: import_zod7.z.array(
-    import_zod7.z.object({
+  attemptScheduling: import_zod8.z.literal(true).optional(),
+  suites: import_zod8.z.array(EvalSuiteSchema),
+  responders: import_zod8.z.array(
+    import_zod8.z.object({
       name: id2,
       schemaId: id2.optional(),
       schemaVersion: id2.optional()
     }).strict()
   ),
-  references: import_zod7.z.array(id2),
-  paramsSchema: import_zod7.z.record(import_zod7.z.string(), import_zod7.z.unknown()).optional(),
+  references: import_zod8.z.array(id2),
+  paramsSchema: import_zod8.z.record(import_zod8.z.string(), import_zod8.z.unknown()).optional(),
   judge: judge.optional()
 }).strict();
-var EvalProgressSchema = import_zod7.z.discriminatedUnion("type", [
-  import_zod7.z.object({
-    type: import_zod7.z.literal("case-runtime-associated"),
+var EvalProgressSchema = import_zod8.z.discriminatedUnion("type", [
+  import_zod8.z.object({
+    type: import_zod8.z.literal("case-dispatch"),
     caseId: id2,
-    repetition: import_zod7.z.number().int().positive(),
-    attemptId: import_zod7.z.uuid(),
+    repetition: import_zod8.z.number().int().positive(),
+    status: import_zod8.z.enum(["queued", "retrying", "reconnecting"]),
+    retryAt: import_zod8.z.iso.datetime().optional(),
+    httpStatus: import_zod8.z.number().int().min(100).max(599).optional(),
+    executionId: import_zod8.z.uuid().optional(),
+    generation: import_zod8.z.uuid().optional()
+  }).strict(),
+  import_zod8.z.object({
+    type: import_zod8.z.literal("case-runtime-associated"),
+    caseId: id2,
+    repetition: import_zod8.z.number().int().positive(),
+    attemptId: import_zod8.z.uuid(),
     execution: runtimeExecution
   }).strict(),
-  import_zod7.z.object({
-    type: import_zod7.z.literal("run-started"),
-    caseIds: import_zod7.z.array(id2).min(1),
-    repetitions: import_zod7.z.number().int().positive(),
-    concurrency: import_zod7.z.number().int().positive()
+  import_zod8.z.object({
+    type: import_zod8.z.literal("run-started"),
+    caseIds: import_zod8.z.array(id2).min(1),
+    repetitions: import_zod8.z.number().int().positive(),
+    concurrency: import_zod8.z.number().int().positive()
   }).strict(),
-  import_zod7.z.object({
-    type: import_zod7.z.literal("case-progress"),
+  import_zod8.z.object({
+    type: import_zod8.z.literal("case-progress"),
     caseId: id2,
-    repetition: import_zod7.z.number().int().positive(),
+    repetition: import_zod8.z.number().int().positive(),
     phase: issue.shape.phase,
-    stepIndex: import_zod7.z.number().int().nonnegative().optional()
+    stepIndex: import_zod8.z.number().int().nonnegative().optional()
   }).strict(),
-  import_zod7.z.object({
-    type: import_zod7.z.literal("case-started"),
-    attemptId: import_zod7.z.uuid().optional(),
+  import_zod8.z.object({
+    type: import_zod8.z.literal("case-started"),
+    attemptId: import_zod8.z.uuid().optional(),
     caseId: id2,
-    repetition: import_zod7.z.number().int().positive(),
-    sessionId: import_zod7.z.string()
+    repetition: import_zod8.z.number().int().positive(),
+    sessionId: import_zod8.z.string()
   }).strict(),
-  import_zod7.z.object({
-    type: import_zod7.z.literal("step-completed"),
+  import_zod8.z.object({
+    type: import_zod8.z.literal("step-completed"),
     caseId: id2,
-    repetition: import_zod7.z.number().int().positive(),
+    repetition: import_zod8.z.number().int().positive(),
     step: EvalStepResultSchema
   }).strict(),
-  import_zod7.z.object({ type: import_zod7.z.literal("case-completed"), result: EvalCaseResultSchema }).strict()
+  import_zod8.z.object({ type: import_zod8.z.literal("case-completed"), result: EvalCaseResultSchema }).strict()
 ]);
-var EvalRemoteRunRequestSchema = import_zod7.z.object({
+var EvalRemoteRunRequestSchema = import_zod8.z.object({
   promptSnapshot: import_browser2.PromptSnapshotSchema.optional(),
-  promptGroupName: import_zod7.z.string().optional(),
+  promptGroupName: import_zod8.z.string().optional(),
   suiteId: id2,
-  suiteRevision: import_zod7.z.string().regex(/^[a-f0-9]{64}$/),
-  grading: import_zod7.z.enum(["app", "studio"]).optional(),
+  suiteRevision: import_zod8.z.string().regex(/^[a-f0-9]{64}$/),
+  grading: import_zod8.z.enum(["app", "studio"]).optional(),
   judge: EvalJudgeIdentitySchema.optional(),
-  caseIds: import_zod7.z.array(id2).min(1).max(100).optional(),
-  repetitions: import_zod7.z.number().int().min(1).max(20).default(1),
-  concurrency: import_zod7.z.number().int().min(1).max(20).default(1),
-  attempt: import_zod7.z.object({
+  caseIds: import_zod8.z.array(id2).min(1).max(100).optional(),
+  repetitions: import_zod8.z.number().int().min(1).max(20).default(1),
+  concurrency: import_zod8.z.number().int().min(1).max(20).default(1),
+  attempt: import_zod8.z.object({
     caseId: id2,
-    repetition: import_zod7.z.number().int().min(1).max(20),
-    evaluationId: import_zod7.z.uuid()
+    repetition: import_zod8.z.number().int().min(1).max(20),
+    evaluationId: import_zod8.z.uuid()
   }).strict().optional()
 }).strict();
 var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
@@ -11061,22 +11171,40 @@ var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
   promptGroupName: true,
   attempt: true
 }).extend({
-  promptSelection: import_zod7.z.discriminatedUnion("type", [
-    import_zod7.z.object({ type: import_zod7.z.literal("live") }).strict(),
-    import_zod7.z.object({
-      type: import_zod7.z.literal("single"),
-      id: import_zod7.z.string(),
-      version: import_zod7.z.number().int().positive()
+  promptSelection: import_zod8.z.discriminatedUnion("type", [
+    import_zod8.z.object({ type: import_zod8.z.literal("live") }).strict(),
+    import_zod8.z.object({
+      type: import_zod8.z.literal("single"),
+      id: import_zod8.z.string(),
+      version: import_zod8.z.number().int().positive()
     }).strict(),
-    import_zod7.z.object({ type: import_zod7.z.literal("group"), groupId: import_zod7.z.uuid() }).strict()
+    import_zod8.z.object({ type: import_zod8.z.literal("group"), groupId: import_zod8.z.uuid() }).strict()
   ]).optional(),
-  targetId: import_zod7.z.string().min(1).max(128),
-  judge: import_zod7.z.enum(["studio", "app"]).default("studio")
+  targetId: import_zod8.z.string().min(1).max(128),
+  judge: import_zod8.z.enum(["studio", "app"]).default("studio")
 });
-var EvalWireEventSchema = import_zod7.z.discriminatedUnion("type", [
-  import_zod7.z.object({ type: import_zod7.z.literal("progress"), event: EvalProgressSchema }).strict(),
-  import_zod7.z.object({ type: import_zod7.z.literal("result"), result: EvalRunResultSchema }).strict(),
-  import_zod7.z.object({ type: import_zod7.z.literal("error"), message: import_zod7.z.string() }).strict()
+var EvalWireEventSchema = import_zod8.z.discriminatedUnion("type", [
+  import_zod8.z.object({
+    type: import_zod8.z.literal("execution-status"),
+    state: import_zod8.z.enum(["queued", "running", "completed", "cancelled", "unknown"])
+  }).strict(),
+  import_zod8.z.object({
+    type: import_zod8.z.literal("progress"),
+    event: EvalProgressSchema,
+    sequence: import_zod8.z.number().int().positive().optional()
+  }).strict(),
+  import_zod8.z.object({
+    type: import_zod8.z.literal("result"),
+    result: EvalRunResultSchema,
+    sequence: import_zod8.z.number().int().positive().optional()
+  }).strict(),
+  import_zod8.z.object({
+    type: import_zod8.z.literal("error"),
+    message: import_zod8.z.string(),
+    failure: failureDescriptor.optional(),
+    diagnosticId: import_zod8.z.uuid().optional(),
+    sequence: import_zod8.z.number().int().positive().optional()
+  }).strict()
 ]);
 var EvalConfigurationError = class extends Error {
   name = "EvalConfigurationError";
@@ -11107,7 +11235,41 @@ function parseEvalSuite(value) {
 var import_node_crypto5 = require("crypto");
 var import_eval_attribution2 = require("@kortyx/core/eval-attribution");
 var import_usage_telemetry = require("@kortyx/usage-telemetry");
-var import_zod8 = require("zod");
+var import_zod10 = require("zod");
+
+// src/evals/errors.ts
+var import_errors9 = require("@kortyx/core/errors");
+var import_zod9 = require("zod");
+var messages = {
+  params: "Case parameters failed validation.",
+  setup: "Case setup failed.",
+  execute: "Agent execution failed or returned an invalid observation.",
+  responder: "Interrupt response failed or was incompatible with the waiting contract.",
+  reference: "Reference resolution failed or returned non-JSON data.",
+  grading: "Grading failed or returned an invalid verdict.",
+  verification: "Case verification failed or returned an invalid verdict.",
+  cleanup: "Case cleanup failed.",
+  reporting: "Progress reporting failed."
+};
+function createEvalIssue(phase, error, options = {}) {
+  const control = (0, import_errors9.isControlFlowError)(error);
+  const declared = !control && ((0, import_errors9.isFailureDescriptor)(error) || (0, import_errors9.isFailureDescriptor)((0, import_errors9.errorProperty)(error, "failure")));
+  const failure = (0, import_errors9.serializeFailure)(control ? void 0 : error);
+  let diagnosticId = options.diagnosticId ?? (0, import_errors9.errorProperty)(error, "diagnosticId");
+  if (!control && !diagnosticId && options.reportError) {
+    try {
+      diagnosticId = options.reportError(error, options.correlation);
+    } catch {
+    }
+  }
+  return {
+    phase,
+    code: declared ? failure.code : options.code ?? "EVAL_HOOK_FAILED",
+    message: declared ? failure.message : messages[phase],
+    failure,
+    ...typeof diagnosticId === "string" && import_zod9.z.uuid().safeParse(diagnosticId).success ? { diagnosticId } : {}
+  };
+}
 
 // src/evals/evidence.ts
 var object = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
@@ -11234,10 +11396,12 @@ function canonical(value) {
 var getEvalSuiteRevision = (suite) => (0, import_node_crypto4.createHash)("sha256").update(canonical(suite)).digest("hex");
 
 // src/evals/stream.ts
+var import_errors11 = require("@kortyx/core/errors");
 var import_prompts = require("@kortyx/prompts");
 var import_stream3 = require("@kortyx/stream");
 
 // src/evals/stream-evidence.ts
+var import_errors10 = require("@kortyx/core/errors");
 function toEvalJson(value) {
   return JSON.parse(JSON.stringify(value));
 }
@@ -11272,7 +11436,9 @@ function captureEvalEvent(chunk) {
         ...source,
         tool: chunk.tool,
         toolCallId: chunk.toolCallId,
-        message: chunk.message
+        message: chunk.message,
+        ...chunk.failure ? { failure: (0, import_errors10.serializeFailure)(chunk.failure) } : {},
+        ...chunk.diagnosticId ? { diagnosticId: chunk.diagnosticId } : {}
       });
     case "tool-result":
       return toEvalJson({
@@ -11348,7 +11514,12 @@ function captureEvalEvent(chunk) {
     case "transition":
       return { type: chunk.type, transitionTo: chunk.transitionTo };
     case "error":
-      return { type: chunk.type, message: chunk.message };
+      return toEvalJson({
+        type: chunk.type,
+        message: chunk.failure ? (0, import_errors10.serializeFailure)(chunk.failure).message : chunk.message,
+        ...chunk.failure ? { failure: (0, import_errors10.serializeFailure)(chunk.failure) } : {},
+        ...chunk.diagnosticId ? { diagnosticId: chunk.diagnosticId } : {}
+      });
     case "cancelled":
       return toEvalJson({
         type: chunk.type,
@@ -11402,14 +11573,14 @@ async function executeEvalChat(args) {
     return { observation: { type: "cancelled", text: "", structured: [] } };
   }
   const pending = waiting?.[0];
-  const messages = [...args.messages ?? args.history];
+  const messages2 = [...args.messages ?? args.history];
   if (args.command.type === "message")
-    messages.push({ role: "user", content: args.command.message });
+    messages2.push({ role: "user", content: args.command.message });
   else {
     if (!pending?.resumeToken || !pending.requestId)
       throw new Error("No waiting interrupt to resume.");
     const response = args.command.type === "cancel" ? { type: "cancel" } : args.command.response;
-    messages.push({
+    messages2.push({
       role: "user",
       content: response.type === "text" ? response.text : response.type === "select" ? response.ids.join(", ") : response.type === "value" ? JSON.stringify(response.value) : "",
       metadata: {
@@ -11431,6 +11602,7 @@ async function executeEvalChat(args) {
   const events = [];
   let previousChunk;
   let error = false;
+  let terminalFailure;
   let cancelled = false;
   let sawDone = false;
   let completedWithData = false;
@@ -11452,7 +11624,7 @@ async function executeEvalChat(args) {
     ...args.context ? { context: args.context } : {}
   };
   try {
-    for await (const chunk of await args.agent.streamChat(messages, options)) {
+    for await (const chunk of await args.agent.streamChat(messages2, options)) {
       accumulator.apply(chunk);
       const event = captureEvalEvent(chunk);
       if (event !== void 0) {
@@ -11466,7 +11638,13 @@ async function executeEvalChat(args) {
       }
       previousChunk = chunk;
       if (chunk.type === "interrupt") interrupts.set(chunk.requestId, chunk);
-      if (chunk.type === "error") error = true;
+      if (chunk.type === "error") {
+        error = true;
+        terminalFailure = {
+          ...chunk.failure ? { failure: (0, import_errors11.serializeFailure)(chunk.failure) } : {},
+          ...chunk.diagnosticId ? { diagnosticId: chunk.diagnosticId } : {}
+        };
+      }
       if (chunk.type === "cancelled") cancelled = true;
       if (chunk.type === "trace") runId = chunk.runId;
       if (chunk.type === "done") {
@@ -11486,6 +11664,7 @@ async function executeEvalChat(args) {
   const observedRunId = finalized?.runId ?? runId;
   const type = error || finalized?.status === "failed" || interrupts.size > 1 ? "error" : cancelled || finalized?.status === "cancelled" ? "cancelled" : interrupts.size ? "interrupt" : "answer";
   const observation = {
+    ...terminalFailure,
     type,
     text: message.content,
     structured: message.contentPieces.filter((item) => item.type === "structured").map((item) => toEvalJson(item.data)),
@@ -11527,21 +11706,6 @@ var positive = (value, label) => {
   return value;
 };
 var handlerRef = (value) => value !== null && typeof value === "object" && !Array.isArray(value) && typeof value.using === "string" && Object.keys(value).every((key) => key === "using" || key === "params");
-var phases = {
-  params: "Case parameters failed validation.",
-  setup: "Case setup failed.",
-  execute: "Agent execution failed or returned an invalid observation.",
-  responder: "Interrupt response failed or was incompatible with the waiting contract.",
-  reference: "Reference resolution failed or returned non-JSON data.",
-  grading: "Grading failed or returned an invalid verdict.",
-  cleanup: "Case cleanup failed.",
-  reporting: "Progress reporting failed."
-};
-var failure = (phase, code = "EVAL_HOOK_FAILED") => ({
-  phase,
-  code,
-  message: phases[phase]
-});
 var statusFor = (counts, errors) => errors.length || counts.error ? "error" : counts.cancelled ? "cancelled" : counts.failed ? "failed" : counts.ungraded ? "ungraded" : "passed";
 function createEvals(options) {
   const defaultEvidence = EvalEvidencePolicySchema.parse(
@@ -11591,11 +11755,25 @@ function createEvals(options) {
       options.defaults?.caseTimeoutMs ?? 6e4,
       "caseTimeoutMs"
     ),
+    setupTimeoutMs: positive(
+      options.defaults?.setupTimeoutMs ?? options.defaults?.caseTimeoutMs ?? 6e4,
+      "setupTimeoutMs"
+    ),
+    verificationTimeoutMs: positive(
+      options.defaults?.verificationTimeoutMs ?? options.defaults?.caseTimeoutMs ?? 6e4,
+      "verificationTimeoutMs"
+    ),
     cleanupTimeoutMs: positive(
       options.defaults?.cleanupTimeoutMs ?? 1e4,
       "cleanupTimeoutMs"
     )
   };
+  const reportError = options.reportError ?? options.agent.reportError?.bind(options.agent);
+  const issueFor = (phase, error, correlation, code) => createEvalIssue(phase, error, {
+    reportError,
+    correlation,
+    ...code ? { code } : {}
+  });
   return {
     listSuites() {
       return clone(suites);
@@ -11616,7 +11794,7 @@ function createEvals(options) {
         })),
         references: Object.keys(references),
         ...options.paramsSchema ? {
-          paramsSchema: import_zod8.z.toJSONSchema(options.paramsSchema, {
+          paramsSchema: import_zod10.z.toJSONSchema(options.paramsSchema, {
             io: "input"
           })
         } : {},
@@ -11695,19 +11873,24 @@ function createEvals(options) {
             void 0,
             () => args.onProgress(clone(event))
           );
-        } catch {
+        } catch (error) {
           reporting = false;
-          result.errors.push(failure("reporting"));
+          result.errors.push(issueFor("reporting", error));
         }
       };
       const attempt = async (item, repetition) => {
         const started = Date.now();
         const controller = new AbortController();
         let timedOut = false;
-        const timer = setTimeout(() => {
-          timedOut = true;
-          controller.abort();
-        }, defaults.caseTimeoutMs);
+        let timer;
+        const deadline = (milliseconds) => {
+          clearTimeout(timer);
+          timer = setTimeout(() => {
+            timedOut = true;
+            controller.abort();
+          }, milliseconds);
+        };
+        deadline(defaults.setupTimeoutMs);
         const signal = args.signal ? AbortSignal.any([args.signal, controller.signal]) : controller.signal;
         const attemptId = (0, import_node_crypto5.randomUUID)();
         const runtimeExecutions = [];
@@ -11720,14 +11903,31 @@ function createEvals(options) {
           status: "passed",
           durationMs: 0,
           steps: [],
-          errors: []
+          errors: [],
+          phaseDurationsMs: {}
         };
         let associationProgress = Promise.resolve();
         let acceptingExecutions = true;
         const executeAttempt = async () => {
           let phase = "params";
+          let phaseStarted = Date.now();
+          const recordPhase = () => {
+            const now = Date.now();
+            const durations = attemptResult.phaseDurationsMs;
+            durations[phase] = (durations[phase] ?? 0) + now - phaseStarted;
+            phaseStarted = now;
+          };
+          const phaseIssue = (error, code) => issueFor(
+            phase,
+            error,
+            {
+              correlation: { sessionId: attemptResult.sessionId, attemptId }
+            },
+            code
+          );
           let stepIndex;
           const reportPhase = async (next) => {
+            recordPhase();
             phase = next;
             if (args.includeActivity)
               await emit({
@@ -11800,6 +12000,7 @@ function createEvals(options) {
             const prepared = options.setup ? await options.setup(base) : void 0;
             context = { ...base, prepared };
             signal.throwIfAborted();
+            deadline(defaults.caseTimeoutMs);
             for (const [index, step] of item.steps.entries()) {
               stepIndex = index;
               activeStep = void 0;
@@ -11846,8 +12047,22 @@ function createEvals(options) {
               attemptResult.steps.push(evaluated);
               activeStep = evaluated;
               signal.throwIfAborted();
-              if (observation.type === "error")
-                throw new Error("Agent failed.");
+              if (observation.type === "error") {
+                const issue2 = createEvalIssue("execute", observation.failure, {
+                  ...observation.diagnosticId ? { diagnosticId: observation.diagnosticId } : {}
+                });
+                attemptResult.errors.push(issue2);
+                attemptResult.status = evaluated.status = "error";
+                evaluated.reason = issue2.message;
+                await emit({
+                  type: "step-completed",
+                  caseId: item.id,
+                  repetition,
+                  step: evaluated
+                });
+                activeStepReported = true;
+                break;
+              }
               if (observation.type === "cancelled") {
                 evaluated.status = "cancelled";
                 attemptResult.status = "cancelled";
@@ -11875,7 +12090,7 @@ function createEvals(options) {
                   const reference = step.expect.reference;
                   if (reference !== void 0) {
                     await reportPhase("reference");
-                    evaluated.reference = handlerRef(reference) ? import_zod8.z.json().parse(
+                    evaluated.reference = handlerRef(reference) ? import_zod10.z.json().parse(
                       await references[reference.using]({
                         ...context,
                         observation: clone(observation),
@@ -11943,18 +12158,32 @@ function createEvals(options) {
                 { role: "assistant", content: observation.text }
               );
             }
-          } catch {
+            if (options.verify && attemptResult.status !== "cancelled" && attemptResult.status !== "error") {
+              stepIndex = void 0;
+              deadline(defaults.verificationTimeoutMs);
+              await reportPhase("verification");
+              const verification = EvalVerdictSchema.parse(
+                await options.verify({
+                  ...context,
+                  result: clone(attemptResult)
+                })
+              );
+              signal.throwIfAborted();
+              attemptResult.verification = verification;
+              if (!verification.passed) attemptResult.status = "failed";
+            }
+          } catch (error) {
             if (signal.aborted) {
               attemptResult.status = timedOut ? "error" : "cancelled";
               if (timedOut)
                 attemptResult.errors.push({
-                  phase,
-                  code: "EVAL_TIMEOUT",
-                  message: "Case exceeded its configured time limit."
+                  ...phaseIssue(error),
+                  code: phase === "params" || phase === "setup" ? "EVAL_SETUP_TIMEOUT" : phase === "verification" ? "EVAL_VERIFICATION_TIMEOUT" : "EVAL_TIMEOUT",
+                  message: phase === "params" || phase === "setup" ? "Case setup exceeded its configured time limit." : phase === "verification" ? "Case verification exceeded its configured time limit." : "Case execution exceeded its configured time limit."
                 });
             } else {
               attemptResult.status = "error";
-              attemptResult.errors.push(failure(phase));
+              attemptResult.errors.push(phaseIssue(error));
             }
             if (activeStep && !activeStepReported) {
               activeStep.status = attemptResult.status;
@@ -11985,12 +12214,21 @@ function createEvals(options) {
                     );
                     if (cancelled.observation.type === "error" || cancelled.continuation !== void 0)
                       attemptResult.errors.push(
-                        failure("cleanup", "EVAL_INTERRUPT_CLEANUP_FAILED")
+                        createEvalIssue(
+                          "cleanup",
+                          cancelled.observation.failure,
+                          {
+                            code: "EVAL_INTERRUPT_CLEANUP_FAILED",
+                            ...cancelled.observation.diagnosticId ? {
+                              diagnosticId: cancelled.observation.diagnosticId
+                            } : {}
+                          }
+                        )
                       );
                     cleanup.signal.throwIfAborted();
-                  } catch {
+                  } catch (error) {
                     attemptResult.errors.push(
-                      failure("cleanup", "EVAL_INTERRUPT_CLEANUP_FAILED")
+                      phaseIssue(error, "EVAL_INTERRUPT_CLEANUP_FAILED")
                     );
                   }
                 }
@@ -12001,8 +12239,8 @@ function createEvals(options) {
                     result: clone(attemptResult)
                   });
                   cleanup.signal.throwIfAborted();
-                } catch {
-                  attemptResult.errors.push(failure("cleanup"));
+                } catch (error) {
+                  attemptResult.errors.push(phaseIssue(error));
                 }
               } finally {
                 clearTimeout(cleanupTimer);
@@ -12010,6 +12248,7 @@ function createEvals(options) {
               if (attemptResult.errors.some((error) => error.phase === "cleanup"))
                 attemptResult.status = "error";
             }
+            recordPhase();
             attemptResult.durationMs = Date.now() - started;
           }
           acceptingExecutions = false;
@@ -12081,23 +12320,23 @@ function defineSuite(suite) {
 }
 
 // src/evals/evaluation-contracts.ts
-var import_zod10 = require("zod");
+var import_zod12 = require("zod");
 
 // src/evals/studio-contracts.ts
 var import_browser3 = require("@kortyx/prompts/browser");
-var import_zod9 = require("zod");
-var StudioEvalTargetsResponseSchema = import_zod9.z.object({
-  canRun: import_zod9.z.boolean(),
+var import_zod11 = require("zod");
+var StudioEvalTargetsResponseSchema = import_zod11.z.object({
+  canRun: import_zod11.z.boolean(),
   studioJudge: EvalJudgeIdentitySchema.nullable().optional(),
-  targets: import_zod9.z.array(
-    import_zod9.z.object({
-      id: import_zod9.z.string(),
-      name: import_zod9.z.string(),
-      environment: import_zod9.z.string(),
-      error: import_zod9.z.string().nullable(),
+  targets: import_zod11.z.array(
+    import_zod11.z.object({
+      id: import_zod11.z.string(),
+      name: import_zod11.z.string(),
+      environment: import_zod11.z.string(),
+      error: import_zod11.z.string().nullable(),
       // Safe discovery diagnostics; optional for compatibility with older APIs.
-      diagnostic: import_zod9.z.object({
-        code: import_zod9.z.enum([
+      diagnostic: import_zod11.z.object({
+        code: import_zod11.z.enum([
           "environment_forbidden",
           "environment_unavailable",
           "endpoint_not_found",
@@ -12106,44 +12345,44 @@ var StudioEvalTargetsResponseSchema = import_zod9.z.object({
           "endpoint_unreachable",
           "manifest_invalid"
         ]),
-        httpStatus: import_zod9.z.number().int().min(100).max(599).optional()
+        httpStatus: import_zod11.z.number().int().min(100).max(599).optional()
       }).nullable().optional(),
       manifest: EvalManifestSchema.nullable(),
-      revisions: import_zod9.z.record(import_zod9.z.string(), import_zod9.z.string())
+      revisions: import_zod11.z.record(import_zod11.z.string(), import_zod11.z.string())
     })
   )
 });
-var EvalCostAmountSchema = import_zod9.z.object({
-  costSummary: import_zod9.z.object({
-    amounts: import_zod9.z.array(
-      import_zod9.z.object({
-        amount: import_zod9.z.number().finite().nonnegative(),
-        currency: import_zod9.z.string().nullable()
+var EvalCostAmountSchema = import_zod11.z.object({
+  costSummary: import_zod11.z.object({
+    amounts: import_zod11.z.array(
+      import_zod11.z.object({
+        amount: import_zod11.z.number().finite().nonnegative(),
+        currency: import_zod11.z.string().nullable()
       })
     ),
-    complete: import_zod9.z.boolean()
+    complete: import_zod11.z.boolean()
   }).optional(),
-  amount: import_zod9.z.number().finite().nonnegative().nullable(),
-  currency: import_zod9.z.string().nullable(),
-  status: import_zod9.z.enum(["complete", "partial", "unavailable"]),
-  calls: import_zod9.z.number().int().nonnegative(),
-  unpricedCalls: import_zod9.z.number().int().nonnegative(),
-  estimated: import_zod9.z.boolean()
+  amount: import_zod11.z.number().finite().nonnegative().nullable(),
+  currency: import_zod11.z.string().nullable(),
+  status: import_zod11.z.enum(["complete", "partial", "unavailable"]),
+  calls: import_zod11.z.number().int().nonnegative(),
+  unpricedCalls: import_zod11.z.number().int().nonnegative(),
+  estimated: import_zod11.z.boolean()
 });
-var EvalCostsSchema = import_zod9.z.object({
+var EvalCostsSchema = import_zod11.z.object({
   workflow: EvalCostAmountSchema,
   judge: EvalCostAmountSchema,
   total: EvalCostAmountSchema
 });
-var StudioEvalRunSummarySchema = import_zod9.z.object({
-  id: import_zod9.z.uuid(),
-  evaluationId: import_zod9.z.uuid().nullable().optional(),
-  targetId: import_zod9.z.string(),
-  targetName: import_zod9.z.string(),
-  environment: import_zod9.z.string(),
-  suiteId: import_zod9.z.string(),
-  suiteRevision: import_zod9.z.string(),
-  status: import_zod9.z.enum([
+var StudioEvalRunSummarySchema = import_zod11.z.object({
+  id: import_zod11.z.uuid(),
+  evaluationId: import_zod11.z.uuid().nullable().optional(),
+  targetId: import_zod11.z.string(),
+  targetName: import_zod11.z.string(),
+  environment: import_zod11.z.string(),
+  suiteId: import_zod11.z.string(),
+  suiteRevision: import_zod11.z.string(),
+  status: import_zod11.z.enum([
     "queued",
     "running",
     "passed",
@@ -12151,49 +12390,49 @@ var StudioEvalRunSummarySchema = import_zod9.z.object({
     "error",
     "cancelled"
   ]),
-  createdAt: import_zod9.z.string(),
-  startedAt: import_zod9.z.string().nullable(),
-  endedAt: import_zod9.z.string().nullable(),
-  error: import_zod9.z.string().nullable(),
-  cancelRequestedAt: import_zod9.z.string().nullable(),
+  createdAt: import_zod11.z.string(),
+  startedAt: import_zod11.z.string().nullable(),
+  endedAt: import_zod11.z.string().nullable(),
+  error: import_zod11.z.string().nullable(),
+  cancelRequestedAt: import_zod11.z.string().nullable(),
   costs: EvalCostsSchema.optional(),
-  counts: import_zod9.z.object({
-    passed: import_zod9.z.number(),
-    failed: import_zod9.z.number(),
-    error: import_zod9.z.number(),
-    cancelled: import_zod9.z.number()
+  counts: import_zod11.z.object({
+    passed: import_zod11.z.number(),
+    failed: import_zod11.z.number(),
+    error: import_zod11.z.number(),
+    cancelled: import_zod11.z.number()
   }).nullable().optional(),
-  suiteName: import_zod9.z.string().nullable().optional(),
-  totalAttempts: import_zod9.z.number().int().nonnegative().optional(),
-  completedAttempts: import_zod9.z.number().int().nonnegative().optional(),
-  phase: import_zod9.z.string().optional()
+  suiteName: import_zod11.z.string().nullable().optional(),
+  totalAttempts: import_zod11.z.number().int().nonnegative().optional(),
+  completedAttempts: import_zod11.z.number().int().nonnegative().optional(),
+  phase: import_zod11.z.string().optional()
 });
-var StudioEvalHistorySchema = import_zod9.z.object({
-  runs: import_zod9.z.array(StudioEvalRunSummarySchema)
+var StudioEvalHistorySchema = import_zod11.z.object({
+  runs: import_zod11.z.array(StudioEvalRunSummarySchema)
 });
-var StudioEvalDetailSchema = import_zod9.z.object({
+var StudioEvalDetailSchema = import_zod11.z.object({
   run: StudioEvalRunSummarySchema.extend({
-    caseCosts: import_zod9.z.record(import_zod9.z.string(), EvalCostsSchema).optional(),
-    request: import_zod9.z.object({
-      caseIds: import_zod9.z.array(import_zod9.z.string()).optional(),
-      grading: import_zod9.z.enum(["app", "studio"]).optional(),
+    caseCosts: import_zod11.z.record(import_zod11.z.string(), EvalCostsSchema).optional(),
+    request: import_zod11.z.object({
+      caseIds: import_zod11.z.array(import_zod11.z.string()).optional(),
+      grading: import_zod11.z.enum(["app", "studio"]).optional(),
       judge: EvalJudgeIdentitySchema.optional(),
-      repetitions: import_zod9.z.number().int().positive(),
-      concurrency: import_zod9.z.number().optional(),
+      repetitions: import_zod11.z.number().int().positive(),
+      concurrency: import_zod11.z.number().optional(),
       promptSnapshot: import_browser3.PromptSnapshotSchema.optional(),
-      promptGroupName: import_zod9.z.string().optional()
+      promptGroupName: import_zod11.z.string().optional()
     }).optional(),
     suite: EvalSuiteSchema,
     result: EvalRunResultSchema.nullable(),
-    events: import_zod9.z.array(import_zod9.z.object({ id: import_zod9.z.number(), event: EvalProgressSchema }))
+    events: import_zod11.z.array(import_zod11.z.object({ id: import_zod11.z.number(), event: EvalProgressSchema }))
   })
 });
 
 // src/evals/evaluation-contracts.ts
-var EvaluationMetadataSchema = import_zod10.z.object({
-  source: import_zod10.z.enum(["manual", "deployment", "schedule", "ci"]).default("manual"),
-  commit: import_zod10.z.string().trim().min(1).max(100).optional(),
-  deploymentUrl: import_zod10.z.url().refine((value) => {
+var EvaluationMetadataSchema = import_zod12.z.object({
+  source: import_zod12.z.enum(["manual", "deployment", "schedule", "ci"]).default("manual"),
+  commit: import_zod12.z.string().trim().min(1).max(100).optional(),
+  deploymentUrl: import_zod12.z.url().refine((value) => {
     try {
       const url = new URL(value);
       return ["https:", "http:"].includes(url.protocol) && !url.username && !url.password;
@@ -12202,67 +12441,67 @@ var EvaluationMetadataSchema = import_zod10.z.object({
     }
   }, "Expected an HTTP(S) URL without credentials").optional()
 }).strict();
-var StudioEvaluationStartRequestSchema = import_zod10.z.object({
-  targetId: import_zod10.z.string().trim().min(1),
-  name: import_zod10.z.string().trim().min(1).max(200).optional(),
+var StudioEvaluationStartRequestSchema = import_zod12.z.object({
+  targetId: import_zod12.z.string().trim().min(1),
+  name: import_zod12.z.string().trim().min(1).max(200).optional(),
   promptSelection: StudioEvalStartRequestSchema.shape.promptSelection,
-  selection: import_zod10.z.enum(["all", "selected"]),
-  suites: import_zod10.z.array(
-    import_zod10.z.object({
-      suiteId: import_zod10.z.string().min(1),
-      suiteRevision: import_zod10.z.string().regex(/^[a-f0-9]{64}$/),
-      caseIds: import_zod10.z.array(import_zod10.z.string().min(1)).min(1).max(100).optional()
+  selection: import_zod12.z.enum(["all", "selected"]),
+  suites: import_zod12.z.array(
+    import_zod12.z.object({
+      suiteId: import_zod12.z.string().min(1),
+      suiteRevision: import_zod12.z.string().regex(/^[a-f0-9]{64}$/),
+      caseIds: import_zod12.z.array(import_zod12.z.string().min(1)).min(1).max(100).optional()
     }).strict()
   ).min(1).max(100),
-  judge: import_zod10.z.enum(["studio", "app"]).default("studio"),
-  repetitions: import_zod10.z.number().int().min(1).max(20).default(1),
-  concurrency: import_zod10.z.number().int().min(1).max(20).default(1),
+  judge: import_zod12.z.enum(["studio", "app"]).default("studio"),
+  repetitions: import_zod12.z.number().int().min(1).max(20).default(1),
+  concurrency: import_zod12.z.number().int().min(1).max(20).default(1),
   metadata: EvaluationMetadataSchema.default({ source: "manual" }),
-  idempotencyKey: import_zod10.z.string().trim().min(1).max(200).optional()
+  idempotencyKey: import_zod12.z.string().trim().min(1).max(200).optional()
 }).strict();
-var StudioEvaluationSummarySchema = import_zod10.z.object({
-  id: import_zod10.z.uuid(),
-  name: import_zod10.z.string(),
-  targetId: import_zod10.z.string(),
-  targetName: import_zod10.z.string(),
-  environment: import_zod10.z.string(),
-  selection: import_zod10.z.enum(["all", "selected"]),
+var StudioEvaluationSummarySchema = import_zod12.z.object({
+  id: import_zod12.z.uuid(),
+  name: import_zod12.z.string(),
+  targetId: import_zod12.z.string(),
+  targetName: import_zod12.z.string(),
+  environment: import_zod12.z.string(),
+  selection: import_zod12.z.enum(["all", "selected"]),
   metadata: EvaluationMetadataSchema,
   judge: EvalJudgeIdentitySchema.optional(),
   status: StudioEvalRunSummarySchema.shape.status,
-  createdAt: import_zod10.z.string(),
-  startedAt: import_zod10.z.string().nullable(),
-  endedAt: import_zod10.z.string().nullable(),
-  cancelRequestedAt: import_zod10.z.string().nullable(),
-  suiteCount: import_zod10.z.number().int().nonnegative(),
-  completedSuites: import_zod10.z.number().int().nonnegative(),
-  totalAttempts: import_zod10.z.number().int().nonnegative(),
-  completedAttempts: import_zod10.z.number().int().nonnegative(),
-  counts: import_zod10.z.object({
-    passed: import_zod10.z.number(),
-    failed: import_zod10.z.number(),
-    error: import_zod10.z.number(),
-    cancelled: import_zod10.z.number()
+  createdAt: import_zod12.z.string(),
+  startedAt: import_zod12.z.string().nullable(),
+  endedAt: import_zod12.z.string().nullable(),
+  cancelRequestedAt: import_zod12.z.string().nullable(),
+  suiteCount: import_zod12.z.number().int().nonnegative(),
+  completedSuites: import_zod12.z.number().int().nonnegative(),
+  totalAttempts: import_zod12.z.number().int().nonnegative(),
+  completedAttempts: import_zod12.z.number().int().nonnegative(),
+  counts: import_zod12.z.object({
+    passed: import_zod12.z.number(),
+    failed: import_zod12.z.number(),
+    error: import_zod12.z.number(),
+    cancelled: import_zod12.z.number()
   }),
   costs: EvalCostsSchema.optional(),
-  legacy: import_zod10.z.boolean().optional()
+  legacy: import_zod12.z.boolean().optional()
 });
-var StudioEvaluationHistorySchema = import_zod10.z.object({
-  runs: import_zod10.z.array(StudioEvaluationSummarySchema)
+var StudioEvaluationHistorySchema = import_zod12.z.object({
+  runs: import_zod12.z.array(StudioEvaluationSummarySchema)
 });
-var StudioEvaluationDetailSchema = import_zod10.z.object({
+var StudioEvaluationDetailSchema = import_zod12.z.object({
   run: StudioEvaluationSummarySchema.extend({
-    suites: import_zod10.z.array(StudioEvalRunSummarySchema)
+    suites: import_zod12.z.array(StudioEvalRunSummarySchema)
   })
 });
-var StudioEvaluationResultsSchema = import_zod10.z.object({
+var StudioEvaluationResultsSchema = import_zod12.z.object({
   run: StudioEvaluationSummarySchema.extend({
-    suites: import_zod10.z.array(StudioEvalDetailSchema.shape.run)
+    suites: import_zod12.z.array(StudioEvalDetailSchema.shape.run)
   })
 });
 
 // src/evals/judge.ts
-var import_zod11 = require("zod");
+var import_zod13 = require("zod");
 function createEvalJudge({
   model,
   id: id3 = `${model.provider.id}/${model.modelId}`,
@@ -12289,7 +12528,7 @@ function createEvalJudge({
         responseFormat: {
           type: "json",
           name: "eval_verdict",
-          schema: import_zod11.z.toJSONSchema(EvalVerdictSchema)
+          schema: import_zod13.z.toJSONSchema(EvalVerdictSchema)
         }
       });
       const result = await judge2.invoke([
@@ -12366,28 +12605,423 @@ function createEvalJudge({
 }
 
 // src/evals/route-handler.ts
-var import_node_crypto6 = require("crypto");
+var import_node_crypto7 = require("crypto");
 var import_prompts2 = require("@kortyx/prompts");
+var import_runtime6 = require("@kortyx/runtime");
+
+// src/evals/route-executions.ts
+var import_node_crypto6 = require("crypto");
+var import_errors13 = require("@kortyx/core/errors");
+var protocol = "jobs-v1";
+var uuid = /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i;
+var wait = (ms, signal) => new Promise((resolve2, reject) => {
+  if (signal.aborted) {
+    reject(signal.reason);
+    return;
+  }
+  const abort = () => {
+    clearTimeout(timer);
+    reject(signal.reason);
+  };
+  const timer = setTimeout(() => {
+    signal.removeEventListener("abort", abort);
+    resolve2();
+  }, ms);
+  signal.addEventListener("abort", abort, { once: true });
+});
+var canonical2 = (value) => {
+  if (Array.isArray(value)) return `[${value.map(canonical2).join(",")}]`;
+  if (value && typeof value === "object")
+    return `{${Object.entries(value).sort(([a], [b]) => a < b ? -1 : 1).map(([key, child]) => `${JSON.stringify(key)}:${canonical2(child)}`).join(",")}}`;
+  return JSON.stringify(value);
+};
+function createEvalExecutionTransport({
+  evals,
+  store,
+  scope,
+  limits,
+  onExecution
+}) {
+  const owner = (0, import_node_crypto6.randomUUID)();
+  const active = /* @__PURE__ */ new Map();
+  let ticking = false;
+  let stopped = false;
+  const key = (id3) => (0, import_node_crypto6.createHash)("sha256").update(`${scope}\0${id3}`).digest("hex");
+  const fail = async (job, error) => {
+    const issue2 = createEvalIssue("reporting", error);
+    await store.append(job.id, owner, {
+      type: "error",
+      message: issue2.message,
+      failure: issue2.failure,
+      ...issue2.diagnosticId ? { diagnosticId: issue2.diagnosticId } : {}
+    });
+  };
+  const run = async (job, controller) => {
+    let renewing = false;
+    const timer2 = setInterval(() => {
+      if (renewing) return;
+      renewing = true;
+      void store.heartbeat(job.id, owner).then(
+        (current) => {
+          if (!current) controller.abort("lease-lost");
+          else if (current.cancelRequested)
+            controller.abort("user-cancelled");
+        },
+        () => controller.abort("lease-lost")
+      ).finally(() => {
+        renewing = false;
+      });
+    }, store.heartbeatIntervalMs);
+    const deadline = setTimeout(
+      () => controller.abort("execution-timeout"),
+      30 * 6e4
+    );
+    try {
+      const body = EvalRemoteRunRequestSchema.parse(job.payload);
+      const current = evals.describe().suites.find((suite) => suite.id === body.suiteId);
+      if (!current || getEvalSuiteRevision(current) !== body.suiteRevision)
+        throw new import_errors13.KortyxError(
+          "EVAL_CONTRACT_CHANGED",
+          "The queued suite changed before execution.",
+          {
+            category: "configuration",
+            retryable: false,
+            safeMessage: "The queued suite changed before execution."
+          }
+        );
+      const result = await evals.run({
+        suiteId: body.suiteId,
+        repetitions: body.repetitions,
+        concurrency: body.concurrency,
+        signal: controller.signal,
+        ...body.attempt ? { attempt: body.attempt } : {},
+        ...body.caseIds ? { caseIds: body.caseIds } : {},
+        ...body.grading ? { grading: body.grading } : {},
+        ...body.judge ? { judgeIdentity: body.judge } : {},
+        ...body.promptSnapshot ? { promptSnapshot: body.promptSnapshot } : {},
+        onProgress: (event) => store.append(job.id, owner, { type: "progress", event })
+      });
+      await store.append(job.id, owner, { type: "result", result });
+      await store.finish(
+        job.id,
+        owner,
+        result.status === "cancelled" ? "cancelled" : "completed"
+      );
+    } catch (error) {
+      try {
+        if (controller.signal.reason === "user-cancelled" && (0, import_errors13.isControlFlowError)(error))
+          error = new import_errors13.KortyxError(
+            "EVAL_EXECUTION_CANCELLED",
+            "Evaluation execution was cancelled.",
+            {
+              category: "request",
+              status: 409,
+              retryable: false,
+              safeMessage: "Evaluation execution was cancelled."
+            }
+          );
+        await fail(job, error);
+        await store.finish(
+          job.id,
+          owner,
+          controller.signal.reason === "user-cancelled" ? "cancelled" : "completed"
+        );
+      } catch {
+      }
+    } finally {
+      clearInterval(timer2);
+      clearTimeout(deadline);
+      active.delete(job.id);
+    }
+  };
+  const tick = async () => {
+    if (stopped || ticking) return;
+    ticking = true;
+    try {
+      while (!stopped) {
+        const job = await store.claim(owner, scope, limits);
+        if (!job) break;
+        const controller = new AbortController();
+        const completion = run(job, controller);
+        active.set(job.id, { controller, completion });
+        try {
+          onExecution?.(completion);
+        } catch {
+        }
+      }
+    } catch {
+    } finally {
+      ticking = false;
+    }
+  };
+  const timer = setInterval(() => {
+    void tick();
+  }, 250);
+  timer.unref?.();
+  void tick();
+  const headers = async (id3) => ({
+    "content-type": "application/x-ndjson",
+    "cache-control": "no-store",
+    "x-accel-buffering": "no",
+    "x-kortyx-eval-protocol": protocol,
+    "x-kortyx-eval-execution-id": id3,
+    "x-kortyx-eval-generation": await store.generation()
+  });
+  const problem = (error, negotiated) => {
+    const failure = (0, import_errors13.serializeFailure)(error);
+    const busy = failure.code === "EVAL_EXECUTOR_BUSY";
+    return Response.json(
+      { error: failure.message, failure, ...busy ? { accepted: false } : {} },
+      {
+        status: busy ? negotiated ? 503 : 429 : !negotiated && failure.code === "EVAL_CONCURRENCY_CHANGED" ? 429 : failure.status ?? 503,
+        headers: {
+          ...negotiated ? { "x-kortyx-eval-protocol": protocol } : {},
+          ...busy ? { "retry-after": "2" } : {}
+        }
+      }
+    );
+  };
+  const observe = async (request, id3, after = 0, negotiated = true) => {
+    const backend = key(id3);
+    const job = await store.status(backend);
+    if (!job)
+      return Response.json(
+        {
+          error: "Evaluation execution is unavailable; do not replay it.",
+          code: "EVAL_EXECUTION_UNKNOWN"
+        },
+        { status: 410 }
+      );
+    if (after > job.sequence)
+      return Response.json(
+        { error: "Invalid execution event cursor." },
+        { status: 409 }
+      );
+    const controller = new AbortController();
+    const disconnect = () => {
+      controller.abort();
+      if (!negotiated) {
+        active.get(backend)?.controller.abort("user-cancelled");
+        void store.cancel(backend).catch(() => {
+        });
+      }
+    };
+    request.signal.addEventListener("abort", disconnect, { once: true });
+    if (request.signal.aborted) disconnect();
+    const encoder = new TextEncoder();
+    const stream = new ReadableStream({
+      start(output) {
+        void (async () => {
+          let cursor = after;
+          let state;
+          try {
+            while (!controller.signal.aborted) {
+              const record = await store.status(backend);
+              if (!record) throw new Error("Execution record expired.");
+              if (negotiated && state !== record.state) {
+                state = record.state;
+                output.enqueue(
+                  encoder.encode(
+                    `${JSON.stringify({ type: "execution-status", state })}
+`
+                  )
+                );
+              }
+              for (const item of await store.events(backend, cursor)) {
+                if (controller.signal.aborted) break;
+                output.enqueue(
+                  encoder.encode(
+                    `${JSON.stringify({ ...item.event, ...negotiated ? { sequence: item.sequence } : {} })}
+`
+                  )
+                );
+                cursor = item.sequence;
+              }
+              if (record.state !== "queued" && record.state !== "running" && cursor >= record.sequence)
+                break;
+              await wait(100, controller.signal);
+            }
+            if (!controller.signal.aborted) output.close();
+          } catch (error) {
+            if (!controller.signal.aborted) output.error(error);
+          } finally {
+            if (controller.signal.aborted) {
+              try {
+                output.close();
+              } catch {
+              }
+            }
+            request.signal.removeEventListener("abort", disconnect);
+          }
+        })();
+      },
+      cancel() {
+        disconnect();
+      }
+    });
+    return new Response(stream, { headers: await headers(id3) });
+  };
+  return {
+    async existing(request) {
+      const url = new URL(request.url);
+      const id3 = url.searchParams.get("executionId");
+      if (!id3) return void 0;
+      if (!uuid.test(id3))
+        return Response.json(
+          { error: "Invalid execution identity." },
+          { status: 400 }
+        );
+      try {
+        const generation = request.headers.get("x-kortyx-eval-generation");
+        if (generation && generation !== await store.generation())
+          return Response.json(
+            {
+              error: "Execution storage changed; execution status is unknown.",
+              code: "EVAL_EXECUTION_STORE_CHANGED"
+            },
+            { status: 409 }
+          );
+        if (request.method === "DELETE") {
+          if (!await store.status(key(id3)))
+            return Response.json(
+              { error: "Unknown execution." },
+              { status: 404 }
+            );
+          await store.cancel(key(id3));
+          active.get(key(id3))?.controller.abort("user-cancelled");
+          return Response.json({ cancelRequested: true });
+        }
+        if (request.method !== "GET")
+          return Response.json(
+            { error: "Unsupported observation method." },
+            { status: 405 }
+          );
+        const raw = url.searchParams.get("after") ?? "0";
+        if (!/^\d{1,8}$/.test(raw))
+          return Response.json(
+            { error: "Invalid execution event cursor." },
+            { status: 400 }
+          );
+        return await observe(request, id3, Number(raw));
+      } catch (error) {
+        return problem(error, true);
+      }
+    },
+    async submit(request, body) {
+      const negotiated = request.headers.get("x-kortyx-eval-protocol") === protocol;
+      if (stopped)
+        return problem(
+          new import_errors13.KortyxError(
+            "EVAL_EXECUTOR_BUSY",
+            "Eval executor is draining; execution was not accepted.",
+            {
+              category: "request",
+              status: 503,
+              retryable: true,
+              safeMessage: "Eval executor is draining; execution was not accepted."
+            }
+          ),
+          negotiated
+        );
+      const id3 = negotiated ? request.headers.get("idempotency-key") : (0, import_node_crypto6.randomUUID)();
+      if (!id3 || !uuid.test(id3))
+        return Response.json(
+          { error: "A UUID execution identity is required." },
+          { status: 400 }
+        );
+      const submitted = request.headers.get("x-kortyx-eval-submitted-at");
+      if (negotiated && (!submitted || !Number.isFinite(Date.parse(submitted)) || Date.parse(submitted) < Date.now() - 864e5 || Date.parse(submitted) > Date.now() + 3e5))
+        return Response.json(
+          {
+            error: "Execution submission window expired; inspect before starting another execution."
+          },
+          { status: 410 }
+        );
+      try {
+        if (negotiated && request.headers.get("x-kortyx-eval-generation") !== await store.generation())
+          return Response.json(
+            {
+              error: "Execution storage changed; execution status is unknown.",
+              code: "EVAL_EXECUTION_STORE_CHANGED"
+            },
+            { status: 409 }
+          );
+        await store.submit(
+          {
+            id: key(id3),
+            scope,
+            fingerprint: (0, import_node_crypto6.createHash)("sha256").update(canonical2(body)).digest("hex"),
+            group: key(body.attempt?.evaluationId ?? id3),
+            concurrency: body.concurrency,
+            weight: body.attempt ? 1 : body.concurrency,
+            payload: body,
+            queued: negotiated
+          },
+          limits
+        );
+        await tick();
+        if (!negotiated) {
+          while ((await store.status(key(id3)))?.state === "queued") {
+            request.signal.throwIfAborted();
+            await wait(10, request.signal);
+            await tick();
+          }
+        }
+        return await observe(request, id3, 0, negotiated);
+      } catch (error) {
+        if (!negotiated && request.signal.aborted) {
+          active.get(key(id3))?.controller.abort("user-cancelled");
+          await store.cancel(key(id3)).catch(() => {
+          });
+        }
+        return problem(error, negotiated);
+      }
+    },
+    async capabilityHeaders() {
+      return {
+        "x-kortyx-eval-protocol": protocol,
+        "x-kortyx-eval-generation": await store.generation()
+      };
+    },
+    async close() {
+      stopped = true;
+      clearInterval(timer);
+      for (const job of active.values()) job.controller.abort("host-stopped");
+      await Promise.all([...active.values()].map((job) => job.completion));
+    }
+  };
+}
+
+// src/evals/route-handler.ts
 function createEvalRouteHandler({
   evals,
   serviceKey,
   maxCases = 100,
-  maxActiveRuns = 1
+  maxActiveRuns = 1,
+  maxActiveAttempts = 20,
+  maxQueuedExecutions = 1e3,
+  executionStore = (0, import_runtime6.createInMemoryEvalExecutionStore)(),
+  onExecution
 }) {
   if (serviceKey.length < 32)
     throw new EvalConfigurationError(
       "Eval service keys must have at least 32 characters."
     );
-  if (!Number.isSafeInteger(maxCases) || maxCases < 1 || !Number.isSafeInteger(maxActiveRuns) || maxActiveRuns < 1)
+  if (!Number.isSafeInteger(maxCases) || maxCases < 1 || !Number.isSafeInteger(maxActiveRuns) || maxActiveRuns < 1 || !Number.isSafeInteger(maxActiveAttempts) || maxActiveAttempts < 1 || !Number.isSafeInteger(maxQueuedExecutions) || maxQueuedExecutions < 1 || !Number.isSafeInteger(executionStore.heartbeatIntervalMs) || executionStore.heartbeatIntervalMs < 1)
     throw new EvalConfigurationError(
       "Eval endpoint limits must be positive integers."
     );
-  const digest = (value) => (0, import_node_crypto6.createHash)("sha256").update(value).digest();
+  const digest = (value) => (0, import_node_crypto7.createHash)("sha256").update(value).digest();
   const expected = digest(`Bearer ${serviceKey}`);
-  let active = 0;
-  const evaluations = /* @__PURE__ */ new Map();
-  return async (request) => {
-    if (!(0, import_node_crypto6.timingSafeEqual)(
+  const execution = createEvalExecutionTransport({
+    evals,
+    store: executionStore,
+    scope: expected.toString("hex"),
+    limits: { maxActiveRuns, maxActiveAttempts, maxQueuedExecutions },
+    ...onExecution ? { onExecution } : {}
+  });
+  const handler3 = async (request) => {
+    if (!(0, import_node_crypto7.timingSafeEqual)(
       expected,
       digest(request.headers.get("authorization") ?? "")
     ))
@@ -12395,10 +13029,15 @@ function createEvalRouteHandler({
         { error: "Unauthorized eval request." },
         { status: 401 }
       );
+    const observed = await execution.existing(request);
+    if (observed) return observed;
     const manifest = evals.describe();
     if (request.method === "GET")
       return Response.json(manifest, {
-        headers: { "cache-control": "no-store" }
+        headers: {
+          "cache-control": "no-store",
+          ...await execution.capabilityHeaders()
+        }
       });
     if (request.method !== "POST")
       return Response.json({ error: "Unsupported method." }, { status: 405 });
@@ -12469,80 +13108,11 @@ function createEvalRouteHandler({
         { error: "Unsupported or invalid scheduled attempt." },
         { status: 409 }
       );
-    const group = body.attempt && evaluations.get(body.attempt.evaluationId);
-    if (group && (group.concurrency !== body.concurrency || group.active >= group.concurrency))
-      return Response.json(
-        { error: "Evaluation concurrency exhausted or changed." },
-        { status: 429 }
-      );
-    if (!group && active >= maxActiveRuns)
-      return Response.json(
-        { error: "Eval executor is busy." },
-        { status: 429 }
-      );
     if (request.signal.aborted)
       return Response.json({ error: "Request cancelled." }, { status: 409 });
-    if (body.attempt) {
-      const state = group ?? { active: 0, concurrency: body.concurrency };
-      if (!group) {
-        active++;
-        evaluations.set(body.attempt.evaluationId, state);
-      }
-      state.active++;
-    } else active++;
-    const controller = new AbortController();
-    const requestSignal = request.signal;
-    const abortRequest = () => controller.abort();
-    requestSignal.addEventListener("abort", abortRequest, { once: true });
-    const signal = controller.signal;
-    const encoder = new TextEncoder();
-    const stream = new ReadableStream({
-      start(output) {
-        const send = (event) => {
-          if (!signal.aborted)
-            output.enqueue(encoder.encode(`${JSON.stringify(event)}
-`));
-        };
-        void evals.run({
-          suiteId: body.suiteId,
-          ...body.promptSnapshot ? { promptSnapshot: body.promptSnapshot } : {},
-          ...body.attempt ? { attempt: body.attempt } : {},
-          ...body.grading ? { grading: body.grading } : {},
-          ...body.judge ? { judgeIdentity: body.judge } : {},
-          repetitions: body.repetitions,
-          concurrency: body.concurrency,
-          signal,
-          ...body.caseIds ? { caseIds: body.caseIds } : {},
-          onProgress: (event) => send({ type: "progress", event })
-        }).then(
-          (result) => send({ type: "result", result }),
-          () => send({ type: "error", message: "Eval runner failed." })
-        ).finally(() => {
-          requestSignal.removeEventListener("abort", abortRequest);
-          if (body.attempt) {
-            const state = evaluations.get(body.attempt.evaluationId);
-            state.active--;
-            if (!state.active) {
-              evaluations.delete(body.attempt.evaluationId);
-              active--;
-            }
-          } else active--;
-          if (!controller.signal.aborted) output.close();
-        });
-      },
-      cancel() {
-        request.signal.removeEventListener("abort", abortRequest);
-        controller.abort();
-      }
-    });
-    return new Response(stream, {
-      headers: {
-        "content-type": "application/x-ndjson",
-        "cache-control": "no-store",
-        "x-accel-buffering": "no"
-      }
-    });
+    return execution.submit(request, body);
   };
+  return Object.assign(handler3, { close: execution.close });
 }
 
 // src/evals/studio-judge.ts
@@ -12656,6 +13226,7 @@ async function createStudioEvalJudge(options) {
   ChatLifecycleHookError,
   EvalConfigurationError,
   EvalEvidencePolicySchema,
+  EvalIssueSchema,
   EvalJudgeIdentitySchema,
   EvalJudgeUsageSchema,
   EvalManifestSchema,
@@ -12677,6 +13248,7 @@ async function createStudioEvalJudge(options) {
   createAgent,
   createChatRouteHandler,
   createCheckpointRouteHandler,
+  createEvalIssue,
   createEvalJudge,
   createEvalRouteHandler,
   createEvals,