@kortyx/agent 0.37.1 → 0.38.0

package/CHANGELOG.md

@@ -1,5 +1,45 @@
 # Changelog
 
+## Unreleased
+
+- Add bounded, idempotent queued eval execution with shared consumer capacity,
+  event replay, explicit cancellation and optional PostgreSQL recovery.
+
+
+## [0.38.0](https://github.com/kortyx-io/kortyx/compare/agent-v0.37.1...agent-v0.38.0) (2026-10-13)
+
+
+### ⚠ BREAKING CHANGES
+
+* **runtime:** Consumer eval routes cap active attempts at 20 by default. Set maxActiveAttempts before upgrading consumers with higher concurrency. Upgrade API and Studio together for the new case-dispatch progress events.
+* **agent:** Consumer eval routes cap active attempts at 20 by default. Set maxActiveAttempts before upgrading consumers with higher concurrency. Upgrade API and Studio together for the new case-dispatch progress events.
+
+### Features
+
+* **agent:** make concurrent eval execution resilient ([#339](https://github.com/kortyx-io/kortyx/issues/339)) ([84d0e4f](https://github.com/kortyx-io/kortyx/commit/84d0e4fe3d35e5cf80713f7550ef2b1d83e5a2ec))
+* **runtime:** add opt-in workflow data reducers ([#340](https://github.com/kortyx-io/kortyx/issues/340)) ([8be2c81](https://github.com/kortyx-io/kortyx/commit/8be2c81f74c5c51ab98443144f3491d4e8d54a88))
+
+
+### Bug Fixes
+
+* **evals:** preserve diagnostics and separate postcondition failures ([#337](https://github.com/kortyx-io/kortyx/issues/337)) ([d4fce4e](https://github.com/kortyx-io/kortyx/commit/d4fce4e670971d3b755835fc681c8125f67a9c70))
+
+
+### Dependencies
+
+* The following workspace dependencies were updated
+  * dependencies
+    * @kortyx/core bumped to 0.14.0
+    * @kortyx/providers bumped to 0.9.3
+    * @kortyx/runtime bumped to 0.24.0
+    * @kortyx/stream bumped to 0.15.5
+    * @kortyx/utils bumped to 0.3.7
+    * @kortyx/prompts bumped to 0.2.1
+    * @kortyx/usage-telemetry bumped to 0.3.0
+  * devDependencies
+    * @kortyx/hooks bumped to 0.34.2
+    * @kortyx/openai bumped to 0.5.9
+
 ## [0.37.1](https://github.com/kortyx-io/kortyx/compare/agent-v0.37.0...agent-v0.37.1) (2026-10-13)
 
 

package/README.md

@@ -95,3 +95,19 @@ Versions are optional. These deterministic checks run before semantic criteria
 and only match finalized visible outputs from the current step. See the
 [conversation eval guide](https://kortyx.io/docs/guides/conversation-evals) for
 SDK setup, Studio execution and custom executor requirements.
+
+Eval issues preserve declared-safe failures and native diagnostic references.
+Configure native diagnostic capture on the agent to inspect original hook errors
+in Studio. Use `verify` for postconditions (false means Failed), and `teardown` for
+reset (a thrown fault means Error). Setup, execution and verification have separate
+cooperative budgets. See [eval error migration](../../docs/evals/error-contract-migration.md)
+before upgrading a mixed SDK/API/Studio deployment.
+
+## Evaluation execution recovery
+
+`createEvalRouteHandler` supports queued, idempotent execution with cursor-based
+observation and explicit cancellation. Use `createPostgresEvalExecutionStore` from
+`@kortyx/runtime` (or `kortyx`) for shared capacity and recovery across replicas.
+The default memory store only survives within its process. See
+[execution resilience](../../docs/evals/execution-resilience.md) for configuration,
+retention, legacy compatibility and upgrade order.

package/dist/evals.d.ts

@@ -1,10 +1,10 @@
-export { C as CreateEvalsOptions, E as EvalCase, a as EvalCaseResult, b as EvalCaseResultSchema, c as EvalCommand, d as EvalConfigurationError, e as EvalContext, f as EvalCriterion, g as EvalCriterionResult, h as EvalDefaults, i as EvalEvidenceEvent, j as EvalEvidenceEventType, k as EvalEvidenceFilters, l as EvalEvidencePolicy, m as EvalEvidencePolicySchema, n as EvalExecution, o as EvalExecutionStatus, p as EvalExpectation, q as EvalGradeInput, r as EvalHandlerRef, s as EvalInterrupt, t as EvalIssue, u as EvalJson, v as EvalJudge, w as EvalJudgeEvidence, x as EvalJudgeIdentity, y as EvalJudgeIdentitySchema, z as EvalJudgeOptions, A as EvalJudgeUsage, B as EvalJudgeUsageSchema, D as EvalManifest, F as EvalManifestSchema, G as EvalObservation, H as EvalObservationSchema, I as EvalOutputExpectation, J as EvalOutputSelector, K as EvalPhase, L as EvalProgress, M as EvalProgressSchema, N as EvalRemoteRunRequest, O as EvalRemoteRunRequestSchema, P as EvalResponder, Q as EvalResumeResponseSchema, R as EvalRunOptions, S as EvalRunResult, T as EvalRunResultSchema, U as EvalRuntimeExecution, V as EvalSetupContext, W as EvalStatus, X as EvalStep, Y as EvalStepResult, Z as EvalStepResultSchema, _ as EvalSuite, $ as EvalSuiteSchema, a0 as EvalVerdict, a1 as EvalVerdictSchema, a2 as EvalWireEvent, a3 as EvalWireEventSchema, a4 as EvaluationMetadataSchema, a5 as StudioEvalJudgeOptions, a6 as StudioEvalJudgeRequestSchema, a7 as StudioEvalJudgeResponseSchema, a8 as StudioEvalStartRequestSchema, a9 as StudioEvaluationDetail, aa as StudioEvaluationDetailSchema, ab as StudioEvaluationHistorySchema, ac as StudioEvaluationResultsSchema, ad as StudioEvaluationStartRequest, ae as StudioEvaluationStartRequestSchema, af as StudioEvaluationSummary, ag as StudioEvaluationSummarySchema, ah as createStudioEvalJudge, ai as defineSuite, aj as getEvalGradeEvidence, ak as parseEvalSuite } from './studio-judge-Dsz6KDY2.js';
+export { C as CreateEvalsOptions, E as EvalCase, a as EvalCaseResult, b as EvalCaseResultSchema, c as EvalCommand, d as EvalConfigurationError, e as EvalContext, f as EvalCriterion, g as EvalCriterionResult, h as EvalDefaults, i as EvalEvidenceEvent, j as EvalEvidenceEventType, k as EvalEvidenceFilters, l as EvalEvidencePolicy, m as EvalEvidencePolicySchema, n as EvalExecution, o as EvalExecutionStatus, p as EvalExpectation, q as EvalGradeInput, r as EvalHandlerRef, s as EvalInterrupt, t as EvalIssue, u as EvalIssueSchema, v as EvalJson, w as EvalJudge, x as EvalJudgeEvidence, y as EvalJudgeIdentity, z as EvalJudgeIdentitySchema, A as EvalJudgeOptions, B as EvalJudgeUsage, D as EvalJudgeUsageSchema, F as EvalManifest, G as EvalManifestSchema, H as EvalObservation, I as EvalObservationSchema, J as EvalOutputExpectation, K as EvalOutputSelector, L as EvalPhase, M as EvalProgress, N as EvalProgressSchema, O as EvalRemoteRunRequest, P as EvalRemoteRunRequestSchema, Q as EvalResponder, R as EvalResumeResponseSchema, S as EvalRunOptions, T as EvalRunResult, U as EvalRunResultSchema, V as EvalRuntimeExecution, W as EvalSetupContext, X as EvalStatus, Y as EvalStep, Z as EvalStepResult, _ as EvalStepResultSchema, $ as EvalSuite, a0 as EvalSuiteSchema, a1 as EvalVerdict, a2 as EvalVerdictSchema, a3 as EvalWireEvent, a4 as EvalWireEventSchema, a5 as EvaluationMetadataSchema, a6 as StudioEvalJudgeOptions, a7 as StudioEvalJudgeRequestSchema, a8 as StudioEvalJudgeResponseSchema, a9 as StudioEvalStartRequestSchema, aa as StudioEvaluationDetail, ab as StudioEvaluationDetailSchema, ac as StudioEvaluationHistorySchema, ad as StudioEvaluationResultsSchema, ae as StudioEvaluationStartRequest, af as StudioEvaluationStartRequestSchema, ag as StudioEvaluationSummary, ah as StudioEvaluationSummarySchema, ai as createEvalIssue, aj as createStudioEvalJudge, ak as defineSuite, al as getEvalGradeEvidence, am as parseEvalSuite } from './studio-judge-DCAw3v60.js';
+import * as _kortyx_core from '@kortyx/core';
 import { z } from 'zod';
 import '@kortyx/prompts';
-import '@kortyx/providers';
 import '@kortyx/core/errors';
+import '@kortyx/providers';
 import '@kortyx/stream';
-import '@kortyx/core';
 import '@kortyx/runtime';
 import '@kortyx/hooks';
 import './chat-message-D4EsFcr1.js';
@@ -838,6 +838,22 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                 ungraded: z.ZodOptional<z.ZodNumber>;
             }, z.core.$strict>;
             cases: z.ZodArray<z.ZodObject<{
+                verification: z.ZodOptional<z.ZodObject<{
+                    passed: z.ZodBoolean;
+                    reason: z.ZodString;
+                    evidence: z.ZodArray<z.ZodString>;
+                }, z.core.$strict>>;
+                phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
+                    params: "params";
+                    execute: "execute";
+                    setup: "setup";
+                    responder: "responder";
+                    reference: "reference";
+                    grading: "grading";
+                    verification: "verification";
+                    cleanup: "cleanup";
+                    reporting: "reporting";
+                }> & z.core.$partial, z.ZodNumber>>;
                 caseId: z.ZodString;
                 repetition: z.ZodNumber;
                 attemptId: z.ZodOptional<z.ZodUUID>;
@@ -859,6 +875,8 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                         version: z.ZodLiteral<"compact-v1">;
                         history: z.ZodBoolean;
                         observation: z.ZodObject<{
+                            failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
+                            diagnosticId: z.ZodOptional<z.ZodUUID>;
                             promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
                                 id: z.ZodString;
                                 version: z.ZodNumber;
@@ -933,6 +951,8 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                         reference: z.ZodOptional<z.ZodJSONSchema>;
                     }, z.core.$strict>;
                     observation: z.ZodObject<{
+                        failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
+                        diagnosticId: z.ZodOptional<z.ZodUUID>;
                         promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
                             id: z.ZodString;
                             version: z.ZodNumber;
@@ -1017,6 +1037,8 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                     }, z.core.$strict>>;
                 }, z.core.$strict>>;
                 errors: z.ZodArray<z.ZodObject<{
+                    failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
+                    diagnosticId: z.ZodOptional<z.ZodUUID>;
                     phase: z.ZodEnum<{
                         params: "params";
                         execute: "execute";
@@ -1024,6 +1046,7 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                         responder: "responder";
                         reference: "reference";
                         grading: "grading";
+                        verification: "verification";
                         cleanup: "cleanup";
                         reporting: "reporting";
                     }>;
@@ -1032,6 +1055,8 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                 }, z.core.$strict>>;
             }, z.core.$strict>>;
             errors: z.ZodArray<z.ZodObject<{
+                failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
+                diagnosticId: z.ZodOptional<z.ZodUUID>;
                 phase: z.ZodEnum<{
                     params: "params";
                     execute: "execute";
@@ -1039,6 +1064,7 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                     responder: "responder";
                     reference: "reference";
                     grading: "grading";
+                    verification: "verification";
                     cleanup: "cleanup";
                     reporting: "reporting";
                 }>;
@@ -1049,6 +1075,19 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
         events: z.ZodArray<z.ZodObject<{
             id: z.ZodNumber;
             event: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                type: z.ZodLiteral<"case-dispatch">;
+                caseId: z.ZodString;
+                repetition: z.ZodNumber;
+                status: z.ZodEnum<{
+                    queued: "queued";
+                    retrying: "retrying";
+                    reconnecting: "reconnecting";
+                }>;
+                retryAt: z.ZodOptional<z.ZodISODateTime>;
+                httpStatus: z.ZodOptional<z.ZodNumber>;
+                executionId: z.ZodOptional<z.ZodUUID>;
+                generation: z.ZodOptional<z.ZodUUID>;
+            }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"case-runtime-associated">;
                 caseId: z.ZodString;
                 repetition: z.ZodNumber;
@@ -1073,6 +1112,7 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                     responder: "responder";
                     reference: "reference";
                     grading: "grading";
+                    verification: "verification";
                     cleanup: "cleanup";
                     reporting: "reporting";
                 }>;
@@ -1092,6 +1132,8 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                         version: z.ZodLiteral<"compact-v1">;
                         history: z.ZodBoolean;
                         observation: z.ZodObject<{
+                            failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
+                            diagnosticId: z.ZodOptional<z.ZodUUID>;
                             promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
                                 id: z.ZodString;
                                 version: z.ZodNumber;
@@ -1166,6 +1208,8 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                         reference: z.ZodOptional<z.ZodJSONSchema>;
                     }, z.core.$strict>;
                     observation: z.ZodObject<{
+                        failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
+                        diagnosticId: z.ZodOptional<z.ZodUUID>;
                         promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
                             id: z.ZodString;
                             version: z.ZodNumber;
@@ -1252,6 +1296,22 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
             }, z.core.$strict>, z.ZodObject<{
                 type: z.ZodLiteral<"case-completed">;
                 result: z.ZodObject<{
+                    verification: z.ZodOptional<z.ZodObject<{
+                        passed: z.ZodBoolean;
+                        reason: z.ZodString;
+                        evidence: z.ZodArray<z.ZodString>;
+                    }, z.core.$strict>>;
+                    phaseDurationsMs: z.ZodOptional<z.ZodRecord<z.ZodEnum<{
+                        params: "params";
+                        execute: "execute";
+                        setup: "setup";
+                        responder: "responder";
+                        reference: "reference";
+                        grading: "grading";
+                        verification: "verification";
+                        cleanup: "cleanup";
+                        reporting: "reporting";
+                    }> & z.core.$partial, z.ZodNumber>>;
                     caseId: z.ZodString;
                     repetition: z.ZodNumber;
                     attemptId: z.ZodOptional<z.ZodUUID>;
@@ -1273,6 +1333,8 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                             version: z.ZodLiteral<"compact-v1">;
                             history: z.ZodBoolean;
                             observation: z.ZodObject<{
+                                failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
+                                diagnosticId: z.ZodOptional<z.ZodUUID>;
                                 promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
                                     id: z.ZodString;
                                     version: z.ZodNumber;
@@ -1347,6 +1409,8 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                             reference: z.ZodOptional<z.ZodJSONSchema>;
                         }, z.core.$strict>;
                         observation: z.ZodObject<{
+                            failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
+                            diagnosticId: z.ZodOptional<z.ZodUUID>;
                             promptUsage: z.ZodOptional<z.ZodArray<z.ZodObject<{
                                 id: z.ZodString;
                                 version: z.ZodNumber;
@@ -1431,6 +1495,8 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                         }, z.core.$strict>>;
                     }, z.core.$strict>>;
                     errors: z.ZodArray<z.ZodObject<{
+                        failure: z.ZodOptional<z.ZodPipe<z.ZodCustom<_kortyx_core.FailureDescriptor, _kortyx_core.FailureDescriptor>, z.ZodTransform<_kortyx_core.SerializedFailureDescriptor, _kortyx_core.FailureDescriptor>>>;
+                        diagnosticId: z.ZodOptional<z.ZodUUID>;
                         phase: z.ZodEnum<{
                             params: "params";
                             execute: "execute";
@@ -1438,6 +1504,7 @@ declare const StudioEvalDetailSchema: z.ZodObject<{
                             responder: "responder";
                             reference: "reference";
                             grading: "grading";
+                            verification: "verification";
                             cleanup: "cleanup";
                             reporting: "reporting";
                         }>;