@kortyx/agent 0.37.1 → 0.38.0

package/dist/evals.js

@@ -24,6 +24,7 @@ __export(evals_exports, {
   EvalCostAmountSchema: () => EvalCostAmountSchema,
   EvalCostsSchema: () => EvalCostsSchema,
   EvalEvidencePolicySchema: () => EvalEvidencePolicySchema,
+  EvalIssueSchema: () => EvalIssueSchema,
   EvalJudgeIdentitySchema: () => EvalJudgeIdentitySchema,
   EvalJudgeUsageSchema: () => EvalJudgeUsageSchema,
   EvalManifestSchema: () => EvalManifestSchema,
@@ -49,6 +50,7 @@ __export(evals_exports, {
   StudioEvaluationResultsSchema: () => StudioEvaluationResultsSchema,
   StudioEvaluationStartRequestSchema: () => StudioEvaluationStartRequestSchema,
   StudioEvaluationSummarySchema: () => StudioEvaluationSummarySchema,
+  createEvalIssue: () => createEvalIssue,
   createStudioEvalJudge: () => createStudioEvalJudge,
   defineSuite: () => defineSuite,
   getEvalGradeEvidence: () => getEvalGradeEvidence,
@@ -57,6 +59,7 @@ __export(evals_exports, {
 module.exports = __toCommonJS(evals_exports);
 
 // src/evals/contracts.ts
+var import_errors = require("@kortyx/core/errors");
 var import_browser = require("@kortyx/prompts/browser");
 var import_zod2 = require("zod");
 
@@ -100,6 +103,9 @@ var EvalEvidencePolicySchema = import_zod.z.object({
 
 // src/evals/contracts.ts
 var id2 = import_zod2.z.string().trim().min(1).max(256);
+var failureDescriptor = import_zod2.z.custom(import_errors.isFailureDescriptor).transform(
+  (value) => JSON.parse(JSON.stringify((0, import_errors.serializeFailure)(value)))
+);
 var handler = import_zod2.z.object({ using: id2, params: import_zod2.z.json().optional() }).strict();
 var EvalResumeResponseSchema = import_zod2.z.discriminatedUnion("type", [
   import_zod2.z.object({ type: import_zod2.z.literal("text"), text: import_zod2.z.string() }).strict(),
@@ -162,6 +168,8 @@ var EvalVerdictSchema = import_zod2.z.object({
   evidence: import_zod2.z.array(import_zod2.z.string().min(1))
 }).strict();
 var EvalObservationSchema = import_zod2.z.object({
+  failure: failureDescriptor.optional(),
+  diagnosticId: import_zod2.z.uuid().optional(),
   promptUsage: import_zod2.z.array(
     import_zod2.z.object({
       id: import_zod2.z.string(),
@@ -201,7 +209,9 @@ var EvalObservationSchema = import_zod2.z.object({
     });
 });
 var status = import_zod2.z.enum(["passed", "failed", "error", "cancelled", "ungraded"]);
-var issue = import_zod2.z.object({
+var EvalIssueSchema = import_zod2.z.object({
+  failure: failureDescriptor.optional(),
+  diagnosticId: import_zod2.z.uuid().optional(),
   phase: import_zod2.z.enum([
     "params",
     "setup",
@@ -209,12 +219,14 @@ var issue = import_zod2.z.object({
     "responder",
     "reference",
     "grading",
+    "verification",
     "cleanup",
     "reporting"
   ]),
   code: import_zod2.z.string(),
   message: import_zod2.z.string()
 }).strict();
+var issue = EvalIssueSchema;
 var EvalJudgeUsageSchema = import_zod2.z.object({
   provider: import_zod2.z.string().min(1),
   model: import_zod2.z.string().min(1),
@@ -266,6 +278,8 @@ var EvalStepResultSchema = import_zod2.z.object({
 }).strict();
 var runtimeExecution = import_zod2.z.object({ sessionId: import_zod2.z.string().min(1), runId: import_zod2.z.string().min(1) }).strict();
 var EvalCaseResultSchema = import_zod2.z.object({
+  verification: EvalVerdictSchema.optional(),
+  phaseDurationsMs: import_zod2.z.partialRecord(issue.shape.phase, import_zod2.z.number().finite().nonnegative()).optional(),
   caseId: id2,
   repetition: import_zod2.z.number().int().positive(),
   attemptId: import_zod2.z.uuid().optional(),
@@ -340,6 +354,16 @@ var EvalManifestSchema = import_zod2.z.object({
   judge: judge.optional()
 }).strict();
 var EvalProgressSchema = import_zod2.z.discriminatedUnion("type", [
+  import_zod2.z.object({
+    type: import_zod2.z.literal("case-dispatch"),
+    caseId: id2,
+    repetition: import_zod2.z.number().int().positive(),
+    status: import_zod2.z.enum(["queued", "retrying", "reconnecting"]),
+    retryAt: import_zod2.z.iso.datetime().optional(),
+    httpStatus: import_zod2.z.number().int().min(100).max(599).optional(),
+    executionId: import_zod2.z.uuid().optional(),
+    generation: import_zod2.z.uuid().optional()
+  }).strict(),
   import_zod2.z.object({
     type: import_zod2.z.literal("case-runtime-associated"),
     caseId: id2,
@@ -411,9 +435,27 @@ var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
   judge: import_zod2.z.enum(["studio", "app"]).default("studio")
 });
 var EvalWireEventSchema = import_zod2.z.discriminatedUnion("type", [
-  import_zod2.z.object({ type: import_zod2.z.literal("progress"), event: EvalProgressSchema }).strict(),
-  import_zod2.z.object({ type: import_zod2.z.literal("result"), result: EvalRunResultSchema }).strict(),
-  import_zod2.z.object({ type: import_zod2.z.literal("error"), message: import_zod2.z.string() }).strict()
+  import_zod2.z.object({
+    type: import_zod2.z.literal("execution-status"),
+    state: import_zod2.z.enum(["queued", "running", "completed", "cancelled", "unknown"])
+  }).strict(),
+  import_zod2.z.object({
+    type: import_zod2.z.literal("progress"),
+    event: EvalProgressSchema,
+    sequence: import_zod2.z.number().int().positive().optional()
+  }).strict(),
+  import_zod2.z.object({
+    type: import_zod2.z.literal("result"),
+    result: EvalRunResultSchema,
+    sequence: import_zod2.z.number().int().positive().optional()
+  }).strict(),
+  import_zod2.z.object({
+    type: import_zod2.z.literal("error"),
+    message: import_zod2.z.string(),
+    failure: failureDescriptor.optional(),
+    diagnosticId: import_zod2.z.uuid().optional(),
+    sequence: import_zod2.z.number().int().positive().optional()
+  }).strict()
 ]);
 var EvalConfigurationError = class extends Error {
   name = "EvalConfigurationError";
@@ -445,24 +487,58 @@ function defineSuite(suite) {
   return suite;
 }
 
+// src/evals/errors.ts
+var import_errors2 = require("@kortyx/core/errors");
+var import_zod3 = require("zod");
+var messages = {
+  params: "Case parameters failed validation.",
+  setup: "Case setup failed.",
+  execute: "Agent execution failed or returned an invalid observation.",
+  responder: "Interrupt response failed or was incompatible with the waiting contract.",
+  reference: "Reference resolution failed or returned non-JSON data.",
+  grading: "Grading failed or returned an invalid verdict.",
+  verification: "Case verification failed or returned an invalid verdict.",
+  cleanup: "Case cleanup failed.",
+  reporting: "Progress reporting failed."
+};
+function createEvalIssue(phase, error, options = {}) {
+  const control = (0, import_errors2.isControlFlowError)(error);
+  const declared = !control && ((0, import_errors2.isFailureDescriptor)(error) || (0, import_errors2.isFailureDescriptor)((0, import_errors2.errorProperty)(error, "failure")));
+  const failure = (0, import_errors2.serializeFailure)(control ? void 0 : error);
+  let diagnosticId = options.diagnosticId ?? (0, import_errors2.errorProperty)(error, "diagnosticId");
+  if (!control && !diagnosticId && options.reportError) {
+    try {
+      diagnosticId = options.reportError(error, options.correlation);
+    } catch {
+    }
+  }
+  return {
+    phase,
+    code: declared ? failure.code : options.code ?? "EVAL_HOOK_FAILED",
+    message: declared ? failure.message : messages[phase],
+    failure,
+    ...typeof diagnosticId === "string" && import_zod3.z.uuid().safeParse(diagnosticId).success ? { diagnosticId } : {}
+  };
+}
+
 // src/evals/evaluation-contracts.ts
-var import_zod4 = require("zod");
+var import_zod5 = require("zod");
 
 // src/evals/studio-contracts.ts
 var import_browser2 = require("@kortyx/prompts/browser");
-var import_zod3 = require("zod");
-var StudioEvalTargetsResponseSchema = import_zod3.z.object({
-  canRun: import_zod3.z.boolean(),
+var import_zod4 = require("zod");
+var StudioEvalTargetsResponseSchema = import_zod4.z.object({
+  canRun: import_zod4.z.boolean(),
   studioJudge: EvalJudgeIdentitySchema.nullable().optional(),
-  targets: import_zod3.z.array(
-    import_zod3.z.object({
-      id: import_zod3.z.string(),
-      name: import_zod3.z.string(),
-      environment: import_zod3.z.string(),
-      error: import_zod3.z.string().nullable(),
+  targets: import_zod4.z.array(
+    import_zod4.z.object({
+      id: import_zod4.z.string(),
+      name: import_zod4.z.string(),
+      environment: import_zod4.z.string(),
+      error: import_zod4.z.string().nullable(),
       // Safe discovery diagnostics; optional for compatibility with older APIs.
-      diagnostic: import_zod3.z.object({
-        code: import_zod3.z.enum([
+      diagnostic: import_zod4.z.object({
+        code: import_zod4.z.enum([
           "environment_forbidden",
           "environment_unavailable",
           "endpoint_not_found",
@@ -471,44 +547,44 @@ var StudioEvalTargetsResponseSchema = import_zod3.z.object({
           "endpoint_unreachable",
           "manifest_invalid"
         ]),
-        httpStatus: import_zod3.z.number().int().min(100).max(599).optional()
+        httpStatus: import_zod4.z.number().int().min(100).max(599).optional()
       }).nullable().optional(),
       manifest: EvalManifestSchema.nullable(),
-      revisions: import_zod3.z.record(import_zod3.z.string(), import_zod3.z.string())
+      revisions: import_zod4.z.record(import_zod4.z.string(), import_zod4.z.string())
     })
   )
 });
-var EvalCostAmountSchema = import_zod3.z.object({
-  costSummary: import_zod3.z.object({
-    amounts: import_zod3.z.array(
-      import_zod3.z.object({
-        amount: import_zod3.z.number().finite().nonnegative(),
-        currency: import_zod3.z.string().nullable()
+var EvalCostAmountSchema = import_zod4.z.object({
+  costSummary: import_zod4.z.object({
+    amounts: import_zod4.z.array(
+      import_zod4.z.object({
+        amount: import_zod4.z.number().finite().nonnegative(),
+        currency: import_zod4.z.string().nullable()
       })
     ),
-    complete: import_zod3.z.boolean()
+    complete: import_zod4.z.boolean()
   }).optional(),
-  amount: import_zod3.z.number().finite().nonnegative().nullable(),
-  currency: import_zod3.z.string().nullable(),
-  status: import_zod3.z.enum(["complete", "partial", "unavailable"]),
-  calls: import_zod3.z.number().int().nonnegative(),
-  unpricedCalls: import_zod3.z.number().int().nonnegative(),
-  estimated: import_zod3.z.boolean()
+  amount: import_zod4.z.number().finite().nonnegative().nullable(),
+  currency: import_zod4.z.string().nullable(),
+  status: import_zod4.z.enum(["complete", "partial", "unavailable"]),
+  calls: import_zod4.z.number().int().nonnegative(),
+  unpricedCalls: import_zod4.z.number().int().nonnegative(),
+  estimated: import_zod4.z.boolean()
 });
-var EvalCostsSchema = import_zod3.z.object({
+var EvalCostsSchema = import_zod4.z.object({
   workflow: EvalCostAmountSchema,
   judge: EvalCostAmountSchema,
   total: EvalCostAmountSchema
 });
-var StudioEvalRunSummarySchema = import_zod3.z.object({
-  id: import_zod3.z.uuid(),
-  evaluationId: import_zod3.z.uuid().nullable().optional(),
-  targetId: import_zod3.z.string(),
-  targetName: import_zod3.z.string(),
-  environment: import_zod3.z.string(),
-  suiteId: import_zod3.z.string(),
-  suiteRevision: import_zod3.z.string(),
-  status: import_zod3.z.enum([
+var StudioEvalRunSummarySchema = import_zod4.z.object({
+  id: import_zod4.z.uuid(),
+  evaluationId: import_zod4.z.uuid().nullable().optional(),
+  targetId: import_zod4.z.string(),
+  targetName: import_zod4.z.string(),
+  environment: import_zod4.z.string(),
+  suiteId: import_zod4.z.string(),
+  suiteRevision: import_zod4.z.string(),
+  status: import_zod4.z.enum([
     "queued",
     "running",
     "passed",
@@ -516,49 +592,49 @@ var StudioEvalRunSummarySchema = import_zod3.z.object({
     "error",
     "cancelled"
   ]),
-  createdAt: import_zod3.z.string(),
-  startedAt: import_zod3.z.string().nullable(),
-  endedAt: import_zod3.z.string().nullable(),
-  error: import_zod3.z.string().nullable(),
-  cancelRequestedAt: import_zod3.z.string().nullable(),
+  createdAt: import_zod4.z.string(),
+  startedAt: import_zod4.z.string().nullable(),
+  endedAt: import_zod4.z.string().nullable(),
+  error: import_zod4.z.string().nullable(),
+  cancelRequestedAt: import_zod4.z.string().nullable(),
   costs: EvalCostsSchema.optional(),
-  counts: import_zod3.z.object({
-    passed: import_zod3.z.number(),
-    failed: import_zod3.z.number(),
-    error: import_zod3.z.number(),
-    cancelled: import_zod3.z.number()
+  counts: import_zod4.z.object({
+    passed: import_zod4.z.number(),
+    failed: import_zod4.z.number(),
+    error: import_zod4.z.number(),
+    cancelled: import_zod4.z.number()
   }).nullable().optional(),
-  suiteName: import_zod3.z.string().nullable().optional(),
-  totalAttempts: import_zod3.z.number().int().nonnegative().optional(),
-  completedAttempts: import_zod3.z.number().int().nonnegative().optional(),
-  phase: import_zod3.z.string().optional()
+  suiteName: import_zod4.z.string().nullable().optional(),
+  totalAttempts: import_zod4.z.number().int().nonnegative().optional(),
+  completedAttempts: import_zod4.z.number().int().nonnegative().optional(),
+  phase: import_zod4.z.string().optional()
 });
-var StudioEvalHistorySchema = import_zod3.z.object({
-  runs: import_zod3.z.array(StudioEvalRunSummarySchema)
+var StudioEvalHistorySchema = import_zod4.z.object({
+  runs: import_zod4.z.array(StudioEvalRunSummarySchema)
 });
-var StudioEvalDetailSchema = import_zod3.z.object({
+var StudioEvalDetailSchema = import_zod4.z.object({
   run: StudioEvalRunSummarySchema.extend({
-    caseCosts: import_zod3.z.record(import_zod3.z.string(), EvalCostsSchema).optional(),
-    request: import_zod3.z.object({
-      caseIds: import_zod3.z.array(import_zod3.z.string()).optional(),
-      grading: import_zod3.z.enum(["app", "studio"]).optional(),
+    caseCosts: import_zod4.z.record(import_zod4.z.string(), EvalCostsSchema).optional(),
+    request: import_zod4.z.object({
+      caseIds: import_zod4.z.array(import_zod4.z.string()).optional(),
+      grading: import_zod4.z.enum(["app", "studio"]).optional(),
       judge: EvalJudgeIdentitySchema.optional(),
-      repetitions: import_zod3.z.number().int().positive(),
-      concurrency: import_zod3.z.number().optional(),
+      repetitions: import_zod4.z.number().int().positive(),
+      concurrency: import_zod4.z.number().optional(),
       promptSnapshot: import_browser2.PromptSnapshotSchema.optional(),
-      promptGroupName: import_zod3.z.string().optional()
+      promptGroupName: import_zod4.z.string().optional()
     }).optional(),
     suite: EvalSuiteSchema,
     result: EvalRunResultSchema.nullable(),
-    events: import_zod3.z.array(import_zod3.z.object({ id: import_zod3.z.number(), event: EvalProgressSchema }))
+    events: import_zod4.z.array(import_zod4.z.object({ id: import_zod4.z.number(), event: EvalProgressSchema }))
   })
 });
 
 // src/evals/evaluation-contracts.ts
-var EvaluationMetadataSchema = import_zod4.z.object({
-  source: import_zod4.z.enum(["manual", "deployment", "schedule", "ci"]).default("manual"),
-  commit: import_zod4.z.string().trim().min(1).max(100).optional(),
-  deploymentUrl: import_zod4.z.url().refine((value) => {
+var EvaluationMetadataSchema = import_zod5.z.object({
+  source: import_zod5.z.enum(["manual", "deployment", "schedule", "ci"]).default("manual"),
+  commit: import_zod5.z.string().trim().min(1).max(100).optional(),
+  deploymentUrl: import_zod5.z.url().refine((value) => {
     try {
       const url = new URL(value);
       return ["https:", "http:"].includes(url.protocol) && !url.username && !url.password;
@@ -567,62 +643,62 @@ var EvaluationMetadataSchema = import_zod4.z.object({
     }
   }, "Expected an HTTP(S) URL without credentials").optional()
 }).strict();
-var StudioEvaluationStartRequestSchema = import_zod4.z.object({
-  targetId: import_zod4.z.string().trim().min(1),
-  name: import_zod4.z.string().trim().min(1).max(200).optional(),
+var StudioEvaluationStartRequestSchema = import_zod5.z.object({
+  targetId: import_zod5.z.string().trim().min(1),
+  name: import_zod5.z.string().trim().min(1).max(200).optional(),
   promptSelection: StudioEvalStartRequestSchema.shape.promptSelection,
-  selection: import_zod4.z.enum(["all", "selected"]),
-  suites: import_zod4.z.array(
-    import_zod4.z.object({
-      suiteId: import_zod4.z.string().min(1),
-      suiteRevision: import_zod4.z.string().regex(/^[a-f0-9]{64}$/),
-      caseIds: import_zod4.z.array(import_zod4.z.string().min(1)).min(1).max(100).optional()
+  selection: import_zod5.z.enum(["all", "selected"]),
+  suites: import_zod5.z.array(
+    import_zod5.z.object({
+      suiteId: import_zod5.z.string().min(1),
+      suiteRevision: import_zod5.z.string().regex(/^[a-f0-9]{64}$/),
+      caseIds: import_zod5.z.array(import_zod5.z.string().min(1)).min(1).max(100).optional()
     }).strict()
   ).min(1).max(100),
-  judge: import_zod4.z.enum(["studio", "app"]).default("studio"),
-  repetitions: import_zod4.z.number().int().min(1).max(20).default(1),
-  concurrency: import_zod4.z.number().int().min(1).max(20).default(1),
+  judge: import_zod5.z.enum(["studio", "app"]).default("studio"),
+  repetitions: import_zod5.z.number().int().min(1).max(20).default(1),
+  concurrency: import_zod5.z.number().int().min(1).max(20).default(1),
   metadata: EvaluationMetadataSchema.default({ source: "manual" }),
-  idempotencyKey: import_zod4.z.string().trim().min(1).max(200).optional()
+  idempotencyKey: import_zod5.z.string().trim().min(1).max(200).optional()
 }).strict();
-var StudioEvaluationSummarySchema = import_zod4.z.object({
-  id: import_zod4.z.uuid(),
-  name: import_zod4.z.string(),
-  targetId: import_zod4.z.string(),
-  targetName: import_zod4.z.string(),
-  environment: import_zod4.z.string(),
-  selection: import_zod4.z.enum(["all", "selected"]),
+var StudioEvaluationSummarySchema = import_zod5.z.object({
+  id: import_zod5.z.uuid(),
+  name: import_zod5.z.string(),
+  targetId: import_zod5.z.string(),
+  targetName: import_zod5.z.string(),
+  environment: import_zod5.z.string(),
+  selection: import_zod5.z.enum(["all", "selected"]),
   metadata: EvaluationMetadataSchema,
   judge: EvalJudgeIdentitySchema.optional(),
   status: StudioEvalRunSummarySchema.shape.status,
-  createdAt: import_zod4.z.string(),
-  startedAt: import_zod4.z.string().nullable(),
-  endedAt: import_zod4.z.string().nullable(),
-  cancelRequestedAt: import_zod4.z.string().nullable(),
-  suiteCount: import_zod4.z.number().int().nonnegative(),
-  completedSuites: import_zod4.z.number().int().nonnegative(),
-  totalAttempts: import_zod4.z.number().int().nonnegative(),
-  completedAttempts: import_zod4.z.number().int().nonnegative(),
-  counts: import_zod4.z.object({
-    passed: import_zod4.z.number(),
-    failed: import_zod4.z.number(),
-    error: import_zod4.z.number(),
-    cancelled: import_zod4.z.number()
+  createdAt: import_zod5.z.string(),
+  startedAt: import_zod5.z.string().nullable(),
+  endedAt: import_zod5.z.string().nullable(),
+  cancelRequestedAt: import_zod5.z.string().nullable(),
+  suiteCount: import_zod5.z.number().int().nonnegative(),
+  completedSuites: import_zod5.z.number().int().nonnegative(),
+  totalAttempts: import_zod5.z.number().int().nonnegative(),
+  completedAttempts: import_zod5.z.number().int().nonnegative(),
+  counts: import_zod5.z.object({
+    passed: import_zod5.z.number(),
+    failed: import_zod5.z.number(),
+    error: import_zod5.z.number(),
+    cancelled: import_zod5.z.number()
   }),
   costs: EvalCostsSchema.optional(),
-  legacy: import_zod4.z.boolean().optional()
+  legacy: import_zod5.z.boolean().optional()
 });
-var StudioEvaluationHistorySchema = import_zod4.z.object({
-  runs: import_zod4.z.array(StudioEvaluationSummarySchema)
+var StudioEvaluationHistorySchema = import_zod5.z.object({
+  runs: import_zod5.z.array(StudioEvaluationSummarySchema)
 });
-var StudioEvaluationDetailSchema = import_zod4.z.object({
+var StudioEvaluationDetailSchema = import_zod5.z.object({
   run: StudioEvaluationSummarySchema.extend({
-    suites: import_zod4.z.array(StudioEvalRunSummarySchema)
+    suites: import_zod5.z.array(StudioEvalRunSummarySchema)
   })
 });
-var StudioEvaluationResultsSchema = import_zod4.z.object({
+var StudioEvaluationResultsSchema = import_zod5.z.object({
   run: StudioEvaluationSummarySchema.extend({
-    suites: import_zod4.z.array(StudioEvalDetailSchema.shape.run)
+    suites: import_zod5.z.array(StudioEvalDetailSchema.shape.run)
   })
 });
 
@@ -831,6 +907,7 @@ async function createStudioEvalJudge(options) {
   EvalCostAmountSchema,
   EvalCostsSchema,
   EvalEvidencePolicySchema,
+  EvalIssueSchema,
   EvalJudgeIdentitySchema,
   EvalJudgeUsageSchema,
   EvalManifestSchema,
@@ -856,6 +933,7 @@ async function createStudioEvalJudge(options) {
   StudioEvaluationResultsSchema,
   StudioEvaluationStartRequestSchema,
   StudioEvaluationSummarySchema,
+  createEvalIssue,
   createStudioEvalJudge,
   defineSuite,
   getEvalGradeEvidence,