@kortyx/agent 0.31.1 → 0.33.0

package/dist/evals.js

@@ -23,6 +23,7 @@ __export(evals_exports, {
   EvalConfigurationError: () => EvalConfigurationError,
   EvalCostAmountSchema: () => EvalCostAmountSchema,
   EvalCostsSchema: () => EvalCostsSchema,
+  EvalEvidencePolicySchema: () => EvalEvidencePolicySchema,
   EvalJudgeIdentitySchema: () => EvalJudgeIdentitySchema,
   EvalJudgeUsageSchema: () => EvalJudgeUsageSchema,
   EvalManifestSchema: () => EvalManifestSchema,
@@ -50,32 +51,73 @@ __export(evals_exports, {
   StudioEvaluationSummarySchema: () => StudioEvaluationSummarySchema,
   createStudioEvalJudge: () => createStudioEvalJudge,
   defineSuite: () => defineSuite,
+  getEvalGradeEvidence: () => getEvalGradeEvidence,
   parseEvalSuite: () => parseEvalSuite
 });
 module.exports = __toCommonJS(evals_exports);
 
 // src/evals/contracts.ts
+var import_zod2 = require("zod");
+
+// src/evals/evidence-policy.ts
 var import_zod = require("zod");
+var EVAL_EVIDENCE_EVENT_TYPES = [
+  "tool-call-start",
+  "tool-call-result",
+  "tool-call-error",
+  "tool-result",
+  "structured-data",
+  "structured-data-invalidated",
+  "interrupt",
+  "error",
+  "cancelled",
+  "limit-reached"
+];
 var id = import_zod.z.string().trim().min(1).max(256);
-var handler = import_zod.z.object({ using: id, params: import_zod.z.json().optional() }).strict();
-var EvalResumeResponseSchema = import_zod.z.discriminatedUnion("type", [
-  import_zod.z.object({ type: import_zod.z.literal("text"), text: import_zod.z.string() }).strict(),
-  import_zod.z.object({ type: import_zod.z.literal("select"), ids: import_zod.z.array(import_zod.z.string()) }).strict(),
-  import_zod.z.object({ type: import_zod.z.literal("value"), value: import_zod.z.json() }).strict(),
-  import_zod.z.object({ type: import_zod.z.literal("cancel") }).strict()
+var filter = import_zod.z.object({ using: id, params: import_zod.z.json().optional() }).strict();
+var EvalEvidencePolicySchema = import_zod.z.object({
+  history: import_zod.z.boolean().optional(),
+  events: import_zod.z.union([
+    import_zod.z.literal(false),
+    import_zod.z.array(import_zod.z.enum(EVAL_EVIDENCE_EVENT_TYPES)),
+    filter
+  ]).optional(),
+  outputs: import_zod.z.union([
+    import_zod.z.literal(false),
+    import_zod.z.array(
+      import_zod.z.object({
+        dataType: id.optional(),
+        schemaId: id.optional(),
+        schemaVersion: id.optional()
+      }).strict().refine((value) => Boolean(value.dataType || value.schemaId), {
+        message: "Select an output by dataType or schemaId."
+      })
+    ),
+    filter
+  ]).optional()
+}).strict();
+
+// src/evals/contracts.ts
+var id2 = import_zod2.z.string().trim().min(1).max(256);
+var handler = import_zod2.z.object({ using: id2, params: import_zod2.z.json().optional() }).strict();
+var EvalResumeResponseSchema = import_zod2.z.discriminatedUnion("type", [
+  import_zod2.z.object({ type: import_zod2.z.literal("text"), text: import_zod2.z.string() }).strict(),
+  import_zod2.z.object({ type: import_zod2.z.literal("select"), ids: import_zod2.z.array(import_zod2.z.string()) }).strict(),
+  import_zod2.z.object({ type: import_zod2.z.literal("value"), value: import_zod2.z.json() }).strict(),
+  import_zod2.z.object({ type: import_zod2.z.literal("cancel") }).strict()
 ]);
-var expectation = import_zod.z.object({
-  type: import_zod.z.enum(["answer", "interrupt"]),
-  schemaId: id.optional(),
-  schemaVersion: id.optional(),
-  outputs: import_zod.z.array(import_zod.z.object({ schemaId: id, schemaVersion: id.optional() }).strict()).optional(),
-  criteria: import_zod.z.array(
-    import_zod.z.union([
-      import_zod.z.string().trim().min(1),
-      import_zod.z.object({ id, text: import_zod.z.string().trim().min(1) }).strict()
+var expectation = import_zod2.z.object({
+  type: import_zod2.z.enum(["answer", "interrupt"]),
+  schemaId: id2.optional(),
+  schemaVersion: id2.optional(),
+  outputs: import_zod2.z.array(import_zod2.z.object({ schemaId: id2, schemaVersion: id2.optional() }).strict()).optional(),
+  criteria: import_zod2.z.array(
+    import_zod2.z.union([
+      import_zod2.z.string().trim().min(1),
+      import_zod2.z.object({ id: id2, text: import_zod2.z.string().trim().min(1) }).strict()
     ])
   ).optional(),
-  reference: import_zod.z.json().optional()
+  reference: import_zod2.z.json().optional()
 }).strict().superRefine((value, ctx) => {
   if (value.type === "answer" && (value.schemaId || value.schemaVersion))
     ctx.addIssue({
@@ -91,20 +133,21 @@ var expectation = import_zod.z.object({
       message: "Criterion IDs must be unique within a step."
     });
 });
-var EvalSuiteSchema = import_zod.z.object({
-  id,
-  name: import_zod.z.string().optional(),
-  cases: import_zod.z.array(
-    import_zod.z.object({
-      id,
-      name: import_zod.z.string().optional(),
-      params: import_zod.z.json().optional(),
-      workflowId: id.optional(),
-      steps: import_zod.z.array(
-        import_zod.z.union([
-          import_zod.z.object({ message: import_zod.z.string().min(1), expect: expectation }).strict(),
-          import_zod.z.object({
-            resume: import_zod.z.union([EvalResumeResponseSchema, handler]),
+var EvalSuiteSchema = import_zod2.z.object({
+  evidence: EvalEvidencePolicySchema.optional(),
+  id: id2,
+  name: import_zod2.z.string().optional(),
+  cases: import_zod2.z.array(
+    import_zod2.z.object({
+      id: id2,
+      name: import_zod2.z.string().optional(),
+      params: import_zod2.z.json().optional(),
+      workflowId: id2.optional(),
+      steps: import_zod2.z.array(
+        import_zod2.z.union([
+          import_zod2.z.object({ message: import_zod2.z.string().min(1), expect: expectation }).strict(),
+          import_zod2.z.object({
+            resume: import_zod2.z.union([EvalResumeResponseSchema, handler]),
             expect: expectation
           }).strict()
         ])
@@ -112,32 +155,32 @@ var EvalSuiteSchema = import_zod.z.object({
     }).strict()
   ).min(1)
 }).strict();
-var EvalVerdictSchema = import_zod.z.object({
-  passed: import_zod.z.boolean(),
-  reason: import_zod.z.string().trim().min(1),
-  evidence: import_zod.z.array(import_zod.z.string().min(1))
+var EvalVerdictSchema = import_zod2.z.object({
+  passed: import_zod2.z.boolean(),
+  reason: import_zod2.z.string().trim().min(1),
+  evidence: import_zod2.z.array(import_zod2.z.string().min(1))
 }).strict();
-var EvalObservationSchema = import_zod.z.object({
-  type: import_zod.z.enum(["answer", "interrupt", "error", "cancelled"]),
-  text: import_zod.z.string(),
-  structured: import_zod.z.array(import_zod.z.json()),
-  events: import_zod.z.array(import_zod.z.json()).optional(),
-  runId: import_zod.z.string().optional(),
-  checkpointId: import_zod.z.string().optional(),
-  interrupt: import_zod.z.object({
-    requestId: import_zod.z.string(),
-    kind: import_zod.z.enum(["text", "choice", "multi-choice", "custom"]),
-    question: import_zod.z.string().optional(),
-    schemaId: import_zod.z.string().optional(),
-    schemaVersion: import_zod.z.string().optional(),
-    options: import_zod.z.array(
-      import_zod.z.object({
-        id: import_zod.z.string(),
-        label: import_zod.z.string(),
-        description: import_zod.z.string().optional()
+var EvalObservationSchema = import_zod2.z.object({
+  type: import_zod2.z.enum(["answer", "interrupt", "error", "cancelled"]),
+  text: import_zod2.z.string(),
+  structured: import_zod2.z.array(import_zod2.z.json()),
+  events: import_zod2.z.array(import_zod2.z.json()).optional(),
+  runId: import_zod2.z.string().optional(),
+  checkpointId: import_zod2.z.string().optional(),
+  interrupt: import_zod2.z.object({
+    requestId: import_zod2.z.string(),
+    kind: import_zod2.z.enum(["text", "choice", "multi-choice", "custom"]),
+    question: import_zod2.z.string().optional(),
+    schemaId: import_zod2.z.string().optional(),
+    schemaVersion: import_zod2.z.string().optional(),
+    options: import_zod2.z.array(
+      import_zod2.z.object({
+        id: import_zod2.z.string(),
+        label: import_zod2.z.string(),
+        description: import_zod2.z.string().optional()
       }).strict()
     ),
-    request: import_zod.z.json().optional()
+    request: import_zod2.z.json().optional()
   }).strict().optional()
 }).strict().superRefine((value, ctx) => {
   if (value.type === "interrupt" !== Boolean(value.interrupt))
@@ -146,9 +189,9 @@ var EvalObservationSchema = import_zod.z.object({
       message: "Interrupt observations require exactly one interrupt."
     });
 });
-var status = import_zod.z.enum(["passed", "failed", "error", "cancelled", "ungraded"]);
-var issue = import_zod.z.object({
-  phase: import_zod.z.enum([
+var status = import_zod2.z.enum(["passed", "failed", "error", "cancelled", "ungraded"]);
+var issue = import_zod2.z.object({
+  phase: import_zod2.z.enum([
     "params",
     "setup",
     "execute",
@@ -158,164 +201,169 @@ var issue = import_zod.z.object({
     "cleanup",
     "reporting"
   ]),
-  code: import_zod.z.string(),
-  message: import_zod.z.string()
+  code: import_zod2.z.string(),
+  message: import_zod2.z.string()
 }).strict();
-var EvalJudgeUsageSchema = import_zod.z.object({
-  provider: import_zod.z.string().min(1),
-  model: import_zod.z.string().min(1),
-  occurredAt: import_zod.z.iso.datetime(),
-  usage: import_zod.z.object({
-    input: import_zod.z.number().finite().nonnegative().optional(),
-    output: import_zod.z.number().finite().nonnegative().optional(),
-    total: import_zod.z.number().finite().nonnegative().optional(),
-    reasoning: import_zod.z.number().finite().nonnegative().optional(),
-    cacheRead: import_zod.z.number().finite().nonnegative().optional(),
-    cacheWrite: import_zod.z.number().finite().nonnegative().optional(),
-    cacheWrite1h: import_zod.z.number().finite().nonnegative().optional(),
-    outputIncludesReasoning: import_zod.z.boolean().optional(),
-    inputIncludesCacheRead: import_zod.z.boolean().optional(),
-    inputIncludesCacheWrite: import_zod.z.boolean().optional()
+var EvalJudgeUsageSchema = import_zod2.z.object({
+  provider: import_zod2.z.string().min(1),
+  model: import_zod2.z.string().min(1),
+  occurredAt: import_zod2.z.iso.datetime(),
+  usage: import_zod2.z.object({
+    input: import_zod2.z.number().finite().nonnegative().optional(),
+    output: import_zod2.z.number().finite().nonnegative().optional(),
+    total: import_zod2.z.number().finite().nonnegative().optional(),
+    reasoning: import_zod2.z.number().finite().nonnegative().optional(),
+    cacheRead: import_zod2.z.number().finite().nonnegative().optional(),
+    cacheWrite: import_zod2.z.number().finite().nonnegative().optional(),
+    cacheWrite1h: import_zod2.z.number().finite().nonnegative().optional(),
+    outputIncludesReasoning: import_zod2.z.boolean().optional(),
+    inputIncludesCacheRead: import_zod2.z.boolean().optional(),
+    inputIncludesCacheWrite: import_zod2.z.boolean().optional()
   }).strict().optional(),
-  pricingContext: import_zod.z.object({
-    serviceTier: import_zod.z.string().min(1).max(100).optional(),
-    inferenceGeo: import_zod.z.string().min(1).max(100).optional(),
-    speed: import_zod.z.string().min(1).max(100).optional()
+  pricingContext: import_zod2.z.object({
+    serviceTier: import_zod2.z.string().min(1).max(100).optional(),
+    inferenceGeo: import_zod2.z.string().min(1).max(100).optional(),
+    speed: import_zod2.z.string().min(1).max(100).optional()
   }).strict().optional(),
-  pricing: import_zod.z.object({
-    source: import_zod.z.literal("provider"),
-    currency: import_zod.z.literal("USD"),
-    actualCostMicros: import_zod.z.number().int().nonnegative().safe()
+  pricing: import_zod2.z.object({
+    source: import_zod2.z.literal("provider"),
+    currency: import_zod2.z.literal("USD"),
+    actualCostMicros: import_zod2.z.number().int().nonnegative().safe()
   }).strict().optional()
 }).strict();
-var EvalStepResultSchema = import_zod.z.object({
-  index: import_zod.z.number().int().nonnegative(),
-  input: import_zod.z.union([
-    import_zod.z.object({ message: import_zod.z.string() }).strict(),
-    import_zod.z.object({ resume: EvalResumeResponseSchema }).strict()
+var EvalStepResultSchema = import_zod2.z.object({
+  evidence: import_zod2.z.object({
+    version: import_zod2.z.literal("compact-v1"),
+    history: import_zod2.z.boolean(),
+    observation: EvalObservationSchema
+  }).strict().optional(),
+  index: import_zod2.z.number().int().nonnegative(),
+  input: import_zod2.z.union([
+    import_zod2.z.object({ message: import_zod2.z.string() }).strict(),
+    import_zod2.z.object({ resume: EvalResumeResponseSchema }).strict()
   ]),
   expectation,
   observation: EvalObservationSchema,
-  reference: import_zod.z.json().optional(),
+  reference: import_zod2.z.json().optional(),
   status,
-  reason: import_zod.z.string().optional(),
-  judgeCalls: import_zod.z.number().int().nonnegative().optional(),
-  judgeUsage: import_zod.z.array(EvalJudgeUsageSchema).optional(),
-  criteria: import_zod.z.array(
-    EvalVerdictSchema.extend({ id: import_zod.z.string(), text: import_zod.z.string() })
+  reason: import_zod2.z.string().optional(),
+  judgeCalls: import_zod2.z.number().int().nonnegative().optional(),
+  judgeUsage: import_zod2.z.array(EvalJudgeUsageSchema).optional(),
+  criteria: import_zod2.z.array(
+    EvalVerdictSchema.extend({ id: import_zod2.z.string(), text: import_zod2.z.string() })
   )
 }).strict();
-var EvalCaseResultSchema = import_zod.z.object({
-  caseId: id,
-  repetition: import_zod.z.number().int().positive(),
-  sessionId: import_zod.z.string(),
+var EvalCaseResultSchema = import_zod2.z.object({
+  caseId: id2,
+  repetition: import_zod2.z.number().int().positive(),
+  sessionId: import_zod2.z.string(),
   status,
-  durationMs: import_zod.z.number().nonnegative(),
-  steps: import_zod.z.array(EvalStepResultSchema),
-  errors: import_zod.z.array(issue)
+  durationMs: import_zod2.z.number().nonnegative(),
+  steps: import_zod2.z.array(EvalStepResultSchema),
+  errors: import_zod2.z.array(issue)
 }).strict();
-var EvalJudgeIdentitySchema = import_zod.z.object({
-  id,
-  version: id,
-  location: import_zod.z.enum(["app", "studio"]).optional()
+var EvalJudgeIdentitySchema = import_zod2.z.object({
+  id: id2,
+  version: id2,
+  location: import_zod2.z.enum(["app", "studio"]).optional()
 }).strict();
 var judge = EvalJudgeIdentitySchema;
-var StudioEvalJudgeRequestSchema = import_zod.z.object({
-  environment: id,
+var StudioEvalJudgeRequestSchema = import_zod2.z.object({
+  environment: id2,
   judge: EvalJudgeIdentitySchema,
-  criterion: import_zod.z.object({ id, text: import_zod.z.string().trim().min(1).max(16384) }).strict(),
+  criterion: import_zod2.z.object({ id: id2, text: import_zod2.z.string().trim().min(1).max(16384) }).strict(),
   input: EvalStepResultSchema.shape.input,
   observation: EvalObservationSchema,
-  reference: import_zod.z.json().optional(),
-  conversation: import_zod.z.array(EvalStepResultSchema).max(100)
+  reference: import_zod2.z.json().optional(),
+  conversation: import_zod2.z.array(EvalStepResultSchema).max(100)
 }).strict();
-var StudioEvalJudgeResponseSchema = import_zod.z.object({
+var StudioEvalJudgeResponseSchema = import_zod2.z.object({
   judge: EvalJudgeIdentitySchema,
   verdict: EvalVerdictSchema,
-  usage: import_zod.z.array(EvalJudgeUsageSchema).optional()
+  usage: import_zod2.z.array(EvalJudgeUsageSchema).optional()
 }).strict();
-var EvalRunResultSchema = import_zod.z.object({
-  id: import_zod.z.uuid(),
-  suiteId: id,
-  suiteRevision: import_zod.z.string().regex(/^[a-f0-9]{64}$/),
+var EvalRunResultSchema = import_zod2.z.object({
+  id: import_zod2.z.uuid(),
+  suiteId: id2,
+  suiteRevision: import_zod2.z.string().regex(/^[a-f0-9]{64}$/),
   suite: EvalSuiteSchema,
   judge: judge.optional(),
-  startedAt: import_zod.z.iso.datetime(),
-  durationMs: import_zod.z.number().nonnegative(),
+  startedAt: import_zod2.z.iso.datetime(),
+  durationMs: import_zod2.z.number().nonnegative(),
   status,
-  counts: import_zod.z.object({
-    passed: import_zod.z.number().int().nonnegative(),
-    failed: import_zod.z.number().int().nonnegative(),
-    error: import_zod.z.number().int().nonnegative(),
-    cancelled: import_zod.z.number().int().nonnegative(),
-    ungraded: import_zod.z.number().int().nonnegative().optional()
+  counts: import_zod2.z.object({
+    passed: import_zod2.z.number().int().nonnegative(),
+    failed: import_zod2.z.number().int().nonnegative(),
+    error: import_zod2.z.number().int().nonnegative(),
+    cancelled: import_zod2.z.number().int().nonnegative(),
+    ungraded: import_zod2.z.number().int().nonnegative().optional()
   }).strict(),
-  cases: import_zod.z.array(EvalCaseResultSchema),
-  errors: import_zod.z.array(issue)
+  cases: import_zod2.z.array(EvalCaseResultSchema),
+  errors: import_zod2.z.array(issue)
 }).strict();
-var EvalManifestSchema = import_zod.z.object({
-  schemaVersion: import_zod.z.literal(1),
-  studioJudging: import_zod.z.literal(true).optional(),
-  suites: import_zod.z.array(EvalSuiteSchema),
-  responders: import_zod.z.array(
-    import_zod.z.object({
-      name: id,
-      schemaId: id.optional(),
-      schemaVersion: id.optional()
+var EvalManifestSchema = import_zod2.z.object({
+  schemaVersion: import_zod2.z.literal(1),
+  studioJudging: import_zod2.z.literal(true).optional(),
+  suites: import_zod2.z.array(EvalSuiteSchema),
+  responders: import_zod2.z.array(
+    import_zod2.z.object({
+      name: id2,
+      schemaId: id2.optional(),
+      schemaVersion: id2.optional()
     }).strict()
   ),
-  references: import_zod.z.array(id),
-  paramsSchema: import_zod.z.record(import_zod.z.string(), import_zod.z.unknown()).optional(),
+  references: import_zod2.z.array(id2),
+  paramsSchema: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.unknown()).optional(),
   judge: judge.optional()
 }).strict();
-var EvalProgressSchema = import_zod.z.discriminatedUnion("type", [
-  import_zod.z.object({
-    type: import_zod.z.literal("run-started"),
-    caseIds: import_zod.z.array(id).min(1),
-    repetitions: import_zod.z.number().int().positive(),
-    concurrency: import_zod.z.number().int().positive()
+var EvalProgressSchema = import_zod2.z.discriminatedUnion("type", [
+  import_zod2.z.object({
+    type: import_zod2.z.literal("run-started"),
+    caseIds: import_zod2.z.array(id2).min(1),
+    repetitions: import_zod2.z.number().int().positive(),
+    concurrency: import_zod2.z.number().int().positive()
   }).strict(),
-  import_zod.z.object({
-    type: import_zod.z.literal("case-progress"),
-    caseId: id,
-    repetition: import_zod.z.number().int().positive(),
+  import_zod2.z.object({
+    type: import_zod2.z.literal("case-progress"),
+    caseId: id2,
+    repetition: import_zod2.z.number().int().positive(),
     phase: issue.shape.phase,
-    stepIndex: import_zod.z.number().int().nonnegative().optional()
+    stepIndex: import_zod2.z.number().int().nonnegative().optional()
   }).strict(),
-  import_zod.z.object({
-    type: import_zod.z.literal("case-started"),
-    caseId: id,
-    repetition: import_zod.z.number().int().positive(),
-    sessionId: import_zod.z.string()
+  import_zod2.z.object({
+    type: import_zod2.z.literal("case-started"),
+    caseId: id2,
+    repetition: import_zod2.z.number().int().positive(),
+    sessionId: import_zod2.z.string()
   }).strict(),
-  import_zod.z.object({
-    type: import_zod.z.literal("step-completed"),
-    caseId: id,
-    repetition: import_zod.z.number().int().positive(),
+  import_zod2.z.object({
+    type: import_zod2.z.literal("step-completed"),
+    caseId: id2,
+    repetition: import_zod2.z.number().int().positive(),
     step: EvalStepResultSchema
   }).strict(),
-  import_zod.z.object({ type: import_zod.z.literal("case-completed"), result: EvalCaseResultSchema }).strict()
+  import_zod2.z.object({ type: import_zod2.z.literal("case-completed"), result: EvalCaseResultSchema }).strict()
 ]);
-var EvalRemoteRunRequestSchema = import_zod.z.object({
-  suiteId: id,
-  suiteRevision: import_zod.z.string().regex(/^[a-f0-9]{64}$/),
-  grading: import_zod.z.enum(["app", "studio"]).optional(),
+var EvalRemoteRunRequestSchema = import_zod2.z.object({
+  suiteId: id2,
+  suiteRevision: import_zod2.z.string().regex(/^[a-f0-9]{64}$/),
+  grading: import_zod2.z.enum(["app", "studio"]).optional(),
   judge: EvalJudgeIdentitySchema.optional(),
-  caseIds: import_zod.z.array(id).min(1).max(100).optional(),
-  repetitions: import_zod.z.number().int().min(1).max(20).default(1),
-  concurrency: import_zod.z.number().int().min(1).max(4).default(1)
+  caseIds: import_zod2.z.array(id2).min(1).max(100).optional(),
+  repetitions: import_zod2.z.number().int().min(1).max(20).default(1),
+  concurrency: import_zod2.z.number().int().min(1).max(4).default(1)
 }).strict();
 var StudioEvalStartRequestSchema = EvalRemoteRunRequestSchema.omit({
   grading: true,
   judge: true
 }).extend({
-  targetId: import_zod.z.string().min(1).max(128),
-  judge: import_zod.z.enum(["studio", "app"]).default("studio")
+  targetId: import_zod2.z.string().min(1).max(128),
+  judge: import_zod2.z.enum(["studio", "app"]).default("studio")
 });
-var EvalWireEventSchema = import_zod.z.discriminatedUnion("type", [
-  import_zod.z.object({ type: import_zod.z.literal("progress"), event: EvalProgressSchema }).strict(),
-  import_zod.z.object({ type: import_zod.z.literal("result"), result: EvalRunResultSchema }).strict(),
-  import_zod.z.object({ type: import_zod.z.literal("error"), message: import_zod.z.string() }).strict()
+var EvalWireEventSchema = import_zod2.z.discriminatedUnion("type", [
+  import_zod2.z.object({ type: import_zod2.z.literal("progress"), event: EvalProgressSchema }).strict(),
+  import_zod2.z.object({ type: import_zod2.z.literal("result"), result: EvalRunResultSchema }).strict(),
+  import_zod2.z.object({ type: import_zod2.z.literal("error"), message: import_zod2.z.string() }).strict()
 ]);
 var EvalConfigurationError = class extends Error {
   name = "EvalConfigurationError";
@@ -348,22 +396,22 @@ function defineSuite(suite) {
 }
 
 // src/evals/evaluation-contracts.ts
-var import_zod3 = require("zod");
+var import_zod4 = require("zod");
 
 // src/evals/studio-contracts.ts
-var import_zod2 = require("zod");
-var StudioEvalTargetsResponseSchema = import_zod2.z.object({
-  canRun: import_zod2.z.boolean(),
+var import_zod3 = require("zod");
+var StudioEvalTargetsResponseSchema = import_zod3.z.object({
+  canRun: import_zod3.z.boolean(),
   studioJudge: EvalJudgeIdentitySchema.nullable().optional(),
-  targets: import_zod2.z.array(
-    import_zod2.z.object({
-      id: import_zod2.z.string(),
-      name: import_zod2.z.string(),
-      environment: import_zod2.z.string(),
-      error: import_zod2.z.string().nullable(),
+  targets: import_zod3.z.array(
+    import_zod3.z.object({
+      id: import_zod3.z.string(),
+      name: import_zod3.z.string(),
+      environment: import_zod3.z.string(),
+      error: import_zod3.z.string().nullable(),
       // Safe discovery diagnostics; optional for compatibility with older APIs.
-      diagnostic: import_zod2.z.object({
-        code: import_zod2.z.enum([
+      diagnostic: import_zod3.z.object({
+        code: import_zod3.z.enum([
           "environment_forbidden",
           "environment_unavailable",
           "endpoint_not_found",
@@ -372,35 +420,35 @@ var StudioEvalTargetsResponseSchema = import_zod2.z.object({
           "endpoint_unreachable",
           "manifest_invalid"
         ]),
-        httpStatus: import_zod2.z.number().int().min(100).max(599).optional()
+        httpStatus: import_zod3.z.number().int().min(100).max(599).optional()
       }).nullable().optional(),
       manifest: EvalManifestSchema.nullable(),
-      revisions: import_zod2.z.record(import_zod2.z.string(), import_zod2.z.string())
+      revisions: import_zod3.z.record(import_zod3.z.string(), import_zod3.z.string())
     })
   )
 });
-var EvalCostAmountSchema = import_zod2.z.object({
-  amount: import_zod2.z.number().finite().nonnegative().nullable(),
-  currency: import_zod2.z.string().nullable(),
-  status: import_zod2.z.enum(["complete", "partial", "unavailable"]),
-  calls: import_zod2.z.number().int().nonnegative(),
-  unpricedCalls: import_zod2.z.number().int().nonnegative(),
-  estimated: import_zod2.z.boolean()
+var EvalCostAmountSchema = import_zod3.z.object({
+  amount: import_zod3.z.number().finite().nonnegative().nullable(),
+  currency: import_zod3.z.string().nullable(),
+  status: import_zod3.z.enum(["complete", "partial", "unavailable"]),
+  calls: import_zod3.z.number().int().nonnegative(),
+  unpricedCalls: import_zod3.z.number().int().nonnegative(),
+  estimated: import_zod3.z.boolean()
 });
-var EvalCostsSchema = import_zod2.z.object({
+var EvalCostsSchema = import_zod3.z.object({
   workflow: EvalCostAmountSchema,
   judge: EvalCostAmountSchema,
   total: EvalCostAmountSchema
 });
-var StudioEvalRunSummarySchema = import_zod2.z.object({
-  id: import_zod2.z.uuid(),
-  evaluationId: import_zod2.z.uuid().nullable().optional(),
-  targetId: import_zod2.z.string(),
-  targetName: import_zod2.z.string(),
-  environment: import_zod2.z.string(),
-  suiteId: import_zod2.z.string(),
-  suiteRevision: import_zod2.z.string(),
-  status: import_zod2.z.enum([
+var StudioEvalRunSummarySchema = import_zod3.z.object({
+  id: import_zod3.z.uuid(),
+  evaluationId: import_zod3.z.uuid().nullable().optional(),
+  targetId: import_zod3.z.string(),
+  targetName: import_zod3.z.string(),
+  environment: import_zod3.z.string(),
+  suiteId: import_zod3.z.string(),
+  suiteRevision: import_zod3.z.string(),
+  status: import_zod3.z.enum([
     "queued",
     "running",
     "passed",
@@ -408,47 +456,47 @@ var StudioEvalRunSummarySchema = import_zod2.z.object({
     "error",
     "cancelled"
   ]),
-  createdAt: import_zod2.z.string(),
-  startedAt: import_zod2.z.string().nullable(),
-  endedAt: import_zod2.z.string().nullable(),
-  error: import_zod2.z.string().nullable(),
-  cancelRequestedAt: import_zod2.z.string().nullable(),
+  createdAt: import_zod3.z.string(),
+  startedAt: import_zod3.z.string().nullable(),
+  endedAt: import_zod3.z.string().nullable(),
+  error: import_zod3.z.string().nullable(),
+  cancelRequestedAt: import_zod3.z.string().nullable(),
   costs: EvalCostsSchema.optional(),
-  counts: import_zod2.z.object({
-    passed: import_zod2.z.number(),
-    failed: import_zod2.z.number(),
-    error: import_zod2.z.number(),
-    cancelled: import_zod2.z.number()
+  counts: import_zod3.z.object({
+    passed: import_zod3.z.number(),
+    failed: import_zod3.z.number(),
+    error: import_zod3.z.number(),
+    cancelled: import_zod3.z.number()
   }).nullable().optional(),
-  suiteName: import_zod2.z.string().nullable().optional(),
-  totalAttempts: import_zod2.z.number().int().nonnegative().optional(),
-  completedAttempts: import_zod2.z.number().int().nonnegative().optional(),
-  phase: import_zod2.z.string().optional()
+  suiteName: import_zod3.z.string().nullable().optional(),
+  totalAttempts: import_zod3.z.number().int().nonnegative().optional(),
+  completedAttempts: import_zod3.z.number().int().nonnegative().optional(),
+  phase: import_zod3.z.string().optional()
 });
-var StudioEvalHistorySchema = import_zod2.z.object({
-  runs: import_zod2.z.array(StudioEvalRunSummarySchema)
+var StudioEvalHistorySchema = import_zod3.z.object({
+  runs: import_zod3.z.array(StudioEvalRunSummarySchema)
 });
-var StudioEvalDetailSchema = import_zod2.z.object({
+var StudioEvalDetailSchema = import_zod3.z.object({
   run: StudioEvalRunSummarySchema.extend({
-    caseCosts: import_zod2.z.record(import_zod2.z.string(), EvalCostsSchema).optional(),
-    request: import_zod2.z.object({
-      caseIds: import_zod2.z.array(import_zod2.z.string()).optional(),
-      grading: import_zod2.z.enum(["app", "studio"]).optional(),
+    caseCosts: import_zod3.z.record(import_zod3.z.string(), EvalCostsSchema).optional(),
+    request: import_zod3.z.object({
+      caseIds: import_zod3.z.array(import_zod3.z.string()).optional(),
+      grading: import_zod3.z.enum(["app", "studio"]).optional(),
       judge: EvalJudgeIdentitySchema.optional(),
-      repetitions: import_zod2.z.number().int().positive(),
-      concurrency: import_zod2.z.number().optional()
+      repetitions: import_zod3.z.number().int().positive(),
+      concurrency: import_zod3.z.number().optional()
     }).optional(),
     suite: EvalSuiteSchema,
     result: EvalRunResultSchema.nullable(),
-    events: import_zod2.z.array(import_zod2.z.object({ id: import_zod2.z.number(), event: EvalProgressSchema }))
+    events: import_zod3.z.array(import_zod3.z.object({ id: import_zod3.z.number(), event: EvalProgressSchema }))
   })
 });
 
 // src/evals/evaluation-contracts.ts
-var EvaluationMetadataSchema = import_zod3.z.object({
-  source: import_zod3.z.enum(["manual", "deployment", "schedule", "ci"]).default("manual"),
-  commit: import_zod3.z.string().trim().min(1).max(100).optional(),
-  deploymentUrl: import_zod3.z.url().refine((value) => {
+var EvaluationMetadataSchema = import_zod4.z.object({
+  source: import_zod4.z.enum(["manual", "deployment", "schedule", "ci"]).default("manual"),
+  commit: import_zod4.z.string().trim().min(1).max(100).optional(),
+  deploymentUrl: import_zod4.z.url().refine((value) => {
     try {
       const url = new URL(value);
       return ["https:", "http:"].includes(url.protocol) && !url.username && !url.password;
@@ -457,64 +505,156 @@ var EvaluationMetadataSchema = import_zod3.z.object({
     }
   }, "Expected an HTTP(S) URL without credentials").optional()
 }).strict();
-var StudioEvaluationStartRequestSchema = import_zod3.z.object({
-  targetId: import_zod3.z.string().trim().min(1),
-  name: import_zod3.z.string().trim().min(1).max(200).optional(),
-  selection: import_zod3.z.enum(["all", "selected"]),
-  suites: import_zod3.z.array(
-    import_zod3.z.object({
-      suiteId: import_zod3.z.string().min(1),
-      suiteRevision: import_zod3.z.string().regex(/^[a-f0-9]{64}$/),
-      caseIds: import_zod3.z.array(import_zod3.z.string().min(1)).min(1).max(100).optional()
+var StudioEvaluationStartRequestSchema = import_zod4.z.object({
+  targetId: import_zod4.z.string().trim().min(1),
+  name: import_zod4.z.string().trim().min(1).max(200).optional(),
+  selection: import_zod4.z.enum(["all", "selected"]),
+  suites: import_zod4.z.array(
+    import_zod4.z.object({
+      suiteId: import_zod4.z.string().min(1),
+      suiteRevision: import_zod4.z.string().regex(/^[a-f0-9]{64}$/),
+      caseIds: import_zod4.z.array(import_zod4.z.string().min(1)).min(1).max(100).optional()
     }).strict()
   ).min(1).max(100),
-  judge: import_zod3.z.enum(["studio", "app"]).default("studio"),
-  repetitions: import_zod3.z.number().int().min(1).max(20).default(1),
-  concurrency: import_zod3.z.number().int().min(1).max(4).default(1),
+  judge: import_zod4.z.enum(["studio", "app"]).default("studio"),
+  repetitions: import_zod4.z.number().int().min(1).max(20).default(1),
+  concurrency: import_zod4.z.number().int().min(1).max(4).default(1),
   metadata: EvaluationMetadataSchema.default({ source: "manual" }),
-  idempotencyKey: import_zod3.z.string().trim().min(1).max(200).optional()
+  idempotencyKey: import_zod4.z.string().trim().min(1).max(200).optional()
 }).strict();
-var StudioEvaluationSummarySchema = import_zod3.z.object({
-  id: import_zod3.z.uuid(),
-  name: import_zod3.z.string(),
-  targetId: import_zod3.z.string(),
-  targetName: import_zod3.z.string(),
-  environment: import_zod3.z.string(),
-  selection: import_zod3.z.enum(["all", "selected"]),
+var StudioEvaluationSummarySchema = import_zod4.z.object({
+  id: import_zod4.z.uuid(),
+  name: import_zod4.z.string(),
+  targetId: import_zod4.z.string(),
+  targetName: import_zod4.z.string(),
+  environment: import_zod4.z.string(),
+  selection: import_zod4.z.enum(["all", "selected"]),
   metadata: EvaluationMetadataSchema,
   judge: EvalJudgeIdentitySchema.optional(),
   status: StudioEvalRunSummarySchema.shape.status,
-  createdAt: import_zod3.z.string(),
-  startedAt: import_zod3.z.string().nullable(),
-  endedAt: import_zod3.z.string().nullable(),
-  cancelRequestedAt: import_zod3.z.string().nullable(),
-  suiteCount: import_zod3.z.number().int().nonnegative(),
-  completedSuites: import_zod3.z.number().int().nonnegative(),
-  totalAttempts: import_zod3.z.number().int().nonnegative(),
-  completedAttempts: import_zod3.z.number().int().nonnegative(),
-  counts: import_zod3.z.object({
-    passed: import_zod3.z.number(),
-    failed: import_zod3.z.number(),
-    error: import_zod3.z.number(),
-    cancelled: import_zod3.z.number()
+  createdAt: import_zod4.z.string(),
+  startedAt: import_zod4.z.string().nullable(),
+  endedAt: import_zod4.z.string().nullable(),
+  cancelRequestedAt: import_zod4.z.string().nullable(),
+  suiteCount: import_zod4.z.number().int().nonnegative(),
+  completedSuites: import_zod4.z.number().int().nonnegative(),
+  totalAttempts: import_zod4.z.number().int().nonnegative(),
+  completedAttempts: import_zod4.z.number().int().nonnegative(),
+  counts: import_zod4.z.object({
+    passed: import_zod4.z.number(),
+    failed: import_zod4.z.number(),
+    error: import_zod4.z.number(),
+    cancelled: import_zod4.z.number()
   }),
   costs: EvalCostsSchema.optional(),
-  legacy: import_zod3.z.boolean().optional()
+  legacy: import_zod4.z.boolean().optional()
 });
-var StudioEvaluationHistorySchema = import_zod3.z.object({
-  runs: import_zod3.z.array(StudioEvaluationSummarySchema)
+var StudioEvaluationHistorySchema = import_zod4.z.object({
+  runs: import_zod4.z.array(StudioEvaluationSummarySchema)
 });
-var StudioEvaluationDetailSchema = import_zod3.z.object({
+var StudioEvaluationDetailSchema = import_zod4.z.object({
   run: StudioEvaluationSummarySchema.extend({
-    suites: import_zod3.z.array(StudioEvalRunSummarySchema)
+    suites: import_zod4.z.array(StudioEvalRunSummarySchema)
   })
 });
-var StudioEvaluationResultsSchema = import_zod3.z.object({
+var StudioEvaluationResultsSchema = import_zod4.z.object({
   run: StudioEvaluationSummarySchema.extend({
-    suites: import_zod3.z.array(StudioEvalDetailSchema.shape.run)
+    suites: import_zod4.z.array(StudioEvalDetailSchema.shape.run)
   })
 });
 
+// src/evals/evidence.ts
+var object = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+var handler2 = (value) => value !== null && typeof value === "object" && "using" in value;
+function predicate(filter2, value, params) {
+  if (!filter2)
+    throw new EvalConfigurationError("Evidence filter is not registered.");
+  const selected = filter2(
+    structuredClone(value),
+    params === void 0 ? void 0 : structuredClone(params)
+  );
+  if (typeof selected !== "boolean")
+    throw new Error("Evidence filters must return a boolean synchronously.");
+  return selected;
+}
+function compactEvalObservation(observation, policy = {}, filters = {}) {
+  const outputs = policy.outputs;
+  const includeOutput = (output) => {
+    if (outputs === false) return false;
+    if (handler2(outputs))
+      return predicate(
+        filters.outputs?.[outputs.using],
+        output,
+        outputs.params
+      );
+    if (!outputs) return true;
+    return object(output) && outputs.some(
+      (selector) => Object.entries(selector).every(([key, value]) => output[key] === value)
+    );
+  };
+  const selectedOutputs = observation.structured.map((output) => ({
+    output,
+    included: includeOutput(output)
+  }));
+  const structured = selectedOutputs.filter((item) => item.included).map((item) => item.output);
+  const events = observation.events?.filter((value) => {
+    if (!object(value) || !EVAL_EVIDENCE_EVENT_TYPES.some((type) => type === value.type))
+      return false;
+    if (value.type === "structured-data") {
+      if (value.kind !== "final") return false;
+      const current = selectedOutputs.find(
+        ({ output: item }) => object(item) && typeof value.streamId === "string" && item.streamId === value.streamId && item.status === "done" && item.dataType === value.dataType && item.schemaId === value.schemaId && item.schemaVersion === value.schemaVersion && JSON.stringify(item.data) === JSON.stringify(value.data)
+      );
+      if (current) {
+        if (!current.included) return false;
+        if (!observation.events?.some(
+          (event) => object(event) && event.type === "structured-data-invalidated" && event.streamId === value.streamId
+        ))
+          return false;
+      } else {
+        const { type: _type, kind: _kind, ...fields } = value;
+        if (!includeOutput({ ...fields, status: "done" })) return false;
+      }
+    }
+    return true;
+  }).filter((event) => {
+    const selection = policy.events;
+    if (selection === false) return false;
+    if (handler2(selection))
+      return predicate(
+        filters.events?.[selection.using],
+        event,
+        selection.params
+      );
+    return !selection || selection.includes(event.type);
+  });
+  return structuredClone({
+    ...observation,
+    structured,
+    ...events ? { events } : {}
+  });
+}
+function createEvalJudgeEvidence(observation, policy = {}, filters = {}) {
+  return {
+    version: "compact-v1",
+    history: policy.history ?? true,
+    observation: compactEvalObservation(observation, policy, filters)
+  };
+}
+function getEvalGradeEvidence(step, previous) {
+  const evidence = step.evidence ?? createEvalJudgeEvidence(step.observation);
+  return {
+    observation: structuredClone(evidence.observation),
+    conversation: evidence.history ? previous.map((prior) => {
+      const { evidence: selected, ...result } = prior;
+      return structuredClone({
+        ...result,
+        observation: selected?.observation ?? compactEvalObservation(prior.observation)
+      });
+    }) : []
+  };
+}
+
 // src/evals/studio-judge.ts
 async function readJson(response, signal) {
   if (!response.ok || !response.body) {
@@ -583,9 +723,14 @@ async function createStudioEvalJudge(options) {
         judge: identity,
         criterion: input.criterion,
         input: input.input,
-        observation: input.observation,
+        observation: compactEvalObservation(input.observation),
         reference: input.reference,
-        conversation: input.conversation
+        conversation: input.conversation.map(
+          ({ evidence: _evidence, ...step }) => ({
+            ...step,
+            observation: compactEvalObservation(step.observation)
+          })
+        )
       });
       const signal2 = AbortSignal.any([
         input.signal,
@@ -622,6 +767,7 @@ async function createStudioEvalJudge(options) {
   EvalConfigurationError,
   EvalCostAmountSchema,
   EvalCostsSchema,
+  EvalEvidencePolicySchema,
   EvalJudgeIdentitySchema,
   EvalJudgeUsageSchema,
   EvalManifestSchema,
@@ -649,6 +795,7 @@ async function createStudioEvalJudge(options) {
   StudioEvaluationSummarySchema,
   createStudioEvalJudge,
   defineSuite,
+  getEvalGradeEvidence,
   parseEvalSuite
 });
 //# sourceMappingURL=evals.js.map