@anvia/core 1.0.0-rc.1 → 1.0.0-rc.3

package/dist/evals/index.d.ts

@@ -1,22 +1,15 @@
-import { U as Usage, l as JsonValue, C as CompletionModel, M as Message } from '../types-B3A4YxaA.js';
+import { U as Usage, J as JsonObject, C as CompletionModel, M as Message } from '../types-D9rKQ_as.js';
 import { Z as ZodSchema } from '../zod-schema-C7F4clpm.js';
-import { A as Agent } from '../agent-DGYY_onS.js';
-import { l as AgentResponse } from '../index-DxB74ybJ.js';
-import { E as EmbeddingModel } from '../types-p55Jc-sr.js';
+import { a as AgentInteractionRequest, b as AgentInteractionResponse } from '../interactions-csssHYcN.js';
+import { t as AgentSuspendedResult, j as AgentResponse, l as AgentRunOptions, k as AgentResult } from '../types-CElwurHZ.js';
+import { E as EmbeddingModel } from '../types-Cr4uiYo5.js';
+import '../model-call-options-CZkSw_xN.js';
 import 'zod';
-import '../guardrails/index.js';
 import '../type-utils-CtHVDRn_.js';
-import '../types-BBEGKB9m.js';
-import '../middleware-KynLg_RW.js';
-import '../tool-C_8GudXn.js';
-import '../types-DulRCYdB.js';
-import '@modelcontextprotocol/sdk/client/sse.js';
-import '@modelcontextprotocol/sdk/client/stdio.js';
-import '@modelcontextprotocol/sdk/client/streamableHttp.js';
-import '../types-DhQfwiZn.js';
-import '../dynamic-tools-i9woysFa.js';
-import '../types-DT3nEemY.js';
-import '../retry-D3Ruy-ba.js';
+import '../retry-CjvSlKGW.js';
+import '../types-CIPaIbZo.js';
+import '../types-DbolvLtZ.js';
+import '../tool-r62gf3QY.js';
 
 type EvalOutcome<Score = unknown> = {
     outcome: "pass";
@@ -57,7 +50,8 @@ declare const EvalOutcome: {
     }): EvalOutcome<Score>;
 };
 
-type EvalMetadata = Record<string, JsonValue | undefined>;
+type EvalMetadata = JsonObject;
+type EvalReporterErrorPolicy = "collect" | "throw";
 type EvalRunOptions = {
     id?: string | undefined;
     datasetName?: string | undefined;
@@ -85,6 +79,7 @@ type EvalTurn = {
     metadata?: EvalMetadata | undefined;
 };
 type EvalTraceRef = {
+    observer?: string | undefined;
     traceId: string;
     observationId?: string | undefined;
     responseId?: string | undefined;
@@ -255,7 +250,7 @@ type RunEvalSuiteOptions<Input, Output, Expected = unknown, Metrics extends read
     concurrency?: number | undefined;
     trace?: EvalTraceSelector<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>> | undefined;
     reporters?: readonly EvalReporter<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>>[] | undefined;
-    failOnReporterError?: boolean | undefined;
+    reporterErrorPolicy?: EvalReporterErrorPolicy | undefined;
     targetUsage?: EvalTargetUsageSelector<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>> | undefined;
     cost?: EvalCostOptions<NoInfer<Input>, NoInfer<Output>, NoInfer<Expected>> | undefined;
 };
@@ -388,12 +383,41 @@ declare function knowledgeRetention<Input, Output, Expected = unknown, const Nam
 }): EvalMetric<Input, Output, number, Expected, Name>;
 type ConversationSource = EvalTurn[] | Message[];
 
-type AgentEvalTargetOptions<Input, Output = AgentResponse, Expected = unknown> = {
-    prompt?: ((input: Input, testCase: EvalCase<Input, Expected>) => string | Message) | undefined;
-    output?: ((response: AgentResponse, testCase: EvalCase<Input, Expected>) => Output) | undefined;
-};
-declare function agentEvalTarget<Input, Expected = unknown>(agent: Agent, options?: AgentEvalTargetOptions<Input, AgentResponse, Expected>): EvalTarget<Input, AgentResponse, Expected>;
-declare function agentEvalTarget<Input, Output, Expected = unknown>(agent: Agent, options: AgentEvalTargetOptions<Input, Output, Expected>): EvalTarget<Input, Output, Expected>;
+type EvaluableAgent<Output> = {
+    generate(input: AgentRunOptions<Output>): Promise<AgentResult<Output>>;
+};
+type IsAny<Value> = 0 extends 1 & Value ? true : false;
+type IsExactly<Left, Right> = IsAny<Left> extends true ? false : [Left] extends [Right] ? [Right] extends [Left] ? true : false : false;
+type AgentEvalTargetOptions<Input, AgentOutput = string, Output = AgentResponse<AgentOutput>, Expected = unknown> = {
+    agent: EvaluableAgent<AgentOutput>;
+    request(args: {
+        input: Input;
+        testCase: EvalCase<Input, Expected>;
+    }): AgentRunOptions<AgentOutput> | Promise<AgentRunOptions<AgentOutput>>;
+    interactions?: {
+        maxResponses?: number | undefined;
+        respond(args: {
+            interaction: AgentInteractionRequest;
+            testCase: EvalCase<Input, Expected>;
+            phase: number;
+        }): AgentInteractionResponse | Promise<AgentInteractionResponse>;
+    } | undefined;
+} & (IsExactly<Output, AgentResponse<AgentOutput>> extends true ? {
+    output?(args: {
+        response: AgentResponse<AgentOutput>;
+        testCase: EvalCase<Input, Expected>;
+    }): Output | Promise<Output>;
+} : {
+    output(args: {
+        response: AgentResponse<AgentOutput>;
+        testCase: EvalCase<Input, Expected>;
+    }): Output | Promise<Output>;
+});
+declare class AgentEvalSuspensionError extends Error {
+    readonly result: AgentSuspendedResult;
+    constructor(result: AgentSuspendedResult, message?: string);
+}
+declare function agentEvalTarget<Input, AgentOutput = string, Output = AgentResponse<AgentOutput>, Expected = unknown>(options: AgentEvalTargetOptions<Input, AgentOutput, Output, Expected>): EvalTarget<Input, Output, Expected>;
 
 type EvalOutputFormat = "pretty" | "json" | "quiet";
 type EvalExpectedTotals = Partial<EvalTotals> & {
@@ -569,6 +593,10 @@ declare function resolveEvalTraceRef(args: {
 }): EvalTraceRef | undefined;
 declare function defaultEvalTraceSelector<Input, Output, Expected>(args: EvalTraceSelectorArgs<Input, Output, Expected>): EvalTraceRef | undefined;
 
+declare class EvalReporterDispatchError extends AggregateError {
+    readonly phase: string;
+    constructor(phase: string, errors: readonly unknown[]);
+}
 declare function runEvalSuite<Input, Output, Expected = unknown, const Metrics extends readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[] = readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[]>(options: RunEvalSuiteOptions<Input, Output, Expected, Metrics>): Promise<EvalSuiteResult<Input, Output, Expected, Metrics>>;
 
 declare function selectPromptOutput(args: EvalMetricArgs<unknown, unknown, unknown>): string;
@@ -640,4 +668,4 @@ declare function defineEvalSuite<const Cases extends readonly EvalCaseLike[], co
     target: Target;
 };
 
-export { type AbstentionCategory, type AbstentionOptions, type AgentEvalTargetOptions, type AnswerRelevancyOptions, type AnyEvalMetric, type ContainsAllOptions, type ContainsAnyOptions, type ContainsListOptions, type ContainsOptions, type DefaultEvalActual, type DefinedEvalSuite, type DoesNotMatchOptions, EvalAssertionError, type EvalCase, type EvalCaseRequirements, type EvalCaseResult, type EvalCasesExpected, type EvalCasesForMetrics, type EvalCasesInput, type EvalCostCalculatorArgs, type EvalCostOptions, type EvalCostSummary, type EvalDataType, type EvalExpectations, type EvalExpectedOutcomes, type EvalExpectedTotals, type EvalMetadata, type EvalMetric, type EvalMetricArgs, type EvalMetricDescriptor, type EvalMetricResult, type EvalMetricResultFor, type EvalMetricScore, EvalOutcome, type EvalOutcomeStatus, type EvalOutputFormat, type EvalOutputWriters, type EvalReportArgs, type EvalReporter, type EvalRunContext, type EvalRunEndArgs, type EvalRunOptions, type EvalRunStartArgs, type EvalScoreDirection, type EvalScoreMap, type EvalScoreProjection, type EvalSuiteResult, type EvalTarget, type EvalTargetUsageSelector, type EvalTotals, type EvalTraceCarrier, type EvalTraceRef, type EvalTraceSelector, type EvalTraceSelectorArgs, type EvalTurn, type EvalUsageSummary, type ExactMatchOptions, type FaithfulnessOptions, type GEvalOptions, type GEvalParameter, type GEvalRubric, type HallucinationOptions, type JsonCorrectnessOptions, type KnowledgeRetentionOptions, type LlmJudgeOptions, type LlmScoreMetricScore, type LlmScoreOptions, type MatchesOptions, type MaxLengthOptions, type NotContainsOptions, type PrintEvalResultOptions, type PromptAlignmentOptions, type RequiredFieldsOptions, type RunEvalCliOptions, type RunEvalSuiteOptions, type SelectorOrValue, type SemanticSimilarityOptions, type SummarizationOptions, type TurnRelevancyOptions, type ValueSelector, abstention, agentEvalTarget, answerRelevancy, assertEvalOutcomes, assertEvalTotals, contains, containsAll, containsAny, defaultEvalTraceSelector, defineEvalCases, defineEvalSuite, defineMetric, doesNotMatch, evalExitCode, exactMatch, faithfulness, gEval, hallucination, jsonCorrectness, knowledgeRetention, llmJudge, llmScore, matches, maxLength, notContains, printEvalResult, projectEvalOutcome, promptAlignment, requiredFields, resolveEvalTraceRef, runEvalCli, runEvalSuite, selectPromptOutput, semanticSimilarity, summarization, turnRelevancy };
+export { type AbstentionCategory, type AbstentionOptions, AgentEvalSuspensionError, type AgentEvalTargetOptions, type AnswerRelevancyOptions, type AnyEvalMetric, type ContainsAllOptions, type ContainsAnyOptions, type ContainsListOptions, type ContainsOptions, type DefaultEvalActual, type DefinedEvalSuite, type DoesNotMatchOptions, EvalAssertionError, type EvalCase, type EvalCaseRequirements, type EvalCaseResult, type EvalCasesExpected, type EvalCasesForMetrics, type EvalCasesInput, type EvalCostCalculatorArgs, type EvalCostOptions, type EvalCostSummary, type EvalDataType, type EvalExpectations, type EvalExpectedOutcomes, type EvalExpectedTotals, type EvalMetadata, type EvalMetric, type EvalMetricArgs, type EvalMetricDescriptor, type EvalMetricResult, type EvalMetricResultFor, type EvalMetricScore, EvalOutcome, type EvalOutcomeStatus, type EvalOutputFormat, type EvalOutputWriters, type EvalReportArgs, type EvalReporter, EvalReporterDispatchError, type EvalReporterErrorPolicy, type EvalRunContext, type EvalRunEndArgs, type EvalRunOptions, type EvalRunStartArgs, type EvalScoreDirection, type EvalScoreMap, type EvalScoreProjection, type EvalSuiteResult, type EvalTarget, type EvalTargetUsageSelector, type EvalTotals, type EvalTraceCarrier, type EvalTraceRef, type EvalTraceSelector, type EvalTraceSelectorArgs, type EvalTurn, type EvalUsageSummary, type ExactMatchOptions, type FaithfulnessOptions, type GEvalOptions, type GEvalParameter, type GEvalRubric, type HallucinationOptions, type JsonCorrectnessOptions, type KnowledgeRetentionOptions, type LlmJudgeOptions, type LlmScoreMetricScore, type LlmScoreOptions, type MatchesOptions, type MaxLengthOptions, type NotContainsOptions, type PrintEvalResultOptions, type PromptAlignmentOptions, type RequiredFieldsOptions, type RunEvalCliOptions, type RunEvalSuiteOptions, type SelectorOrValue, type SemanticSimilarityOptions, type SummarizationOptions, type TurnRelevancyOptions, type ValueSelector, abstention, agentEvalTarget, answerRelevancy, assertEvalOutcomes, assertEvalTotals, contains, containsAll, containsAny, defaultEvalTraceSelector, defineEvalCases, defineEvalSuite, defineMetric, doesNotMatch, evalExitCode, exactMatch, faithfulness, gEval, hallucination, jsonCorrectness, knowledgeRetention, llmJudge, llmScore, matches, maxLength, notContains, printEvalResult, projectEvalOutcome, promptAlignment, requiredFields, resolveEvalTraceRef, runEvalCli, runEvalSuite, selectPromptOutput, semanticSimilarity, summarization, turnRelevancy };

package/dist/evals/index.js

@@ -1,33 +1,29 @@
 import {
-  cancelAgentApproval
-} from "../chunk-IL3ZSNJI.js";
-import "../chunk-YK4WAAS4.js";
-import "../chunk-6WGGTJE6.js";
-import "../chunk-R2LJSQUW.js";
-import "../chunk-5XKDW36Z.js";
+  AgentRunBlockedError
+} from "../chunk-TRQ3XRFD.js";
 import {
   cosineSimilarity,
-  embedText,
+  embedText
+} from "../chunk-GQL5KRAH.js";
+import {
   mapWithConcurrency
-} from "../chunk-DZ3CKYTH.js";
+} from "../chunk-3RM57ZT2.js";
 import {
-  Extractor
-} from "../chunk-J6GNQART.js";
-import "../chunk-I6G7BC42.js";
-import "../chunk-M3IOWB4Y.js";
-import "../chunk-RFDVKCIE.js";
+  extract
+} from "../chunk-ZCPOIDAQ.js";
+import "../chunk-MGNTXGTX.js";
 import {
   Usage
-} from "../chunk-ADH7NNCS.js";
-import "../chunk-A3UBYTJT.js";
-import "../chunk-FNQB2JEH.js";
+} from "../chunk-D2ECOWN5.js";
+import "../chunk-PK5LKOFL.js";
+import "../chunk-FN7HLLAZ.js";
 
 // src/evals/advanced-metrics.ts
 import { z } from "zod";
 
 // src/evals/format.ts
 function defaultOutputValue(output) {
-  if (typeof output === "object" && output !== null && "output" in output && typeof output.output === "string") {
+  if (typeof output === "object" && output !== null && "output" in output) {
     return output.output;
   }
   return output;
@@ -54,16 +50,15 @@ function errorMessage(error) {
 
 // src/evals/judge.ts
 async function runJudge(args) {
-  const extractor = new Extractor({
+  const result = await extract({
     model: args.model,
     outputSchema: args.schema,
-    instructions: args.instructions
-  });
-  const result = await extractor.extractResult(args.prompt, {
+    instructions: args.instructions,
+    text: args.prompt,
     temperature: 0,
     retries: args.retries <= 0 ? void 0 : { maxAttempts: Math.trunc(args.retries) + 1 }
   });
-  return { data: result.data, usage: result.usage };
+  return { data: result.output, usage: result.usage };
 }
 function addUsage(...values) {
   return values.reduce((total, usage) => Usage.add(total, usage), Usage.empty());
@@ -1120,17 +1115,61 @@ function contentText(content) {
 }
 
 // src/evals/agent-target.ts
-function agentEvalTarget(agent, options = {}) {
+var AgentEvalSuspensionError = class extends Error {
+  constructor(result, message = "Agent eval target suspended without an interaction responder.") {
+    super(message);
+    this.result = result;
+    this.name = "AgentEvalSuspensionError";
+  }
+  result;
+};
+function agentEvalTarget(options) {
   return async (input, testCase) => {
-    const prompt = options.prompt?.(input, testCase) ?? String(input);
-    const response = await agent.generate(prompt);
-    if (response.status === "approval_required") {
-      await cancelAgentApproval(response, "Agent eval targets cannot suspend for tool approval.");
-      throw new Error("Agent eval targets cannot suspend for tool approval.");
+    const maxResponses = options.interactions?.maxResponses ?? 10;
+    if (!Number.isSafeInteger(maxResponses) || maxResponses < 1) {
+      throw new TypeError("Agent eval interactions.maxResponses must be a positive integer.");
+    }
+    const request = await options.request({ input, testCase });
+    const runSettings = agentRunSettings(request);
+    let response = await options.agent.generate(request);
+    let phase = 0;
+    while (response.status === "suspended") {
+      if (options.interactions === void 0) {
+        throw new AgentEvalSuspensionError(response);
+      }
+      if (phase >= maxResponses) {
+        throw new AgentEvalSuspensionError(
+          response,
+          `Agent eval target exceeded the interaction response limit of ${maxResponses}.`
+        );
+      }
+      phase += 1;
+      const interactionResponse = await options.interactions.respond({
+        interaction: response.interaction,
+        testCase,
+        phase
+      });
+      response = await options.agent.generate({
+        continuation: response.continuation,
+        response: interactionResponse,
+        ...runSettings
+      });
     }
-    return options.output === void 0 ? response : options.output(response, testCase);
+    if (response.status === "blocked") throw new AgentRunBlockedError(response);
+    return options.output === void 0 ? response : await options.output({ response, testCase });
   };
 }
+function agentRunSettings(request) {
+  const {
+    prompt: _prompt,
+    messages: _messages,
+    session: _session,
+    continuation: _continuation,
+    response: _response,
+    ...settings
+  } = request;
+  return settings;
+}
 
 // src/evals/reporting.ts
 function projectEvalOutcome(outcome, dataType, projectScore) {
@@ -1193,6 +1232,7 @@ function traceFromCarrier(value) {
 function traceFromMetadata(metadata) {
   if (metadata === void 0) return void 0;
   return readTraceRef({
+    observer: metadata.traceObserver,
     traceId: metadata.traceId,
     observationId: metadata.observationId,
     responseId: metadata.responseId
@@ -1202,9 +1242,11 @@ function readTraceRef(value) {
   if (typeof value !== "object" || value === null) return void 0;
   const traceId = value.traceId;
   if (typeof traceId !== "string" || traceId.length === 0) return void 0;
+  const observer = value.observer;
   const observationId = value.observationId;
   const responseId = value.responseId;
   const trace = { traceId };
+  if (typeof observer === "string" && observer.length > 0) trace.observer = observer;
   if (typeof observationId === "string" && observationId.length > 0) {
     trace.observationId = observationId;
   }
@@ -1213,6 +1255,14 @@ function readTraceRef(value) {
 }
 
 // src/evals/runner.ts
+var EvalReporterDispatchError = class extends AggregateError {
+  phase;
+  constructor(phase, errors) {
+    super(errors, `Evaluation reporter ${phase} failed ${errors.length} time(s).`);
+    this.name = "EvalReporterDispatchError";
+    this.phase = phase;
+  }
+};
 async function runEvalSuite(options) {
   validateSuiteOptions(options);
   const startedAtMs = Date.now();
@@ -1229,7 +1279,7 @@ async function runEvalSuite(options) {
     reporterErrors = await notifyRunStart(
       reporters,
       lifecycle,
-      options.failOnReporterError === true
+      options.reporterErrorPolicy ?? "collect"
     );
   } catch (error) {
     await notifyRunEnd(reporters, {
@@ -1264,25 +1314,26 @@ async function runEvalSuite(options) {
     metrics: aggregates.metrics,
     cases: aggregates.cases,
     usage: aggregates.usage,
-    ...aggregates.cost === void 0 ? {} : { cost: aggregates.cost },
     durationMs: Date.now() - startedAtMs,
     reporterErrors
   };
+  if (aggregates.cost !== void 0) {
+    result.cost = aggregates.cost;
+  }
+  const runEndArgs = {
+    ...lifecycle,
+    status: "completed",
+    completedAt,
+    durationMs: result.durationMs,
+    metrics: result.metrics,
+    cases: result.cases,
+    usage: result.usage
+  };
+  if (result.cost !== void 0) {
+    runEndArgs.cost = result.cost;
+  }
   result.reporterErrors.push(
-    ...await notifyRunEnd(
-      reporters,
-      {
-        ...lifecycle,
-        status: "completed",
-        completedAt,
-        durationMs: result.durationMs,
-        metrics: result.metrics,
-        cases: result.cases,
-        usage: result.usage,
-        ...result.cost === void 0 ? {} : { cost: result.cost }
-      },
-      options.failOnReporterError === true
-    )
+    ...await notifyRunEnd(reporters, runEndArgs, options.reporterErrorPolicy ?? "collect")
   );
   return result;
 }
@@ -1335,7 +1386,7 @@ async function runEvalCase(options, testCase, run) {
       trace: traceResult.trace,
       traceError: traceResult.error,
       reporters: options.reporters ?? [],
-      failOnReporterError: options.failOnReporterError === true
+      reporterErrorPolicy: options.reporterErrorPolicy ?? "collect"
     });
     const metricResult = {
       metricName: metric.name,
@@ -1388,9 +1439,7 @@ async function safeEvaluate(suiteName, testCase, output, metric) {
 async function reportOutcome(args) {
   const errors = [];
   if (args.traceError !== void 0) {
-    if (args.failOnReporterError) throw args.traceError;
     errors.push(args.traceError);
-    return errors;
   }
   for (const reporter of args.reporters) {
     try {
@@ -1405,12 +1454,10 @@ async function reportOutcome(args) {
         outcome: args.outcome
       });
     } catch (error) {
-      if (args.failOnReporterError) {
-        throw error;
-      }
       errors.push(error);
     }
   }
+  throwReporterErrors("report", errors, args.reporterErrorPolicy);
   return errors;
 }
 function resolveRun(options, startedAtMs) {
@@ -1426,40 +1473,46 @@ function resolveRun(options, startedAtMs) {
       throw new TypeError(`Evaluation run ${label} must contain 1 to 256 characters`);
     }
   }
-  return {
+  const run = {
     id,
-    startedAt: new Date(startedAtMs).toISOString(),
-    ...options.run?.datasetName === void 0 ? {} : { datasetName: options.run.datasetName },
-    ...options.run?.datasetVersion === void 0 ? {} : { datasetVersion: options.run.datasetVersion },
-    ...options.run?.metadata === void 0 ? {} : { metadata: options.run.metadata }
+    startedAt: new Date(startedAtMs).toISOString()
   };
+  if (options.run?.datasetName !== void 0) run.datasetName = options.run.datasetName;
+  if (options.run?.datasetVersion !== void 0) run.datasetVersion = options.run.datasetVersion;
+  if (options.run?.metadata !== void 0) run.metadata = options.run.metadata;
+  return run;
 }
-async function notifyRunStart(reporters, args, failOnReporterError) {
+async function notifyRunStart(reporters, args, errorPolicy) {
   const errors = [];
   for (const reporter of reporters) {
     if (reporter.onRunStart === void 0) continue;
     try {
       await reporter.onRunStart(args);
     } catch (error) {
-      if (failOnReporterError) throw error;
       errors.push(error);
     }
   }
+  throwReporterErrors("onRunStart", errors, errorPolicy);
   return errors;
 }
-async function notifyRunEnd(reporters, args, failOnReporterError = false) {
+async function notifyRunEnd(reporters, args, errorPolicy = "collect") {
   const errors = [];
   for (const reporter of reporters) {
     if (reporter.onRunEnd === void 0) continue;
     try {
       await reporter.onRunEnd(args);
     } catch (error) {
-      if (failOnReporterError) throw error;
       errors.push(error);
     }
   }
+  throwReporterErrors("onRunEnd", errors, errorPolicy);
   return errors;
 }
+function throwReporterErrors(phase, errors, errorPolicy) {
+  if (errorPolicy === "throw" && errors.length > 0) {
+    throw new EvalReporterDispatchError(phase, errors);
+  }
+}
 function countMetricOutcomes(results) {
   const totals = emptyTotals();
   for (const result of results) {
@@ -1714,18 +1767,21 @@ function jsonResult(result) {
 }
 function expectationMismatches(result, expectations) {
   if (expectations === void 0) return [];
-  return [
-    ...expectations.totals === void 0 ? [] : totalMismatches(result, expectations.totals),
-    ...expectations.outcomes === void 0 ? [] : outcomeMismatches(result, expectations.outcomes)
-  ];
+  const mismatches = [];
+  if (expectations.totals !== void 0) {
+    mismatches.push(...totalMismatches(result, expectations.totals));
+  }
+  if (expectations.outcomes !== void 0) {
+    mismatches.push(...outcomeMismatches(result, expectations.outcomes));
+  }
+  return mismatches;
 }
 function totalMismatches(result, expected) {
-  const directMetrics = {
-    ...expected.total === void 0 ? {} : { total: expected.total },
-    ...expected.passed === void 0 ? {} : { passed: expected.passed },
-    ...expected.failed === void 0 ? {} : { failed: expected.failed },
-    ...expected.invalid === void 0 ? {} : { invalid: expected.invalid }
-  };
+  const directMetrics = {};
+  if (expected.total !== void 0) directMetrics.total = expected.total;
+  if (expected.passed !== void 0) directMetrics.passed = expected.passed;
+  if (expected.failed !== void 0) directMetrics.failed = expected.failed;
+  if (expected.invalid !== void 0) directMetrics.invalid = expected.invalid;
   return [
     ...totalsGroupMismatches("metrics", result.metrics, {
       ...directMetrics,
@@ -2004,9 +2060,9 @@ function semanticSimilarity(options) {
       if (typeof expected !== "string") {
         return EvalOutcome.invalid("Semantic similarity expected value must be a string.");
       }
-      const [actualEmbedding, expectedEmbedding] = await Promise.all([
-        embedText(options.model, actual),
-        embedText(options.model, expected)
+      const [{ embedding: actualEmbedding }, { embedding: expectedEmbedding }] = await Promise.all([
+        embedText({ model: options.model, text: actual }),
+        embedText({ model: options.model, text: expected })
       ]);
       const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);
       return score >= options.threshold ? EvalOutcome.pass(score) : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });
@@ -2014,23 +2070,19 @@ function semanticSimilarity(options) {
   };
 }
 function llmJudge(options) {
-  const extractor = new Extractor({
-    model: options.model,
-    outputSchema: options.schema,
-    instructions: options.instructions ?? "Judge the eval case by the requested schema. Submit the judgment using the schema."
-  });
   return {
     name: options.name ?? "llm_judge",
     required: options.required ?? true,
     async evaluate(args) {
       try {
-        const result = await extractor.extractResult(
-          await resolveJudgePrompt(options.prompt, args),
-          {
-            retries: evalExtractionRetries(options.retries)
-          }
-        );
-        return options.passes(result.data) ? EvalOutcome.pass(result.data, { usage: result.usage }) : EvalOutcome.fail(result.data, { usage: result.usage });
+        const result = await extract({
+          model: options.model,
+          outputSchema: options.schema,
+          instructions: options.instructions ?? "Judge the eval case by the requested schema. Submit the judgment using the schema.",
+          text: await resolveJudgePrompt(options.prompt, args),
+          retries: evalExtractionRetries(options.retries)
+        });
+        return options.passes(result.output) ? EvalOutcome.pass(result.output, { usage: result.usage }) : EvalOutcome.fail(result.output, { usage: result.usage });
       } catch (error) {
         return EvalOutcome.invalid(errorMessage(error));
       }
@@ -2039,17 +2091,6 @@ function llmJudge(options) {
 }
 function llmScore(options) {
   const criteria = Array.isArray(options.criteria) ? options.criteria.join("\n") : options.criteria;
-  const extractor = new Extractor({
-    model: options.model,
-    outputSchema: z2.object({
-      score: z2.number(),
-      feedback: z2.string()
-    }),
-    instructions: options.instructions ?? `Score the eval case against these criteria:
-${criteria}
-
-Return a score between 0 and 1 and brief feedback.`
-  });
   return {
     name: options.name ?? "llm_score",
     required: options.required ?? true,
@@ -2059,13 +2100,20 @@ Return a score between 0 and 1 and brief feedback.`
     threshold: options.threshold,
     async evaluate(args) {
       try {
-        const result = await extractor.extractResult(
-          await resolveJudgePrompt(options.prompt, args),
-          {
-            retries: evalExtractionRetries(options.retries)
-          }
-        );
-        const score = result.data;
+        const result = await extract({
+          model: options.model,
+          outputSchema: z2.object({
+            score: z2.number(),
+            feedback: z2.string()
+          }),
+          instructions: options.instructions ?? `Score the eval case against these criteria:
+${criteria}
+
+Return a score between 0 and 1 and brief feedback.`,
+          text: await resolveJudgePrompt(options.prompt, args),
+          retries: evalExtractionRetries(options.retries)
+        });
+        const score = result.output;
         if (score.score < 0 || score.score > 1) {
           return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {
             score,
@@ -2100,8 +2148,10 @@ function defineEvalSuite(options) {
   };
 }
 export {
+  AgentEvalSuspensionError,
   EvalAssertionError,
   EvalOutcome,
+  EvalReporterDispatchError,
   abstention,
   agentEvalTarget,
   answerRelevancy,