@anvia/core 0.26.0 → 1.0.0-rc.10

package/dist/evals/index.js

@@ -1,37 +1,33 @@
 import {
-  ExtractorBuilder
-} from "../chunk-YLMDBQO4.js";
-import "../chunk-JI5YZWNT.js";
-import "../chunk-DALYP4PX.js";
-import "../chunk-BALBBVI3.js";
-import "../chunk-YK4WAAS4.js";
-import "../chunk-XUUY2L2D.js";
-import "../chunk-HYUYZEAJ.js";
-import "../chunk-MELYDKWQ.js";
-import "../chunk-2ODTMRHP.js";
-import "../chunk-UQX6EXWG.js";
-import "../chunk-MRQLEK2B.js";
+  AgentRunBlockedError
+} from "../chunk-X4LK4ODN.js";
 import {
   cosineSimilarity,
   embedText
-} from "../chunk-MNWK2USR.js";
+} from "../chunk-ZA564323.js";
 import {
   mapWithConcurrency
-} from "../chunk-OIMLU4SF.js";
-import "../chunk-Q25IWLBT.js";
-import "../chunk-MC3CCKEB.js";
+} from "../chunk-3RM57ZT2.js";
+import {
+  extract
+} from "../chunk-LTXABCOO.js";
+import "../chunk-D4PACGPA.js";
 import {
   Usage
-} from "../chunk-ADH7NNCS.js";
-import "../chunk-WQKHFADH.js";
-import "../chunk-CWUJUSOS.js";
+} from "../chunk-7PHMK7W4.js";
+import "../chunk-7JLAIN6E.js";
+import "../chunk-QGX73TSQ.js";
+import {
+  isJsonValue
+} from "../chunk-3XQGVDU5.js";
+import "../chunk-3RWESPUG.js";
 
 // src/evals/advanced-metrics.ts
 import { z } from "zod";
 
 // src/evals/format.ts
 function defaultOutputValue(output) {
-  if (typeof output === "object" && output !== null && "output" in output && typeof output.output === "string") {
+  if (typeof output === "object" && output !== null && "output" in output) {
     return output.output;
   }
   return output;
@@ -58,8 +54,15 @@ function errorMessage(error) {
 
 // src/evals/judge.ts
 async function runJudge(args) {
-  const result = await new ExtractorBuilder(args.model, args.schema).instructions(args.instructions).temperature(0).retries(args.retries).build().extractWithUsage(args.prompt);
-  return { data: result.data, usage: result.usage };
+  const result = await extract({
+    model: args.model,
+    outputSchema: args.schema,
+    instructions: args.instructions,
+    text: args.prompt,
+    temperature: 0,
+    retries: args.retries <= 0 ? void 0 : { maxAttempts: Math.trunc(args.retries) + 1 }
+  });
+  return { data: result.output, usage: result.usage };
 }
 function addUsage(...values) {
   return values.reduce((total, usage) => Usage.add(total, usage), Usage.empty());
@@ -238,13 +241,14 @@ ${actual}`,
         usage = addUsage(usage, verdictResult.usage);
       }
       const score = verdicts.length === 0 ? 1 : verdicts.filter((verdict) => verdict.verdict !== "no").length / verdicts.length;
+      const serializedVerdicts = serializeVerdicts(verdicts);
       const reasonResult = await maybeReason({
         model: options.model,
         includeReason: config.includeReason,
         retries: config.retries,
         metric: "answer relevancy",
         score,
-        evidence: { input, verdicts }
+        evidence: { input, verdicts: serializedVerdicts }
       });
       usage = addUsage(usage, reasonResult.usage);
       return higherOutcome({
@@ -252,7 +256,7 @@ ${actual}`,
         threshold: config.threshold,
         strictMode: config.strictMode,
         comment: reasonResult.reason,
-        details: { statements, verdicts },
+        details: { statements, verdicts: serializedVerdicts },
         usage
       });
     } catch (error) {
@@ -320,6 +324,9 @@ function jsonCorrectness(options) {
         let validationError;
         try {
           parsed = JSON.parse(actual);
+          if (!isJsonValue(parsed)) {
+            throw new TypeError("Generated output is not a JSON value.");
+          }
           const result = options.schema.safeParse(parsed);
           if (!result.success) {
             validationError = z.prettifyError(result.error);
@@ -452,13 +459,14 @@ ${actual}`,
         (verdict) => verdict.verdict === "yes" || verdict.verdict === "idk" && !penalizeAmbiguousClaims
       ).length;
       const score = verdicts.length === 0 ? 1 : supported / verdicts.length;
+      const serializedVerdicts = serializeVerdicts(verdicts);
       const reasonResult = await maybeReason({
         model: options.model,
         includeReason: config.includeReason,
         retries: config.retries,
         metric: "faithfulness",
         score,
-        evidence: { verdicts, penalizeAmbiguousClaims }
+        evidence: { verdicts: serializedVerdicts, penalizeAmbiguousClaims }
       });
       usage = addUsage(usage, reasonResult.usage);
       return higherOutcome({
@@ -466,7 +474,7 @@ ${actual}`,
         threshold: config.threshold,
         strictMode: config.strictMode,
         comment: reasonResult.reason,
-        details: { truths, claims, verdicts, penalizeAmbiguousClaims },
+        details: { truths, claims, verdicts: serializedVerdicts, penalizeAmbiguousClaims },
         usage
       });
     } catch (error) {
@@ -630,13 +638,19 @@ ${actual}`,
       });
       const coverageScore = coverageTotal === 0 ? 0 : coverageMatched / coverageTotal;
       const score = Math.min(alignmentScore, coverageScore);
+      const serializedAlignmentVerdicts = serializeVerdicts(alignmentVerdicts);
       const reasonResult = await maybeReason({
         model: options.model,
         includeReason: config.includeReason,
         retries: config.retries,
         metric: "summarization",
         score,
-        evidence: { alignmentVerdicts, coverageVerdicts, alignmentScore, coverageScore }
+        evidence: {
+          alignmentVerdicts: serializedAlignmentVerdicts,
+          coverageVerdicts,
+          alignmentScore,
+          coverageScore
+        }
       });
       const usage = addUsage(
         truthResult.usage,
@@ -656,7 +670,7 @@ ${actual}`,
           truths,
           claims,
           assessmentQuestions: questions,
-          alignmentVerdicts,
+          alignmentVerdicts: serializedAlignmentVerdicts,
           coverageVerdicts,
           scoreBreakdown: { alignment: alignmentScore, coverage: coverageScore }
         },
@@ -1042,15 +1056,33 @@ function assertSameLength(label, inputs, outputs) {
     throw new Error(`${label} count ${outputs.length} did not match input count ${inputs.length}.`);
   }
 }
+function serializeVerdicts(verdicts) {
+  return verdicts.map((verdict) => {
+    const serialized = { verdict: verdict.verdict };
+    if (verdict.reason !== void 0) serialized.reason = verdict.reason;
+    return serialized;
+  });
+}
 function jsonPrompt(value) {
   return JSON.stringify(value, null, 2);
 }
 function toJsonValue(value) {
-  try {
-    return JSON.parse(JSON.stringify(value));
-  } catch {
-    return String(value);
-  }
+  if (!isJsonValue(value)) {
+    throw new TypeError("G-Eval expectedOutput must be a JSON value.");
+  }
+  return cloneJsonValue(value);
+}
+function cloneJsonValue(value) {
+  if (value === null || typeof value !== "object") return value;
+  if (Array.isArray(value)) return value.map(cloneJsonValue);
+  return Object.fromEntries(
+    Object.entries(value).map(([key, item]) => {
+      if (item === void 0) {
+        throw new TypeError("G-Eval expectedOutput must be a JSON value.");
+      }
+      return [key, cloneJsonValue(item)];
+    })
+  );
 }
 function normalizeEvalTurns(value) {
   const source = conversationArray(value);
@@ -1116,13 +1148,61 @@ function contentText(content) {
 }
 
 // src/evals/agent-target.ts
-function agentEvalTarget(agent, options = {}) {
+var AgentEvalSuspensionError = class extends Error {
+  constructor(result, message = "Agent eval target suspended without an interaction responder.") {
+    super(message);
+    this.result = result;
+    this.name = "AgentEvalSuspensionError";
+  }
+  result;
+};
+function agentEvalTarget(options) {
   return async (input, testCase) => {
-    const prompt = options.prompt?.(input, testCase) ?? String(input);
-    const response = await agent.prompt(prompt).send();
-    return options.output === void 0 ? response : options.output(response, testCase);
+    const maxResponses = options.interactions?.maxResponses ?? 10;
+    if (!Number.isSafeInteger(maxResponses) || maxResponses < 1) {
+      throw new TypeError("Agent eval interactions.maxResponses must be a positive integer.");
+    }
+    const request = await options.request({ input, testCase });
+    const runSettings = agentRunSettings(request);
+    let response = await options.agent.generate(request);
+    let phase = 0;
+    while (response.status === "suspended") {
+      if (options.interactions === void 0) {
+        throw new AgentEvalSuspensionError(response);
+      }
+      if (phase >= maxResponses) {
+        throw new AgentEvalSuspensionError(
+          response,
+          `Agent eval target exceeded the interaction response limit of ${maxResponses}.`
+        );
+      }
+      phase += 1;
+      const interactionResponse = await options.interactions.respond({
+        interaction: response.interaction,
+        testCase,
+        phase
+      });
+      response = await options.agent.generate({
+        continuation: response.continuation,
+        response: interactionResponse,
+        ...runSettings
+      });
+    }
+    if (response.status === "blocked") throw new AgentRunBlockedError(response);
+    return options.output === void 0 ? response : await options.output({ response, testCase });
   };
 }
+function agentRunSettings(request) {
+  const {
+    prompt: _prompt,
+    messages: _messages,
+    session: _session,
+    continuation: _continuation,
+    response: _response,
+    ...settings
+  } = request;
+  return settings;
+}
 
 // src/evals/reporting.ts
 function projectEvalOutcome(outcome, dataType, projectScore) {
@@ -1185,6 +1265,7 @@ function traceFromCarrier(value) {
 function traceFromMetadata(metadata) {
   if (metadata === void 0) return void 0;
   return readTraceRef({
+    observer: metadata.traceObserver,
     traceId: metadata.traceId,
     observationId: metadata.observationId,
     responseId: metadata.responseId
@@ -1194,9 +1275,11 @@ function readTraceRef(value) {
   if (typeof value !== "object" || value === null) return void 0;
   const traceId = value.traceId;
   if (typeof traceId !== "string" || traceId.length === 0) return void 0;
+  const observer = value.observer;
   const observationId = value.observationId;
   const responseId = value.responseId;
   const trace = { traceId };
+  if (typeof observer === "string" && observer.length > 0) trace.observer = observer;
   if (typeof observationId === "string" && observationId.length > 0) {
     trace.observationId = observationId;
   }
@@ -1205,6 +1288,14 @@ function readTraceRef(value) {
 }
 
 // src/evals/runner.ts
+var EvalReporterDispatchError = class extends AggregateError {
+  phase;
+  constructor(phase, errors) {
+    super(errors, `Evaluation reporter ${phase} failed ${errors.length} time(s).`);
+    this.name = "EvalReporterDispatchError";
+    this.phase = phase;
+  }
+};
 async function runEvalSuite(options) {
   validateSuiteOptions(options);
   const startedAtMs = Date.now();
@@ -1221,7 +1312,7 @@ async function runEvalSuite(options) {
     reporterErrors = await notifyRunStart(
       reporters,
       lifecycle,
-      options.failOnReporterError === true
+      options.reporterErrorPolicy ?? "collect"
     );
   } catch (error) {
     await notifyRunEnd(reporters, {
@@ -1256,25 +1347,26 @@ async function runEvalSuite(options) {
     metrics: aggregates.metrics,
     cases: aggregates.cases,
     usage: aggregates.usage,
-    ...aggregates.cost === void 0 ? {} : { cost: aggregates.cost },
     durationMs: Date.now() - startedAtMs,
     reporterErrors
   };
+  if (aggregates.cost !== void 0) {
+    result.cost = aggregates.cost;
+  }
+  const runEndArgs = {
+    ...lifecycle,
+    status: "completed",
+    completedAt,
+    durationMs: result.durationMs,
+    metrics: result.metrics,
+    cases: result.cases,
+    usage: result.usage
+  };
+  if (result.cost !== void 0) {
+    runEndArgs.cost = result.cost;
+  }
   result.reporterErrors.push(
-    ...await notifyRunEnd(
-      reporters,
-      {
-        ...lifecycle,
-        status: "completed",
-        completedAt,
-        durationMs: result.durationMs,
-        metrics: result.metrics,
-        cases: result.cases,
-        usage: result.usage,
-        ...result.cost === void 0 ? {} : { cost: result.cost }
-      },
-      options.failOnReporterError === true
-    )
+    ...await notifyRunEnd(reporters, runEndArgs, options.reporterErrorPolicy ?? "collect")
   );
   return result;
 }
@@ -1327,7 +1419,7 @@ async function runEvalCase(options, testCase, run) {
       trace: traceResult.trace,
       traceError: traceResult.error,
       reporters: options.reporters ?? [],
-      failOnReporterError: options.failOnReporterError === true
+      reporterErrorPolicy: options.reporterErrorPolicy ?? "collect"
     });
     const metricResult = {
       metricName: metric.name,
@@ -1380,9 +1472,7 @@ async function safeEvaluate(suiteName, testCase, output, metric) {
 async function reportOutcome(args) {
   const errors = [];
   if (args.traceError !== void 0) {
-    if (args.failOnReporterError) throw args.traceError;
     errors.push(args.traceError);
-    return errors;
   }
   for (const reporter of args.reporters) {
     try {
@@ -1397,12 +1487,10 @@ async function reportOutcome(args) {
         outcome: args.outcome
       });
     } catch (error) {
-      if (args.failOnReporterError) {
-        throw error;
-      }
       errors.push(error);
     }
   }
+  throwReporterErrors("report", errors, args.reporterErrorPolicy);
   return errors;
 }
 function resolveRun(options, startedAtMs) {
@@ -1418,40 +1506,46 @@ function resolveRun(options, startedAtMs) {
       throw new TypeError(`Evaluation run ${label} must contain 1 to 256 characters`);
     }
   }
-  return {
+  const run = {
     id,
-    startedAt: new Date(startedAtMs).toISOString(),
-    ...options.run?.datasetName === void 0 ? {} : { datasetName: options.run.datasetName },
-    ...options.run?.datasetVersion === void 0 ? {} : { datasetVersion: options.run.datasetVersion },
-    ...options.run?.metadata === void 0 ? {} : { metadata: options.run.metadata }
+    startedAt: new Date(startedAtMs).toISOString()
   };
+  if (options.run?.datasetName !== void 0) run.datasetName = options.run.datasetName;
+  if (options.run?.datasetVersion !== void 0) run.datasetVersion = options.run.datasetVersion;
+  if (options.run?.metadata !== void 0) run.metadata = options.run.metadata;
+  return run;
 }
-async function notifyRunStart(reporters, args, failOnReporterError) {
+async function notifyRunStart(reporters, args, errorPolicy) {
   const errors = [];
   for (const reporter of reporters) {
     if (reporter.onRunStart === void 0) continue;
     try {
       await reporter.onRunStart(args);
     } catch (error) {
-      if (failOnReporterError) throw error;
       errors.push(error);
     }
   }
+  throwReporterErrors("onRunStart", errors, errorPolicy);
   return errors;
 }
-async function notifyRunEnd(reporters, args, failOnReporterError = false) {
+async function notifyRunEnd(reporters, args, errorPolicy = "collect") {
   const errors = [];
   for (const reporter of reporters) {
     if (reporter.onRunEnd === void 0) continue;
     try {
       await reporter.onRunEnd(args);
     } catch (error) {
-      if (failOnReporterError) throw error;
       errors.push(error);
     }
   }
+  throwReporterErrors("onRunEnd", errors, errorPolicy);
   return errors;
 }
+function throwReporterErrors(phase, errors, errorPolicy) {
+  if (errorPolicy === "throw" && errors.length > 0) {
+    throw new EvalReporterDispatchError(phase, errors);
+  }
+}
 function countMetricOutcomes(results) {
   const totals = emptyTotals();
   for (const result of results) {
@@ -1706,18 +1800,21 @@ function jsonResult(result) {
 }
 function expectationMismatches(result, expectations) {
   if (expectations === void 0) return [];
-  return [
-    ...expectations.totals === void 0 ? [] : totalMismatches(result, expectations.totals),
-    ...expectations.outcomes === void 0 ? [] : outcomeMismatches(result, expectations.outcomes)
-  ];
+  const mismatches = [];
+  if (expectations.totals !== void 0) {
+    mismatches.push(...totalMismatches(result, expectations.totals));
+  }
+  if (expectations.outcomes !== void 0) {
+    mismatches.push(...outcomeMismatches(result, expectations.outcomes));
+  }
+  return mismatches;
 }
 function totalMismatches(result, expected) {
-  const directMetrics = {
-    ...expected.total === void 0 ? {} : { total: expected.total },
-    ...expected.passed === void 0 ? {} : { passed: expected.passed },
-    ...expected.failed === void 0 ? {} : { failed: expected.failed },
-    ...expected.invalid === void 0 ? {} : { invalid: expected.invalid }
-  };
+  const directMetrics = {};
+  if (expected.total !== void 0) directMetrics.total = expected.total;
+  if (expected.passed !== void 0) directMetrics.passed = expected.passed;
+  if (expected.failed !== void 0) directMetrics.failed = expected.failed;
+  if (expected.invalid !== void 0) directMetrics.invalid = expected.invalid;
   return [
     ...totalsGroupMismatches("metrics", result.metrics, {
       ...directMetrics,
@@ -1996,9 +2093,9 @@ function semanticSimilarity(options) {
       if (typeof expected !== "string") {
         return EvalOutcome.invalid("Semantic similarity expected value must be a string.");
       }
-      const [actualEmbedding, expectedEmbedding] = await Promise.all([
-        embedText(options.model, actual),
-        embedText(options.model, expected)
+      const [{ embedding: actualEmbedding }, { embedding: expectedEmbedding }] = await Promise.all([
+        embedText({ model: options.model, text: actual }),
+        embedText({ model: options.model, text: expected })
       ]);
       const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);
       return score >= options.threshold ? EvalOutcome.pass(score) : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });
@@ -2006,18 +2103,19 @@ function semanticSimilarity(options) {
   };
 }
 function llmJudge(options) {
-  const extractor = new ExtractorBuilder(options.model, options.schema).instructions(
-    options.instructions ?? "Judge the eval case by the requested schema. Submit the judgment using the schema."
-  ).retries(options.retries ?? 0).build();
   return {
     name: options.name ?? "llm_judge",
     required: options.required ?? true,
     async evaluate(args) {
       try {
-        const result = await extractor.extractWithUsage(
-          await resolveJudgePrompt(options.prompt, args)
-        );
-        return options.passes(result.data) ? EvalOutcome.pass(result.data, { usage: result.usage }) : EvalOutcome.fail(result.data, { usage: result.usage });
+        const result = await extract({
+          model: options.model,
+          outputSchema: options.schema,
+          instructions: options.instructions ?? "Judge the eval case by the requested schema. Submit the judgment using the schema.",
+          text: await resolveJudgePrompt(options.prompt, args),
+          retries: evalExtractionRetries(options.retries)
+        });
+        return options.passes(result.output) ? EvalOutcome.pass(result.output, { usage: result.usage }) : EvalOutcome.fail(result.output, { usage: result.usage });
       } catch (error) {
         return EvalOutcome.invalid(errorMessage(error));
       }
@@ -2026,18 +2124,6 @@ function llmJudge(options) {
 }
 function llmScore(options) {
   const criteria = Array.isArray(options.criteria) ? options.criteria.join("\n") : options.criteria;
-  const extractor = new ExtractorBuilder(
-    options.model,
-    z2.object({
-      score: z2.number(),
-      feedback: z2.string()
-    })
-  ).instructions(
-    options.instructions ?? `Score the eval case against these criteria:
-${criteria}
-
-Return a score between 0 and 1 and brief feedback.`
-  ).retries(options.retries ?? 0).build();
   return {
     name: options.name ?? "llm_score",
     required: options.required ?? true,
@@ -2047,10 +2133,20 @@ Return a score between 0 and 1 and brief feedback.`
     threshold: options.threshold,
     async evaluate(args) {
       try {
-        const result = await extractor.extractWithUsage(
-          await resolveJudgePrompt(options.prompt, args)
-        );
-        const score = result.data;
+        const result = await extract({
+          model: options.model,
+          outputSchema: z2.object({
+            score: z2.number(),
+            feedback: z2.string()
+          }),
+          instructions: options.instructions ?? `Score the eval case against these criteria:
+${criteria}
+
+Return a score between 0 and 1 and brief feedback.`,
+          text: await resolveJudgePrompt(options.prompt, args),
+          retries: evalExtractionRetries(options.retries)
+        });
+        const score = result.output;
         if (score.score < 0 || score.score > 1) {
           return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {
             score,
@@ -2064,6 +2160,13 @@ Return a score between 0 and 1 and brief feedback.`
     }
   };
 }
+function evalExtractionRetries(retries) {
+  const retryCount = Math.max(0, Math.trunc(retries ?? 0));
+  if (retryCount === 0) {
+    return void 0;
+  }
+  return { maxAttempts: retryCount + 1 };
+}
 
 // src/evals/suite.ts
 function defineEvalCases(cases) {
@@ -2078,8 +2181,10 @@ function defineEvalSuite(options) {
   };
 }
 export {
+  AgentEvalSuspensionError,
   EvalAssertionError,
   EvalOutcome,
+  EvalReporterDispatchError,
   abstention,
   agentEvalTarget,
   answerRelevancy,