@anvia/core 0.26.0 → 1.0.0-rc.2

package/dist/evals/index.js

@@ -1,37 +1,36 @@
 import {
-  ExtractorBuilder
-} from "../chunk-YLMDBQO4.js";
-import "../chunk-JI5YZWNT.js";
-import "../chunk-DALYP4PX.js";
-import "../chunk-BALBBVI3.js";
+  AgentRunBlockedError,
+  cancelAgentApproval
+} from "../chunk-DBSCKTUL.js";
 import "../chunk-YK4WAAS4.js";
-import "../chunk-XUUY2L2D.js";
-import "../chunk-HYUYZEAJ.js";
-import "../chunk-MELYDKWQ.js";
-import "../chunk-2ODTMRHP.js";
-import "../chunk-UQX6EXWG.js";
-import "../chunk-MRQLEK2B.js";
+import "../chunk-EINTGEGY.js";
+import "../chunk-CQNNSZPG.js";
+import "../chunk-D37WMSP6.js";
+import "../chunk-77AIYARB.js";
+import "../chunk-OJH6HWHX.js";
+import "../chunk-CWM7B2OO.js";
 import {
   cosineSimilarity,
-  embedText
-} from "../chunk-MNWK2USR.js";
-import {
+  embedText,
   mapWithConcurrency
-} from "../chunk-OIMLU4SF.js";
-import "../chunk-Q25IWLBT.js";
-import "../chunk-MC3CCKEB.js";
+} from "../chunk-FMHLROPQ.js";
+import {
+  extract
+} from "../chunk-4WI3VGZK.js";
+import "../chunk-MGNTXGTX.js";
 import {
   Usage
-} from "../chunk-ADH7NNCS.js";
-import "../chunk-WQKHFADH.js";
-import "../chunk-CWUJUSOS.js";
+} from "../chunk-KR3RCCHJ.js";
+import "../chunk-TIZYTPJO.js";
+import "../chunk-KIDGIJM7.js";
+import "../chunk-FN7HLLAZ.js";
 
 // src/evals/advanced-metrics.ts
 import { z } from "zod";
 
 // src/evals/format.ts
 function defaultOutputValue(output) {
-  if (typeof output === "object" && output !== null && "output" in output && typeof output.output === "string") {
+  if (typeof output === "object" && output !== null && "output" in output) {
     return output.output;
   }
   return output;
@@ -58,8 +57,15 @@ function errorMessage(error) {
 
 // src/evals/judge.ts
 async function runJudge(args) {
-  const result = await new ExtractorBuilder(args.model, args.schema).instructions(args.instructions).temperature(0).retries(args.retries).build().extractWithUsage(args.prompt);
-  return { data: result.data, usage: result.usage };
+  const result = await extract({
+    model: args.model,
+    outputSchema: args.schema,
+    instructions: args.instructions,
+    text: args.prompt,
+    temperature: 0,
+    retries: args.retries <= 0 ? void 0 : { maxAttempts: Math.trunc(args.retries) + 1 }
+  });
+  return { data: result.output, usage: result.usage };
 }
 function addUsage(...values) {
   return values.reduce((total, usage) => Usage.add(total, usage), Usage.empty());
@@ -1116,11 +1122,24 @@ function contentText(content) {
 }
 
 // src/evals/agent-target.ts
-function agentEvalTarget(agent, options = {}) {
+var AgentEvalApprovalError = class extends Error {
+  constructor(result) {
+    super("Agent eval targets cannot suspend for tool approval.");
+    this.result = result;
+    this.name = "AgentEvalApprovalError";
+  }
+  result;
+};
+function agentEvalTarget(options) {
   return async (input, testCase) => {
-    const prompt = options.prompt?.(input, testCase) ?? String(input);
-    const response = await agent.prompt(prompt).send();
-    return options.output === void 0 ? response : options.output(response, testCase);
+    const request = await options.request({ input, testCase });
+    const response = await options.agent.generate(request);
+    if (response.status === "approval_required") {
+      await cancelAgentApproval(response, "Agent eval targets cannot suspend for tool approval.");
+      throw new AgentEvalApprovalError(response);
+    }
+    if (response.status === "blocked") throw new AgentRunBlockedError(response);
+    return options.output === void 0 ? response : await options.output({ response, testCase });
   };
 }
 
@@ -1185,6 +1204,7 @@ function traceFromCarrier(value) {
 function traceFromMetadata(metadata) {
   if (metadata === void 0) return void 0;
   return readTraceRef({
+    observer: metadata.traceObserver,
     traceId: metadata.traceId,
     observationId: metadata.observationId,
     responseId: metadata.responseId
@@ -1194,9 +1214,11 @@ function readTraceRef(value) {
   if (typeof value !== "object" || value === null) return void 0;
   const traceId = value.traceId;
   if (typeof traceId !== "string" || traceId.length === 0) return void 0;
+  const observer = value.observer;
   const observationId = value.observationId;
   const responseId = value.responseId;
   const trace = { traceId };
+  if (typeof observer === "string" && observer.length > 0) trace.observer = observer;
   if (typeof observationId === "string" && observationId.length > 0) {
     trace.observationId = observationId;
   }
@@ -1205,6 +1227,14 @@ function readTraceRef(value) {
 }
 
 // src/evals/runner.ts
+var EvalReporterDispatchError = class extends AggregateError {
+  phase;
+  constructor(phase, errors) {
+    super(errors, `Evaluation reporter ${phase} failed ${errors.length} time(s).`);
+    this.name = "EvalReporterDispatchError";
+    this.phase = phase;
+  }
+};
 async function runEvalSuite(options) {
   validateSuiteOptions(options);
   const startedAtMs = Date.now();
@@ -1221,7 +1251,7 @@ async function runEvalSuite(options) {
     reporterErrors = await notifyRunStart(
       reporters,
       lifecycle,
-      options.failOnReporterError === true
+      options.reporterErrorPolicy ?? "collect"
     );
   } catch (error) {
     await notifyRunEnd(reporters, {
@@ -1273,7 +1303,7 @@ async function runEvalSuite(options) {
         usage: result.usage,
         ...result.cost === void 0 ? {} : { cost: result.cost }
       },
-      options.failOnReporterError === true
+      options.reporterErrorPolicy ?? "collect"
     )
   );
   return result;
@@ -1327,7 +1357,7 @@ async function runEvalCase(options, testCase, run) {
       trace: traceResult.trace,
       traceError: traceResult.error,
       reporters: options.reporters ?? [],
-      failOnReporterError: options.failOnReporterError === true
+      reporterErrorPolicy: options.reporterErrorPolicy ?? "collect"
     });
     const metricResult = {
       metricName: metric.name,
@@ -1380,9 +1410,7 @@ async function safeEvaluate(suiteName, testCase, output, metric) {
 async function reportOutcome(args) {
   const errors = [];
   if (args.traceError !== void 0) {
-    if (args.failOnReporterError) throw args.traceError;
     errors.push(args.traceError);
-    return errors;
   }
   for (const reporter of args.reporters) {
     try {
@@ -1397,12 +1425,10 @@ async function reportOutcome(args) {
         outcome: args.outcome
       });
     } catch (error) {
-      if (args.failOnReporterError) {
-        throw error;
-      }
       errors.push(error);
     }
   }
+  throwReporterErrors("report", errors, args.reporterErrorPolicy);
   return errors;
 }
 function resolveRun(options, startedAtMs) {
@@ -1426,32 +1452,37 @@ function resolveRun(options, startedAtMs) {
     ...options.run?.metadata === void 0 ? {} : { metadata: options.run.metadata }
   };
 }
-async function notifyRunStart(reporters, args, failOnReporterError) {
+async function notifyRunStart(reporters, args, errorPolicy) {
   const errors = [];
   for (const reporter of reporters) {
     if (reporter.onRunStart === void 0) continue;
     try {
       await reporter.onRunStart(args);
     } catch (error) {
-      if (failOnReporterError) throw error;
       errors.push(error);
     }
   }
+  throwReporterErrors("onRunStart", errors, errorPolicy);
   return errors;
 }
-async function notifyRunEnd(reporters, args, failOnReporterError = false) {
+async function notifyRunEnd(reporters, args, errorPolicy = "collect") {
   const errors = [];
   for (const reporter of reporters) {
     if (reporter.onRunEnd === void 0) continue;
     try {
       await reporter.onRunEnd(args);
     } catch (error) {
-      if (failOnReporterError) throw error;
       errors.push(error);
     }
   }
+  throwReporterErrors("onRunEnd", errors, errorPolicy);
   return errors;
 }
+function throwReporterErrors(phase, errors, errorPolicy) {
+  if (errorPolicy === "throw" && errors.length > 0) {
+    throw new EvalReporterDispatchError(phase, errors);
+  }
+}
 function countMetricOutcomes(results) {
   const totals = emptyTotals();
   for (const result of results) {
@@ -1996,9 +2027,9 @@ function semanticSimilarity(options) {
       if (typeof expected !== "string") {
         return EvalOutcome.invalid("Semantic similarity expected value must be a string.");
       }
-      const [actualEmbedding, expectedEmbedding] = await Promise.all([
-        embedText(options.model, actual),
-        embedText(options.model, expected)
+      const [{ embedding: actualEmbedding }, { embedding: expectedEmbedding }] = await Promise.all([
+        embedText({ model: options.model, text: actual }),
+        embedText({ model: options.model, text: expected })
       ]);
       const score = cosineSimilarity(actualEmbedding.vector, expectedEmbedding.vector);
       return score >= options.threshold ? EvalOutcome.pass(score) : EvalOutcome.fail(score, { comment: `Similarity below threshold ${options.threshold}.` });
@@ -2006,18 +2037,19 @@ function semanticSimilarity(options) {
   };
 }
 function llmJudge(options) {
-  const extractor = new ExtractorBuilder(options.model, options.schema).instructions(
-    options.instructions ?? "Judge the eval case by the requested schema. Submit the judgment using the schema."
-  ).retries(options.retries ?? 0).build();
   return {
     name: options.name ?? "llm_judge",
     required: options.required ?? true,
     async evaluate(args) {
       try {
-        const result = await extractor.extractWithUsage(
-          await resolveJudgePrompt(options.prompt, args)
-        );
-        return options.passes(result.data) ? EvalOutcome.pass(result.data, { usage: result.usage }) : EvalOutcome.fail(result.data, { usage: result.usage });
+        const result = await extract({
+          model: options.model,
+          outputSchema: options.schema,
+          instructions: options.instructions ?? "Judge the eval case by the requested schema. Submit the judgment using the schema.",
+          text: await resolveJudgePrompt(options.prompt, args),
+          retries: evalExtractionRetries(options.retries)
+        });
+        return options.passes(result.output) ? EvalOutcome.pass(result.output, { usage: result.usage }) : EvalOutcome.fail(result.output, { usage: result.usage });
       } catch (error) {
         return EvalOutcome.invalid(errorMessage(error));
       }
@@ -2026,18 +2058,6 @@ function llmJudge(options) {
 }
 function llmScore(options) {
   const criteria = Array.isArray(options.criteria) ? options.criteria.join("\n") : options.criteria;
-  const extractor = new ExtractorBuilder(
-    options.model,
-    z2.object({
-      score: z2.number(),
-      feedback: z2.string()
-    })
-  ).instructions(
-    options.instructions ?? `Score the eval case against these criteria:
-${criteria}
-
-Return a score between 0 and 1 and brief feedback.`
-  ).retries(options.retries ?? 0).build();
   return {
     name: options.name ?? "llm_score",
     required: options.required ?? true,
@@ -2047,10 +2067,20 @@ Return a score between 0 and 1 and brief feedback.`
     threshold: options.threshold,
     async evaluate(args) {
       try {
-        const result = await extractor.extractWithUsage(
-          await resolveJudgePrompt(options.prompt, args)
-        );
-        const score = result.data;
+        const result = await extract({
+          model: options.model,
+          outputSchema: z2.object({
+            score: z2.number(),
+            feedback: z2.string()
+          }),
+          instructions: options.instructions ?? `Score the eval case against these criteria:
+${criteria}
+
+Return a score between 0 and 1 and brief feedback.`,
+          text: await resolveJudgePrompt(options.prompt, args),
+          retries: evalExtractionRetries(options.retries)
+        });
+        const score = result.output;
         if (score.score < 0 || score.score > 1) {
           return EvalOutcome.invalid(`Score ${score.score} outside valid range [0, 1].`, {
             score,
@@ -2064,6 +2094,13 @@ Return a score between 0 and 1 and brief feedback.`
     }
   };
 }
+function evalExtractionRetries(retries) {
+  const retryCount = Math.max(0, Math.trunc(retries ?? 0));
+  if (retryCount === 0) {
+    return void 0;
+  }
+  return { maxAttempts: retryCount + 1 };
+}
 
 // src/evals/suite.ts
 function defineEvalCases(cases) {
@@ -2078,8 +2115,10 @@ function defineEvalSuite(options) {
   };
 }
 export {
+  AgentEvalApprovalError,
   EvalAssertionError,
   EvalOutcome,
+  EvalReporterDispatchError,
   abstention,
   agentEvalTarget,
   answerRelevancy,