@anvia/core 1.1.2 → 1.2.1

package/dist/chunk-QUS6S2NB.js

@@ -0,0 +1,663 @@
+import {
+  defaultEvalTraceSelector
+} from "./chunk-PHLGGRHC.js";
+import {
+  errorMessage
+} from "./chunk-ZYXMWEL4.js";
+import {
+  EvalOutcome
+} from "./chunk-EJR6VVB3.js";
+import {
+  EvalAbortError,
+  EvalTimeoutError,
+  abortable,
+  createConcurrencyLimiter,
+  createEvalCaseSignal
+} from "./chunk-JMMZICU7.js";
+import {
+  Usage
+} from "./chunk-IDNL6IH4.js";
+
+// src/evals/runner.ts
+var EvalReporterDispatchError = class extends AggregateError {
+  phase;
+  constructor(phase, errors) {
+    super(errors, `Evaluation reporter ${phase} failed ${errors.length} time(s).`);
+    this.name = "EvalReporterDispatchError";
+    this.phase = phase;
+  }
+};
+var EvalFailFastError = class extends Error {
+  constructor(caseId, outcome) {
+    super(`Evaluation stopped after case ${caseId} produced a required ${outcome} outcome.`);
+    this.caseId = caseId;
+    this.outcome = outcome;
+    this.name = "EvalFailFastError";
+  }
+  caseId;
+  outcome;
+};
+async function runEvalSuite(options) {
+  validateSuiteOptions(options);
+  const selectedCases = selectCases(options);
+  const selectedOptions = {
+    ...options,
+    cases: selectedCases
+  };
+  const startedAtMs = Date.now();
+  const run = resolveRun(selectedOptions, startedAtMs);
+  const reporters = options.reporters ?? [];
+  const lifecycle = {
+    run,
+    suiteName: options.name,
+    caseCount: selectedCases.length,
+    metricNames: options.metrics.map((metric) => metric.name)
+  };
+  let reporterErrors;
+  try {
+    reporterErrors = await notifyRunStart(
+      reporters,
+      lifecycle,
+      options.reporterErrorPolicy ?? "collect"
+    );
+  } catch (error) {
+    await notifyRunEnd(reporters, {
+      ...lifecycle,
+      status: "failed",
+      completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      durationMs: Date.now() - startedAtMs,
+      error
+    });
+    throw error;
+  }
+  let results;
+  let aggregates;
+  try {
+    results = await runEvalCases(selectedOptions, run);
+    aggregates = await aggregateResult(selectedOptions, results);
+  } catch (error) {
+    await notifyRunEnd(reporters, {
+      ...lifecycle,
+      status: "failed",
+      completedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      durationMs: Date.now() - startedAtMs,
+      error
+    });
+    throw error;
+  }
+  const completedAt = (/* @__PURE__ */ new Date()).toISOString();
+  const result = {
+    name: options.name,
+    run: { ...run, completedAt },
+    results,
+    metrics: aggregates.metrics,
+    cases: aggregates.cases,
+    usage: aggregates.usage,
+    durationMs: Date.now() - startedAtMs,
+    reporterErrors
+  };
+  result.reporterErrors.push(
+    ...results.flatMap(
+      (caseResult) => caseResult.metrics.flatMap((metricResult) => metricResult.reporterErrors)
+    )
+  );
+  if (aggregates.cost !== void 0) {
+    result.cost = aggregates.cost;
+  }
+  const runEndArgs = {
+    ...lifecycle,
+    status: "completed",
+    completedAt,
+    durationMs: result.durationMs,
+    metrics: result.metrics,
+    cases: result.cases,
+    usage: result.usage
+  };
+  if (result.cost !== void 0) {
+    runEndArgs.cost = result.cost;
+  }
+  result.reporterErrors.push(
+    ...await notifyRunEnd(reporters, runEndArgs, options.reporterErrorPolicy ?? "collect")
+  );
+  return result;
+}
+async function runEvalCases(options, run) {
+  const targetConcurrency = options.targetConcurrency ?? options.concurrency ?? 1;
+  const metricConcurrency = options.metricConcurrency ?? options.concurrency ?? 1;
+  const workerConcurrency = Math.max(targetConcurrency, metricConcurrency);
+  const targetLimit = createConcurrencyLimiter(targetConcurrency);
+  const metricLimit = createConcurrencyLimiter(metricConcurrency);
+  const results = Array(options.cases.length);
+  let nextIndex = 0;
+  let completedCases = 0;
+  let failure;
+  async function worker() {
+    while (failure === void 0 && nextIndex < options.cases.length) {
+      if (isAborted(options.signal)) {
+        failure ??= { error: suiteAbortReason(options.signal) };
+        break;
+      }
+      const index = nextIndex;
+      nextIndex += 1;
+      try {
+        results[index] = await runEvalCase(
+          options,
+          options.cases[index],
+          run,
+          targetLimit,
+          metricLimit,
+          () => completedCases
+        );
+        completedCases += 1;
+        if (isAborted(options.signal)) {
+          failure ??= { error: suiteAbortReason(options.signal) };
+        }
+        if (options.failFast === true && results[index]?.outcome !== "pass") {
+          const result = results[index];
+          if (result !== void 0 && result.outcome !== "pass") {
+            failure ??= { error: new EvalFailFastError(result.case.id, result.outcome) };
+          }
+        }
+      } catch (error) {
+        failure ??= { error };
+      }
+    }
+  }
+  await Promise.all(
+    Array.from({ length: Math.min(workerConcurrency, options.cases.length) }, () => worker())
+  );
+  if (failure !== void 0) throw failure.error;
+  return results;
+}
+async function runEvalCase(options, testCase, run, targetLimit, metricLimit, completedCases) {
+  const caseStartedAt = performance.now();
+  const caseSignal = createEvalCaseSignal(options.signal, options.caseTimeoutMs);
+  try {
+    const progress = (event) => notifyProgress(options, event);
+    await progress({
+      type: "case-start",
+      suiteName: options.name,
+      case: testCase,
+      completedCases: completedCases(),
+      totalCases: options.cases.length
+    });
+    let output;
+    let targetError;
+    let targetStatus = "succeeded";
+    let targetDurationMs = 0;
+    try {
+      output = await targetLimit(async () => {
+        const targetStartedAt = performance.now();
+        try {
+          return await (caseSignal.signal.aborted ? Promise.reject(caseSignal.signal.reason) : abortable(
+            caseSignal.signal,
+            Promise.resolve(
+              options.target(testCase.input, testCase, { signal: caseSignal.signal })
+            )
+          ));
+        } finally {
+          targetDurationMs = performance.now() - targetStartedAt;
+        }
+      });
+    } catch (error) {
+      if (options.signal?.aborted === true) {
+        throw options.signal.reason ?? new EvalAbortError();
+      }
+      targetStatus = "failed";
+      targetError = error;
+    }
+    await progress({
+      type: "target-complete",
+      suiteName: options.name,
+      case: testCase,
+      targetStatus,
+      output,
+      error: targetError,
+      durationMs: targetDurationMs,
+      completedCases: completedCases(),
+      totalCases: options.cases.length
+    });
+    const traceResult = await resolveTrace(options, testCase, output, targetError, targetStatus);
+    const metrics = await Promise.all(
+      options.metrics.map(
+        (metric) => metricLimit(async () => {
+          const metricStartedAt = performance.now();
+          const outcome = targetStatus === "succeeded" ? await safeEvaluate(
+            options.name,
+            testCase,
+            output,
+            metric,
+            caseSignal.signal
+          ) : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`, {
+            kind: targetError instanceof EvalTimeoutError ? "timeout" : "target",
+            error: targetError
+          });
+          const durationMs = performance.now() - metricStartedAt;
+          const reporterErrors = await reportOutcome({
+            run,
+            suiteName: options.name,
+            testCase,
+            output,
+            targetError,
+            targetStatus,
+            metric,
+            outcome,
+            trace: traceResult.trace,
+            traceError: traceResult.error,
+            reporters: options.reporters ?? [],
+            reporterErrorPolicy: options.reporterErrorPolicy ?? "collect"
+          });
+          await progress({
+            type: "metric-complete",
+            suiteName: options.name,
+            case: testCase,
+            metricName: metric.name,
+            outcome,
+            durationMs,
+            completedCases: completedCases(),
+            totalCases: options.cases.length
+          });
+          const metricResult = {
+            metricName: metric.name,
+            required: metric.required ?? true,
+            outcome,
+            durationMs,
+            reporterErrors
+          };
+          if (metric.direction !== void 0) metricResult.direction = metric.direction;
+          if (metric.threshold !== void 0) metricResult.threshold = metric.threshold;
+          return metricResult;
+        })
+      )
+    );
+    const scores = Object.fromEntries(
+      metrics.map((metric) => [metric.metricName, metric.outcome])
+    );
+    const result = {
+      case: testCase,
+      outcome: caseOutcome(targetStatus, metrics),
+      targetStatus,
+      targetDurationMs,
+      durationMs: performance.now() - caseStartedAt,
+      metrics,
+      scores,
+      usage: emptyUsageSummary()
+    };
+    if (targetStatus === "succeeded") {
+      result.output = output;
+    }
+    if (targetStatus === "failed") {
+      result.targetError = targetError;
+    }
+    await progress({
+      type: "case-complete",
+      suiteName: options.name,
+      result,
+      completedCases: completedCases() + 1,
+      totalCases: options.cases.length
+    });
+    return result;
+  } finally {
+    caseSignal.dispose();
+  }
+}
+async function resolveTrace(options, testCase, output, targetError, targetStatus) {
+  try {
+    const selector = options.trace ?? defaultEvalTraceSelector;
+    const trace = await selector({
+      suiteName: options.name,
+      case: testCase,
+      output,
+      targetError,
+      targetStatus
+    });
+    return trace === void 0 ? {} : { trace };
+  } catch (error) {
+    return { error };
+  }
+}
+async function safeEvaluate(suiteName, testCase, output, metric, signal) {
+  if (signal.aborted) {
+    return EvalOutcome.fromError(
+      signal.reason,
+      signal.reason instanceof EvalTimeoutError ? "timeout" : "metric"
+    );
+  }
+  try {
+    return await abortable(
+      signal,
+      Promise.resolve(metric.evaluate({ suiteName, case: testCase, output, signal }))
+    );
+  } catch (error) {
+    return EvalOutcome.fromError(error, error instanceof EvalTimeoutError ? "timeout" : "metric");
+  }
+}
+async function reportOutcome(args) {
+  const errors = [];
+  if (args.traceError !== void 0) {
+    errors.push(args.traceError);
+  }
+  for (const reporter of args.reporters) {
+    try {
+      await reporter.report({
+        run: args.run,
+        suiteName: args.suiteName,
+        case: args.testCase,
+        output: args.output,
+        targetError: args.targetError,
+        targetStatus: args.targetStatus,
+        trace: args.trace,
+        metric: args.metric,
+        outcome: args.outcome
+      });
+    } catch (error) {
+      errors.push(error);
+    }
+  }
+  throwReporterErrors("report", errors, args.reporterErrorPolicy);
+  return errors;
+}
+function resolveRun(options, startedAtMs) {
+  const id = options.run?.id ?? globalThis.crypto.randomUUID();
+  if (id.trim().length === 0 || id.length > 128) {
+    throw new TypeError("Evaluation run id must contain 1 to 128 characters");
+  }
+  for (const [label, value] of [
+    ["dataset name", options.run?.datasetName],
+    ["dataset version", options.run?.datasetVersion]
+  ]) {
+    if (value !== void 0 && (value.trim().length === 0 || value.length > 256)) {
+      throw new TypeError(`Evaluation run ${label} must contain 1 to 256 characters`);
+    }
+  }
+  const run = {
+    id,
+    startedAt: new Date(startedAtMs).toISOString()
+  };
+  if (options.run?.datasetName !== void 0) run.datasetName = options.run.datasetName;
+  if (options.run?.datasetVersion !== void 0) run.datasetVersion = options.run.datasetVersion;
+  if (options.run?.promptRef !== void 0) run.promptRef = options.run.promptRef;
+  if (options.run?.metadata !== void 0) run.metadata = options.run.metadata;
+  return run;
+}
+async function notifyRunStart(reporters, args, errorPolicy) {
+  const errors = [];
+  for (const reporter of reporters) {
+    if (reporter.onRunStart === void 0) continue;
+    try {
+      await reporter.onRunStart(args);
+    } catch (error) {
+      errors.push(error);
+    }
+  }
+  throwReporterErrors("onRunStart", errors, errorPolicy);
+  return errors;
+}
+async function notifyRunEnd(reporters, args, errorPolicy = "collect") {
+  const errors = [];
+  for (const reporter of reporters) {
+    if (reporter.onRunEnd === void 0) continue;
+    try {
+      await reporter.onRunEnd(args);
+    } catch (error) {
+      errors.push(error);
+    }
+  }
+  throwReporterErrors("onRunEnd", errors, errorPolicy);
+  return errors;
+}
+function throwReporterErrors(phase, errors, errorPolicy) {
+  if (errorPolicy === "throw" && errors.length > 0) {
+    throw new EvalReporterDispatchError(phase, errors);
+  }
+}
+function countMetricOutcomes(results) {
+  const totals = emptyTotals();
+  for (const result of results) {
+    for (const metric of result.metrics) {
+      totals.total += 1;
+      totals[statusKey(metric.outcome.outcome)] += 1;
+    }
+  }
+  return totals;
+}
+function countCaseOutcomes(results) {
+  const totals = emptyTotals();
+  for (const result of results) {
+    totals.total += 1;
+    totals[statusKey(result.outcome)] += 1;
+  }
+  return totals;
+}
+function emptyTotals() {
+  return { total: 0, passed: 0, failed: 0, invalid: 0 };
+}
+function emptyUsageSummary() {
+  return {
+    target: Usage.empty(),
+    evaluation: Usage.empty(),
+    total: Usage.empty()
+  };
+}
+function statusKey(status) {
+  if (status === "pass") return "passed";
+  if (status === "fail") return "failed";
+  return "invalid";
+}
+function caseOutcome(targetStatus, metrics) {
+  if (targetStatus === "failed") return "invalid";
+  const required = metrics.filter((metric) => metric.required);
+  if (required.some((metric) => metric.outcome.outcome === "invalid")) return "invalid";
+  if (required.some((metric) => metric.outcome.outcome === "fail")) return "fail";
+  return "pass";
+}
+async function aggregateResult(options, results) {
+  let targetUsage = Usage.empty();
+  let evaluationUsage = Usage.empty();
+  let targetCost = 0;
+  let evaluationCost = 0;
+  for (const result of results) {
+    let caseTargetUsage = Usage.empty();
+    let caseEvaluationUsage = Usage.empty();
+    let caseTargetCost = 0;
+    let caseEvaluationCost = 0;
+    if (result.targetStatus === "succeeded") {
+      const usage2 = await resolveTargetUsage(options, result.case, result.output);
+      if (usage2 !== void 0) {
+        caseTargetUsage = usage2;
+        targetUsage = Usage.add(targetUsage, usage2);
+        if (options.cost !== void 0) {
+          caseTargetCost = await calculateCost(
+            options.cost.calculate({
+              kind: "target",
+              suiteName: options.name,
+              case: result.case,
+              output: result.output,
+              usage: usage2
+            })
+          );
+          targetCost += caseTargetCost;
+        }
+      }
+    }
+    for (const metricResult of result.metrics) {
+      const usage2 = metricResult.outcome.usage;
+      if (usage2 === void 0) continue;
+      assertUsage(usage2, `Evaluation usage for metric ${metricResult.metricName}`);
+      caseEvaluationUsage = Usage.add(caseEvaluationUsage, usage2);
+      evaluationUsage = Usage.add(evaluationUsage, usage2);
+      if (options.cost !== void 0 && result.targetStatus === "succeeded") {
+        const metric = options.metrics.find(
+          (candidate) => candidate.name === metricResult.metricName
+        );
+        if (metric !== void 0) {
+          const metricCost = await calculateCost(
+            options.cost.calculate({
+              kind: "evaluation",
+              suiteName: options.name,
+              case: result.case,
+              output: result.output,
+              metric,
+              usage: usage2
+            })
+          );
+          metricResult.cost = metricCost;
+          caseEvaluationCost += metricCost;
+          evaluationCost += metricCost;
+        }
+      }
+    }
+    result.usage = {
+      target: caseTargetUsage,
+      evaluation: caseEvaluationUsage,
+      total: Usage.add(caseTargetUsage, caseEvaluationUsage)
+    };
+    if (options.cost !== void 0) {
+      result.cost = {
+        currency: options.cost.currency,
+        target: caseTargetCost,
+        evaluation: caseEvaluationCost,
+        total: caseTargetCost + caseEvaluationCost
+      };
+    }
+  }
+  const usage = {
+    target: targetUsage,
+    evaluation: evaluationUsage,
+    total: Usage.add(targetUsage, evaluationUsage)
+  };
+  const aggregates = {
+    metrics: countMetricOutcomes(results),
+    cases: countCaseOutcomes(results),
+    usage
+  };
+  if (options.cost !== void 0) {
+    aggregates.cost = {
+      currency: options.cost.currency,
+      target: targetCost,
+      evaluation: evaluationCost,
+      total: targetCost + evaluationCost
+    };
+  }
+  return aggregates;
+}
+async function resolveTargetUsage(options, testCase, output) {
+  const usage = options.targetUsage === void 0 ? usageFromOutput(output) : await options.targetUsage({
+    suiteName: options.name,
+    case: testCase,
+    output,
+    signal: new AbortController().signal
+  });
+  if (usage !== void 0) assertUsage(usage, `Target usage for case ${testCase.id}`);
+  return usage;
+}
+function usageFromOutput(output) {
+  if (typeof output !== "object" || output === null || !("usage" in output)) return void 0;
+  return output.usage;
+}
+function assertUsage(usage, label) {
+  for (const [key, value] of Object.entries(usage)) {
+    if (key === "details") continue;
+    if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
+      throw new TypeError(`${label} must contain finite, non-negative token counts`);
+    }
+  }
+}
+async function calculateCost(value) {
+  const cost = await value;
+  if (!Number.isFinite(cost) || cost < 0) {
+    throw new TypeError("Evaluation cost calculator must return a finite, non-negative number");
+  }
+  return cost;
+}
+function validateSuiteOptions(options) {
+  if (options.name.trim().length === 0) {
+    throw new TypeError("Evaluation suite name must not be empty");
+  }
+  if (options.cases.length === 0) {
+    throw new TypeError("Evaluation suite must contain at least one case");
+  }
+  if (options.metrics.length === 0) {
+    throw new TypeError("Evaluation suite must contain at least one metric");
+  }
+  for (const testCase of options.cases) {
+    if (testCase.id.trim().length === 0) {
+      throw new TypeError("Evaluation case id must not be empty");
+    }
+  }
+  for (const metric of options.metrics) {
+    if (metric.name.trim().length === 0) {
+      throw new TypeError("Evaluation metric name must not be empty");
+    }
+  }
+  assertUnique(
+    options.cases.map((testCase) => testCase.id),
+    "Evaluation case id"
+  );
+  assertUnique(
+    options.metrics.map((metric) => metric.name),
+    "Evaluation metric name"
+  );
+  if (options.cost !== void 0 && options.cost.currency.trim().length === 0) {
+    throw new TypeError("Evaluation cost currency must not be empty");
+  }
+  for (const [label, value] of [
+    ["concurrency", options.concurrency],
+    ["targetConcurrency", options.targetConcurrency],
+    ["metricConcurrency", options.metricConcurrency]
+  ]) {
+    if (value !== void 0 && (!Number.isSafeInteger(value) || value < 1)) {
+      throw new RangeError(`Evaluation ${label} must be a positive integer`);
+    }
+  }
+  if (options.caseTimeoutMs !== void 0 && (!Number.isSafeInteger(options.caseTimeoutMs) || options.caseTimeoutMs < 1)) {
+    throw new RangeError("Evaluation caseTimeoutMs must be a positive integer");
+  }
+  if (options.shard !== void 0) {
+    if (!Number.isSafeInteger(options.shard.count) || options.shard.count < 1) {
+      throw new RangeError("Evaluation shard count must be a positive integer");
+    }
+    if (!Number.isSafeInteger(options.shard.index) || options.shard.index < 0 || options.shard.index >= options.shard.count) {
+      throw new RangeError("Evaluation shard index must be between 0 and count - 1");
+    }
+  }
+}
+function assertUnique(values, label) {
+  const seen = /* @__PURE__ */ new Set();
+  for (const value of values) {
+    if (seen.has(value)) throw new TypeError(`${label} must be unique: ${value}`);
+    seen.add(value);
+  }
+}
+function selectCases(options) {
+  const requested = options.caseIds === void 0 ? void 0 : new Set(options.caseIds);
+  if (requested !== void 0) {
+    assertUnique([...options.caseIds], "Evaluation selected case id");
+    const available = new Set(options.cases.map((testCase) => testCase.id));
+    for (const id of requested) {
+      if (!available.has(id))
+        throw new TypeError(`Evaluation selected case id was not found: ${id}`);
+    }
+  }
+  const filtered = options.cases.filter(
+    (testCase, index) => (requested === void 0 || requested.has(testCase.id)) && (options.caseFilter === void 0 || options.caseFilter(testCase, index))
+  );
+  if (options.shard === void 0) return filtered;
+  return filtered.filter((_, index) => index % options.shard.count === options.shard.index);
+}
+async function notifyProgress(options, event) {
+  await options.onProgress?.(event);
+}
+function isAborted(signal) {
+  return signal?.aborted === true;
+}
+function suiteAbortReason(signal) {
+  return signal?.reason ?? new EvalAbortError();
+}
+
+export {
+  EvalReporterDispatchError,
+  EvalFailFastError,
+  runEvalSuite
+};
+//# sourceMappingURL=chunk-QUS6S2NB.js.map