@anvia/core 1.1.2 → 1.2.1

package/dist/chunk-QUS6S2NB.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/evals/runner.ts"],"sourcesContent":["import { type Usage, Usage as UsageValue } from \"../completion\";\nimport {\n  abortable,\n  type ConcurrencyLimiter,\n  createConcurrencyLimiter,\n  createEvalCaseSignal,\n  EvalAbortError,\n  EvalTimeoutError,\n} from \"./execution\";\nimport { errorMessage } from \"./format\";\nimport { EvalOutcome, type EvalOutcome as EvalOutcomeType } from \"./outcome\";\nimport { defaultEvalTraceSelector } from \"./reporting\";\nimport type {\n  EvalCase,\n  EvalCaseResult,\n  EvalCostSummary,\n  EvalMetric,\n  EvalMetricResult,\n  EvalReporter,\n  EvalRunContext,\n  EvalRunEndArgs,\n  EvalRunStartArgs,\n  EvalSuiteResult,\n  EvalTotals,\n  EvalTraceRef,\n  EvalProgressEvent,\n  RunEvalSuiteOptions,\n} from \"./types\";\n\nexport class EvalReporterDispatchError extends AggregateError {\n  readonly phase: string;\n\n  constructor(phase: string, errors: readonly unknown[]) {\n    super(errors, `Evaluation reporter ${phase} failed ${errors.length} time(s).`);\n    this.name = \"EvalReporterDispatchError\";\n    this.phase = phase;\n  }\n}\n\nexport class EvalFailFastError extends Error {\n  constructor(\n    readonly caseId: string,\n    readonly outcome: \"fail\" | \"invalid\",\n  ) {\n    super(`Evaluation stopped after case ${caseId} produced a required ${outcome} outcome.`);\n    this.name = \"EvalFailFastError\";\n  }\n}\n\nexport async function runEvalSuite<\n  Input,\n  Output,\n  Expected = unknown,\n  const Metrics extends readonly EvalMetric<\n    NoInfer<Input>,\n    NoInfer<Output>,\n    unknown,\n    NoInfer<Expected>,\n    string\n  >[] = readonly EvalMetric<NoInfer<Input>, NoInfer<Output>, unknown, NoInfer<Expected>, string>[],\n>(\n  options: RunEvalSuiteOptions<Input, Output, Expected, Metrics>,\n): Promise<EvalSuiteResult<Input, Output, Expected, Metrics>>;\nexport async function runEvalSuite<\n  Input,\n  Output,\n  Expected,\n  Metrics extends readonly EvalMetric<Input, Output, unknown, Expected, string>[],\n>(\n  options: RunEvalSuiteOptions<Input, Output, Expected, Metrics>,\n): Promise<EvalSuiteResult<Input, Output, Expected, Metrics>> {\n  validateSuiteOptions(options);\n  const selectedCases = selectCases(options);\n  const selectedOptions = {\n    ...options,\n    cases: selectedCases,\n  } as unknown as RunEvalSuiteOptions<Input, Output, Expected, Metrics>;\n  const startedAtMs = Date.now();\n  const run = resolveRun(selectedOptions, startedAtMs);\n  const reporters = options.reporters ?? [];\n  const lifecycle = {\n    run,\n    suiteName: options.name,\n    caseCount: selectedCases.length,\n    metricNames: options.metrics.map((metric) => metric.name),\n  } satisfies EvalRunStartArgs;\n  let reporterErrors: unknown[];\n  try {\n    reporterErrors = await notifyRunStart(\n      reporters,\n      lifecycle,\n      options.reporterErrorPolicy ?? \"collect\",\n    );\n  } catch (error) {\n    await notifyRunEnd(reporters, {\n      ...lifecycle,\n      status: \"failed\",\n      completedAt: new Date().toISOString(),\n      durationMs: Date.now() - startedAtMs,\n      error,\n    });\n    throw error;\n  }\n  let results: Array<EvalCaseResult<Input, Output, Expected, Metrics>>;\n  let aggregates: Awaited<ReturnType<typeof aggregateResult>>;\n  try {\n    results = await runEvalCases(selectedOptions, run);\n    aggregates = await aggregateResult(selectedOptions, results);\n  } catch (error) {\n    await notifyRunEnd(reporters, {\n      ...lifecycle,\n      status: \"failed\",\n      completedAt: new Date().toISOString(),\n      durationMs: Date.now() - startedAtMs,\n      error,\n    });\n    throw error;\n  }\n  const completedAt = new Date().toISOString();\n  const result: EvalSuiteResult<Input, Output, Expected, Metrics> = {\n    name: options.name,\n    run: { ...run, completedAt },\n    results,\n    metrics: aggregates.metrics,\n    cases: aggregates.cases,\n    usage: aggregates.usage,\n    durationMs: Date.now() - startedAtMs,\n    reporterErrors,\n  };\n  result.reporterErrors.push(\n    ...results.flatMap((caseResult) =>\n      caseResult.metrics.flatMap((metricResult) => metricResult.reporterErrors),\n    ),\n  );\n  if (aggregates.cost !== undefined) {\n    result.cost = aggregates.cost;\n  }\n  const runEndArgs: EvalRunEndArgs = {\n    ...lifecycle,\n    status: \"completed\",\n    completedAt,\n    durationMs: result.durationMs,\n    metrics: result.metrics,\n    cases: result.cases,\n    usage: result.usage,\n  };\n  if (result.cost !== undefined) {\n    runEndArgs.cost = result.cost;\n  }\n  result.reporterErrors.push(\n    ...(await notifyRunEnd(reporters, runEndArgs, options.reporterErrorPolicy ?? \"collect\")),\n  );\n  return result;\n}\n\nasync function runEvalCases<\n  Input,\n  Output,\n  Expected,\n  Metrics extends readonly EvalMetric<Input, Output, unknown, Expected, string>[],\n>(\n  options: RunEvalSuiteOptions<Input, Output, Expected, Metrics>,\n  run: EvalRunContext,\n): Promise<Array<EvalCaseResult<Input, Output, Expected, Metrics>>> {\n  const targetConcurrency = options.targetConcurrency ?? options.concurrency ?? 1;\n  const metricConcurrency = options.metricConcurrency ?? options.concurrency ?? 1;\n  const workerConcurrency = Math.max(targetConcurrency, metricConcurrency);\n  const targetLimit = createConcurrencyLimiter(targetConcurrency);\n  const metricLimit = createConcurrencyLimiter(metricConcurrency);\n  const results = Array<EvalCaseResult<Input, Output, Expected, Metrics>>(options.cases.length);\n  let nextIndex = 0;\n  let completedCases = 0;\n  let failure: { error: unknown } | undefined;\n\n  async function worker(): Promise<void> {\n    while (failure === undefined && nextIndex < options.cases.length) {\n      if (isAborted(options.signal)) {\n        failure ??= { error: suiteAbortReason(options.signal) };\n        break;\n      }\n      const index = nextIndex;\n      nextIndex += 1;\n      try {\n        results[index] = await runEvalCase(\n          options,\n          options.cases[index] as EvalCase<Input, Expected>,\n          run,\n          targetLimit,\n          metricLimit,\n          () => completedCases,\n        );\n        completedCases += 1;\n        if (isAborted(options.signal)) {\n          failure ??= { error: suiteAbortReason(options.signal) };\n        }\n        if (options.failFast === true && results[index]?.outcome !== \"pass\") {\n          const result = results[index];\n          if (result !== undefined && result.outcome !== \"pass\") {\n            failure ??= { error: new EvalFailFastError(result.case.id, result.outcome) };\n          }\n        }\n      } catch (error) {\n        failure ??= { error };\n      }\n    }\n  }\n\n  await Promise.all(\n    Array.from({ length: Math.min(workerConcurrency, options.cases.length) }, () => worker()),\n  );\n  if (failure !== undefined) throw failure.error;\n  return results;\n}\n\nasync function runEvalCase<\n  Input,\n  Output,\n  Expected,\n  Metrics extends readonly EvalMetric<Input, Output, unknown, Expected, string>[],\n>(\n  options: RunEvalSuiteOptions<Input, Output, Expected, Metrics>,\n  testCase: EvalCase<Input, Expected>,\n  run: EvalRunContext,\n  targetLimit: ConcurrencyLimiter,\n  metricLimit: ConcurrencyLimiter,\n  completedCases: () => number,\n): Promise<EvalCaseResult<Input, Output, Expected, Metrics>> {\n  const caseStartedAt = performance.now();\n  const caseSignal = createEvalCaseSignal(options.signal, options.caseTimeoutMs);\n  try {\n    const progress = (event: EvalProgressEvent<Input, Output, Expected>) =>\n      notifyProgress(options, event);\n    await progress({\n      type: \"case-start\",\n      suiteName: options.name,\n      case: testCase,\n      completedCases: completedCases(),\n      totalCases: options.cases.length,\n    });\n    let output: Output | undefined;\n    let targetError: unknown;\n    let targetStatus: \"succeeded\" | \"failed\" = \"succeeded\";\n    let targetDurationMs = 0;\n    try {\n      output = await targetLimit(async () => {\n        const targetStartedAt = performance.now();\n        try {\n          return await (caseSignal.signal.aborted\n            ? Promise.reject(caseSignal.signal.reason)\n            : abortable(\n                caseSignal.signal,\n                Promise.resolve(\n                  options.target(testCase.input, testCase, { signal: caseSignal.signal }),\n                ),\n              ));\n        } finally {\n          targetDurationMs = performance.now() - targetStartedAt;\n        }\n      });\n    } catch (error) {\n      if (options.signal?.aborted === true) {\n        throw options.signal.reason ?? new EvalAbortError();\n      }\n      targetStatus = \"failed\";\n      targetError = error;\n    }\n    await progress({\n      type: \"target-complete\",\n      suiteName: options.name,\n      case: testCase,\n      targetStatus,\n      output,\n      error: targetError,\n      durationMs: targetDurationMs,\n      completedCases: completedCases(),\n      totalCases: options.cases.length,\n    });\n    const traceResult = await resolveTrace(options, testCase, output, targetError, targetStatus);\n\n    const metrics = await Promise.all(\n      options.metrics.map((metric) =>\n        metricLimit(async () => {\n          const metricStartedAt = performance.now();\n          const outcome =\n            targetStatus === \"succeeded\"\n              ? await safeEvaluate(\n                  options.name,\n                  testCase,\n                  output as Output,\n                  metric,\n                  caseSignal.signal,\n                )\n              : EvalOutcome.invalid(`Target failed: ${errorMessage(targetError)}`, {\n                  kind: targetError instanceof EvalTimeoutError ? \"timeout\" : \"target\",\n                  error: targetError,\n                });\n          const durationMs = performance.now() - metricStartedAt;\n          const reporterErrors = await reportOutcome({\n            run,\n            suiteName: options.name,\n            testCase,\n            output,\n            targetError,\n            targetStatus,\n            metric,\n            outcome,\n            trace: traceResult.trace,\n            traceError: traceResult.error,\n            reporters: options.reporters ?? [],\n            reporterErrorPolicy: options.reporterErrorPolicy ?? \"collect\",\n          });\n          await progress({\n            type: \"metric-complete\",\n            suiteName: options.name,\n            case: testCase,\n            metricName: metric.name,\n            outcome,\n            durationMs,\n            completedCases: completedCases(),\n            totalCases: options.cases.length,\n          });\n          const metricResult: EvalMetricResult = {\n            metricName: metric.name,\n            required: metric.required ?? true,\n            outcome,\n            durationMs,\n            reporterErrors,\n          };\n          if (metric.direction !== undefined) metricResult.direction = metric.direction;\n          if (metric.threshold !== undefined) metricResult.threshold = metric.threshold;\n          return metricResult;\n        }),\n      ),\n    );\n\n    const scores = Object.fromEntries(\n      metrics.map((metric) => [metric.metricName, metric.outcome]),\n    ) as EvalCaseResult<Input, Output, Expected, Metrics>[\"scores\"];\n    const result: EvalCaseResult<Input, Output, Expected, Metrics> = {\n      case: testCase,\n      outcome: caseOutcome(targetStatus, metrics),\n      targetStatus,\n      targetDurationMs,\n      durationMs: performance.now() - caseStartedAt,\n      metrics: metrics as EvalCaseResult<Input, Output, Expected, Metrics>[\"metrics\"],\n      scores,\n      usage: emptyUsageSummary(),\n    };\n    if (targetStatus === \"succeeded\") {\n      result.output = output;\n    }\n    if (targetStatus === \"failed\") {\n      result.targetError = targetError;\n    }\n    await progress({\n      type: \"case-complete\",\n      suiteName: options.name,\n      result,\n      completedCases: completedCases() + 1,\n      totalCases: options.cases.length,\n    });\n    return result;\n  } finally {\n    caseSignal.dispose();\n  }\n}\n\nasync function resolveTrace<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  testCase: EvalCase<Input, Expected>,\n  output: Output | undefined,\n  targetError: unknown,\n  targetStatus: \"succeeded\" | \"failed\",\n): Promise<{ trace?: EvalTraceRef | undefined; error?: unknown }> {\n  try {\n    const selector = options.trace ?? defaultEvalTraceSelector;\n    const trace = await selector({\n      suiteName: options.name,\n      case: testCase,\n      output,\n      targetError,\n      targetStatus,\n    });\n    return trace === undefined ? {} : { trace };\n  } catch (error) {\n    return { error };\n  }\n}\n\nasync function safeEvaluate<Input, Output, Expected>(\n  suiteName: string,\n  testCase: EvalCase<Input, Expected>,\n  output: Output,\n  metric: EvalMetric<Input, Output, unknown, Expected>,\n  signal: AbortSignal,\n): Promise<EvalOutcomeType> {\n  if (signal.aborted) {\n    return EvalOutcome.fromError(\n      signal.reason,\n      signal.reason instanceof EvalTimeoutError ? \"timeout\" : \"metric\",\n    );\n  }\n  try {\n    return await abortable(\n      signal,\n      Promise.resolve(metric.evaluate({ suiteName, case: testCase, output, signal })),\n    );\n  } catch (error) {\n    return EvalOutcome.fromError(error, error instanceof EvalTimeoutError ? \"timeout\" : \"metric\");\n  }\n}\n\nasync function reportOutcome<Input, Output, Expected>(args: {\n  run: EvalRunContext;\n  suiteName: string;\n  testCase: EvalCase<Input, Expected>;\n  output: Output | undefined;\n  targetError: unknown;\n  targetStatus: \"succeeded\" | \"failed\";\n  metric: EvalMetric<Input, Output, unknown, Expected>;\n  outcome: EvalOutcomeType;\n  trace: EvalTraceRef | undefined;\n  traceError: unknown;\n  reporters: readonly EvalReporter<Input, Output, Expected>[];\n  reporterErrorPolicy: \"collect\" | \"throw\";\n}): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  if (args.traceError !== undefined) {\n    errors.push(args.traceError);\n  }\n  for (const reporter of args.reporters) {\n    try {\n      await reporter.report({\n        run: args.run,\n        suiteName: args.suiteName,\n        case: args.testCase,\n        output: args.output,\n        targetError: args.targetError,\n        targetStatus: args.targetStatus,\n        trace: args.trace,\n        metric: args.metric,\n        outcome: args.outcome,\n      });\n    } catch (error) {\n      errors.push(error);\n    }\n  }\n  throwReporterErrors(\"report\", errors, args.reporterErrorPolicy);\n  return errors;\n}\n\nfunction resolveRun<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  startedAtMs: number,\n): EvalRunContext {\n  const id = options.run?.id ?? globalThis.crypto.randomUUID();\n  if (id.trim().length === 0 || id.length > 128) {\n    throw new TypeError(\"Evaluation run id must contain 1 to 128 characters\");\n  }\n  for (const [label, value] of [\n    [\"dataset name\", options.run?.datasetName],\n    [\"dataset version\", options.run?.datasetVersion],\n  ] as const) {\n    if (value !== undefined && (value.trim().length === 0 || value.length > 256)) {\n      throw new TypeError(`Evaluation run ${label} must contain 1 to 256 characters`);\n    }\n  }\n  const run: EvalRunContext = {\n    id,\n    startedAt: new Date(startedAtMs).toISOString(),\n  };\n  if (options.run?.datasetName !== undefined) run.datasetName = options.run.datasetName;\n  if (options.run?.datasetVersion !== undefined) run.datasetVersion = options.run.datasetVersion;\n  if (options.run?.promptRef !== undefined) run.promptRef = options.run.promptRef;\n  if (options.run?.metadata !== undefined) run.metadata = options.run.metadata;\n  return run;\n}\n\nasync function notifyRunStart<Input, Output, Expected>(\n  reporters: readonly EvalReporter<Input, Output, Expected>[],\n  args: EvalRunStartArgs,\n  errorPolicy: \"collect\" | \"throw\",\n): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  for (const reporter of reporters) {\n    if (reporter.onRunStart === undefined) continue;\n    try {\n      await reporter.onRunStart(args);\n    } catch (error) {\n      errors.push(error);\n    }\n  }\n  throwReporterErrors(\"onRunStart\", errors, errorPolicy);\n  return errors;\n}\n\nasync function notifyRunEnd<Input, Output, Expected>(\n  reporters: readonly EvalReporter<Input, Output, Expected>[],\n  args: EvalRunEndArgs,\n  errorPolicy: \"collect\" | \"throw\" = \"collect\",\n): Promise<unknown[]> {\n  const errors: unknown[] = [];\n  for (const reporter of reporters) {\n    if (reporter.onRunEnd === undefined) continue;\n    try {\n      await reporter.onRunEnd(args);\n    } catch (error) {\n      errors.push(error);\n    }\n  }\n  throwReporterErrors(\"onRunEnd\", errors, errorPolicy);\n  return errors;\n}\n\nfunction throwReporterErrors(\n  phase: string,\n  errors: readonly unknown[],\n  errorPolicy: \"collect\" | \"throw\",\n): void {\n  if (errorPolicy === \"throw\" && errors.length > 0) {\n    throw new EvalReporterDispatchError(phase, errors);\n  }\n}\n\nfunction countMetricOutcomes(\n  results: Array<EvalCaseResult<unknown, unknown, unknown>>,\n): EvalTotals {\n  const totals = emptyTotals();\n  for (const result of results) {\n    for (const metric of result.metrics) {\n      totals.total += 1;\n      totals[statusKey(metric.outcome.outcome)] += 1;\n    }\n  }\n  return totals;\n}\n\nfunction countCaseOutcomes(results: Array<EvalCaseResult<unknown, unknown, unknown>>): EvalTotals {\n  const totals = emptyTotals();\n  for (const result of results) {\n    totals.total += 1;\n    totals[statusKey(result.outcome)] += 1;\n  }\n  return totals;\n}\n\nfunction emptyTotals(): EvalTotals {\n  return { total: 0, passed: 0, failed: 0, invalid: 0 };\n}\n\nfunction emptyUsageSummary() {\n  return {\n    target: UsageValue.empty(),\n    evaluation: UsageValue.empty(),\n    total: UsageValue.empty(),\n  };\n}\n\nfunction statusKey(status: \"pass\" | \"fail\" | \"invalid\"): \"passed\" | \"failed\" | \"invalid\" {\n  if (status === \"pass\") return \"passed\";\n  if (status === \"fail\") return \"failed\";\n  return \"invalid\";\n}\n\nfunction caseOutcome(\n  targetStatus: \"succeeded\" | \"failed\",\n  metrics: EvalMetricResult[],\n): \"pass\" | \"fail\" | \"invalid\" {\n  if (targetStatus === \"failed\") return \"invalid\";\n  const required = metrics.filter((metric) => metric.required);\n  if (required.some((metric) => metric.outcome.outcome === \"invalid\")) return \"invalid\";\n  if (required.some((metric) => metric.outcome.outcome === \"fail\")) return \"fail\";\n  return \"pass\";\n}\n\nasync function aggregateResult<\n  Input,\n  Output,\n  Expected,\n  Metrics extends readonly EvalMetric<Input, Output, unknown, Expected, string>[],\n>(\n  options: RunEvalSuiteOptions<Input, Output, Expected, Metrics>,\n  results: Array<EvalCaseResult<Input, Output, Expected, Metrics>>,\n): Promise<{\n  metrics: EvalTotals;\n  cases: EvalTotals;\n  usage: { target: Usage; evaluation: Usage; total: Usage };\n  cost?: EvalCostSummary | undefined;\n}> {\n  let targetUsage = UsageValue.empty();\n  let evaluationUsage = UsageValue.empty();\n  let targetCost = 0;\n  let evaluationCost = 0;\n\n  for (const result of results) {\n    let caseTargetUsage = UsageValue.empty();\n    let caseEvaluationUsage = UsageValue.empty();\n    let caseTargetCost = 0;\n    let caseEvaluationCost = 0;\n    if (result.targetStatus === \"succeeded\") {\n      const usage = await resolveTargetUsage(options, result.case, result.output as Output);\n      if (usage !== undefined) {\n        caseTargetUsage = usage;\n        targetUsage = UsageValue.add(targetUsage, usage);\n        if (options.cost !== undefined) {\n          caseTargetCost = await calculateCost(\n            options.cost.calculate({\n              kind: \"target\",\n              suiteName: options.name,\n              case: result.case,\n              output: result.output as Output,\n              usage,\n            }),\n          );\n          targetCost += caseTargetCost;\n        }\n      }\n    }\n    for (const metricResult of result.metrics) {\n      const usage = metricResult.outcome.usage;\n      if (usage === undefined) continue;\n      assertUsage(usage, `Evaluation usage for metric ${metricResult.metricName}`);\n      caseEvaluationUsage = UsageValue.add(caseEvaluationUsage, usage);\n      evaluationUsage = UsageValue.add(evaluationUsage, usage);\n      if (options.cost !== undefined && result.targetStatus === \"succeeded\") {\n        const metric = options.metrics.find(\n          (candidate) => candidate.name === metricResult.metricName,\n        );\n        if (metric !== undefined) {\n          const metricCost = await calculateCost(\n            options.cost.calculate({\n              kind: \"evaluation\",\n              suiteName: options.name,\n              case: result.case,\n              output: result.output as Output,\n              metric,\n              usage,\n            }),\n          );\n          metricResult.cost = metricCost;\n          caseEvaluationCost += metricCost;\n          evaluationCost += metricCost;\n        }\n      }\n    }\n    result.usage = {\n      target: caseTargetUsage,\n      evaluation: caseEvaluationUsage,\n      total: UsageValue.add(caseTargetUsage, caseEvaluationUsage),\n    };\n    if (options.cost !== undefined) {\n      result.cost = {\n        currency: options.cost.currency,\n        target: caseTargetCost,\n        evaluation: caseEvaluationCost,\n        total: caseTargetCost + caseEvaluationCost,\n      };\n    }\n  }\n\n  const usage = {\n    target: targetUsage,\n    evaluation: evaluationUsage,\n    total: UsageValue.add(targetUsage, evaluationUsage),\n  };\n  const aggregates: {\n    metrics: EvalTotals;\n    cases: EvalTotals;\n    usage: typeof usage;\n    cost?: EvalCostSummary | undefined;\n  } = {\n    metrics: countMetricOutcomes(results),\n    cases: countCaseOutcomes(results),\n    usage,\n  };\n  if (options.cost !== undefined) {\n    aggregates.cost = {\n      currency: options.cost.currency,\n      target: targetCost,\n      evaluation: evaluationCost,\n      total: targetCost + evaluationCost,\n    };\n  }\n  return aggregates;\n}\n\nasync function resolveTargetUsage<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  testCase: EvalCase<Input, Expected>,\n  output: Output,\n): Promise<Usage | undefined> {\n  const usage =\n    options.targetUsage === undefined\n      ? usageFromOutput(output)\n      : await options.targetUsage({\n          suiteName: options.name,\n          case: testCase,\n          output,\n          signal: new AbortController().signal,\n        });\n  if (usage !== undefined) assertUsage(usage, `Target usage for case ${testCase.id}`);\n  return usage;\n}\n\nfunction usageFromOutput(output: unknown): Usage | undefined {\n  if (typeof output !== \"object\" || output === null || !(\"usage\" in output)) return undefined;\n  return (output as { usage?: Usage | undefined }).usage;\n}\n\nfunction assertUsage(usage: Usage, label: string): void {\n  for (const [key, value] of Object.entries(usage)) {\n    if (key === \"details\") continue;\n    if (typeof value !== \"number\" || !Number.isFinite(value) || value < 0) {\n      throw new TypeError(`${label} must contain finite, non-negative token counts`);\n    }\n  }\n}\n\nasync function calculateCost(value: number | Promise<number>): Promise<number> {\n  const cost = await value;\n  if (!Number.isFinite(cost) || cost < 0) {\n    throw new TypeError(\"Evaluation cost calculator must return a finite, non-negative number\");\n  }\n  return cost;\n}\n\nfunction validateSuiteOptions<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n): void {\n  if (options.name.trim().length === 0) {\n    throw new TypeError(\"Evaluation suite name must not be empty\");\n  }\n  if (options.cases.length === 0) {\n    throw new TypeError(\"Evaluation suite must contain at least one case\");\n  }\n  if (options.metrics.length === 0) {\n    throw new TypeError(\"Evaluation suite must contain at least one metric\");\n  }\n  for (const testCase of options.cases) {\n    if (testCase.id.trim().length === 0) {\n      throw new TypeError(\"Evaluation case id must not be empty\");\n    }\n  }\n  for (const metric of options.metrics) {\n    if (metric.name.trim().length === 0) {\n      throw new TypeError(\"Evaluation metric name must not be empty\");\n    }\n  }\n  assertUnique(\n    options.cases.map((testCase) => testCase.id),\n    \"Evaluation case id\",\n  );\n  assertUnique(\n    options.metrics.map((metric) => metric.name),\n    \"Evaluation metric name\",\n  );\n  if (options.cost !== undefined && options.cost.currency.trim().length === 0) {\n    throw new TypeError(\"Evaluation cost currency must not be empty\");\n  }\n  for (const [label, value] of [\n    [\"concurrency\", options.concurrency],\n    [\"targetConcurrency\", options.targetConcurrency],\n    [\"metricConcurrency\", options.metricConcurrency],\n  ] as const) {\n    if (value !== undefined && (!Number.isSafeInteger(value) || value < 1)) {\n      throw new RangeError(`Evaluation ${label} must be a positive integer`);\n    }\n  }\n  if (\n    options.caseTimeoutMs !== undefined &&\n    (!Number.isSafeInteger(options.caseTimeoutMs) || options.caseTimeoutMs < 1)\n  ) {\n    throw new RangeError(\"Evaluation caseTimeoutMs must be a positive integer\");\n  }\n  if (options.shard !== undefined) {\n    if (!Number.isSafeInteger(options.shard.count) || options.shard.count < 1) {\n      throw new RangeError(\"Evaluation shard count must be a positive integer\");\n    }\n    if (\n      !Number.isSafeInteger(options.shard.index) ||\n      options.shard.index < 0 ||\n      options.shard.index >= options.shard.count\n    ) {\n      throw new RangeError(\"Evaluation shard index must be between 0 and count - 1\");\n    }\n  }\n}\n\nfunction assertUnique(values: string[], label: string): void {\n  const seen = new Set<string>();\n  for (const value of values) {\n    if (seen.has(value)) throw new TypeError(`${label} must be unique: ${value}`);\n    seen.add(value);\n  }\n}\n\nfunction selectCases<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n): readonly EvalCase<Input, Expected>[] {\n  const requested = options.caseIds === undefined ? undefined : new Set(options.caseIds);\n  if (requested !== undefined) {\n    assertUnique([...options.caseIds!], \"Evaluation selected case id\");\n    const available = new Set(options.cases.map((testCase) => testCase.id));\n    for (const id of requested) {\n      if (!available.has(id))\n        throw new TypeError(`Evaluation selected case id was not found: ${id}`);\n    }\n  }\n  const filtered = options.cases.filter(\n    (testCase, index) =>\n      (requested === undefined || requested.has(testCase.id)) &&\n      (options.caseFilter === undefined || options.caseFilter(testCase, index)),\n  );\n  if (options.shard === undefined) return filtered;\n  return filtered.filter((_, index) => index % options.shard!.count === options.shard!.index);\n}\n\nasync function notifyProgress<Input, Output, Expected>(\n  options: RunEvalSuiteOptions<Input, Output, Expected>,\n  event: EvalProgressEvent<Input, Output, Expected>,\n): Promise<void> {\n  await options.onProgress?.(event);\n}\n\nfunction isAborted(signal: AbortSignal | undefined): boolean {\n  return signal?.aborted === true;\n}\n\nfunction suiteAbortReason(signal: AbortSignal | undefined): unknown {\n  return signal?.reason ?? new EvalAbortError();\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;AA6BO,IAAM,4BAAN,cAAwC,eAAe;AAAA,EACnD;AAAA,EAET,YAAY,OAAe,QAA4B;AACrD,UAAM,QAAQ,uBAAuB,KAAK,WAAW,OAAO,MAAM,WAAW;AAC7E,SAAK,OAAO;AACZ,SAAK,QAAQ;AAAA,EACf;AACF;AAEO,IAAM,oBAAN,cAAgC,MAAM;AAAA,EAC3C,YACW,QACA,SACT;AACA,UAAM,iCAAiC,MAAM,wBAAwB,OAAO,WAAW;AAH9E;AACA;AAGT,SAAK,OAAO;AAAA,EACd;AAAA,EALW;AAAA,EACA;AAKb;AAgBA,eAAsB,aAMpB,SAC4D;AAC5D,uBAAqB,OAAO;AAC5B,QAAM,gBAAgB,YAAY,OAAO;AACzC,QAAM,kBAAkB;AAAA,IACtB,GAAG;AAAA,IACH,OAAO;AAAA,EACT;AACA,QAAM,cAAc,KAAK,IAAI;AAC7B,QAAM,MAAM,WAAW,iBAAiB,WAAW;AACnD,QAAM,YAAY,QAAQ,aAAa,CAAC;AACxC,QAAM,YAAY;AAAA,IAChB;AAAA,IACA,WAAW,QAAQ;AAAA,IACnB,WAAW,cAAc;AAAA,IACzB,aAAa,QAAQ,QAAQ,IAAI,CAAC,WAAW,OAAO,IAAI;AAAA,EAC1D;AACA,MAAI;AACJ,MAAI;AACF,qBAAiB,MAAM;AAAA,MACrB;AAAA,MACA;AAAA,MACA,QAAQ,uBAAuB;AAAA,IACjC;AAAA,EACF,SAAS,OAAO;AACd,UAAM,aAAa,WAAW;AAAA,MAC5B,GAAG;AAAA,MACH,QAAQ;AAAA,MACR,cAAa,oBAAI,KAAK,GAAE,YAAY;AAAA,MACpC,YAAY,KAAK,IAAI,IAAI;AAAA,MACzB;AAAA,IACF,CAAC;AACD,UAAM;AAAA,EACR;AACA,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,cAAU,MAAM,aAAa,iBAAiB,GAAG;AACjD,iBAAa,MAAM,gBAAgB,iBAAiB,OAAO;AAAA,EAC7D,SAAS,OAAO;AACd,UAAM,aAAa,WAAW;AAAA,MAC5B,GAAG;AAAA,MACH,QAAQ;AAAA,MACR,cAAa,oBAAI,KAAK,GAAE,YAAY;AAAA,MACpC,YAAY,KAAK,IAAI,IAAI;AAAA,MACzB;AAAA,IACF,CAAC;AACD,UAAM;AAAA,EACR;AACA,QAAM,eAAc,oBAAI,KAAK,GAAE,YAAY;AAC3C,QAAM,SAA4D;AAAA,IAChE,MAAM,QAAQ;AAAA,IACd,KAAK,EAAE,GAAG,KAAK,YAAY;AAAA,IAC3B;AAAA,IACA,SAAS,WAAW;AAAA,IACpB,OAAO,WAAW;AAAA,IAClB,OAAO,WAAW;AAAA,IAClB,YAAY,KAAK,IAAI,IAAI;AAAA,IACzB;AAAA,EACF;AACA,SAAO,eAAe;AAAA,IACpB,GAAG,QAAQ;AAAA,MAAQ,CAAC,eAClB,WAAW,QAAQ,QAAQ,CAAC,iBAAiB,aAAa,cAAc;AAAA,IAC1E;AAAA,EACF;AACA,MAAI,WAAW,SAAS,QAAW;AACjC,WAAO,OAAO,WAAW;AAAA,EAC3B;AACA,QAAM,aAA6B;AAAA,IACjC,GAAG;AAAA,IACH,QAAQ;AAAA,IACR;AAAA,IACA,YAAY,OAAO;AAAA,IACnB,SAAS,OAAO;AAAA,IAChB,OAAO,OAAO;AAAA,IACd,OAAO,OAAO;AAAA,EAChB;AACA,MAAI,OAAO,SAAS,QAAW;AAC7B,eAAW,OAAO,OAAO;AAAA,EAC3B;AACA,SAAO,eAAe;AAAA,IACpB,GAAI,MAAM,aAAa,WAAW,YAAY,QAAQ,uBAAuB,SAAS;AAAA,EACxF;AACA,SAAO;AACT;AAEA,eAAe,aAMb,SACA,KACkE;AAClE,QAAM,oBAAoB,QAAQ,qBAAqB,QAAQ,eAAe;AAC9E,QAAM,oBAAoB,QAAQ,qBAAqB,QAAQ,eAAe;AAC9E,QAAM,oBAAoB,KAAK,IAAI,mBAAmB,iBAAiB;AACvE,QAAM,cAAc,yBAAyB,iBAAiB;AAC9D,QAAM,cAAc,yBAAyB,iBAAiB;AAC9D,QAAM,UAAU,MAAwD,QAAQ,MAAM,MAAM;AAC5F,MAAI,YAAY;AAChB,MAAI,iBAAiB;AACrB,MAAI;AAEJ,iBAAe,SAAwB;AACrC,WAAO,YAAY,UAAa,YAAY,QAAQ,MAAM,QAAQ;AAChE,UAAI,UAAU,QAAQ,MAAM,GAAG;AAC7B,oBAAY,EAAE,OAAO,iBAAiB,QAAQ,MAAM,EAAE;AACtD;AAAA,MACF;AACA,YAAM,QAAQ;AACd,mBAAa;AACb,UAAI;AACF,gBAAQ,KAAK,IAAI,MAAM;AAAA,UACrB;AAAA,UACA,QAAQ,MAAM,KAAK;AAAA,UACnB;AAAA,UACA;AAAA,UACA;AAAA,UACA,MAAM;AAAA,QACR;AACA,0BAAkB;AAClB,YAAI,UAAU,QAAQ,MAAM,GAAG;AAC7B,sBAAY,EAAE,OAAO,iBAAiB,QAAQ,MAAM,EAAE;AAAA,QACxD;AACA,YAAI,QAAQ,aAAa,QAAQ,QAAQ,KAAK,GAAG,YAAY,QAAQ;AACnE,gBAAM,SAAS,QAAQ,KAAK;AAC5B,cAAI,WAAW,UAAa,OAAO,YAAY,QAAQ;AACrD,wBAAY,EAAE,OAAO,IAAI,kBAAkB,OAAO,KAAK,IAAI,OAAO,OAAO,EAAE;AAAA,UAC7E;AAAA,QACF;AAAA,MACF,SAAS,OAAO;AACd,oBAAY,EAAE,MAAM;AAAA,MACtB;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAQ;AAAA,IACZ,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,mBAAmB,QAAQ,MAAM,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC;AAAA,EAC1F;AACA,MAAI,YAAY,OAAW,OAAM,QAAQ;AACzC,SAAO;AACT;AAEA,eAAe,YAMb,SACA,UACA,KACA,aACA,aACA,gBAC2D;AAC3D,QAAM,gBAAgB,YAAY,IAAI;AACtC,QAAM,aAAa,qBAAqB,QAAQ,QAAQ,QAAQ,aAAa;AAC7E,MAAI;AACF,UAAM,WAAW,CAAC,UAChB,eAAe,SAAS,KAAK;AAC/B,UAAM,SAAS;AAAA,MACb,MAAM;AAAA,MACN,WAAW,QAAQ;AAAA,MACnB,MAAM;AAAA,MACN,gBAAgB,eAAe;AAAA,MAC/B,YAAY,QAAQ,MAAM;AAAA,IAC5B,CAAC;AACD,QAAI;AACJ,QAAI;AACJ,QAAI,eAAuC;AAC3C,QAAI,mBAAmB;AACvB,QAAI;AACF,eAAS,MAAM,YAAY,YAAY;AACrC,cAAM,kBAAkB,YAAY,IAAI;AACxC,YAAI;AACF,iBAAO,OAAO,WAAW,OAAO,UAC5B,QAAQ,OAAO,WAAW,OAAO,MAAM,IACvC;AAAA,YACE,WAAW;AAAA,YACX,QAAQ;AAAA,cACN,QAAQ,OAAO,SAAS,OAAO,UAAU,EAAE,QAAQ,WAAW,OAAO,CAAC;AAAA,YACxE;AAAA,UACF;AAAA,QACN,UAAE;AACA,6BAAmB,YAAY,IAAI,IAAI;AAAA,QACzC;AAAA,MACF,CAAC;AAAA,IACH,SAAS,OAAO;AACd,UAAI,QAAQ,QAAQ,YAAY,MAAM;AACpC,cAAM,QAAQ,OAAO,UAAU,IAAI,eAAe;AAAA,MACpD;AACA,qBAAe;AACf,oBAAc;AAAA,IAChB;AACA,UAAM,SAAS;AAAA,MACb,MAAM;AAAA,MACN,WAAW,QAAQ;AAAA,MACnB,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA,OAAO;AAAA,MACP,YAAY;AAAA,MACZ,gBAAgB,eAAe;AAAA,MAC/B,YAAY,QAAQ,MAAM;AAAA,IAC5B,CAAC;AACD,UAAM,cAAc,MAAM,aAAa,SAAS,UAAU,QAAQ,aAAa,YAAY;AAE3F,UAAM,UAAU,MAAM,QAAQ;AAAA,MAC5B,QAAQ,QAAQ;AAAA,QAAI,CAAC,WACnB,YAAY,YAAY;AACtB,gBAAM,kBAAkB,YAAY,IAAI;AACxC,gBAAM,UACJ,iBAAiB,cACb,MAAM;AAAA,YACJ,QAAQ;AAAA,YACR;AAAA,YACA;AAAA,YACA;AAAA,YACA,WAAW;AAAA,UACb,IACA,YAAY,QAAQ,kBAAkB,aAAa,WAAW,CAAC,IAAI;AAAA,YACjE,MAAM,uBAAuB,mBAAmB,YAAY;AAAA,YAC5D,OAAO;AAAA,UACT,CAAC;AACP,gBAAM,aAAa,YAAY,IAAI,IAAI;AACvC,gBAAM,iBAAiB,MAAM,cAAc;AAAA,YACzC;AAAA,YACA,WAAW,QAAQ;AAAA,YACnB;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA;AAAA,YACA,OAAO,YAAY;AAAA,YACnB,YAAY,YAAY;AAAA,YACxB,WAAW,QAAQ,aAAa,CAAC;AAAA,YACjC,qBAAqB,QAAQ,uBAAuB;AAAA,UACtD,CAAC;AACD,gBAAM,SAAS;AAAA,YACb,MAAM;AAAA,YACN,WAAW,QAAQ;AAAA,YACnB,MAAM;AAAA,YACN,YAAY,OAAO;AAAA,YACnB;AAAA,YACA;AAAA,YACA,gBAAgB,eAAe;AAAA,YAC/B,YAAY,QAAQ,MAAM;AAAA,UAC5B,CAAC;AACD,gBAAM,eAAiC;AAAA,YACrC,YAAY,OAAO;AAAA,YACnB,UAAU,OAAO,YAAY;AAAA,YAC7B;AAAA,YACA;AAAA,YACA;AAAA,UACF;AACA,cAAI,OAAO,cAAc,OAAW,cAAa,YAAY,OAAO;AACpE,cAAI,OAAO,cAAc,OAAW,cAAa,YAAY,OAAO;AACpE,iBAAO;AAAA,QACT,CAAC;AAAA,MACH;AAAA,IACF;AAEA,UAAM,SAAS,OAAO;AAAA,MACpB,QAAQ,IAAI,CAAC,WAAW,CAAC,OAAO,YAAY,OAAO,OAAO,CAAC;AAAA,IAC7D;AACA,UAAM,SAA2D;AAAA,MAC/D,MAAM;AAAA,MACN,SAAS,YAAY,cAAc,OAAO;AAAA,MAC1C;AAAA,MACA;AAAA,MACA,YAAY,YAAY,IAAI,IAAI;AAAA,MAChC;AAAA,MACA;AAAA,MACA,OAAO,kBAAkB;AAAA,IAC3B;AACA,QAAI,iBAAiB,aAAa;AAChC,aAAO,SAAS;AAAA,IAClB;AACA,QAAI,iBAAiB,UAAU;AAC7B,aAAO,cAAc;AAAA,IACvB;AACA,UAAM,SAAS;AAAA,MACb,MAAM;AAAA,MACN,WAAW,QAAQ;AAAA,MACnB;AAAA,MACA,gBAAgB,eAAe,IAAI;AAAA,MACnC,YAAY,QAAQ,MAAM;AAAA,IAC5B,CAAC;AACD,WAAO;AAAA,EACT,UAAE;AACA,eAAW,QAAQ;AAAA,EACrB;AACF;AAEA,eAAe,aACb,SACA,UACA,QACA,aACA,cACgE;AAChE,MAAI;AACF,UAAM,WAAW,QAAQ,SAAS;AAClC,UAAM,QAAQ,MAAM,SAAS;AAAA,MAC3B,WAAW,QAAQ;AAAA,MACnB,MAAM;AAAA,MACN;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AACD,WAAO,UAAU,SAAY,CAAC,IAAI,EAAE,MAAM;AAAA,EAC5C,SAAS,OAAO;AACd,WAAO,EAAE,MAAM;AAAA,EACjB;AACF;AAEA,eAAe,aACb,WACA,UACA,QACA,QACA,QAC0B;AAC1B,MAAI,OAAO,SAAS;AAClB,WAAO,YAAY;AAAA,MACjB,OAAO;AAAA,MACP,OAAO,kBAAkB,mBAAmB,YAAY;AAAA,IAC1D;AAAA,EACF;AACA,MAAI;AACF,WAAO,MAAM;AAAA,MACX;AAAA,MACA,QAAQ,QAAQ,OAAO,SAAS,EAAE,WAAW,MAAM,UAAU,QAAQ,OAAO,CAAC,CAAC;AAAA,IAChF;AAAA,EACF,SAAS,OAAO;AACd,WAAO,YAAY,UAAU,OAAO,iBAAiB,mBAAmB,YAAY,QAAQ;AAAA,EAC9F;AACF;AAEA,eAAe,cAAuC,MAa/B;AACrB,QAAM,SAAoB,CAAC;AAC3B,MAAI,KAAK,eAAe,QAAW;AACjC,WAAO,KAAK,KAAK,UAAU;AAAA,EAC7B;AACA,aAAW,YAAY,KAAK,WAAW;AACrC,QAAI;AACF,YAAM,SAAS,OAAO;AAAA,QACpB,KAAK,KAAK;AAAA,QACV,WAAW,KAAK;AAAA,QAChB,MAAM,KAAK;AAAA,QACX,QAAQ,KAAK;AAAA,QACb,aAAa,KAAK;AAAA,QAClB,cAAc,KAAK;AAAA,QACnB,OAAO,KAAK;AAAA,QACZ,QAAQ,KAAK;AAAA,QACb,SAAS,KAAK;AAAA,MAChB,CAAC;AAAA,IACH,SAAS,OAAO;AACd,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,sBAAoB,UAAU,QAAQ,KAAK,mBAAmB;AAC9D,SAAO;AACT;AAEA,SAAS,WACP,SACA,aACgB;AAChB,QAAM,KAAK,QAAQ,KAAK,MAAM,WAAW,OAAO,WAAW;AAC3D,MAAI,GAAG,KAAK,EAAE,WAAW,KAAK,GAAG,SAAS,KAAK;AAC7C,UAAM,IAAI,UAAU,oDAAoD;AAAA,EAC1E;AACA,aAAW,CAAC,OAAO,KAAK,KAAK;AAAA,IAC3B,CAAC,gBAAgB,QAAQ,KAAK,WAAW;AAAA,IACzC,CAAC,mBAAmB,QAAQ,KAAK,cAAc;AAAA,EACjD,GAAY;AACV,QAAI,UAAU,WAAc,MAAM,KAAK,EAAE,WAAW,KAAK,MAAM,SAAS,MAAM;AAC5E,YAAM,IAAI,UAAU,kBAAkB,KAAK,mCAAmC;AAAA,IAChF;AAAA,EACF;AACA,QAAM,MAAsB;AAAA,IAC1B;AAAA,IACA,WAAW,IAAI,KAAK,WAAW,EAAE,YAAY;AAAA,EAC/C;AACA,MAAI,QAAQ,KAAK,gBAAgB,OAAW,KAAI,cAAc,QAAQ,IAAI;AAC1E,MAAI,QAAQ,KAAK,mBAAmB,OAAW,KAAI,iBAAiB,QAAQ,IAAI;AAChF,MAAI,QAAQ,KAAK,cAAc,OAAW,KAAI,YAAY,QAAQ,IAAI;AACtE,MAAI,QAAQ,KAAK,aAAa,OAAW,KAAI,WAAW,QAAQ,IAAI;AACpE,SAAO;AACT;AAEA,eAAe,eACb,WACA,MACA,aACoB;AACpB,QAAM,SAAoB,CAAC;AAC3B,aAAW,YAAY,WAAW;AAChC,QAAI,SAAS,eAAe,OAAW;AACvC,QAAI;AACF,YAAM,SAAS,WAAW,IAAI;AAAA,IAChC,SAAS,OAAO;AACd,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,sBAAoB,cAAc,QAAQ,WAAW;AACrD,SAAO;AACT;AAEA,eAAe,aACb,WACA,MACA,cAAmC,WACf;AACpB,QAAM,SAAoB,CAAC;AAC3B,aAAW,YAAY,WAAW;AAChC,QAAI,SAAS,aAAa,OAAW;AACrC,QAAI;AACF,YAAM,SAAS,SAAS,IAAI;AAAA,IAC9B,SAAS,OAAO;AACd,aAAO,KAAK,KAAK;AAAA,IACnB;AAAA,EACF;AACA,sBAAoB,YAAY,QAAQ,WAAW;AACnD,SAAO;AACT;AAEA,SAAS,oBACP,OACA,QACA,aACM;AACN,MAAI,gBAAgB,WAAW,OAAO,SAAS,GAAG;AAChD,UAAM,IAAI,0BAA0B,OAAO,MAAM;AAAA,EACnD;AACF;AAEA,SAAS,oBACP,SACY;AACZ,QAAM,SAAS,YAAY;AAC3B,aAAW,UAAU,SAAS;AAC5B,eAAW,UAAU,OAAO,SAAS;AACnC,aAAO,SAAS;AAChB,aAAO,UAAU,OAAO,QAAQ,OAAO,CAAC,KAAK;AAAA,IAC/C;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,SAAuE;AAChG,QAAM,SAAS,YAAY;AAC3B,aAAW,UAAU,SAAS;AAC5B,WAAO,SAAS;AAChB,WAAO,UAAU,OAAO,OAAO,CAAC,KAAK;AAAA,EACvC;AACA,SAAO;AACT;AAEA,SAAS,cAA0B;AACjC,SAAO,EAAE,OAAO,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,EAAE;AACtD;AAEA,SAAS,oBAAoB;AAC3B,SAAO;AAAA,IACL,QAAQ,MAAW,MAAM;AAAA,IACzB,YAAY,MAAW,MAAM;AAAA,IAC7B,OAAO,MAAW,MAAM;AAAA,EAC1B;AACF;AAEA,SAAS,UAAU,QAAsE;AACvF,MAAI,WAAW,OAAQ,QAAO;AAC9B,MAAI,WAAW,OAAQ,QAAO;AAC9B,SAAO;AACT;AAEA,SAAS,YACP,cACA,SAC6B;AAC7B,MAAI,iBAAiB,SAAU,QAAO;AACtC,QAAM,WAAW,QAAQ,OAAO,CAAC,WAAW,OAAO,QAAQ;AAC3D,MAAI,SAAS,KAAK,CAAC,WAAW,OAAO,QAAQ,YAAY,SAAS,EAAG,QAAO;AAC5E,MAAI,SAAS,KAAK,CAAC,WAAW,OAAO,QAAQ,YAAY,MAAM,EAAG,QAAO;AACzE,SAAO;AACT;AAEA,eAAe,gBAMb,SACA,SAMC;AACD,MAAI,cAAc,MAAW,MAAM;AACnC,MAAI,kBAAkB,MAAW,MAAM;AACvC,MAAI,aAAa;AACjB,MAAI,iBAAiB;AAErB,aAAW,UAAU,SAAS;AAC5B,QAAI,kBAAkB,MAAW,MAAM;AACvC,QAAI,sBAAsB,MAAW,MAAM;AAC3C,QAAI,iBAAiB;AACrB,QAAI,qBAAqB;AACzB,QAAI,OAAO,iBAAiB,aAAa;AACvC,YAAMA,SAAQ,MAAM,mBAAmB,SAAS,OAAO,MAAM,OAAO,MAAgB;AACpF,UAAIA,WAAU,QAAW;AACvB,0BAAkBA;AAClB,sBAAc,MAAW,IAAI,aAAaA,MAAK;AAC/C,YAAI,QAAQ,SAAS,QAAW;AAC9B,2BAAiB,MAAM;AAAA,YACrB,QAAQ,KAAK,UAAU;AAAA,cACrB,MAAM;AAAA,cACN,WAAW,QAAQ;AAAA,cACnB,MAAM,OAAO;AAAA,cACb,QAAQ,OAAO;AAAA,cACf,OAAAA;AAAA,YACF,CAAC;AAAA,UACH;AACA,wBAAc;AAAA,QAChB;AAAA,MACF;AAAA,IACF;AACA,eAAW,gBAAgB,OAAO,SAAS;AACzC,YAAMA,SAAQ,aAAa,QAAQ;AACnC,UAAIA,WAAU,OAAW;AACzB,kBAAYA,QAAO,+BAA+B,aAAa,UAAU,EAAE;AAC3E,4BAAsB,MAAW,IAAI,qBAAqBA,MAAK;AAC/D,wBAAkB,MAAW,IAAI,iBAAiBA,MAAK;AACvD,UAAI,QAAQ,SAAS,UAAa,OAAO,iBAAiB,aAAa;AACrE,cAAM,SAAS,QAAQ,QAAQ;AAAA,UAC7B,CAAC,cAAc,UAAU,SAAS,aAAa;AAAA,QACjD;AACA,YAAI,WAAW,QAAW;AACxB,gBAAM,aAAa,MAAM;AAAA,YACvB,QAAQ,KAAK,UAAU;AAAA,cACrB,MAAM;AAAA,cACN,WAAW,QAAQ;AAAA,cACnB,MAAM,OAAO;AAAA,cACb,QAAQ,OAAO;AAAA,cACf;AAAA,cACA,OAAAA;AAAA,YACF,CAAC;AAAA,UACH;AACA,uBAAa,OAAO;AACpB,gCAAsB;AACtB,4BAAkB;AAAA,QACpB;AAAA,MACF;AAAA,IACF;AACA,WAAO,QAAQ;AAAA,MACb,QAAQ;AAAA,MACR,YAAY;AAAA,MACZ,OAAO,MAAW,IAAI,iBAAiB,mBAAmB;AAAA,IAC5D;AACA,QAAI,QAAQ,SAAS,QAAW;AAC9B,aAAO,OAAO;AAAA,QACZ,UAAU,QAAQ,KAAK;AAAA,QACvB,QAAQ;AAAA,QACR,YAAY;AAAA,QACZ,OAAO,iBAAiB;AAAA,MAC1B;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAQ;AAAA,IACZ,QAAQ;AAAA,IACR,YAAY;AAAA,IACZ,OAAO,MAAW,IAAI,aAAa,eAAe;AAAA,EACpD;AACA,QAAM,aAKF;AAAA,IACF,SAAS,oBAAoB,OAAO;AAAA,IACpC,OAAO,kBAAkB,OAAO;AAAA,IAChC;AAAA,EACF;AACA,MAAI,QAAQ,SAAS,QAAW;AAC9B,eAAW,OAAO;AAAA,MAChB,UAAU,QAAQ,KAAK;AAAA,MACvB,QAAQ;AAAA,MACR,YAAY;AAAA,MACZ,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AACA,SAAO;AACT;AAEA,eAAe,mBACb,SACA,UACA,QAC4B;AAC5B,QAAM,QACJ,QAAQ,gBAAgB,SACpB,gBAAgB,MAAM,IACtB,MAAM,QAAQ,YAAY;AAAA,IACxB,WAAW,QAAQ;AAAA,IACnB,MAAM;AAAA,IACN;AAAA,IACA,QAAQ,IAAI,gBAAgB,EAAE;AAAA,EAChC,CAAC;AACP,MAAI,UAAU,OAAW,aAAY,OAAO,yBAAyB,SAAS,EAAE,EAAE;AAClF,SAAO;AACT;AAEA,SAAS,gBAAgB,QAAoC;AAC3D,MAAI,OAAO,WAAW,YAAY,WAAW,QAAQ,EAAE,WAAW,QAAS,QAAO;AAClF,SAAQ,OAAyC;AACnD;AAEA,SAAS,YAAY,OAAc,OAAqB;AACtD,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,KAAK,GAAG;AAChD,QAAI,QAAQ,UAAW;AACvB,QAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GAAG;AACrE,YAAM,IAAI,UAAU,GAAG,KAAK,iDAAiD;AAAA,IAC/E;AAAA,EACF;AACF;AAEA,eAAe,cAAc,OAAkD;AAC7E,QAAM,OAAO,MAAM;AACnB,MAAI,CAAC,OAAO,SAAS,IAAI,KAAK,OAAO,GAAG;AACtC,UAAM,IAAI,UAAU,sEAAsE;AAAA,EAC5F;AACA,SAAO;AACT;AAEA,SAAS,qBACP,SACM;AACN,MAAI,QAAQ,KAAK,KAAK,EAAE,WAAW,GAAG;AACpC,UAAM,IAAI,UAAU,yCAAyC;AAAA,EAC/D;AACA,MAAI,QAAQ,MAAM,WAAW,GAAG;AAC9B,UAAM,IAAI,UAAU,iDAAiD;AAAA,EACvE;AACA,MAAI,QAAQ,QAAQ,WAAW,GAAG;AAChC,UAAM,IAAI,UAAU,mDAAmD;AAAA,EACzE;AACA,aAAW,YAAY,QAAQ,OAAO;AACpC,QAAI,SAAS,GAAG,KAAK,EAAE,WAAW,GAAG;AACnC,YAAM,IAAI,UAAU,sCAAsC;AAAA,IAC5D;AAAA,EACF;AACA,aAAW,UAAU,QAAQ,SAAS;AACpC,QAAI,OAAO,KAAK,KAAK,EAAE,WAAW,GAAG;AACnC,YAAM,IAAI,UAAU,0CAA0C;AAAA,IAChE;AAAA,EACF;AACA;AAAA,IACE,QAAQ,MAAM,IAAI,CAAC,aAAa,SAAS,EAAE;AAAA,IAC3C;AAAA,EACF;AACA;AAAA,IACE,QAAQ,QAAQ,IAAI,CAAC,WAAW,OAAO,IAAI;AAAA,IAC3C;AAAA,EACF;AACA,MAAI,QAAQ,SAAS,UAAa,QAAQ,KAAK,SAAS,KAAK,EAAE,WAAW,GAAG;AAC3E,UAAM,IAAI,UAAU,4CAA4C;AAAA,EAClE;AACA,aAAW,CAAC,OAAO,KAAK,KAAK;AAAA,IAC3B,CAAC,eAAe,QAAQ,WAAW;AAAA,IACnC,CAAC,qBAAqB,QAAQ,iBAAiB;AAAA,IAC/C,CAAC,qBAAqB,QAAQ,iBAAiB;AAAA,EACjD,GAAY;AACV,QAAI,UAAU,WAAc,CAAC,OAAO,cAAc,KAAK,KAAK,QAAQ,IAAI;AACtE,YAAM,IAAI,WAAW,cAAc,KAAK,6BAA6B;AAAA,IACvE;AAAA,EACF;AACA,MACE,QAAQ,kBAAkB,WACzB,CAAC,OAAO,cAAc,QAAQ,aAAa,KAAK,QAAQ,gBAAgB,IACzE;AACA,UAAM,IAAI,WAAW,qDAAqD;AAAA,EAC5E;AACA,MAAI,QAAQ,UAAU,QAAW;AAC/B,QAAI,CAAC,OAAO,cAAc,QAAQ,MAAM,KAAK,KAAK,QAAQ,MAAM,QAAQ,GAAG;AACzE,YAAM,IAAI,WAAW,mDAAmD;AAAA,IAC1E;AACA,QACE,CAAC,OAAO,cAAc,QAAQ,MAAM,KAAK,KACzC,QAAQ,MAAM,QAAQ,KACtB,QAAQ,MAAM,SAAS,QAAQ,MAAM,OACrC;AACA,YAAM,IAAI,WAAW,wDAAwD;AAAA,IAC/E;AAAA,EACF;AACF;AAEA,SAAS,aAAa,QAAkB,OAAqB;AAC3D,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,SAAS,QAAQ;AAC1B,QAAI,KAAK,IAAI,KAAK,EAAG,OAAM,IAAI,UAAU,GAAG,KAAK,oBAAoB,KAAK,EAAE;AAC5E,SAAK,IAAI,KAAK;AAAA,EAChB;AACF;AAEA,SAAS,YACP,SACsC;AACtC,QAAM,YAAY,QAAQ,YAAY,SAAY,SAAY,IAAI,IAAI,QAAQ,OAAO;AACrF,MAAI,cAAc,QAAW;AAC3B,iBAAa,CAAC,GAAG,QAAQ,OAAQ,GAAG,6BAA6B;AACjE,UAAM,YAAY,IAAI,IAAI,QAAQ,MAAM,IAAI,CAAC,aAAa,SAAS,EAAE,CAAC;AACtE,eAAW,MAAM,WAAW;AAC1B,UAAI,CAAC,UAAU,IAAI,EAAE;AACnB,cAAM,IAAI,UAAU,8CAA8C,EAAE,EAAE;AAAA,IAC1E;AAAA,EACF;AACA,QAAM,WAAW,QAAQ,MAAM;AAAA,IAC7B,CAAC,UAAU,WACR,cAAc,UAAa,UAAU,IAAI,SAAS,EAAE,OACpD,QAAQ,eAAe,UAAa,QAAQ,WAAW,UAAU,KAAK;AAAA,EAC3E;AACA,MAAI,QAAQ,UAAU,OAAW,QAAO;AACxC,SAAO,SAAS,OAAO,CAAC,GAAG,UAAU,QAAQ,QAAQ,MAAO,UAAU,QAAQ,MAAO,KAAK;AAC5F;AAEA,eAAe,eACb,SACA,OACe;AACf,QAAM,QAAQ,aAAa,KAAK;AAClC;AAEA,SAAS,UAAU,QAA0C;AAC3D,SAAO,QAAQ,YAAY;AAC7B;AAEA,SAAS,iBAAiB,QAA0C;AAClE,SAAO,QAAQ,UAAU,IAAI,eAAe;AAC9C;","names":["usage"]}

package/dist/chunk-R6722CIU.js

@@ -0,0 +1,28 @@
+// src/pipeline/errors.ts
+var PipelineAgentSuspensionError = class extends Error {
+  constructor(result) {
+    super("Pipeline agent stage suspended for human interaction.");
+    this.result = result;
+    this.name = "PipelineAgentSuspensionError";
+  }
+  result;
+};
+var PipelineObserverDispatchError = class extends AggregateError {
+  phase;
+  failures;
+  constructor(phase, failures) {
+    super(
+      failures.map((failure) => failure.error),
+      `Pipeline observer ${phase} failed for ${failures.map((failure) => failure.observer).join(", ")}.`
+    );
+    this.name = "PipelineObserverDispatchError";
+    this.phase = phase;
+    this.failures = Object.freeze([...failures]);
+  }
+};
+
+export {
+  PipelineAgentSuspensionError,
+  PipelineObserverDispatchError
+};
+//# sourceMappingURL=chunk-R6722CIU.js.map

package/dist/chunk-R6722CIU.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/pipeline/errors.ts"],"sourcesContent":["import type { AgentInteractionOutcome } from \"../agent/run-types\";\n\nexport class PipelineAgentSuspensionError extends Error {\n  constructor(readonly result: AgentInteractionOutcome) {\n    super(\"Pipeline agent stage suspended for human interaction.\");\n    this.name = \"PipelineAgentSuspensionError\";\n  }\n}\n\nexport type PipelineObserverFailure = {\n  readonly observer: string;\n  readonly error: unknown;\n};\n\nexport class PipelineObserverDispatchError extends AggregateError {\n  readonly phase: string;\n  readonly failures: readonly PipelineObserverFailure[];\n\n  constructor(phase: string, failures: readonly PipelineObserverFailure[]) {\n    super(\n      failures.map((failure) => failure.error),\n      `Pipeline observer ${phase} failed for ${failures\n        .map((failure) => failure.observer)\n        .join(\", \")}.`,\n    );\n    this.name = \"PipelineObserverDispatchError\";\n    this.phase = phase;\n    this.failures = Object.freeze([...failures]);\n  }\n}\n"],"mappings":";AAEO,IAAM,+BAAN,cAA2C,MAAM;AAAA,EACtD,YAAqB,QAAiC;AACpD,UAAM,uDAAuD;AAD1C;AAEnB,SAAK,OAAO;AAAA,EACd;AAAA,EAHqB;AAIvB;AAOO,IAAM,gCAAN,cAA4C,eAAe;AAAA,EACvD;AAAA,EACA;AAAA,EAET,YAAY,OAAe,UAA8C;AACvE;AAAA,MACE,SAAS,IAAI,CAAC,YAAY,QAAQ,KAAK;AAAA,MACvC,qBAAqB,KAAK,eAAe,SACtC,IAAI,CAAC,YAAY,QAAQ,QAAQ,EACjC,KAAK,IAAI,CAAC;AAAA,IACf;AACA,SAAK,OAAO;AACZ,SAAK,QAAQ;AACb,SAAK,WAAW,OAAO,OAAO,CAAC,GAAG,QAAQ,CAAC;AAAA,EAC7C;AACF;","names":[]}

package/dist/chunk-RA4YVN43.js

@@ -0,0 +1 @@
+//# sourceMappingURL=chunk-RA4YVN43.js.map

package/dist/chunk-RA4YVN43.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}