@axlsdk/eval 0.22.2 → 0.23.0

package/dist/index.d.cts

@@ -273,6 +273,93 @@ type ScorerDetail = {
      */
     skipped?: boolean;
 };
+/**
+ * Provider-call latency sums for one model within one eval item, rolled up from
+ * the `timing` block on `agent_call_end` (see `CallTiming` in `@axlsdk/axl`).
+ *
+ * The point of the split is that `EvalItem.duration` is wall clock for the whole
+ * workflow — tools, gates, the SDK's own rate-limiter queue and all — so it
+ * cannot be compared across models or across runs with different fan-out.
+ * `wireMs` is the provider's time; `queuedMs` is the SDK's self-imposed pacing;
+ * `retryMs` is failed attempts and their backoff.
+ *
+ * Keyed by the effective model URI (`openai:gpt-4o`), the same key
+ * `metadata.modelCallCounts` uses, so the two can be joined.
+ */
+type ItemModelTiming = {
+    /** Provider calls that REPORTED timing. May be lower than this model's
+     *  `modelCallCounts` entry — an uninstrumented custom provider and the error
+     *  path both contribute a call with no timing. Divide the sums by this, never
+     *  by the call count. */
+    calls: number;
+    /** Sum of `CallTiming.queuedMs` — waiting on the SDK's own rate limiter. */
+    queuedMs: number;
+    /** Sum of `CallTiming.retryMs` — failed attempts plus their backoff. */
+    retryMs: number;
+    /** Sum of `CallTiming.wireMs` — time attributable to the provider. Across
+     *  concurrent calls this can exceed the item's `duration`; that is expected. */
+    wireMs: number;
+    /** Sum of `CallTiming.firstTokenMs` over the STREAMING calls that reported
+     *  one. Absent when no call did, so a non-streaming model reports no
+     *  misleading `0`. Its denominator is `firstTokenCalls`, NOT `calls`. */
+    firstTokenMs?: number;
+    /** Timed calls that reported a `firstTokenMs`. Present exactly when
+     *  `firstTokenMs` is. It exists because a model can mix streamed and
+     *  non-streamed calls, and dividing the first-token sum by `calls` would then
+     *  report a latency no call ever achieved. */
+    firstTokenCalls?: number;
+};
+/** Distribution shape shared by the wall-clock and per-model timing stats. */
+type TimingStats = {
+    mean: number;
+    min: number;
+    max: number;
+    p50: number;
+    p95: number;
+};
+/**
+ * Per-model latency stats across a run, on `EvalSummary.modelTiming`.
+ *
+ * Every field here is a distribution over **per-call** values, pooled across all
+ * of the run's successful items — the same population `calls` counts. One
+ * provider call is one sample, so an item that makes ten calls weighs ten times
+ * an item that makes one. That is the right weighting for a model comparison:
+ * these numbers describe the model's latency, not the item's.
+ *
+ * This is deliberately a different weighting from the wall-clock
+ * `EvalSummary.timing`, which samples once per item because it describes the
+ * workflow. Read them side by side; do not expect them to agree.
+ */
+type ModelTimingStats = {
+    /** Total SUCCESSFUL timed provider calls for this model across every item —
+     *  the sample size behind every distribution below. May be lower than this
+     *  model's total call count: an uninstrumented provider reports no timing, and
+     *  a failed call is excluded even when it does report one, so these stats
+     *  describe answers rather than a blend of answers and failures. */
+    calls: number;
+    /** Per-call wire latency (ms) — time attributable to the provider. The
+     *  primary model-comparison figure alongside `firstTokenMs`. */
+    wireMs: TimingStats;
+    /** Per-call queue wait (ms) in Axl's own rate limiter. Self-imposed wait, so
+     *  it never distorts a comparison of the models themselves. */
+    queuedMs: TimingStats;
+    /** Per-call retry time (ms) — failed attempts and their backoff. A model
+     *  throttled hard on the day of the run shows it here, which is what keeps
+     *  `wireMs` an honest comparison. */
+    retryMs: TimingStats;
+    /** Per-call time to first content delta (ms), over the STREAMING calls that
+     *  reported one — non-streaming calls are excluded from the sample rather
+     *  than entered as `0`. Absent when no call reported one. This is the
+     *  model-discriminating figure: response headers arrive at roughly one round
+     *  trip regardless of model, first token does not. */
+    firstTokenMs?: TimingStats;
+    /** How many calls that `firstTokenMs` sample covers. It earns its place
+     *  because a `TimingStats` carries no sample size, so on a model that mixes
+     *  streamed and non-streamed calls there is otherwise no way to tell a
+     *  first-token figure drawn from one call apart from one drawn from all of
+     *  them. Present exactly when `firstTokenMs` is. */
+    firstTokenCalls?: number;
+};
 type EvalItem = {
     input: unknown;
     annotations?: unknown;
@@ -288,6 +375,20 @@ type EvalItem = {
     scoreDetails?: Record<string, ScorerDetail>;
     /** Execution metadata forwarded from the runtime (models, tokens, agentCalls, etc). */
     metadata?: Record<string, unknown>;
+    /** Per-model provider-call latency for this item, rolled up from
+     *  `agent_call_end.timing`. Absent when the item made no timed provider call
+     *  — an item with no ask, an uninstrumented custom provider, or a runtime
+     *  without `trackExecution`. Distinct from `duration` (workflow wall clock),
+     *  which is unchanged.
+     *
+     *  These are per-item SUMS, kept compact so a persisted artifact stays small.
+     *  `summary.modelTiming` is NOT derived from them — it is computed from the
+     *  raw per-call blocks while the run is in memory, so its percentiles are
+     *  real per-call percentiles rather than percentiles of item means. A
+     *  consumer holding only a saved artifact can therefore recover per-item
+     *  sums and per-model totals from `item.timing`, but not the distribution.
+     *  (`rescore` already reports no timing stats at all, so nothing regresses.) */
+    timing?: Record<string, ItemModelTiming>;
     /** Trace events captured during this item's execution. Only populated when
      *  `runEval` was called with `{ captureTraces: true }`. Verbose-mode
      *  `agent_call_start.data.messages` snapshots are stripped to keep memory bounded;
@@ -327,6 +428,8 @@ type EvalSummary = {
          */
         skipped?: number;
     }>;
+    /** Wall-clock stats over per-item `duration`. Unchanged by the per-model
+     *  timing rollup — this still measures the whole workflow, queue and all. */
     timing?: {
         mean: number;
         min: number;
@@ -334,6 +437,9 @@ type EvalSummary = {
         p50: number;
         p95: number;
     };
+    /** Per-model provider-latency stats, present only when at least one item
+     *  reported timing. Read alongside `timing`, never instead of it. */
+    modelTiming?: Record<string, ModelTimingStats>;
     /**
      * Populated by `runEval` ONLY when `EvalConfig.failOnScorerErrorRate` is set
      * and one or more scorers exceeded tolerance. `runEval` never throws on this
@@ -633,4 +739,4 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type ItemModelTiming, type LlmScorerConfig, type ModelTimingStats, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };

package/dist/index.d.ts

@@ -273,6 +273,93 @@ type ScorerDetail = {
      */
     skipped?: boolean;
 };
+/**
+ * Provider-call latency sums for one model within one eval item, rolled up from
+ * the `timing` block on `agent_call_end` (see `CallTiming` in `@axlsdk/axl`).
+ *
+ * The point of the split is that `EvalItem.duration` is wall clock for the whole
+ * workflow — tools, gates, the SDK's own rate-limiter queue and all — so it
+ * cannot be compared across models or across runs with different fan-out.
+ * `wireMs` is the provider's time; `queuedMs` is the SDK's self-imposed pacing;
+ * `retryMs` is failed attempts and their backoff.
+ *
+ * Keyed by the effective model URI (`openai:gpt-4o`), the same key
+ * `metadata.modelCallCounts` uses, so the two can be joined.
+ */
+type ItemModelTiming = {
+    /** Provider calls that REPORTED timing. May be lower than this model's
+     *  `modelCallCounts` entry — an uninstrumented custom provider and the error
+     *  path both contribute a call with no timing. Divide the sums by this, never
+     *  by the call count. */
+    calls: number;
+    /** Sum of `CallTiming.queuedMs` — waiting on the SDK's own rate limiter. */
+    queuedMs: number;
+    /** Sum of `CallTiming.retryMs` — failed attempts plus their backoff. */
+    retryMs: number;
+    /** Sum of `CallTiming.wireMs` — time attributable to the provider. Across
+     *  concurrent calls this can exceed the item's `duration`; that is expected. */
+    wireMs: number;
+    /** Sum of `CallTiming.firstTokenMs` over the STREAMING calls that reported
+     *  one. Absent when no call did, so a non-streaming model reports no
+     *  misleading `0`. Its denominator is `firstTokenCalls`, NOT `calls`. */
+    firstTokenMs?: number;
+    /** Timed calls that reported a `firstTokenMs`. Present exactly when
+     *  `firstTokenMs` is. It exists because a model can mix streamed and
+     *  non-streamed calls, and dividing the first-token sum by `calls` would then
+     *  report a latency no call ever achieved. */
+    firstTokenCalls?: number;
+};
+/** Distribution shape shared by the wall-clock and per-model timing stats. */
+type TimingStats = {
+    mean: number;
+    min: number;
+    max: number;
+    p50: number;
+    p95: number;
+};
+/**
+ * Per-model latency stats across a run, on `EvalSummary.modelTiming`.
+ *
+ * Every field here is a distribution over **per-call** values, pooled across all
+ * of the run's successful items — the same population `calls` counts. One
+ * provider call is one sample, so an item that makes ten calls weighs ten times
+ * an item that makes one. That is the right weighting for a model comparison:
+ * these numbers describe the model's latency, not the item's.
+ *
+ * This is deliberately a different weighting from the wall-clock
+ * `EvalSummary.timing`, which samples once per item because it describes the
+ * workflow. Read them side by side; do not expect them to agree.
+ */
+type ModelTimingStats = {
+    /** Total SUCCESSFUL timed provider calls for this model across every item —
+     *  the sample size behind every distribution below. May be lower than this
+     *  model's total call count: an uninstrumented provider reports no timing, and
+     *  a failed call is excluded even when it does report one, so these stats
+     *  describe answers rather than a blend of answers and failures. */
+    calls: number;
+    /** Per-call wire latency (ms) — time attributable to the provider. The
+     *  primary model-comparison figure alongside `firstTokenMs`. */
+    wireMs: TimingStats;
+    /** Per-call queue wait (ms) in Axl's own rate limiter. Self-imposed wait, so
+     *  it never distorts a comparison of the models themselves. */
+    queuedMs: TimingStats;
+    /** Per-call retry time (ms) — failed attempts and their backoff. A model
+     *  throttled hard on the day of the run shows it here, which is what keeps
+     *  `wireMs` an honest comparison. */
+    retryMs: TimingStats;
+    /** Per-call time to first content delta (ms), over the STREAMING calls that
+     *  reported one — non-streaming calls are excluded from the sample rather
+     *  than entered as `0`. Absent when no call reported one. This is the
+     *  model-discriminating figure: response headers arrive at roughly one round
+     *  trip regardless of model, first token does not. */
+    firstTokenMs?: TimingStats;
+    /** How many calls that `firstTokenMs` sample covers. It earns its place
+     *  because a `TimingStats` carries no sample size, so on a model that mixes
+     *  streamed and non-streamed calls there is otherwise no way to tell a
+     *  first-token figure drawn from one call apart from one drawn from all of
+     *  them. Present exactly when `firstTokenMs` is. */
+    firstTokenCalls?: number;
+};
 type EvalItem = {
     input: unknown;
     annotations?: unknown;
@@ -288,6 +375,20 @@ type EvalItem = {
     scoreDetails?: Record<string, ScorerDetail>;
     /** Execution metadata forwarded from the runtime (models, tokens, agentCalls, etc). */
     metadata?: Record<string, unknown>;
+    /** Per-model provider-call latency for this item, rolled up from
+     *  `agent_call_end.timing`. Absent when the item made no timed provider call
+     *  — an item with no ask, an uninstrumented custom provider, or a runtime
+     *  without `trackExecution`. Distinct from `duration` (workflow wall clock),
+     *  which is unchanged.
+     *
+     *  These are per-item SUMS, kept compact so a persisted artifact stays small.
+     *  `summary.modelTiming` is NOT derived from them — it is computed from the
+     *  raw per-call blocks while the run is in memory, so its percentiles are
+     *  real per-call percentiles rather than percentiles of item means. A
+     *  consumer holding only a saved artifact can therefore recover per-item
+     *  sums and per-model totals from `item.timing`, but not the distribution.
+     *  (`rescore` already reports no timing stats at all, so nothing regresses.) */
+    timing?: Record<string, ItemModelTiming>;
     /** Trace events captured during this item's execution. Only populated when
      *  `runEval` was called with `{ captureTraces: true }`. Verbose-mode
      *  `agent_call_start.data.messages` snapshots are stripped to keep memory bounded;
@@ -327,6 +428,8 @@ type EvalSummary = {
          */
         skipped?: number;
     }>;
+    /** Wall-clock stats over per-item `duration`. Unchanged by the per-model
+     *  timing rollup — this still measures the whole workflow, queue and all. */
     timing?: {
         mean: number;
         min: number;
@@ -334,6 +437,9 @@ type EvalSummary = {
         p50: number;
         p95: number;
     };
+    /** Per-model provider-latency stats, present only when at least one item
+     *  reported timing. Read alongside `timing`, never instead of it. */
+    modelTiming?: Record<string, ModelTimingStats>;
     /**
      * Populated by `runEval` ONLY when `EvalConfig.failOnScorerErrorRate` is set
      * and one or more scorers exceeded tolerance. `runEval` never throws on this
@@ -633,4 +739,4 @@ type MultiRunSummary = {
  */
 declare function aggregateRuns(runs: EvalResult[]): MultiRunSummary;
 
-export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type LlmScorerConfig, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };
+export { type BootstrapCIResult, type Dataset, type DatasetConfig, type DatasetItem, type DegradedScorer, type EvalCompareOptions, type EvalComparison, type EvalComparisonPartial, type EvalConfig, type EvalImprovement, type EvalItem, type EvalProgressEvent, type EvalRegression, type EvalResult, type EvalSummary, type ExtraKeyPolicy, type ItemModelTiming, type LlmScorerConfig, type ModelTimingStats, type MultiRunSummary, type RescoreOptions, type RunEvalOptions, type Scorer, type ScorerApplies, type ScorerConfig, type ScorerContext, type ScorerDetail, type ScorerFn, type ScorerResult, type ScorerToleranceVerdict, aggregateRuns, dataset, defineEval, evalCompare, evaluateScorerErrorRateGate, evaluateScorerTolerance, llmScorer, normalizeScorerResult, pairedBootstrapCI, rescore, runEval, scorer };

package/dist/index.js

@@ -9,7 +9,7 @@ import {
   rescore,
   runEval,
   scorer
-} from "./chunk-C7CICWIA.js";
+} from "./chunk-FQHKBSZD.js";
 
 // src/dataset.ts
 import { readFile } from "fs/promises";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@axlsdk/eval",
-  "version": "0.22.2",
+  "version": "0.23.0",
   "description": "Evaluation framework for Axl agentic workflows",
   "type": "module",
   "main": "./dist/index.cjs",
@@ -25,7 +25,7 @@
     "dist"
   ],
   "dependencies": {
-    "@axlsdk/axl": "0.22.2"
+    "@axlsdk/axl": "0.23.0"
   },
   "peerDependencies": {
     "zod": "^4.0.0",

package/dist/chunk-C7CICWIA.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/bootstrap.ts","../src/utils.ts","../src/compare.ts","../src/scorer.ts","../src/score-item.ts","../src/runner.ts","../src/rescore.ts","../src/multi-run.ts"],"sourcesContent":["export type BootstrapCIResult = {\n  /** Lower bound of the confidence interval. */\n  lower: number;\n  /** Upper bound of the confidence interval. */\n  upper: number;\n  /** Mean of the original differences. */\n  mean: number;\n  /** Proportion of resamples where the mean difference is negative (probability of regression). */\n  pRegression: number;\n  /** Proportion of resamples where the mean difference is positive (probability of improvement). */\n  pImprovement: number;\n};\n\n/**\n * Compute a bootstrap confidence interval on paired differences.\n *\n * Resamples the differences array with replacement, computes the mean of each\n * resample, and returns the percentile-based confidence interval.\n */\nexport function pairedBootstrapCI(\n  differences: number[],\n  options?: { nResamples?: number; alpha?: number; seed?: number },\n): BootstrapCIResult {\n  const n = differences.length;\n  if (n === 0) return { lower: 0, upper: 0, mean: 0, pRegression: 0, pImprovement: 0 };\n\n  const nResamples = options?.nResamples ?? 1000;\n  const alpha = options?.alpha ?? 0.05;\n\n  const mean = differences.reduce((a, b) => a + b, 0) / n;\n\n  if (n === 1)\n    return {\n      lower: differences[0],\n      upper: differences[0],\n      mean,\n      pRegression: mean < 0 ? 1 : 0,\n      pImprovement: mean > 0 ? 1 : 0,\n    };\n\n  if (nResamples <= 0)\n    return {\n      lower: round(mean),\n      upper: round(mean),\n      mean: round(mean),\n      pRegression: mean < 0 ? 1 : 0,\n      pImprovement: mean > 0 ? 1 : 0,\n    };\n\n  // Seeded or random number generator\n  const rng = options?.seed != null ? xorshift32(options.seed) : () => Math.random();\n\n  const resampleMeans: number[] = new Array(nResamples);\n  for (let r = 0; r < nResamples; r++) {\n    let sum = 0;\n    for (let i = 0; i < n; i++) {\n      sum += differences[Math.floor(rng() * n)];\n    }\n    resampleMeans[r] = sum / n;\n  }\n\n  resampleMeans.sort((a, b) => a - b);\n\n  const lowerIdx = Math.floor((alpha / 2) * nResamples);\n  const upperIdx = Math.floor((1 - alpha / 2) * nResamples) - 1;\n\n  // Count resamples below/above zero for directional probabilities\n  // resampleMeans is already sorted, so binary search for zero\n  let belowZero = 0;\n  for (let i = 0; i < nResamples; i++) {\n    if (resampleMeans[i] < 0) belowZero++;\n    else break;\n  }\n  let aboveZero = 0;\n  for (let i = nResamples - 1; i >= 0; i--) {\n    if (resampleMeans[i] > 0) aboveZero++;\n    else break;\n  }\n\n  return {\n    lower: round(resampleMeans[Math.max(0, lowerIdx)]),\n    upper: round(resampleMeans[Math.min(nResamples - 1, upperIdx)]),\n    mean: round(mean),\n    pRegression: round(belowZero / nResamples),\n    pImprovement: round(aboveZero / nResamples),\n  };\n}\n\nfunction round(n: number): number {\n  return Math.round(n * 10000) / 10000;\n}\n\n/** Simple xorshift32 PRNG for deterministic test behavior. Returns values in [0, 1). */\nfunction xorshift32(seed: number): () => number {\n  let state = seed | 0 || 1; // Ensure non-zero\n  return () => {\n    state ^= state << 13;\n    state ^= state >> 17;\n    state ^= state << 5;\n    return (state >>> 0) / 0x100000000;\n  };\n}\n","export function computeStats(scores: number[]): {\n  mean: number;\n  min: number;\n  max: number;\n  p50: number;\n  p95: number;\n} {\n  if (scores.length === 0) return { mean: 0, min: 0, max: 0, p50: 0, p95: 0 };\n  const sorted = [...scores].sort((a, b) => a - b);\n  const mean = sorted.reduce((a, b) => a + b, 0) / sorted.length;\n  const min = sorted[0];\n  const max = sorted[sorted.length - 1];\n  const p50 = sorted[Math.floor(sorted.length * 0.5)];\n  const p95 = sorted[Math.floor(sorted.length * 0.95)];\n  return { mean: round(mean), min: round(min), max: round(max), p50: round(p50), p95: round(p95) };\n}\n\nexport function round(n: number): number {\n  return Math.round(n * 1000) / 1000;\n}\n\n/**\n * Count, for one scorer across a set of items, how many produced a valid score\n * (`scored`), ran-and-failed (`failed`), or were deliberately skipped by the\n * scorer's `applies` predicate (`skipped`). The single source of truth for the\n * failure-rate trust signal, shared by `runEval`, `rescore`, and `evalCompare`.\n *\n * Three discriminators on the per-scorer `scoreDetails` entry, checked in\n * precedence order so a positive `skipped` marker always wins over the\n * duration heuristic:\n *  - a non-null `scores[name]` ⇒ `scored`;\n *  - else `skipped === true` (the `applies` predicate returned `false`) ⇒\n *    `skipped` — deliberately NOT run, so it belongs in neither the mean nor\n *    the failure-rate denominator;\n *  - else a `duration` ⇒ `failed` (`scoreItem` stamps it whenever a scorer\n *    actually executes: success, throw, or out-of-range);\n *  - else (no score, no marker, no duration) the scorer was skipped by\n *    cancellation (signal aborted before it started) and belongs in no bucket.\n *\n * So `scored + failed` is the honest \"attempted\" denominator for a failure rate\n * — it excludes both `applies`-skips and cancellations and can be `<` the\n * eligible item count.\n *\n * Items whose WORKFLOW errored (`i.error`) are excluded entirely — the scorers\n * never ran for them, so they're not part of the scorer's sample either way.\n */\nexport function scorerCounts(\n  items: readonly {\n    error?: string;\n    scores: Record<string, number | null>;\n    scoreDetails?: Record<string, { duration?: number; skipped?: boolean }>;\n  }[],\n  name: string,\n): { scored: number; failed: number; skipped: number } {\n  let scored = 0;\n  let failed = 0;\n  let skipped = 0;\n  for (const i of items) {\n    if (i.error) continue;\n    if (i.scores[name] != null) scored++;\n    else if (i.scoreDetails?.[name]?.skipped === true) skipped++;\n    else if (i.scoreDetails?.[name]?.duration != null) failed++;\n  }\n  return { scored, failed, skipped };\n}\n\n/** A scorer's failure-rate verdict against a tolerance — see {@link evaluateScorerTolerance}. */\nexport type ScorerToleranceVerdict = {\n  /** `scored + failed` — the attempted denominator (excludes cancelled/never-run). */\n  attempted: number;\n  /** `failed / attempted`, or `0` when nothing was attempted. */\n  rate: number;\n  /** Over tolerance AND something was actually attempted. */\n  exceeds: boolean;\n  /** Nothing was attempted (`attempted === 0`) — no basis to certify a rate. */\n  zeroSample: boolean;\n};\n\n/**\n * Single source of truth for the type-aware failure-rate decision, shared by the\n * source-side gate (`runEval` → `summary.degraded`) and the gate-side compare\n * gate (`evalCompare` consumers). Deterministic scorers tolerate ZERO failures\n * (a deterministic throw is a bug, not noise); LLM scorers tolerate up to `limit`\n * (`failed / attempted`). `exceeds` is never true when nothing was attempted —\n * callers decide their own policy for `zeroSample` (the runner skips it; the\n * compare gate refuses to certify a zero-sample scorer).\n */\nexport function evaluateScorerTolerance(\n  scored: number,\n  failed: number,\n  type: 'llm' | 'deterministic',\n  limit: number,\n): ScorerToleranceVerdict {\n  const attempted = scored + failed;\n  const rate = attempted === 0 ? 0 : failed / attempted;\n  const exceeds = attempted > 0 && (type === 'deterministic' ? failed > 0 : rate > limit);\n  return { attempted, rate, exceeds, zeroSample: attempted === 0 };\n}\n\n/**\n * Run `task` over `items` with a bounded worker pool, preserving input order.\n *\n * Results are written to a pre-allocated array at the item's original index, so\n * `results[i]` always corresponds to `items[i]` regardless of completion order.\n *\n * The helper does NOT catch — each `task` must own its try/catch. This is how\n * callers get `Promise.allSettled` semantics (one failing task never rejects\n * the whole batch) while still using a simple worker-pool: a task that resolves\n * (even after internally handling an error) lets the pool continue draining.\n *\n * Concurrency is clamped to `[1, items.length]` in exactly one place here, and\n * non-finite / fractional values are coerced (NaN/Infinity/`2.5` → a sane\n * integer). Callers must NOT pre-clamp with `Math.min(concurrency, items.length)`\n * and pass the result in — a user/env-supplied `0` (or `NaN` from a config like\n * `Number(process.env.X)`) would otherwise yield zero workers while items\n * remain, so `Promise.all` resolves immediately leaving the results array full\n * of holes (and the caller crashes on the `undefined` entries).\n */\nexport async function mapWithConcurrency<T, R = void>(\n  items: readonly T[],\n  concurrency: number,\n  task: (item: T, index: number) => Promise<R>,\n): Promise<R[]> {\n  const results = new Array<R>(items.length);\n  // Coerce non-finite (NaN/Infinity) to the floor of 1, and floor fractional\n  // values, BEFORE clamping — `Math.max(1, NaN)` is `NaN`, which would make\n  // `Array.from({ length: NaN })` empty and silently run zero workers.\n  const requested = Number.isFinite(concurrency) ? Math.floor(concurrency) : 1;\n  const workerCount = items.length === 0 ? 0 : Math.min(Math.max(1, requested), items.length);\n\n  let next = 0;\n  async function runNext(): Promise<void> {\n    while (next < items.length) {\n      const i = next++;\n      results[i] = await task(items[i], i);\n    }\n  }\n\n  await Promise.all(Array.from({ length: workerCount }, () => runNext()));\n  return results;\n}\n","import type {\n  EvalResult,\n  EvalComparison,\n  EvalCompareOptions,\n  EvalRegression,\n  EvalImprovement,\n} from './types.js';\nimport { pairedBootstrapCI } from './bootstrap.js';\nimport { scorerCounts, evaluateScorerTolerance, round } from './utils.js';\n\nconst DEFAULT_LLM_THRESHOLD = 0.05;\nconst DEFAULT_DETERMINISTIC_THRESHOLD = 0;\nconst LEGACY_THRESHOLD = 0.1;\n\nfunction resolveThreshold(\n  scorerName: string,\n  options: EvalCompareOptions | undefined,\n  metadata: Record<string, unknown>,\n): number {\n  // Explicit global threshold\n  if (typeof options?.thresholds === 'number') return options.thresholds;\n\n  // Per-scorer threshold map\n  if (options?.thresholds && typeof options.thresholds === 'object') {\n    const perScorer = options.thresholds[scorerName];\n    if (perScorer != null) return perScorer;\n    // Fall through to auto-calibration if scorer not in map\n  }\n\n  // Auto-calibrate from scorerTypes metadata\n  const scorerTypes = metadata.scorerTypes as Record<string, string> | undefined;\n  if (scorerTypes && scorerName in scorerTypes) {\n    return scorerTypes[scorerName] === 'llm'\n      ? DEFAULT_LLM_THRESHOLD\n      : DEFAULT_DETERMINISTIC_THRESHOLD;\n  }\n\n  // Legacy fallback for results without scorerTypes\n  return LEGACY_THRESHOLD;\n}\n\nexport function evalCompare(\n  baseline: EvalResult | EvalResult[],\n  candidate: EvalResult | EvalResult[],\n  options?: EvalCompareOptions,\n): EvalComparison {\n  const baselineRunsRaw = Array.isArray(baseline) ? baseline : [baseline];\n  const candidateRunsRaw = Array.isArray(candidate) ? candidate : [candidate];\n\n  if (baselineRunsRaw.length === 0 || candidateRunsRaw.length === 0) {\n    throw new Error('Cannot compare empty eval result arrays');\n  }\n\n  // Truncate both sides to a common run count BEFORE computing anything. The\n  // paired bootstrap CI was already restricted to `min(baseline, candidate)`\n  // runs (a CI over paired diffs requires equal-length sides), but means /\n  // regressions / timing / cost previously used the full-length pool on each\n  // side. That hybrid produced an internally inconsistent view: a 5-run\n  // baseline vs 2-run candidate showed `delta = mean(5) - mean(2)` next to\n  // a CI computed over only 2 paired diffs, so the user couldn't tell the\n  // numbers were drawn from different samples. Truncating symmetrically\n  // here keeps every per-scorer figure aligned with the CI's sample size,\n  // and the `runCount` field on each side surfaces the truncation\n  // explicitly so the UI can render \"n=2 of 5 pooled\".\n  const runCount = Math.min(baselineRunsRaw.length, candidateRunsRaw.length);\n  const baselineRuns = baselineRunsRaw.slice(0, runCount);\n  const candidateRuns = candidateRunsRaw.slice(0, runCount);\n\n  // Use the first run as representative for metadata, dataset, scorers.\n  // `partial` detection downstream still walks the FULL pool so\n  // pre-truncation history (`metadata.batchAttempted` from any pooled run)\n  // is honored.\n  const baselineRef = baselineRunsRaw[0];\n  const candidateRef = candidateRunsRaw[0];\n\n  if (baselineRef.dataset !== candidateRef.dataset) {\n    throw new Error(\n      `Cannot compare evals from different datasets: \"${baselineRef.dataset}\" vs \"${candidateRef.dataset}\"`,\n    );\n  }\n\n  const baselineScorerNames = Object.keys(baselineRef.summary.scorers).sort();\n  const candidateScorerNames = Object.keys(candidateRef.summary.scorers).sort();\n  if (JSON.stringify(baselineScorerNames) !== JSON.stringify(candidateScorerNames)) {\n    throw new Error(`Cannot compare evals with different scorers`);\n  }\n\n  // Collect per-item paired differences for each scorer across all runs (used for bootstrap CI).\n  // For multi-run: pool differences from all (baselineRun[r].items[i], candidateRun[r].items[i]) pairs.\n  const pairedDiffs: Record<string, number[]> = {};\n  for (const name of baselineScorerNames) {\n    pairedDiffs[name] = [];\n  }\n  for (let r = 0; r < runCount; r++) {\n    const bRun = baselineRuns[r];\n    const cRun = candidateRuns[r];\n    const minLength = Math.min(bRun.items.length, cRun.items.length);\n    for (let i = 0; i < minLength; i++) {\n      const bItem = bRun.items[i];\n      const cItem = cRun.items[i];\n      if (bItem.error || cItem.error) continue;\n      for (const name of baselineScorerNames) {\n        const bScore = bItem.scores[name];\n        const cScore = cItem.scores[name];\n        if (bScore != null && cScore != null) {\n          pairedDiffs[name].push(cScore - bScore);\n        }\n      }\n    }\n  }\n\n  // Compute aggregate means across the truncated runs (matches the CI's sample).\n  const baselineMeans: Record<string, number> = {};\n  const candidateMeans: Record<string, number> = {};\n  for (const name of baselineScorerNames) {\n    baselineMeans[name] =\n      baselineRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n      baselineRuns.length;\n    candidateMeans[name] =\n      candidateRuns.reduce((sum, r) => sum + (r.summary.scorers[name]?.mean ?? 0), 0) /\n      candidateRuns.length;\n  }\n\n  const scorers: EvalComparison['scorers'] = {};\n  for (const name of baselineScorerNames) {\n    const bMeanRaw = baselineMeans[name];\n    const cMeanRaw = candidateMeans[name];\n    const deltaRaw = cMeanRaw - bMeanRaw;\n    const bMean = round(bMeanRaw);\n    const cMean = round(cMeanRaw);\n    const delta = round(deltaRaw);\n    const deltaPercent = bMeanRaw > 0 ? round((deltaRaw / bMeanRaw) * 100) : 0;\n    const threshold = resolveThreshold(name, options, baselineRef.metadata);\n\n    // Per-side scored/failed over the SAME truncated pool the means/CI use, so\n    // a gate (`--max-scorer-error-rate`) reads a sample consistent with the\n    // displayed numbers. Sum across this side's truncated runs (the\n    // `scorerCounts` helper excludes workflow-errored items and counts a\n    // ran-and-failed scorer via its `scoreDetails.duration` discriminator).\n    let baselineScored = 0;\n    let baselineFailed = 0;\n    let baselineSkipped = 0;\n    for (const run of baselineRuns) {\n      const c = scorerCounts(run.items, name);\n      baselineScored += c.scored;\n      baselineFailed += c.failed;\n      baselineSkipped += c.skipped;\n    }\n    let candidateScored = 0;\n    let candidateFailed = 0;\n    let candidateSkipped = 0;\n    for (const run of candidateRuns) {\n      const c = scorerCounts(run.items, name);\n      candidateScored += c.scored;\n      candidateFailed += c.failed;\n      candidateSkipped += c.skipped;\n    }\n\n    const entry: EvalComparison['scorers'][string] = {\n      baselineMean: bMean,\n      candidateMean: cMean,\n      delta,\n      deltaPercent,\n      baselineScored,\n      baselineFailed,\n      candidateScored,\n      candidateFailed,\n      baselineSkipped,\n      candidateSkipped,\n    };\n\n    // `n` is the PAIRED sample size — items scored on BOTH sides, the only\n    // items a difference can be computed for. Always set it (even 0/1), because\n    // that's exactly the asymmetric case the labeling needs to surface: the\n    // displayed `delta` is built from each side's INDEPENDENT mean (over that\n    // side's own scored items), while the CI below is paired. When `n` is small\n    // relative to `{baseline,candidate}Scored`, the two rest on different\n    // samples. The CI/significance machinery still requires >= 2 paired diffs.\n    const diffs = pairedDiffs[name];\n    entry.n = diffs.length;\n    if (diffs.length >= 2) {\n      const ci = pairedBootstrapCI(diffs);\n      entry.ci = { lower: ci.lower, upper: ci.upper };\n      entry.pRegression = ci.pRegression;\n      entry.pImprovement = ci.pImprovement;\n      // Significant when CI excludes zero AND delta exceeds practical threshold\n      const ciExcludesZero = ci.lower > 0 || ci.upper < 0;\n      entry.significant = ciExcludesZero && Math.abs(ci.mean) >= threshold;\n    }\n\n    scorers[name] = entry;\n  }\n\n  // Per-item regressions/improvements.\n  // For multi-run: average each item's score across runs to get a stable per-item comparison.\n  const regressions: EvalRegression[] = [];\n  const improvements: EvalImprovement[] = [];\n\n  const itemCount = Math.min(\n    ...baselineRuns.map((r) => r.items.length),\n    ...candidateRuns.map((r) => r.items.length),\n  );\n\n  for (let i = 0; i < itemCount; i++) {\n    // Check if any run has an error for this item\n    const hasBaselineError = baselineRuns.some((r) => r.items[i]?.error);\n    const hasCandidateError = candidateRuns.some((r) => r.items[i]?.error);\n    if (hasBaselineError || hasCandidateError) continue;\n\n    for (const name of baselineScorerNames) {\n      // Average this item's score across all runs on each side\n      const bScores = baselineRuns\n        .map((r) => r.items[i]?.scores[name])\n        .filter((s): s is number => s != null);\n      const cScores = candidateRuns\n        .map((r) => r.items[i]?.scores[name])\n        .filter((s): s is number => s != null);\n      if (bScores.length === 0 || cScores.length === 0) continue;\n\n      const bAvg = bScores.reduce((a, b) => a + b, 0) / bScores.length;\n      const cAvg = cScores.reduce((a, b) => a + b, 0) / cScores.length;\n      const delta = round(cAvg - bAvg);\n      const threshold = resolveThreshold(name, options, baselineRef.metadata);\n      if (delta < -threshold)\n        regressions.push({\n          itemIndex: i,\n          input: baselineRef.items[i]?.input,\n          scorer: name,\n          baselineScore: round(bAvg),\n          candidateScore: round(cAvg),\n          delta,\n        });\n      else if (delta > threshold)\n        improvements.push({\n          itemIndex: i,\n          input: baselineRef.items[i]?.input,\n          scorer: name,\n          baselineScore: round(bAvg),\n          candidateScore: round(cAvg),\n          delta,\n        });\n    }\n  }\n\n  // Timing comparison (average across all runs for multi-run)\n  let timing: EvalComparison['timing'];\n  const baselineTimings = baselineRuns\n    .filter((r) => r.summary.timing)\n    .map((r) => r.summary.timing!.mean);\n  const candidateTimings = candidateRuns\n    .filter((r) => r.summary.timing)\n    .map((r) => r.summary.timing!.mean);\n  if (baselineTimings.length > 0 && candidateTimings.length > 0) {\n    const bMean = baselineTimings.reduce((a, b) => a + b, 0) / baselineTimings.length;\n    const cMean = candidateTimings.reduce((a, b) => a + b, 0) / candidateTimings.length;\n    const delta = round(cMean - bMean);\n    const deltaPercent = bMean > 0 ? round(((cMean - bMean) / bMean) * 100) : 0;\n    timing = { baselineMean: round(bMean), candidateMean: round(cMean), delta, deltaPercent };\n  }\n\n  // Cost comparison (per-run average for multi-run)\n  let cost: EvalComparison['cost'];\n  const baselineAvgCost =\n    baselineRuns.reduce((sum, r) => sum + r.totalCost, 0) / baselineRuns.length;\n  const candidateAvgCost =\n    candidateRuns.reduce((sum, r) => sum + r.totalCost, 0) / candidateRuns.length;\n  if (baselineAvgCost > 0 || candidateAvgCost > 0) {\n    const deltaRaw = candidateAvgCost - baselineAvgCost;\n    const delta = round(deltaRaw);\n    const deltaPercent = baselineAvgCost > 0 ? round((deltaRaw / baselineAvgCost) * 100) : 0;\n    cost = {\n      baselineTotal: round(baselineAvgCost),\n      candidateTotal: round(candidateAvgCost),\n      delta,\n      deltaPercent,\n    };\n  }\n\n  const parts: string[] = [];\n  for (const name of baselineScorerNames) {\n    const s = scorers[name];\n    if (Math.abs(s.delta) > 0.001) {\n      const direction = s.delta > 0 ? 'improves' : 'regresses';\n      const sign = s.delta > 0 ? '+' : '';\n      let sigLabel = '';\n      if (s.significant === true) sigLabel = ' (significant)';\n      else if (s.significant === false) sigLabel = ' (not significant)';\n      parts.push(\n        `'${name}' ${direction} by ${sign}${s.deltaPercent.toFixed(1)}% (${s.baselineMean.toFixed(2)} -> ${s.candidateMean.toFixed(2)})${sigLabel}`,\n      );\n    }\n  }\n  if (timing && Math.abs(timing.deltaPercent) > 1) {\n    const dir = timing.delta > 0 ? 'slower' : 'faster';\n    parts.push(`${Math.abs(timing.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  if (cost && Math.abs(cost.deltaPercent) > 1) {\n    const dir = cost.delta > 0 ? 'more expensive' : 'cheaper';\n    parts.push(`${Math.abs(cost.deltaPercent).toFixed(0)}% ${dir}`);\n  }\n  const summaryStr = `candidate ${parts.length > 0 ? parts.join(', ') : 'no meaningful changes'} with ${regressions.length} regressions and ${improvements.length} improvements`;\n\n  // Run partial detection against the FULL pre-truncation pool. The user's\n  // original selection (e.g. 5 of an attempted 10 runs) determines partial\n  // status; the truncation we did above is a stats-consistency concern, not\n  // a partial-batch one.\n  const baselinePartial = detectPartial(baselineRunsRaw);\n  const candidatePartial = detectPartial(candidateRunsRaw);\n\n  return {\n    baseline: {\n      id: baselineRef.id,\n      metadata: baselineRef.metadata,\n      runCount,\n      ...(baselinePartial ? { partial: baselinePartial } : {}),\n    },\n    candidate: {\n      id: candidateRef.id,\n      metadata: candidateRef.metadata,\n      runCount,\n      ...(candidatePartial ? { partial: candidatePartial } : {}),\n    },\n    scorers,\n    timing,\n    cost,\n    regressions,\n    improvements,\n    summary: summaryStr,\n  };\n}\n\n/**\n * Detect whether one side of a comparison was pooled from a partial batch\n * (or a user-selected subset). Returns `{ completed, attempted }` when the\n * pooled run count is less than the original batch's planned run count\n * (read from `metadata.batchAttempted` on any run in the pool); `undefined`\n * otherwise.\n *\n * The signal is the same regardless of cause (batch failed mid-way vs user\n * cherry-picked from the picker) — both result in a smaller-N pool that\n * the consumer should mark visually so the comparison isn't mistaken for\n * an apples-to-apples run.\n *\n * Walks every run looking for the first finite `batchAttempted` rather than\n * trusting `runs[0]` alone — when a user cherry-picks runs from the compare\n * picker, the array order is the user's order, so the first item may be a\n * legacy single-run with no batch metadata while a sibling carries it.\n *\n * If `runs.length > attempted` (more results than were planned), treats the\n * data as suspicious and returns `undefined` — silently marking it \"complete\"\n * would mask data corruption.\n */\nfunction detectPartial(runs: EvalResult[]): { completed: number; attempted: number } | undefined {\n  if (runs.length === 0) return undefined;\n  let attempted: number | undefined;\n  for (const r of runs) {\n    const v = r.metadata?.batchAttempted;\n    if (typeof v === 'number' && Number.isFinite(v)) {\n      attempted = v;\n      break;\n    }\n  }\n  if (attempted === undefined) return undefined;\n  if (runs.length >= attempted) return undefined;\n  return { completed: runs.length, attempted };\n}\n\n/**\n * Decide whether a comparison should be REFUSED under a gate-side scorer\n * failure-rate limit (`axl-eval compare --max-scorer-error-rate`). Pure and\n * testable — returns a human-readable refusal reason, or `null` to allow.\n *\n * Type-aware via the baseline's `scorerTypes` metadata: a scorer EXPLICITLY\n * marked `'deterministic'` tolerates ZERO failures (a deterministic throw is a\n * bug); everything else — including an unknown/absent type on a pre-`scorerTypes`\n * artifact — uses the LLM `limit`. Defaulting unknown types to LLM (the\n * permissive tier) is deliberate: classifying an old artifact's scorers as\n * deterministic would hard-fail them on the first flake, contradicting the\n * `--max-scorer-error-rate <n>` the user passed. Scorer NAMES are identical\n * across sides (enforced by `evalCompare`), so reading types from the baseline\n * is sufficient.\n *\n * Also refuses a scorer that produced ZERO scored items on a side — a mean over\n * an empty sample can't be certified. Counts come from the same truncated pool\n * `evalCompare` used for the means/CI (`baseline/candidateScored/Failed`).\n */\nexport function evaluateScorerErrorRateGate(\n  comparison: EvalComparison,\n  maxScorerErrorRate: number,\n): string | null {\n  const scorerTypes = (comparison.baseline.metadata?.scorerTypes ?? {}) as Record<string, string>;\n  for (const name of Object.keys(comparison.scorers)) {\n    const s = comparison.scorers[name];\n    const type = scorerTypes[name] === 'deterministic' ? 'deterministic' : 'llm';\n    const limit = type === 'deterministic' ? 0 : maxScorerErrorRate;\n    for (const side of ['baseline', 'candidate'] as const) {\n      const scored = (side === 'baseline' ? s.baselineScored : s.candidateScored) ?? 0;\n      const failed = (side === 'baseline' ? s.baselineFailed : s.candidateFailed) ?? 0;\n      const skipped = (side === 'baseline' ? s.baselineSkipped : s.candidateSkipped) ?? 0;\n      const verdict = evaluateScorerTolerance(scored, failed, type, limit);\n      if (verdict.zeroSample) {\n        // attempted === 0 means the scorer produced no valid scores AND recorded\n        // no ran-and-failed attempts on this side, so a mean over an empty sample\n        // can't be certified. Distinguish the likely cause: a fully `applies`-\n        // skipped scorer (N/A on every item — a scope/config issue, not a fault)\n        // vs. items that errored in the workflow.\n        const cause =\n          skipped > 0\n            ? `every one of its items was N/A (the \\`applies\\` predicate skipped all ${skipped})`\n            : `its items errored or produced no score`;\n        return `scorer \"${name}\" has no scored items on ${side} (${cause}) — cannot certify a mean over an empty sample.`;\n      }\n      if (verdict.exceeds) {\n        const limitStr =\n          type === 'deterministic'\n            ? 'deterministic zero-tolerance'\n            : `${(limit * 100).toFixed(1)}%`;\n        return `scorer \"${name}\" failed ${(verdict.rate * 100).toFixed(1)}% on ${side} (${failed}/${verdict.attempted}), over the ${limitStr} limit.`;\n      }\n    }\n  }\n  return null;\n}\n","import type { Provider } from '@axlsdk/axl';\n\n/** Result from a scorer that includes metadata beyond the numeric score. */\nexport type ScorerResult = {\n  score: number;\n  metadata?: Record<string, unknown>;\n  cost?: number;\n};\n\n/** Normalize a scorer return value to a ScorerResult. */\nexport function normalizeScorerResult(result: number | ScorerResult): ScorerResult {\n  return typeof result === 'number' ? { score: result } : result;\n}\n\n/**\n * Cost contract for thrown scorer errors. A scorer (e.g. `llmScorer`) that\n * incurs provider cost *before* failing attaches that cost to the thrown error\n * via {@link attachScorerErrorCost} so the eval runner still bills it. The\n * cost rides as an enumerable `cost` property (a small, stable public contract\n * custom scorers can also use). These helpers replace scattered `(err as any).cost`\n * reads/writes with one typed, finite-guarded surface.\n */\nexport function attachScorerErrorCost(err: unknown, cost: number | undefined): void {\n  if (\n    err &&\n    typeof err === 'object' &&\n    typeof cost === 'number' &&\n    Number.isFinite(cost) &&\n    cost >= 0\n  ) {\n    (err as { cost?: number }).cost = cost;\n  }\n}\n\n/** Read a non-negative finite cost off a thrown scorer error, or `undefined`. */\nexport function extractScorerErrorCost(err: unknown): number | undefined {\n  const v = err && typeof err === 'object' ? (err as { cost?: unknown }).cost : undefined;\n  return typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : undefined;\n}\n\n/** Context passed to scorers by the eval runner. */\nexport type ScorerContext = {\n  /** Resolve a provider:model URI to a provider instance and model name. */\n  resolveProvider: (modelUri: string) => { provider: Provider; model: string };\n  /** Abort signal forwarded from the eval runner; scorers should pass it into provider calls. */\n  signal?: AbortSignal;\n};\n\nexport type ScorerFn<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => number | ScorerResult;\n\n/**\n * Applicability predicate for a conditional scorer. Returns `true` if this\n * scorer should run for the given item, `false` to skip it.\n *\n * The eval runner runs every scorer against every item, so a scorer that only\n * applies to a subset (a refusal judge for refusal-expected items, a constraint\n * judge for constrained items, …) declares its scope here. A skipped item is\n * counted as NEITHER `scored` NOR `failed` — it's excluded from the mean AND\n * from the failure-rate denominator, so a conditional scorer stays honest to\n * both. The predicate runs BEFORE the scorer body, so an inapplicable\n * `llmScorer` never makes its provider call.\n *\n * Signature mirrors {@link ScorerFn} (`output, input, annotations`). Applicability\n * usually keys off `input`/`annotations` (e.g. a flag or exercise type), but\n * `output` is available too (e.g. \"only score outputs that refused\"). A\n * predicate that THROWS is a bug, not a skip — it's recorded as a scorer\n * failure, never silently swallowed.\n *\n * The contract is `=> boolean`: return `true` to run, `false` to skip. The\n * runtime is defensively lenient — it coerces with `!verdict`, so a stray falsy\n * value (a predicate that forgot to `return`) skips rather than crashes — but\n * authors should return a real boolean. When deriving from a truthy value,\n * coerce it yourself: `(o, i, a) => !!a.constraints?.length`, not\n * `=> a.constraints?.length` (which is `number | undefined`, a type error). A\n * predicate that accidentally never returns therefore skips EVERY item — but\n * that surfaces conspicuously as a non-zero `skipped` count (the Studio \"N/A\"\n * chip), not as a silently green mean.\n */\nexport type ScorerApplies<TOutput, TInput, TAnnotations> = (\n  output: TOutput,\n  input: TInput,\n  annotations?: TAnnotations,\n) => boolean;\n\nexport type ScorerConfig<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  name: string;\n  description: string;\n  score: ScorerFn<TOutput, TInput, TAnnotations>;\n  /** Optional applicability predicate — see {@link ScorerApplies}. When it\n   *  returns `false`, the scorer is skipped for that item (counted as neither\n   *  scored nor failed). Omit to run on every item (the default).\n   *\n   *  Declared with **method syntax** (not a function-valued property) for parity\n   *  with {@link Scorer.applies}, where the syntax is load-bearing: see the note\n   *  there and `__tests__/scorer-assignability.test-d.ts`. */\n  applies?(output: TOutput, input: TInput, annotations?: TAnnotations): boolean;\n};\n\nexport type Scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown> = {\n  readonly name: string;\n  readonly description: string;\n  readonly isLlm: boolean;\n  score(\n    output: TOutput,\n    input: TInput,\n    annotations?: TAnnotations,\n    context?: ScorerContext,\n  ): number | ScorerResult | Promise<number | ScorerResult>;\n  /** See {@link ScorerApplies}. Evaluated by the runner before {@link score}.\n   *\n   *  Declared with **method syntax** (not a function-valued `ScorerApplies`\n   *  property) so its parameters are checked bivariantly, exactly like\n   *  {@link score}. `EvalConfig.scorers` erases the generics to\n   *  `Scorer<unknown, unknown, unknown>[]`; under `strictFunctionTypes` a\n   *  property would make those params contravariant, so a concretely-typed\n   *  `Scorer<…, TInput, TAnnotations>` would no longer be assignable to that\n   *  array (the 0.18.1 regression). `__tests__/scorer-assignability.test-d.ts`\n   *  locks this in and is compiled by the `typecheck` gate. */\n  applies?(output: TOutput, input: TInput, annotations?: TAnnotations): boolean;\n};\n\nexport function scorer<TOutput = unknown, TInput = unknown, TAnnotations = unknown>(\n  config: ScorerConfig<TOutput, TInput, TAnnotations>,\n): Scorer<TOutput, TInput, TAnnotations> {\n  return {\n    name: config.name,\n    description: config.description,\n    isLlm: false,\n    score: config.score,\n    applies: config.applies,\n  };\n}\n","import type { Scorer, ScorerContext } from './scorer.js';\nimport { normalizeScorerResult, extractScorerErrorCost } from './scorer.js';\nimport type { EvalItem } from './types.js';\nimport { round, mapWithConcurrency } from './utils.js';\n\n/** The subset of an `EvalItem` that `scoreItem` reads and mutates. */\ntype ScorableItem = Pick<\n  EvalItem,\n  'input' | 'output' | 'annotations' | 'scores' | 'scoreDetails' | 'scorerErrors' | 'scorerCost'\n>;\n\n/**\n * Score one item's `output` with every scorer in `scorers`, running up to\n * `scorerConcurrency` of them at once. Mutates `item.scores`,\n * `item.scoreDetails`, `item.scorerErrors`, and `item.scorerCost` in place, and\n * returns the per-item scorer cost so the caller can fold it into its own\n * running total (`totalCost += await scoreItem(...)`).\n *\n * This is the single source of truth for the scoring inner loop, shared by\n * `runEval` and `rescore` — both previously carried a byte-identical copy, so a\n * determinism or cancellation fix had to be made twice.\n *\n * Determinism (independent of scorer completion order):\n *  - `scores`/`scoreDetails` keys are pre-seeded in `scorers` order, so JSON key\n *    order is stable and a scorer skipped by cancellation is deterministically\n *    `null` rather than absent.\n *  - `scorerErrors` are collected name-keyed and flattened in `scorers` order.\n *\n * Cancellation: a scorer not yet started when `signal` aborts is skipped; an\n * in-flight scorer whose call rejects with an `AbortError` is treated as\n * cancellation (leaves its pre-seeded `null`, records no error) — narrowed to\n * the error identity so a genuine bug in another scorer that happens to throw\n * while a sibling triggered the abort is still reported, not swallowed.\n */\nexport async function scoreItem(\n  item: ScorableItem,\n  scorers: readonly Scorer<unknown, unknown, unknown>[],\n  scorerConcurrency: number,\n  scorerContext: ScorerContext,\n  signal?: AbortSignal,\n): Promise<number> {\n  const scoreDetails = (item.scoreDetails ??= {});\n  for (const s of scorers) {\n    item.scores[s.name] = null;\n    scoreDetails[s.name] = { score: null };\n  }\n\n  let itemScorerCost = 0;\n  const scorerErrorsByName: Record<string, string> = {};\n\n  await mapWithConcurrency(scorers, scorerConcurrency, async (scorer) => {\n    if (signal?.aborted) return;\n    const scorerStart = Date.now();\n    try {\n      // Applicability gate: a conditional scorer declares which items it applies\n      // to via `applies`. A `false` verdict skips it entirely — no provider call\n      // for an llmScorer — and records it as NEITHER scored nor failed via the\n      // positive `skipped` marker (no `duration`, so the duration heuristic in\n      // `scorerCounts` can't misread it as a failure). A predicate that THROWS is\n      // a bug, not a skip: it falls through to the catch below and is recorded as\n      // a scorer failure (with a duration), never silently swallowed.\n      if (scorer.applies && !scorer.applies(item.output, item.input, item.annotations)) {\n        item.scores[scorer.name] = null;\n        scoreDetails[scorer.name] = { score: null, skipped: true };\n        return;\n      }\n\n      const raw = await scorer.score(item.output, item.input, item.annotations, scorerContext);\n      const scorerResult = normalizeScorerResult(raw);\n\n      if (scorerResult.cost != null) {\n        itemScorerCost += scorerResult.cost;\n      }\n\n      const scorerDuration = Date.now() - scorerStart;\n\n      if (\n        !Number.isFinite(scorerResult.score) ||\n        scorerResult.score < 0 ||\n        scorerResult.score > 1\n      ) {\n        scorerErrorsByName[scorer.name] =\n          `Scorer \"${scorer.name}\" returned out-of-range score ${scorerResult.score} for input ${JSON.stringify(item.input)}`;\n        item.scores[scorer.name] = null;\n        scoreDetails[scorer.name] = {\n          score: null,\n          metadata: scorerResult.metadata,\n          duration: scorerDuration,\n          cost: scorerResult.cost,\n        };\n      } else {\n        item.scores[scorer.name] = round(scorerResult.score);\n        scoreDetails[scorer.name] = {\n          score: round(scorerResult.score),\n          metadata: scorerResult.metadata,\n          duration: scorerDuration,\n          cost: scorerResult.cost,\n        };\n      }\n    } catch (err) {\n      // A cancelled call (AbortError) is not a scoring failure — leave the\n      // pre-seeded null and record nothing.\n      if ((err as { name?: string })?.name === 'AbortError') return;\n      const errCost = extractScorerErrorCost(err);\n      if (errCost != null) itemScorerCost += errCost;\n      scorerErrorsByName[scorer.name] =\n        `Scorer \"${scorer.name}\" threw: ${err instanceof Error ? err.message : String(err)}`;\n      item.scores[scorer.name] = null;\n      scoreDetails[scorer.name] = {\n        score: null,\n        duration: Date.now() - scorerStart,\n        cost: errCost,\n      };\n    }\n  });\n\n  const orderedErrors = scorers\n    .map((s) => scorerErrorsByName[s.name])\n    .filter((e): e is string => e != null);\n  if (orderedErrors.length > 0) item.scorerErrors = orderedErrors;\n  item.scorerCost = itemScorerCost > 0 ? itemScorerCost : undefined;\n  return itemScorerCost;\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalConfig, EvalResult, EvalItem, EvalSummary, RunEvalOptions } from './types.js';\nimport type { ScorerContext } from './scorer.js';\nimport type { DegradedScorer } from './types.js';\nimport {\n  computeStats,\n  mapWithConcurrency,\n  scorerCounts,\n  evaluateScorerTolerance,\n} from './utils.js';\nimport { scoreItem } from './score-item.js';\nimport { randomUUID } from 'node:crypto';\n\nfunction parseCost(cost: string): number {\n  const match = cost.match(/^\\$?([\\d.]+)$/);\n  if (!match) throw new Error(`Invalid cost format: \"${cost}\"`);\n  return parseFloat(match[1]);\n}\n\n/**\n * Extract a user-returned cost only if it's a non-negative finite number.\n * Guards against workflows that return `{ cost: 'free' }`, `{ cost: NaN }`,\n * `{ cost: -1 }`, `{ cost: Infinity }`, etc. — the TS type says `cost?: number`\n * but at runtime we can't trust that.\n *\n * Negative costs are rejected because (a) cost is a USD amount and negative\n * values are nonsensical, and (b) a negative cost would silently shrink\n * `totalCost` below the budget limit check at the processItem level,\n * letting a buggy/malicious workflow run unbounded. `0` is preserved\n * because free operations on paid models are valid.\n *\n * When a user-supplied value is rejected (present but invalid), we log\n * once per item via `console.warn` so the type violation is visible\n * rather than silently swallowed.\n */\nfunction extractUserCost(result: unknown, label = 'executeWorkflow'): number | undefined {\n  if (result === null || typeof result !== 'object') return undefined;\n  const raw = (result as { cost?: unknown }).cost;\n  if (raw === undefined) return undefined;\n  if (typeof raw === 'number' && Number.isFinite(raw) && raw >= 0) return raw;\n\n  console.warn(\n    `[axl-eval] Ignoring invalid \\`cost\\` from ${label} return: expected non-negative finite number, got ${typeof raw === 'number' ? String(raw) : typeof raw}. Falling back to tracked cost (or undefined).`,\n  );\n  return undefined;\n}\n\n/**\n * Extract a user-returned metadata record only if it's a plain object.\n * Rejects arrays, null, scalars, and exotic object types (Date, Map, Set,\n * Error, class instances) that would satisfy a loose `typeof === 'object'`\n * check but break `Record<string, unknown>` assumptions in downstream\n * consumers (spread, Object.entries, property access).\n */\nfunction extractUserMetadata(\n  result: unknown,\n  label = 'executeWorkflow',\n): Record<string, unknown> | undefined {\n  if (result === null || typeof result !== 'object') return undefined;\n  const meta = (result as { metadata?: unknown }).metadata;\n  if (meta === undefined) return undefined;\n  if (meta === null || typeof meta !== 'object' || Array.isArray(meta)) {\n    console.warn(\n      `[axl-eval] Ignoring invalid \\`metadata\\` from ${label} return: expected plain object, got ${meta === null ? 'null' : Array.isArray(meta) ? 'array' : typeof meta}.`,\n    );\n    return undefined;\n  }\n  // Reject exotic objects (Date, Map, Set, Error, class instances) whose\n  // prototype chain differs from Object.prototype. Those pass `typeof ===\n  // 'object'` but don't behave like `Record<string, unknown>`.\n  const proto = Object.getPrototypeOf(meta);\n  if (proto !== Object.prototype && proto !== null) {\n    console.warn(\n      `[axl-eval] Ignoring invalid \\`metadata\\` from ${label} return: expected plain object, got ${(proto?.constructor?.name as string | undefined) ?? 'exotic object'}.`,\n    );\n    return undefined;\n  }\n  return meta as Record<string, unknown>;\n}\n\nexport async function runEval(\n  config: EvalConfig,\n  executeWorkflow: (\n    input: unknown,\n    runtime: AxlRuntime,\n  ) => Promise<{ output: unknown; cost?: number; metadata?: Record<string, unknown> }>,\n  runtime: AxlRuntime,\n  options?: RunEvalOptions,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const id = randomUUID();\n  const items = await config.dataset.getItems();\n  // Snapshot dataset-load diagnostics (e.g. annotation keys the schema dropped)\n  // so we can surface them on EvalResult.metadata for any consumer — mirrors the\n  // console.warn the dataset already emits. Read synchronously after getItems()\n  // (no await between); defensive against hand-rolled Dataset objects that don't\n  // expose the field.\n  const droppedAnnotationKeys = [...(config.dataset.droppedAnnotationKeys ?? [])];\n  const concurrency = config.concurrency ?? 5;\n  // Per-item scorer fan-out. Defaults to the same value as item `concurrency`\n  // (parallel-by-default), so the worst-case concurrent judge calls is\n  // `concurrency × scorerConcurrency`. The provider layer backs off on\n  // 429/503/529; users who need a tighter ceiling lower item `concurrency`.\n  const scorerConcurrency = config.scorerConcurrency ?? 5;\n  const budgetLimit = config.budget ? parseCost(config.budget) : undefined;\n\n  // Create a scorer context that LLM scorers use to resolve providers.\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n    signal: options?.signal,\n  };\n\n  const evalItems: EvalItem[] = new Array(items.length);\n  let totalCost = 0;\n  let budgetExceeded = false;\n\n  async function processItem(item: (typeof items)[0], itemIndex: number): Promise<void> {\n    if (options?.signal?.aborted) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Cancelled',\n        scores: {},\n      };\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    if (budgetExceeded) {\n      evalItems[itemIndex] = {\n        input: item.input,\n        annotations: item.annotations,\n        output: null,\n        error: 'Budget exceeded',\n        scores: {},\n      };\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    const evalItem: EvalItem = {\n      input: item.input,\n      annotations: item.annotations,\n      output: null,\n      scores: {},\n    };\n    const itemStart = Date.now();\n    try {\n      // When captureTraces is on, wrap the user's executeWorkflow in\n      // trackExecution so we get a per-item TraceEvent[] scoped via\n      // AsyncLocalStorage. trackExecution's scope walks the parent chain, so\n      // any nested trackExecution (e.g. from the CLI) continues to work.\n      if (options?.captureTraces) {\n        const tracked = await runtime.trackExecution(\n          async () => executeWorkflow(item.input, runtime),\n          { captureTraces: true },\n        );\n        evalItem.duration = Date.now() - itemStart;\n        evalItem.output = tracked.result.output;\n        // Prefer user-returned cost/metadata; fall back to tracked values.\n        // Type-guarded so a non-number cost (e.g. `{ cost: 'free' }`) or a\n        // non-object metadata (e.g. an array or scalar) doesn't silently\n        // corrupt downstream math or shape expectations.\n        evalItem.cost = extractUserCost(tracked.result) ?? tracked.cost;\n        if (tracked.unpriced) evalItem.unpriced = true;\n        evalItem.metadata = extractUserMetadata(tracked.result) ?? tracked.metadata;\n        if (tracked.traces && tracked.traces.length > 0) {\n          evalItem.traces = tracked.traces;\n        }\n        if (evalItem.cost != null) {\n          totalCost += evalItem.cost;\n        }\n      } else {\n        const result = await executeWorkflow(item.input, runtime);\n        evalItem.duration = Date.now() - itemStart;\n        evalItem.output = result.output;\n        evalItem.cost = extractUserCost(result);\n        const meta = extractUserMetadata(result);\n        if (meta) evalItem.metadata = meta;\n        if (evalItem.cost != null) {\n          totalCost += evalItem.cost;\n        }\n      }\n    } catch (err) {\n      evalItem.duration = Date.now() - itemStart;\n      evalItem.error = err instanceof Error ? err.message : String(err);\n      // Failed items are exactly when per-item traces are most valuable —\n      // recover them from the captured-traces side-channel `trackExecution`\n      // attaches to the error on the failure path.\n      if (options?.captureTraces) {\n        const captured = (err as { axlCapturedTraces?: unknown }).axlCapturedTraces;\n        if (Array.isArray(captured) && captured.length > 0) {\n          evalItem.traces = captured as EvalItem['traces'];\n        }\n      }\n      evalItems[itemIndex] = evalItem;\n      options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n      return;\n    }\n\n    if (budgetLimit != null && totalCost > budgetLimit) {\n      budgetExceeded = true;\n    }\n\n    // Score the item's output with all scorers (bounded fan-out). scoreItem\n    // owns the determinism + cancellation + cost logic shared with rescore();\n    // it returns the per-item scorer cost to fold into the running total.\n    // NB: keep the await on its own line — `totalCost += await …` would read\n    // totalCost BEFORE suspending, losing updates across concurrent items.\n    const itemScorerCost = await scoreItem(\n      evalItem,\n      config.scorers,\n      scorerConcurrency,\n      scorerContext,\n      options?.signal,\n    );\n    totalCost += itemScorerCost;\n    evalItems[itemIndex] = evalItem;\n    options?.onProgress?.({ type: 'item_done', itemIndex, totalItems: items.length });\n  }\n\n  // processItem writes into the pre-allocated `evalItems` closure array (the\n  // source of truth) and returns void — the pool's returned array is ignored.\n  await mapWithConcurrency(items, concurrency, (item, i) => processItem(item, i));\n\n  const failures = evalItems.filter((i) => i.error).length;\n  const scorerNames = config.scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = evalItems\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    const { scored, failed, skipped } = scorerCounts(evalItems, name);\n    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };\n  }\n\n  const scorerTypes: Record<string, string> = {};\n  for (const s of config.scorers) {\n    scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n  }\n\n  // Failure-rate trust signal (opt-in). When `failOnScorerErrorRate` is set, a\n  // scorer is degraded when it's deterministic and failed at all (a failure is\n  // a bug, not noise), OR it's an LLM judge whose failure rate over its\n  // ATTEMPTED items exceeds tolerance. We never throw — the result is still\n  // useful; we flag it and let the CLI/consumer decide the exit code. Invalid\n  // limits are ignored loudly so a typo can't silently disable the gate.\n  let degraded: DegradedScorer[] | undefined;\n  const rawLimit = config.failOnScorerErrorRate;\n  if (rawLimit != null) {\n    if (!Number.isFinite(rawLimit) || rawLimit < 0 || rawLimit > 1) {\n      console.warn(\n        `[axl-eval] Ignoring invalid failOnScorerErrorRate (${rawLimit}); expected a number in [0, 1].`,\n      );\n    } else {\n      for (const name of scorerNames) {\n        const stats = scorerStats[name];\n        const scored = stats.scored ?? 0;\n        const failed = stats.failed ?? 0;\n        const type = scorerTypes[name] === 'llm' ? 'llm' : 'deterministic';\n        const limit = type === 'deterministic' ? 0 : rawLimit;\n        const verdict = evaluateScorerTolerance(scored, failed, type, limit);\n        // A scorer that never ran (zeroSample) isn't degraded at the source — the\n        // run produced no basis to judge it. (The compare gate treats zero-sample\n        // differently: it refuses to certify it.)\n        if (verdict.exceeds) {\n          (degraded ??= []).push({ scorer: name, rate: verdict.rate, limit, type, scored, failed });\n        }\n      }\n    }\n  }\n\n  const durations = evalItems.filter((i) => !i.error && i.duration != null).map((i) => i.duration!);\n  const timing = durations.length > 0 ? computeStats(durations) : undefined;\n\n  // Aggregate per-model LLM call counts across all items\n  const totalModelCalls = new Map<string, number>();\n  for (const item of evalItems) {\n    const itemCounts = item.metadata?.modelCallCounts;\n    if (itemCounts && typeof itemCounts === 'object') {\n      for (const [m, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n        if (typeof count === 'number')\n          totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + count);\n      }\n    } else {\n      // Fallback: count unique models per item (for executeWorkflow that doesn't provide call counts)\n      const itemModels = item.metadata?.models;\n      if (Array.isArray(itemModels)) {\n        for (const m of itemModels) {\n          if (typeof m === 'string') totalModelCalls.set(m, (totalModelCalls.get(m) ?? 0) + 1);\n        }\n      }\n    }\n  }\n\n  // models: unique list sorted by total calls (most-called first)\n  // modelCounts: total LLM calls per model (e.g., { \"openai:gpt-4o\": 12, \"openai:gpt-4o-mini\": 12 })\n  const modelsMeta: Record<string, unknown> = {};\n  if (totalModelCalls.size > 0) {\n    const sorted = [...totalModelCalls.entries()].sort((a, b) => b[1] - a[1]);\n    modelsMeta.models = sorted.map(([m]) => m);\n    modelsMeta.modelCounts = Object.fromEntries(sorted);\n  }\n\n  // Aggregate per-workflow call counts across all items (parallel to models).\n  // Workflows come from trackExecution's trace-event collection — callers\n  // don't specify workflow names anywhere, they just appear because the\n  // runtime emits workflow_start events for every execute() call.\n  const totalWorkflowCalls = new Map<string, number>();\n  for (const item of evalItems) {\n    const itemCounts = item.metadata?.workflowCallCounts;\n    if (itemCounts && typeof itemCounts === 'object') {\n      for (const [w, count] of Object.entries(itemCounts as Record<string, unknown>)) {\n        if (typeof count === 'number')\n          totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + count);\n      }\n    } else {\n      // Fallback: count unique workflows per item\n      const itemWorkflows = item.metadata?.workflows;\n      if (Array.isArray(itemWorkflows)) {\n        for (const w of itemWorkflows) {\n          if (typeof w === 'string')\n            totalWorkflowCalls.set(w, (totalWorkflowCalls.get(w) ?? 0) + 1);\n        }\n      }\n    }\n  }\n\n  // Fall back to config.workflow when the callback bypassed the runtime's\n  // execute() path entirely (e.g. AxlTestRuntime-based tests). This keeps the\n  // metadata.workflows array non-empty for the common test-harness case.\n  const workflowsMeta: Record<string, unknown> = {};\n  if (totalWorkflowCalls.size > 0) {\n    const sorted = [...totalWorkflowCalls.entries()].sort((a, b) => b[1] - a[1]);\n    workflowsMeta.workflows = sorted.map(([w]) => w);\n    workflowsMeta.workflowCounts = Object.fromEntries(sorted);\n  } else if (config.workflow) {\n    workflowsMeta.workflows = [config.workflow];\n    workflowsMeta.workflowCounts = { [config.workflow]: items.length };\n  }\n\n  options?.onProgress?.({ type: 'run_done', totalItems: items.length, failures });\n\n  return {\n    id,\n    dataset: config.dataset.name,\n    metadata: {\n      ...config.metadata,\n      scorerTypes,\n      ...modelsMeta,\n      ...workflowsMeta,\n      ...(droppedAnnotationKeys.length > 0 ? { droppedAnnotationKeys } : {}),\n    },\n    timestamp: new Date().toISOString(),\n    totalCost,\n    ...(evalItems.some((item) => item.unpriced) ? { unpriced: true } : {}),\n    duration: Date.now() - startTime,\n    items: evalItems,\n    summary: {\n      count: items.length,\n      failures,\n      scorers: scorerStats,\n      timing,\n      ...(degraded ? { degraded } : {}),\n    },\n  };\n}\n","import type { AxlRuntime } from '@axlsdk/axl';\nimport type { EvalResult, EvalItem, EvalSummary } from './types.js';\nimport type { Scorer, ScorerContext } from './scorer.js';\nimport { computeStats, mapWithConcurrency, scorerCounts } from './utils.js';\nimport { scoreItem } from './score-item.js';\nimport { randomUUID } from 'node:crypto';\n\nexport type RescoreOptions = {\n  /** Item-level worker-pool size (how many saved items rescore in parallel). Default 5. */\n  concurrency?: number;\n  /** Per-item scorer fan-out (how many scorers run concurrently within one item).\n   *  Default 5 — matches `EvalConfig.scorerConcurrency`. Worst-case concurrent\n   *  judge calls is `concurrency × scorerConcurrency`. */\n  scorerConcurrency?: number;\n  /** Abort signal forwarded into ScorerContext so in-flight LLM scorer calls can\n   *  be cancelled mid-flight. Also checked between items to short-circuit\n   *  remaining work. Mirrors `RunEvalOptions.signal`. */\n  signal?: AbortSignal;\n};\n\n/**\n * Re-run scorers on the saved outputs of an existing eval result.\n * Preserves original input/output/annotations. Only re-runs scoring.\n */\nexport async function rescore(\n  result: EvalResult,\n  scorers: Scorer[],\n  runtime: AxlRuntime,\n  options?: RescoreOptions,\n): Promise<EvalResult> {\n  const startTime = Date.now();\n  const concurrency = options?.concurrency ?? 5;\n  const scorerConcurrency = options?.scorerConcurrency ?? 5;\n\n  const scorerContext: ScorerContext = {\n    resolveProvider: (uri: string) => {\n      if (typeof runtime.resolveProvider !== 'function') {\n        throw new Error(\n          `LLM scorers require a runtime with resolveProvider(). ` +\n            `Ensure you are using a real AxlRuntime instance, not a mock.`,\n        );\n      }\n      return runtime.resolveProvider(uri);\n    },\n    signal: options?.signal,\n  };\n\n  const rescored: EvalItem[] = new Array(result.items.length);\n  let totalCost = 0;\n\n  async function rescoreItem(original: EvalItem, itemIndex: number): Promise<void> {\n    // Short-circuit if the rescore has been cancelled — matches runEval's\n    // between-items signal check (runner.ts) so cancellation behaves the same\n    // in both paths. In-flight LLM scorer calls additionally abort via\n    // scorerContext.signal → provider.chat({ signal }).\n    if (options?.signal?.aborted) {\n      rescored[itemIndex] = {\n        input: original.input,\n        annotations: original.annotations,\n        output: original.output,\n        error: 'cancelled',\n        scores: {},\n      };\n      return;\n    }\n\n    // Pass through error items without scoring\n    if (original.error) {\n      rescored[itemIndex] = {\n        input: original.input,\n        annotations: original.annotations,\n        output: original.output,\n        error: original.error,\n        scores: {},\n      };\n      return;\n    }\n\n    // Preserve per-item traces from the original run: rescore re-runs scorers\n    // but leaves the workflow untouched, so the original execution traces\n    // remain accurate and useful for diagnosing score changes.\n    const item: EvalItem = {\n      input: original.input,\n      annotations: original.annotations,\n      output: original.output,\n      metadata: original.metadata,\n      traces: original.traces,\n      scores: {},\n      scoreDetails: {},\n    };\n\n    // Same shared scoring path as runEval — determinism, cancellation, and cost\n    // accounting all live in scoreItem. Keep the await off the `+=` line so the\n    // read-modify-write isn't split by suspension across concurrent items.\n    const itemScorerCost = await scoreItem(\n      item,\n      scorers,\n      scorerConcurrency,\n      scorerContext,\n      options?.signal,\n    );\n    totalCost += itemScorerCost;\n    rescored[itemIndex] = item;\n  }\n\n  // rescoreItem writes into the pre-allocated `rescored` closure array (the\n  // source of truth) and returns void — the pool's returned array is ignored.\n  await mapWithConcurrency(result.items, concurrency, (item, i) => rescoreItem(item, i));\n\n  const failures = rescored.filter((i) => i.error).length;\n  const scorerNames = scorers.map((s) => s.name);\n  const scorerStats: EvalSummary['scorers'] = {};\n  for (const name of scorerNames) {\n    const scores = rescored\n      .filter((i) => !i.error && i.scores[name] != null)\n      .map((i) => i.scores[name] as number);\n    // Same scored/failed surfacing as runEval (table + Studio parity). NOTE:\n    // rescore deliberately does NOT support `failOnScorerErrorRate` — it takes\n    // RescoreOptions, not EvalConfig, so there's no degradation gate here. The\n    // counts are informational; gating belongs to the run that produced output.\n    const { scored, failed, skipped } = scorerCounts(rescored, name);\n    scorerStats[name] = { ...computeStats(scores), scored, failed, skipped };\n  }\n\n  const scorerTypes: Record<string, string> = {};\n  for (const s of scorers) {\n    scorerTypes[s.name] = s.isLlm ? 'llm' : 'deterministic';\n  }\n\n  return {\n    id: randomUUID(),\n    dataset: result.dataset,\n    metadata: (() => {\n      // Strip run group membership — rescored results are independent evaluations.\n      // metadata.workflows is preserved via ...rest so the rescored result keeps\n      // the same workflow attribution as the original.\n      //\n      // Also strip the scorer-filtered stamp: whether THIS rescore ran a subset\n      // is a property of the rescore invocation, not the source run. A full\n      // rescore of a `--scorers`-filtered run would otherwise inherit a stale\n      // `scorerFiltered: true` + a `scorersRun` listing scorers it didn't even\n      // use — falsely tripping the compare gate and the Studio banner.\n      const rest: Record<string, unknown> = { ...result.metadata };\n      delete rest.runGroupId;\n      delete rest.runIndex;\n      delete rest.scorerFiltered;\n      delete rest.scorersRun;\n      const merged: Record<string, unknown> = {\n        ...rest,\n        rescored: true,\n        originalId: result.id,\n        scorerTypes,\n      };\n      // Backward compatibility: pre-0.14 EvalResult artifacts had `workflow`\n      // as a top-level string field with no `metadata.workflows`. Migrate it\n      // forward so rescored results from old artifacts retain their workflow\n      // attribution under the modern shape.\n      if (!Array.isArray(merged.workflows)) {\n        const legacyWorkflow = (result as { workflow?: unknown }).workflow;\n        if (typeof legacyWorkflow === 'string' && legacyWorkflow) {\n          merged.workflows = [legacyWorkflow];\n          merged.workflowCounts = { [legacyWorkflow]: result.items.length };\n        }\n      }\n      return merged;\n    })(),\n    timestamp: new Date().toISOString(),\n    totalCost,\n    duration: Date.now() - startTime,\n    items: rescored,\n    summary: { count: result.items.length, failures, scorers: scorerStats },\n  };\n}\n","import type { EvalResult } from './types.js';\nimport { randomUUID } from 'node:crypto';\nimport { round } from './utils.js';\n\nexport type MultiRunSummary = {\n  runGroupId: string;\n  runCount: number;\n  /**\n   * Workflow names observed across all runs in the group, deduped and ordered\n   * by first appearance. Parallel to `EvalResult.metadata.workflows`. A homogeneous\n   * multi-run group produced by `axl-eval --runs N` typically has one entry;\n   * heterogeneous groups (custom callbacks) can have multiple.\n   */\n  workflows: string[];\n  dataset: string;\n  totalCost: number;\n  totalDuration: number;\n  scorers: Record<\n    string,\n    {\n      mean: number;\n      std: number;\n      min: number;\n      max: number;\n      /** Total valid scores summed across all runs in the group. */\n      scored?: number;\n      /** Total ran-and-failed scorer invocations summed across all runs. */\n      failed?: number;\n      /** Total `applies`-skipped (N/A) items summed across all runs. Excluded\n       *  from both the mean and the failure-rate denominator. */\n      skipped?: number;\n    }\n  >;\n  timing?: { mean: number; std: number };\n};\n\nfunction std(values: number[], mean: number): number {\n  if (values.length <= 1) return 0;\n  const sumSqDiffs = values.reduce((sum, v) => sum + (v - mean) ** 2, 0);\n  return Math.sqrt(sumSqDiffs / (values.length - 1));\n}\n\n/**\n * Aggregate multiple eval runs into a summary with mean +/- std per scorer.\n */\nexport function aggregateRuns(runs: EvalResult[]): MultiRunSummary {\n  if (runs.length === 0) throw new Error('Cannot aggregate zero runs');\n\n  const runGroupId = (runs[0].metadata.runGroupId as string) ?? randomUUID();\n  const { dataset } = runs[0];\n\n  // Union workflow names across all runs in the group, first-seen first.\n  // Most groups have one, but custom callbacks can produce heterogeneous ones.\n  const seenWorkflows = new Set<string>();\n  const workflows: string[] = [];\n  for (const run of runs) {\n    const list = run.metadata.workflows;\n    if (Array.isArray(list)) {\n      for (const w of list) {\n        if (typeof w === 'string' && !seenWorkflows.has(w)) {\n          seenWorkflows.add(w);\n          workflows.push(w);\n        }\n      }\n    }\n  }\n\n  const totalCost = runs.reduce((sum, r) => sum + r.totalCost, 0);\n  const totalDuration = runs.reduce((sum, r) => sum + r.duration, 0);\n\n  // Collect per-scorer means from each run\n  const scorerNames = Object.keys(runs[0].summary.scorers);\n  const scorers: MultiRunSummary['scorers'] = {};\n\n  for (const name of scorerNames) {\n    // A run where this scorer scored ZERO items has an empty-sample mean of 0\n    // (computeStats([]) → 0), which would drag the aggregate mean-of-means toward\n    // 0 as if it were a real 0.0 score. Exclude such runs from the mean/min/max/std\n    // — but only when we can tell: `scored` is absent on pre-0.18.0 artifacts, so\n    // those runs are kept (old behavior). If EVERY run scored nothing, fall back to\n    // all runs so we never divide by zero.\n    const contributing = runs.filter((r) => {\n      const sc = r.summary.scorers[name]?.scored;\n      return sc == null || sc > 0;\n    });\n    const meanSource = contributing.length > 0 ? contributing : runs;\n    const means = meanSource.map((r) => r.summary.scorers[name]?.mean ?? 0);\n    const meanOfMeans = means.reduce((a, b) => a + b, 0) / means.length;\n    const minMean = Math.min(...means);\n    const maxMean = Math.max(...means);\n\n    // Sum the per-run sample sizes across ALL runs (they describe the whole\n    // group's thinning, independent of which runs contributed to the mean).\n    // Read with `?? 0` — old artifacts predate these fields.\n    const scored = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.scored ?? 0), 0);\n    const failed = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.failed ?? 0), 0);\n    const skipped = runs.reduce((sum, r) => sum + (r.summary.scorers[name]?.skipped ?? 0), 0);\n\n    scorers[name] = {\n      mean: round(meanOfMeans),\n      std: round(std(means, meanOfMeans)),\n      min: round(minMean),\n      max: round(maxMean),\n      scored,\n      failed,\n      skipped,\n    };\n  }\n\n  // Timing aggregation\n  let timing: MultiRunSummary['timing'];\n  const timingMeans = runs.filter((r) => r.summary.timing).map((r) => r.summary.timing!.mean);\n  if (timingMeans.length === runs.length && timingMeans.length > 0) {\n    const meanTiming = timingMeans.reduce((a, b) => a + b, 0) / timingMeans.length;\n    timing = {\n      mean: round(meanTiming),\n      std: round(std(timingMeans, meanTiming)),\n    };\n  }\n\n  return {\n    runGroupId,\n    runCount: runs.length,\n    workflows,\n    dataset,\n    totalCost: round(totalCost),\n    totalDuration: round(totalDuration),\n    scorers,\n    timing,\n  };\n}\n"],"mappings":";AAmBO,SAAS,kBACd,aACA,SACmB;AACnB,QAAM,IAAI,YAAY;AACtB,MAAI,MAAM,EAAG,QAAO,EAAE,OAAO,GAAG,OAAO,GAAG,MAAM,GAAG,aAAa,GAAG,cAAc,EAAE;AAEnF,QAAM,aAAa,SAAS,cAAc;AAC1C,QAAM,QAAQ,SAAS,SAAS;AAEhC,QAAM,OAAO,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAEtD,MAAI,MAAM;AACR,WAAO;AAAA,MACL,OAAO,YAAY,CAAC;AAAA,MACpB,OAAO,YAAY,CAAC;AAAA,MACpB;AAAA,MACA,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAEF,MAAI,cAAc;AAChB,WAAO;AAAA,MACL,OAAO,MAAM,IAAI;AAAA,MACjB,OAAO,MAAM,IAAI;AAAA,MACjB,MAAM,MAAM,IAAI;AAAA,MAChB,aAAa,OAAO,IAAI,IAAI;AAAA,MAC5B,cAAc,OAAO,IAAI,IAAI;AAAA,IAC/B;AAGF,QAAM,MAAM,SAAS,QAAQ,OAAO,WAAW,QAAQ,IAAI,IAAI,MAAM,KAAK,OAAO;AAEjF,QAAM,gBAA0B,IAAI,MAAM,UAAU;AACpD,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,MAAM;AACV,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,aAAO,YAAY,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC;AAAA,IAC1C;AACA,kBAAc,CAAC,IAAI,MAAM;AAAA,EAC3B;AAEA,gBAAc,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAElC,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAI5D,MAAI,YAAY;AAChB,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AACA,MAAI,YAAY;AAChB,WAAS,IAAI,aAAa,GAAG,KAAK,GAAG,KAAK;AACxC,QAAI,cAAc,CAAC,IAAI,EAAG;AAAA,QACrB;AAAA,EACP;AAEA,SAAO;AAAA,IACL,OAAO,MAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,CAAC,CAAC;AAAA,IACjD,OAAO,MAAM,cAAc,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC,CAAC;AAAA,IAC9D,MAAM,MAAM,IAAI;AAAA,IAChB,aAAa,MAAM,YAAY,UAAU;AAAA,IACzC,cAAc,MAAM,YAAY,UAAU;AAAA,EAC5C;AACF;AAEA,SAAS,MAAM,GAAmB;AAChC,SAAO,KAAK,MAAM,IAAI,GAAK,IAAI;AACjC;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,QAAQ,OAAO,KAAK;AACxB,SAAO,MAAM;AACX,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,aAAS,SAAS;AAClB,YAAQ,UAAU,KAAK;AAAA,EACzB;AACF;;;ACrGO,SAAS,aAAa,QAM3B;AACA,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,EAAE;AAC1E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC/C,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACxD,QAAM,MAAM,OAAO,CAAC;AACpB,QAAM,MAAM,OAAO,OAAO,SAAS,CAAC;AACpC,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,GAAG,CAAC;AAClD,QAAM,MAAM,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI,CAAC;AACnD,SAAO,EAAE,MAAMA,OAAM,IAAI,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,GAAG,KAAKA,OAAM,GAAG,EAAE;AACjG;AAEO,SAASA,OAAM,GAAmB;AACvC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AA2BO,SAAS,aACd,OAKA,MACqD;AACrD,MAAI,SAAS;AACb,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,KAAK,OAAO;AACrB,QAAI,EAAE,MAAO;AACb,QAAI,EAAE,OAAO,IAAI,KAAK,KAAM;AAAA,aACnB,EAAE,eAAe,IAAI,GAAG,YAAY,KAAM;AAAA,aAC1C,EAAE,eAAe,IAAI,GAAG,YAAY,KAAM;AAAA,EACrD;AACA,SAAO,EAAE,QAAQ,QAAQ,QAAQ;AACnC;AAuBO,SAAS,wBACd,QACA,QACA,MACA,OACwB;AACxB,QAAM,YAAY,SAAS;AAC3B,QAAM,OAAO,cAAc,IAAI,IAAI,SAAS;AAC5C,QAAM,UAAU,YAAY,MAAM,SAAS,kBAAkB,SAAS,IAAI,OAAO;AACjF,SAAO,EAAE,WAAW,MAAM,SAAS,YAAY,cAAc,EAAE;AACjE;AAqBA,eAAsB,mBACpB,OACA,aACA,MACc;AACd,QAAM,UAAU,IAAI,MAAS,MAAM,MAAM;AAIzC,QAAM,YAAY,OAAO,SAAS,WAAW,IAAI,KAAK,MAAM,WAAW,IAAI;AAC3E,QAAM,cAAc,MAAM,WAAW,IAAI,IAAI,KAAK,IAAI,KAAK,IAAI,GAAG,SAAS,GAAG,MAAM,MAAM;AAE1F,MAAI,OAAO;AACX,iBAAe,UAAyB;AACtC,WAAO,OAAO,MAAM,QAAQ;AAC1B,YAAM,IAAI;AACV,cAAQ,CAAC,IAAI,MAAM,KAAK,MAAM,CAAC,GAAG,CAAC;AAAA,IACrC;AAAA,EACF;AAEA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,YAAY,GAAG,MAAM,QAAQ,CAAC,CAAC;AACtE,SAAO;AACT;;;AClIA,IAAM,wBAAwB;AAC9B,IAAM,kCAAkC;AACxC,IAAM,mBAAmB;AAEzB,SAAS,iBACP,YACA,SACA,UACQ;AAER,MAAI,OAAO,SAAS,eAAe,SAAU,QAAO,QAAQ;AAG5D,MAAI,SAAS,cAAc,OAAO,QAAQ,eAAe,UAAU;AACjE,UAAM,YAAY,QAAQ,WAAW,UAAU;AAC/C,QAAI,aAAa,KAAM,QAAO;AAAA,EAEhC;AAGA,QAAM,cAAc,SAAS;AAC7B,MAAI,eAAe,cAAc,aAAa;AAC5C,WAAO,YAAY,UAAU,MAAM,QAC/B,wBACA;AAAA,EACN;AAGA,SAAO;AACT;AAEO,SAAS,YACd,UACA,WACA,SACgB;AAChB,QAAM,kBAAkB,MAAM,QAAQ,QAAQ,IAAI,WAAW,CAAC,QAAQ;AACtE,QAAM,mBAAmB,MAAM,QAAQ,SAAS,IAAI,YAAY,CAAC,SAAS;AAE1E,MAAI,gBAAgB,WAAW,KAAK,iBAAiB,WAAW,GAAG;AACjE,UAAM,IAAI,MAAM,yCAAyC;AAAA,EAC3D;AAaA,QAAM,WAAW,KAAK,IAAI,gBAAgB,QAAQ,iBAAiB,MAAM;AACzE,QAAM,eAAe,gBAAgB,MAAM,GAAG,QAAQ;AACtD,QAAM,gBAAgB,iBAAiB,MAAM,GAAG,QAAQ;AAMxD,QAAM,cAAc,gBAAgB,CAAC;AACrC,QAAM,eAAe,iBAAiB,CAAC;AAEvC,MAAI,YAAY,YAAY,aAAa,SAAS;AAChD,UAAM,IAAI;AAAA,MACR,kDAAkD,YAAY,OAAO,SAAS,aAAa,OAAO;AAAA,IACpG;AAAA,EACF;AAEA,QAAM,sBAAsB,OAAO,KAAK,YAAY,QAAQ,OAAO,EAAE,KAAK;AAC1E,QAAM,uBAAuB,OAAO,KAAK,aAAa,QAAQ,OAAO,EAAE,KAAK;AAC5E,MAAI,KAAK,UAAU,mBAAmB,MAAM,KAAK,UAAU,oBAAoB,GAAG;AAChF,UAAM,IAAI,MAAM,6CAA6C;AAAA,EAC/D;AAIA,QAAM,cAAwC,CAAC;AAC/C,aAAW,QAAQ,qBAAqB;AACtC,gBAAY,IAAI,IAAI,CAAC;AAAA,EACvB;AACA,WAAS,IAAI,GAAG,IAAI,UAAU,KAAK;AACjC,UAAM,OAAO,aAAa,CAAC;AAC3B,UAAM,OAAO,cAAc,CAAC;AAC5B,UAAM,YAAY,KAAK,IAAI,KAAK,MAAM,QAAQ,KAAK,MAAM,MAAM;AAC/D,aAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAClC,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,YAAM,QAAQ,KAAK,MAAM,CAAC;AAC1B,UAAI,MAAM,SAAS,MAAM,MAAO;AAChC,iBAAW,QAAQ,qBAAqB;AACtC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,cAAM,SAAS,MAAM,OAAO,IAAI;AAChC,YAAI,UAAU,QAAQ,UAAU,MAAM;AACpC,sBAAY,IAAI,EAAE,KAAK,SAAS,MAAM;AAAA,QACxC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAGA,QAAM,gBAAwC,CAAC;AAC/C,QAAM,iBAAyC,CAAC;AAChD,aAAW,QAAQ,qBAAqB;AACtC,kBAAc,IAAI,IAChB,aAAa,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC7E,aAAa;AACf,mBAAe,IAAI,IACjB,cAAc,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,IAAI,CAAC,IAC9E,cAAc;AAAA,EAClB;AAEA,QAAM,UAAqC,CAAC;AAC5C,aAAW,QAAQ,qBAAqB;AACtC,UAAM,WAAW,cAAc,IAAI;AACnC,UAAM,WAAW,eAAe,IAAI;AACpC,UAAM,WAAW,WAAW;AAC5B,UAAM,QAAQC,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,WAAW,IAAIA,OAAO,WAAW,WAAY,GAAG,IAAI;AACzE,UAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AAOtE,QAAI,iBAAiB;AACrB,QAAI,iBAAiB;AACrB,QAAI,kBAAkB;AACtB,eAAW,OAAO,cAAc;AAC9B,YAAM,IAAI,aAAa,IAAI,OAAO,IAAI;AACtC,wBAAkB,EAAE;AACpB,wBAAkB,EAAE;AACpB,yBAAmB,EAAE;AAAA,IACvB;AACA,QAAI,kBAAkB;AACtB,QAAI,kBAAkB;AACtB,QAAI,mBAAmB;AACvB,eAAW,OAAO,eAAe;AAC/B,YAAM,IAAI,aAAa,IAAI,OAAO,IAAI;AACtC,yBAAmB,EAAE;AACrB,yBAAmB,EAAE;AACrB,0BAAoB,EAAE;AAAA,IACxB;AAEA,UAAM,QAA2C;AAAA,MAC/C,cAAc;AAAA,MACd,eAAe;AAAA,MACf;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AASA,UAAM,QAAQ,YAAY,IAAI;AAC9B,UAAM,IAAI,MAAM;AAChB,QAAI,MAAM,UAAU,GAAG;AACrB,YAAM,KAAK,kBAAkB,KAAK;AAClC,YAAM,KAAK,EAAE,OAAO,GAAG,OAAO,OAAO,GAAG,MAAM;AAC9C,YAAM,cAAc,GAAG;AACvB,YAAM,eAAe,GAAG;AAExB,YAAM,iBAAiB,GAAG,QAAQ,KAAK,GAAG,QAAQ;AAClD,YAAM,cAAc,kBAAkB,KAAK,IAAI,GAAG,IAAI,KAAK;AAAA,IAC7D;AAEA,YAAQ,IAAI,IAAI;AAAA,EAClB;AAIA,QAAM,cAAgC,CAAC;AACvC,QAAM,eAAkC,CAAC;AAEzC,QAAM,YAAY,KAAK;AAAA,IACrB,GAAG,aAAa,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,IACzC,GAAG,cAAc,IAAI,CAAC,MAAM,EAAE,MAAM,MAAM;AAAA,EAC5C;AAEA,WAAS,IAAI,GAAG,IAAI,WAAW,KAAK;AAElC,UAAM,mBAAmB,aAAa,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACnE,UAAM,oBAAoB,cAAc,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,KAAK;AACrE,QAAI,oBAAoB,kBAAmB;AAE3C,eAAW,QAAQ,qBAAqB;AAEtC,YAAM,UAAU,aACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,YAAM,UAAU,cACb,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,IAAI,CAAC,EACnC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,UAAI,QAAQ,WAAW,KAAK,QAAQ,WAAW,EAAG;AAElD,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,OAAO,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,QAAQ;AAC1D,YAAM,QAAQA,OAAM,OAAO,IAAI;AAC/B,YAAM,YAAY,iBAAiB,MAAM,SAAS,YAAY,QAAQ;AACtE,UAAI,QAAQ,CAAC;AACX,oBAAY,KAAK;AAAA,UACf,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,eACM,QAAQ;AACf,qBAAa,KAAK;AAAA,UAChB,WAAW;AAAA,UACX,OAAO,YAAY,MAAM,CAAC,GAAG;AAAA,UAC7B,QAAQ;AAAA,UACR,eAAeA,OAAM,IAAI;AAAA,UACzB,gBAAgBA,OAAM,IAAI;AAAA,UAC1B;AAAA,QACF,CAAC;AAAA,IACL;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,kBAAkB,aACrB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,QAAM,mBAAmB,cACtB,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAC9B,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AACpC,MAAI,gBAAgB,SAAS,KAAK,iBAAiB,SAAS,GAAG;AAC7D,UAAM,QAAQ,gBAAgB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,gBAAgB;AAC3E,UAAM,QAAQ,iBAAiB,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,iBAAiB;AAC7E,UAAM,QAAQA,OAAM,QAAQ,KAAK;AACjC,UAAM,eAAe,QAAQ,IAAIA,QAAQ,QAAQ,SAAS,QAAS,GAAG,IAAI;AAC1E,aAAS,EAAE,cAAcA,OAAM,KAAK,GAAG,eAAeA,OAAM,KAAK,GAAG,OAAO,aAAa;AAAA,EAC1F;AAGA,MAAI;AACJ,QAAM,kBACJ,aAAa,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,aAAa;AACvE,QAAM,mBACJ,cAAc,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC,IAAI,cAAc;AACzE,MAAI,kBAAkB,KAAK,mBAAmB,GAAG;AAC/C,UAAM,WAAW,mBAAmB;AACpC,UAAM,QAAQA,OAAM,QAAQ;AAC5B,UAAM,eAAe,kBAAkB,IAAIA,OAAO,WAAW,kBAAmB,GAAG,IAAI;AACvF,WAAO;AAAA,MACL,eAAeA,OAAM,eAAe;AAAA,MACpC,gBAAgBA,OAAM,gBAAgB;AAAA,MACtC;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAEA,QAAM,QAAkB,CAAC;AACzB,aAAW,QAAQ,qBAAqB;AACtC,UAAM,IAAI,QAAQ,IAAI;AACtB,QAAI,KAAK,IAAI,EAAE,KAAK,IAAI,MAAO;AAC7B,YAAM,YAAY,EAAE,QAAQ,IAAI,aAAa;AAC7C,YAAM,OAAO,EAAE,QAAQ,IAAI,MAAM;AACjC,UAAI,WAAW;AACf,UAAI,EAAE,gBAAgB,KAAM,YAAW;AAAA,eAC9B,EAAE,gBAAgB,MAAO,YAAW;AAC7C,YAAM;AAAA,QACJ,IAAI,IAAI,KAAK,SAAS,OAAO,IAAI,GAAG,EAAE,aAAa,QAAQ,CAAC,CAAC,MAAM,EAAE,aAAa,QAAQ,CAAC,CAAC,OAAO,EAAE,cAAc,QAAQ,CAAC,CAAC,IAAI,QAAQ;AAAA,MAC3I;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAU,KAAK,IAAI,OAAO,YAAY,IAAI,GAAG;AAC/C,UAAM,MAAM,OAAO,QAAQ,IAAI,WAAW;AAC1C,UAAM,KAAK,GAAG,KAAK,IAAI,OAAO,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAClE;AACA,MAAI,QAAQ,KAAK,IAAI,KAAK,YAAY,IAAI,GAAG;AAC3C,UAAM,MAAM,KAAK,QAAQ,IAAI,mBAAmB;AAChD,UAAM,KAAK,GAAG,KAAK,IAAI,KAAK,YAAY,EAAE,QAAQ,CAAC,CAAC,KAAK,GAAG,EAAE;AAAA,EAChE;AACA,QAAM,aAAa,aAAa,MAAM,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,uBAAuB,SAAS,YAAY,MAAM,oBAAoB,aAAa,MAAM;AAM/J,QAAM,kBAAkB,cAAc,eAAe;AACrD,QAAM,mBAAmB,cAAc,gBAAgB;AAEvD,SAAO;AAAA,IACL,UAAU;AAAA,MACR,IAAI,YAAY;AAAA,MAChB,UAAU,YAAY;AAAA,MACtB;AAAA,MACA,GAAI,kBAAkB,EAAE,SAAS,gBAAgB,IAAI,CAAC;AAAA,IACxD;AAAA,IACA,WAAW;AAAA,MACT,IAAI,aAAa;AAAA,MACjB,UAAU,aAAa;AAAA,MACvB;AAAA,MACA,GAAI,mBAAmB,EAAE,SAAS,iBAAiB,IAAI,CAAC;AAAA,IAC1D;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS;AAAA,EACX;AACF;AAuBA,SAAS,cAAc,MAA0E;AAC/F,MAAI,KAAK,WAAW,EAAG,QAAO;AAC9B,MAAI;AACJ,aAAW,KAAK,MAAM;AACpB,UAAM,IAAI,EAAE,UAAU;AACtB,QAAI,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,GAAG;AAC/C,kBAAY;AACZ;AAAA,IACF;AAAA,EACF;AACA,MAAI,cAAc,OAAW,QAAO;AACpC,MAAI,KAAK,UAAU,UAAW,QAAO;AACrC,SAAO,EAAE,WAAW,KAAK,QAAQ,UAAU;AAC7C;AAqBO,SAAS,4BACd,YACA,oBACe;AACf,QAAM,cAAe,WAAW,SAAS,UAAU,eAAe,CAAC;AACnE,aAAW,QAAQ,OAAO,KAAK,WAAW,OAAO,GAAG;AAClD,UAAM,IAAI,WAAW,QAAQ,IAAI;AACjC,UAAM,OAAO,YAAY,IAAI,MAAM,kBAAkB,kBAAkB;AACvE,UAAM,QAAQ,SAAS,kBAAkB,IAAI;AAC7C,eAAW,QAAQ,CAAC,YAAY,WAAW,GAAY;AACrD,YAAM,UAAU,SAAS,aAAa,EAAE,iBAAiB,EAAE,oBAAoB;AAC/E,YAAM,UAAU,SAAS,aAAa,EAAE,iBAAiB,EAAE,oBAAoB;AAC/E,YAAM,WAAW,SAAS,aAAa,EAAE,kBAAkB,EAAE,qBAAqB;AAClF,YAAM,UAAU,wBAAwB,QAAQ,QAAQ,MAAM,KAAK;AACnE,UAAI,QAAQ,YAAY;AAMtB,cAAM,QACJ,UAAU,IACN,yEAAyE,OAAO,MAChF;AACN,eAAO,WAAW,IAAI,4BAA4B,IAAI,KAAK,KAAK;AAAA,MAClE;AACA,UAAI,QAAQ,SAAS;AACnB,cAAM,WACJ,SAAS,kBACL,iCACA,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC;AACjC,eAAO,WAAW,IAAI,aAAa,QAAQ,OAAO,KAAK,QAAQ,CAAC,CAAC,QAAQ,IAAI,KAAK,MAAM,IAAI,QAAQ,SAAS,eAAe,QAAQ;AAAA,MACtI;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;;;AC5ZO,SAAS,sBAAsB,QAA6C;AACjF,SAAO,OAAO,WAAW,WAAW,EAAE,OAAO,OAAO,IAAI;AAC1D;AAUO,SAAS,sBAAsB,KAAc,MAAgC;AAClF,MACE,OACA,OAAO,QAAQ,YACf,OAAO,SAAS,YAChB,OAAO,SAAS,IAAI,KACpB,QAAQ,GACR;AACA,IAAC,IAA0B,OAAO;AAAA,EACpC;AACF;AAGO,SAAS,uBAAuB,KAAkC;AACvE,QAAM,IAAI,OAAO,OAAO,QAAQ,WAAY,IAA2B,OAAO;AAC9E,SAAO,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,KAAK,KAAK,IAAI,IAAI;AACrE;AAuFO,SAAS,OACd,QACuC;AACvC,SAAO;AAAA,IACL,MAAM,OAAO;AAAA,IACb,aAAa,OAAO;AAAA,IACpB,OAAO;AAAA,IACP,OAAO,OAAO;AAAA,IACd,SAAS,OAAO;AAAA,EAClB;AACF;;;ACrGA,eAAsB,UACpB,MACA,SACA,mBACA,eACA,QACiB;AACjB,QAAM,eAAgB,KAAK,iBAAiB,CAAC;AAC7C,aAAW,KAAK,SAAS;AACvB,SAAK,OAAO,EAAE,IAAI,IAAI;AACtB,iBAAa,EAAE,IAAI,IAAI,EAAE,OAAO,KAAK;AAAA,EACvC;AAEA,MAAI,iBAAiB;AACrB,QAAM,qBAA6C,CAAC;AAEpD,QAAM,mBAAmB,SAAS,mBAAmB,OAAOC,YAAW;AACrE,QAAI,QAAQ,QAAS;AACrB,UAAM,cAAc,KAAK,IAAI;AAC7B,QAAI;AAQF,UAAIA,QAAO,WAAW,CAACA,QAAO,QAAQ,KAAK,QAAQ,KAAK,OAAO,KAAK,WAAW,GAAG;AAChF,aAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,qBAAaA,QAAO,IAAI,IAAI,EAAE,OAAO,MAAM,SAAS,KAAK;AACzD;AAAA,MACF;AAEA,YAAM,MAAM,MAAMA,QAAO,MAAM,KAAK,QAAQ,KAAK,OAAO,KAAK,aAAa,aAAa;AACvF,YAAM,eAAe,sBAAsB,GAAG;AAE9C,UAAI,aAAa,QAAQ,MAAM;AAC7B,0BAAkB,aAAa;AAAA,MACjC;AAEA,YAAM,iBAAiB,KAAK,IAAI,IAAI;AAEpC,UACE,CAAC,OAAO,SAAS,aAAa,KAAK,KACnC,aAAa,QAAQ,KACrB,aAAa,QAAQ,GACrB;AACA,2BAAmBA,QAAO,IAAI,IAC5B,WAAWA,QAAO,IAAI,iCAAiC,aAAa,KAAK,cAAc,KAAK,UAAU,KAAK,KAAK,CAAC;AACnH,aAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,qBAAaA,QAAO,IAAI,IAAI;AAAA,UAC1B,OAAO;AAAA,UACP,UAAU,aAAa;AAAA,UACvB,UAAU;AAAA,UACV,MAAM,aAAa;AAAA,QACrB;AAAA,MACF,OAAO;AACL,aAAK,OAAOA,QAAO,IAAI,IAAIC,OAAM,aAAa,KAAK;AACnD,qBAAaD,QAAO,IAAI,IAAI;AAAA,UAC1B,OAAOC,OAAM,aAAa,KAAK;AAAA,UAC/B,UAAU,aAAa;AAAA,UACvB,UAAU;AAAA,UACV,MAAM,aAAa;AAAA,QACrB;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AAGZ,UAAK,KAA2B,SAAS,aAAc;AACvD,YAAM,UAAU,uBAAuB,GAAG;AAC1C,UAAI,WAAW,KAAM,mBAAkB;AACvC,yBAAmBD,QAAO,IAAI,IAC5B,WAAWA,QAAO,IAAI,YAAY,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AACpF,WAAK,OAAOA,QAAO,IAAI,IAAI;AAC3B,mBAAaA,QAAO,IAAI,IAAI;AAAA,QAC1B,OAAO;AAAA,QACP,UAAU,KAAK,IAAI,IAAI;AAAA,QACvB,MAAM;AAAA,MACR;AAAA,IACF;AAAA,EACF,CAAC;AAED,QAAM,gBAAgB,QACnB,IAAI,CAAC,MAAM,mBAAmB,EAAE,IAAI,CAAC,EACrC,OAAO,CAAC,MAAmB,KAAK,IAAI;AACvC,MAAI,cAAc,SAAS,EAAG,MAAK,eAAe;AAClD,OAAK,aAAa,iBAAiB,IAAI,iBAAiB;AACxD,SAAO;AACT;;;AC/GA,SAAS,kBAAkB;AAE3B,SAAS,UAAU,MAAsB;AACvC,QAAM,QAAQ,KAAK,MAAM,eAAe;AACxC,MAAI,CAAC,MAAO,OAAM,IAAI,MAAM,yBAAyB,IAAI,GAAG;AAC5D,SAAO,WAAW,MAAM,CAAC,CAAC;AAC5B;AAkBA,SAAS,gBAAgB,QAAiB,QAAQ,mBAAuC;AACvF,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,MAAO,OAA8B;AAC3C,MAAI,QAAQ,OAAW,QAAO;AAC9B,MAAI,OAAO,QAAQ,YAAY,OAAO,SAAS,GAAG,KAAK,OAAO,EAAG,QAAO;AAExE,UAAQ;AAAA,IACN,6CAA6C,KAAK,qDAAqD,OAAO,QAAQ,WAAW,OAAO,GAAG,IAAI,OAAO,GAAG;AAAA,EAC3J;AACA,SAAO;AACT;AASA,SAAS,oBACP,QACA,QAAQ,mBAC6B;AACrC,MAAI,WAAW,QAAQ,OAAO,WAAW,SAAU,QAAO;AAC1D,QAAM,OAAQ,OAAkC;AAChD,MAAI,SAAS,OAAW,QAAO;AAC/B,MAAI,SAAS,QAAQ,OAAO,SAAS,YAAY,MAAM,QAAQ,IAAI,GAAG;AACpE,YAAQ;AAAA,MACN,iDAAiD,KAAK,uCAAuC,SAAS,OAAO,SAAS,MAAM,QAAQ,IAAI,IAAI,UAAU,OAAO,IAAI;AAAA,IACnK;AACA,WAAO;AAAA,EACT;AAIA,QAAM,QAAQ,OAAO,eAAe,IAAI;AACxC,MAAI,UAAU,OAAO,aAAa,UAAU,MAAM;AAChD,YAAQ;AAAA,MACN,iDAAiD,KAAK,uCAAwC,OAAO,aAAa,QAA+B,eAAe;AAAA,IAClK;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,eAAsB,QACpB,QACA,iBAIA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,KAAK,WAAW;AACtB,QAAM,QAAQ,MAAM,OAAO,QAAQ,SAAS;AAM5C,QAAM,wBAAwB,CAAC,GAAI,OAAO,QAAQ,yBAAyB,CAAC,CAAE;AAC9E,QAAM,cAAc,OAAO,eAAe;AAK1C,QAAM,oBAAoB,OAAO,qBAAqB;AACtD,QAAM,cAAc,OAAO,SAAS,UAAU,OAAO,MAAM,IAAI;AAG/D,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,IACA,QAAQ,SAAS;AAAA,EACnB;AAEA,QAAM,YAAwB,IAAI,MAAM,MAAM,MAAM;AACpD,MAAI,YAAY;AAChB,MAAI,iBAAiB;AAErB,iBAAe,YAAY,MAAyB,WAAkC;AACpF,QAAI,SAAS,QAAQ,SAAS;AAC5B,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,QAAI,gBAAgB;AAClB,gBAAU,SAAS,IAAI;AAAA,QACrB,OAAO,KAAK;AAAA,QACZ,aAAa,KAAK;AAAA,QAClB,QAAQ;AAAA,QACR,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,UAAM,WAAqB;AAAA,MACzB,OAAO,KAAK;AAAA,MACZ,aAAa,KAAK;AAAA,MAClB,QAAQ;AAAA,MACR,QAAQ,CAAC;AAAA,IACX;AACA,UAAM,YAAY,KAAK,IAAI;AAC3B,QAAI;AAKF,UAAI,SAAS,eAAe;AAC1B,cAAM,UAAU,MAAM,QAAQ;AAAA,UAC5B,YAAY,gBAAgB,KAAK,OAAO,OAAO;AAAA,UAC/C,EAAE,eAAe,KAAK;AAAA,QACxB;AACA,iBAAS,WAAW,KAAK,IAAI,IAAI;AACjC,iBAAS,SAAS,QAAQ,OAAO;AAKjC,iBAAS,OAAO,gBAAgB,QAAQ,MAAM,KAAK,QAAQ;AAC3D,YAAI,QAAQ,SAAU,UAAS,WAAW;AAC1C,iBAAS,WAAW,oBAAoB,QAAQ,MAAM,KAAK,QAAQ;AACnE,YAAI,QAAQ,UAAU,QAAQ,OAAO,SAAS,GAAG;AAC/C,mBAAS,SAAS,QAAQ;AAAA,QAC5B;AACA,YAAI,SAAS,QAAQ,MAAM;AACzB,uBAAa,SAAS;AAAA,QACxB;AAAA,MACF,OAAO;AACL,cAAM,SAAS,MAAM,gBAAgB,KAAK,OAAO,OAAO;AACxD,iBAAS,WAAW,KAAK,IAAI,IAAI;AACjC,iBAAS,SAAS,OAAO;AACzB,iBAAS,OAAO,gBAAgB,MAAM;AACtC,cAAM,OAAO,oBAAoB,MAAM;AACvC,YAAI,KAAM,UAAS,WAAW;AAC9B,YAAI,SAAS,QAAQ,MAAM;AACzB,uBAAa,SAAS;AAAA,QACxB;AAAA,MACF;AAAA,IACF,SAAS,KAAK;AACZ,eAAS,WAAW,KAAK,IAAI,IAAI;AACjC,eAAS,QAAQ,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAIhE,UAAI,SAAS,eAAe;AAC1B,cAAM,WAAY,IAAwC;AAC1D,YAAI,MAAM,QAAQ,QAAQ,KAAK,SAAS,SAAS,GAAG;AAClD,mBAAS,SAAS;AAAA,QACpB;AAAA,MACF;AACA,gBAAU,SAAS,IAAI;AACvB,eAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAChF;AAAA,IACF;AAEA,QAAI,eAAe,QAAQ,YAAY,aAAa;AAClD,uBAAiB;AAAA,IACnB;AAOA,UAAM,iBAAiB,MAAM;AAAA,MAC3B;AAAA,MACA,OAAO;AAAA,MACP;AAAA,MACA;AAAA,MACA,SAAS;AAAA,IACX;AACA,iBAAa;AACb,cAAU,SAAS,IAAI;AACvB,aAAS,aAAa,EAAE,MAAM,aAAa,WAAW,YAAY,MAAM,OAAO,CAAC;AAAA,EAClF;AAIA,QAAM,mBAAmB,OAAO,aAAa,CAAC,MAAM,MAAM,YAAY,MAAM,CAAC,CAAC;AAE9E,QAAM,WAAW,UAAU,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AAClD,QAAM,cAAc,OAAO,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AACpD,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,UACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AACtC,UAAM,EAAE,QAAQ,QAAQ,QAAQ,IAAI,aAAa,WAAW,IAAI;AAChE,gBAAY,IAAI,IAAI,EAAE,GAAG,aAAa,MAAM,GAAG,QAAQ,QAAQ,QAAQ;AAAA,EACzE;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,OAAO,SAAS;AAC9B,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAQA,MAAI;AACJ,QAAM,WAAW,OAAO;AACxB,MAAI,YAAY,MAAM;AACpB,QAAI,CAAC,OAAO,SAAS,QAAQ,KAAK,WAAW,KAAK,WAAW,GAAG;AAC9D,cAAQ;AAAA,QACN,sDAAsD,QAAQ;AAAA,MAChE;AAAA,IACF,OAAO;AACL,iBAAW,QAAQ,aAAa;AAC9B,cAAM,QAAQ,YAAY,IAAI;AAC9B,cAAM,SAAS,MAAM,UAAU;AAC/B,cAAM,SAAS,MAAM,UAAU;AAC/B,cAAM,OAAO,YAAY,IAAI,MAAM,QAAQ,QAAQ;AACnD,cAAM,QAAQ,SAAS,kBAAkB,IAAI;AAC7C,cAAM,UAAU,wBAAwB,QAAQ,QAAQ,MAAM,KAAK;AAInE,YAAI,QAAQ,SAAS;AACnB,WAAC,aAAa,CAAC,GAAG,KAAK,EAAE,QAAQ,MAAM,MAAM,QAAQ,MAAM,OAAO,MAAM,QAAQ,OAAO,CAAC;AAAA,QAC1F;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAY,UAAU,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,YAAY,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,QAAS;AAChG,QAAM,SAAS,UAAU,SAAS,IAAI,aAAa,SAAS,IAAI;AAGhE,QAAM,kBAAkB,oBAAI,IAAoB;AAChD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,0BAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MAChE;AAAA,IACF,OAAO;AAEL,YAAM,aAAa,KAAK,UAAU;AAClC,UAAI,MAAM,QAAQ,UAAU,GAAG;AAC7B,mBAAW,KAAK,YAAY;AAC1B,cAAI,OAAO,MAAM,SAAU,iBAAgB,IAAI,IAAI,gBAAgB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QACrF;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAIA,QAAM,aAAsC,CAAC;AAC7C,MAAI,gBAAgB,OAAO,GAAG;AAC5B,UAAM,SAAS,CAAC,GAAG,gBAAgB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE,eAAW,SAAS,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AACzC,eAAW,cAAc,OAAO,YAAY,MAAM;AAAA,EACpD;AAMA,QAAM,qBAAqB,oBAAI,IAAoB;AACnD,aAAW,QAAQ,WAAW;AAC5B,UAAM,aAAa,KAAK,UAAU;AAClC,QAAI,cAAc,OAAO,eAAe,UAAU;AAChD,iBAAW,CAAC,GAAG,KAAK,KAAK,OAAO,QAAQ,UAAqC,GAAG;AAC9E,YAAI,OAAO,UAAU;AACnB,6BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,KAAK;AAAA,MACtE;AAAA,IACF,OAAO;AAEL,YAAM,gBAAgB,KAAK,UAAU;AACrC,UAAI,MAAM,QAAQ,aAAa,GAAG;AAChC,mBAAW,KAAK,eAAe;AAC7B,cAAI,OAAO,MAAM;AACf,+BAAmB,IAAI,IAAI,mBAAmB,IAAI,CAAC,KAAK,KAAK,CAAC;AAAA,QAClE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAKA,QAAM,gBAAyC,CAAC;AAChD,MAAI,mBAAmB,OAAO,GAAG;AAC/B,UAAM,SAAS,CAAC,GAAG,mBAAmB,QAAQ,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC;AAC3E,kBAAc,YAAY,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;AAC/C,kBAAc,iBAAiB,OAAO,YAAY,MAAM;AAAA,EAC1D,WAAW,OAAO,UAAU;AAC1B,kBAAc,YAAY,CAAC,OAAO,QAAQ;AAC1C,kBAAc,iBAAiB,EAAE,CAAC,OAAO,QAAQ,GAAG,MAAM,OAAO;AAAA,EACnE;AAEA,WAAS,aAAa,EAAE,MAAM,YAAY,YAAY,MAAM,QAAQ,SAAS,CAAC;AAE9E,SAAO;AAAA,IACL;AAAA,IACA,SAAS,OAAO,QAAQ;AAAA,IACxB,UAAU;AAAA,MACR,GAAG,OAAO;AAAA,MACV;AAAA,MACA,GAAG;AAAA,MACH,GAAG;AAAA,MACH,GAAI,sBAAsB,SAAS,IAAI,EAAE,sBAAsB,IAAI,CAAC;AAAA,IACtE;AAAA,IACA,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,GAAI,UAAU,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,EAAE,UAAU,KAAK,IAAI,CAAC;AAAA,IACpE,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS;AAAA,MACP,OAAO,MAAM;AAAA,MACb;AAAA,MACA,SAAS;AAAA,MACT;AAAA,MACA,GAAI,WAAW,EAAE,SAAS,IAAI,CAAC;AAAA,IACjC;AAAA,EACF;AACF;;;AClXA,SAAS,cAAAE,mBAAkB;AAmB3B,eAAsB,QACpB,QACA,SACA,SACA,SACqB;AACrB,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,cAAc,SAAS,eAAe;AAC5C,QAAM,oBAAoB,SAAS,qBAAqB;AAExD,QAAM,gBAA+B;AAAA,IACnC,iBAAiB,CAAC,QAAgB;AAChC,UAAI,OAAO,QAAQ,oBAAoB,YAAY;AACjD,cAAM,IAAI;AAAA,UACR;AAAA,QAEF;AAAA,MACF;AACA,aAAO,QAAQ,gBAAgB,GAAG;AAAA,IACpC;AAAA,IACA,QAAQ,SAAS;AAAA,EACnB;AAEA,QAAM,WAAuB,IAAI,MAAM,OAAO,MAAM,MAAM;AAC1D,MAAI,YAAY;AAEhB,iBAAe,YAAY,UAAoB,WAAkC;AAK/E,QAAI,SAAS,QAAQ,SAAS;AAC5B,eAAS,SAAS,IAAI;AAAA,QACpB,OAAO,SAAS;AAAA,QAChB,aAAa,SAAS;AAAA,QACtB,QAAQ,SAAS;AAAA,QACjB,OAAO;AAAA,QACP,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAGA,QAAI,SAAS,OAAO;AAClB,eAAS,SAAS,IAAI;AAAA,QACpB,OAAO,SAAS;AAAA,QAChB,aAAa,SAAS;AAAA,QACtB,QAAQ,SAAS;AAAA,QACjB,OAAO,SAAS;AAAA,QAChB,QAAQ,CAAC;AAAA,MACX;AACA;AAAA,IACF;AAKA,UAAM,OAAiB;AAAA,MACrB,OAAO,SAAS;AAAA,MAChB,aAAa,SAAS;AAAA,MACtB,QAAQ,SAAS;AAAA,MACjB,UAAU,SAAS;AAAA,MACnB,QAAQ,SAAS;AAAA,MACjB,QAAQ,CAAC;AAAA,MACT,cAAc,CAAC;AAAA,IACjB;AAKA,UAAM,iBAAiB,MAAM;AAAA,MAC3B;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS;AAAA,IACX;AACA,iBAAa;AACb,aAAS,SAAS,IAAI;AAAA,EACxB;AAIA,QAAM,mBAAmB,OAAO,OAAO,aAAa,CAAC,MAAM,MAAM,YAAY,MAAM,CAAC,CAAC;AAErF,QAAM,WAAW,SAAS,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE;AACjD,QAAM,cAAc,QAAQ,IAAI,CAAC,MAAM,EAAE,IAAI;AAC7C,QAAM,cAAsC,CAAC;AAC7C,aAAW,QAAQ,aAAa;AAC9B,UAAM,SAAS,SACZ,OAAO,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,OAAO,IAAI,KAAK,IAAI,EAChD,IAAI,CAAC,MAAM,EAAE,OAAO,IAAI,CAAW;AAKtC,UAAM,EAAE,QAAQ,QAAQ,QAAQ,IAAI,aAAa,UAAU,IAAI;AAC/D,gBAAY,IAAI,IAAI,EAAE,GAAG,aAAa,MAAM,GAAG,QAAQ,QAAQ,QAAQ;AAAA,EACzE;AAEA,QAAM,cAAsC,CAAC;AAC7C,aAAW,KAAK,SAAS;AACvB,gBAAY,EAAE,IAAI,IAAI,EAAE,QAAQ,QAAQ;AAAA,EAC1C;AAEA,SAAO;AAAA,IACL,IAAIA,YAAW;AAAA,IACf,SAAS,OAAO;AAAA,IAChB,WAAW,MAAM;AAUf,YAAM,OAAgC,EAAE,GAAG,OAAO,SAAS;AAC3D,aAAO,KAAK;AACZ,aAAO,KAAK;AACZ,aAAO,KAAK;AACZ,aAAO,KAAK;AACZ,YAAM,SAAkC;AAAA,QACtC,GAAG;AAAA,QACH,UAAU;AAAA,QACV,YAAY,OAAO;AAAA,QACnB;AAAA,MACF;AAKA,UAAI,CAAC,MAAM,QAAQ,OAAO,SAAS,GAAG;AACpC,cAAM,iBAAkB,OAAkC;AAC1D,YAAI,OAAO,mBAAmB,YAAY,gBAAgB;AACxD,iBAAO,YAAY,CAAC,cAAc;AAClC,iBAAO,iBAAiB,EAAE,CAAC,cAAc,GAAG,OAAO,MAAM,OAAO;AAAA,QAClE;AAAA,MACF;AACA,aAAO;AAAA,IACT,GAAG;AAAA,IACH,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,IAClC;AAAA,IACA,UAAU,KAAK,IAAI,IAAI;AAAA,IACvB,OAAO;AAAA,IACP,SAAS,EAAE,OAAO,OAAO,MAAM,QAAQ,UAAU,SAAS,YAAY;AAAA,EACxE;AACF;;;AC3KA,SAAS,cAAAC,mBAAkB;AAmC3B,SAAS,IAAI,QAAkB,MAAsB;AACnD,MAAI,OAAO,UAAU,EAAG,QAAO;AAC/B,QAAM,aAAa,OAAO,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC;AACrE,SAAO,KAAK,KAAK,cAAc,OAAO,SAAS,EAAE;AACnD;AAKO,SAAS,cAAc,MAAqC;AACjE,MAAI,KAAK,WAAW,EAAG,OAAM,IAAI,MAAM,4BAA4B;AAEnE,QAAM,aAAc,KAAK,CAAC,EAAE,SAAS,cAAyBC,YAAW;AACzE,QAAM,EAAE,QAAQ,IAAI,KAAK,CAAC;AAI1B,QAAM,gBAAgB,oBAAI,IAAY;AACtC,QAAM,YAAsB,CAAC;AAC7B,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,IAAI,SAAS;AAC1B,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,iBAAW,KAAK,MAAM;AACpB,YAAI,OAAO,MAAM,YAAY,CAAC,cAAc,IAAI,CAAC,GAAG;AAClD,wBAAc,IAAI,CAAC;AACnB,oBAAU,KAAK,CAAC;AAAA,QAClB;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,YAAY,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,WAAW,CAAC;AAC9D,QAAM,gBAAgB,KAAK,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,UAAU,CAAC;AAGjE,QAAM,cAAc,OAAO,KAAK,KAAK,CAAC,EAAE,QAAQ,OAAO;AACvD,QAAM,UAAsC,CAAC;AAE7C,aAAW,QAAQ,aAAa;AAO9B,UAAM,eAAe,KAAK,OAAO,CAAC,MAAM;AACtC,YAAM,KAAK,EAAE,QAAQ,QAAQ,IAAI,GAAG;AACpC,aAAO,MAAM,QAAQ,KAAK;AAAA,IAC5B,CAAC;AACD,UAAM,aAAa,aAAa,SAAS,IAAI,eAAe;AAC5D,UAAM,QAAQ,WAAW,IAAI,CAAC,MAAM,EAAE,QAAQ,QAAQ,IAAI,GAAG,QAAQ,CAAC;AACtE,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,MAAM;AAC7D,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AACjC,UAAM,UAAU,KAAK,IAAI,GAAG,KAAK;AAKjC,UAAM,SAAS,KAAK,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,UAAU,IAAI,CAAC;AACtF,UAAM,SAAS,KAAK,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,UAAU,IAAI,CAAC;AACtF,UAAM,UAAU,KAAK,OAAO,CAAC,KAAK,MAAM,OAAO,EAAE,QAAQ,QAAQ,IAAI,GAAG,WAAW,IAAI,CAAC;AAExF,YAAQ,IAAI,IAAI;AAAA,MACd,MAAMC,OAAM,WAAW;AAAA,MACvB,KAAKA,OAAM,IAAI,OAAO,WAAW,CAAC;AAAA,MAClC,KAAKA,OAAM,OAAO;AAAA,MAClB,KAAKA,OAAM,OAAO;AAAA,MAClB;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAGA,MAAI;AACJ,QAAM,cAAc,KAAK,OAAO,CAAC,MAAM,EAAE,QAAQ,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,OAAQ,IAAI;AAC1F,MAAI,YAAY,WAAW,KAAK,UAAU,YAAY,SAAS,GAAG;AAChE,UAAM,aAAa,YAAY,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,YAAY;AACxE,aAAS;AAAA,MACP,MAAMA,OAAM,UAAU;AAAA,MACtB,KAAKA,OAAM,IAAI,aAAa,UAAU,CAAC;AAAA,IACzC;AAAA,EACF;AAEA,SAAO;AAAA,IACL;AAAA,IACA,UAAU,KAAK;AAAA,IACf;AAAA,IACA;AAAA,IACA,WAAWA,OAAM,SAAS;AAAA,IAC1B,eAAeA,OAAM,aAAa;AAAA,IAClC;AAAA,IACA;AAAA,EACF;AACF;","names":["round","round","scorer","round","randomUUID","randomUUID","randomUUID","round"]}