axiom 0.44.0 → 0.45.0

package/dist/evals.js

@@ -5,8 +5,8 @@ import {
   resolveAxiomConnection,
   startActiveSpan,
   startSpan
-} from "./chunk-U3JDCQ3Y.js";
-import "./chunk-56V2A6IW.js";
+} from "./chunk-QAIGRB75.js";
+import "./chunk-RSHQRY3F.js";
 import "./chunk-T4FBJ252.js";
 import {
   warnScorerDeprecation
@@ -18,7 +18,7 @@ import {
   getGlobalFlagOverrides,
   setGlobalFlagOverrides,
   withEvalContext
-} from "./chunk-7NTVK3F4.js";
+} from "./chunk-R4CARCOX.js";
 import {
   Mean
 } from "./chunk-EQPNZ33J.js";
@@ -30,10 +30,11 @@ import {
   errorToString
 } from "./chunk-S65FSMB3.js";
 import "./chunk-X2LH7XLM.js";
-import "./chunk-HW6E4M7S.js";
+import "./chunk-N3RHEQMQ.js";
 import {
   createScorer
-} from "./chunk-MCKVQ2IB.js";
+} from "./chunk-JSEK2D3F.js";
+import "./chunk-3Q36GUCI.js";
 import {
   Attr
 } from "./chunk-TWE3LIRZ.js";
@@ -578,7 +579,7 @@ async function registerEval(evalName, opts) {
             [Attr.Eval.Version]: evalVersion,
             [Attr.Eval.Type]: "regression",
             // TODO: where to get experiment type value from?
-            [Attr.Eval.Tags]: [],
+            [Attr.Eval.Tags]: JSON.stringify(["offline"]),
             [Attr.Eval.Collection.ID]: "custom",
             // TODO: where to get collection split value from?
             [Attr.Eval.Collection.Name]: "custom",
@@ -660,8 +661,6 @@ async function registerEval(evalName, opts) {
         if (instrumentationError) {
           throw instrumentationError;
         }
-        const tags = ["offline"];
-        suiteSpan?.setAttribute(Attr.Eval.Tags, JSON.stringify(tags));
         const flagSummary = /* @__PURE__ */ new Map();
         for (const flag of allOutOfScopeFlags) {
           if (flagSummary.has(flag.flagPath)) {
@@ -812,6 +811,7 @@ async function registerEval(evalName, opts) {
                               {
                                 attributes: {
                                   [Attr.GenAI.Operation.Name]: "eval.score",
+                                  [Attr.Eval.Tags]: JSON.stringify(["offline"]),
                                   [Attr.Eval.ID]: evalId,
                                   [Attr.Eval.Name]: evalName,
                                   [Attr.Eval.Version]: evalVersion,

package/dist/evals.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/evals.ts","../src/evals/eval.ts","../src/evals/git-info.ts","../src/evals/eval.service.ts","../src/utils/fetcher.ts","../src/util/traces.ts","../src/util/deep-equal.ts","../src/util/tryCatch.ts","../src/evals/scorer.types.ts"],"sourcesContent":["export { Eval } from './evals/eval';\nexport type { EvalTask, EvalParams } from './evals/eval.types';\nexport { AxiomReporter } from './evals/reporter';\n\nexport type { EvalContextData } from './evals/context/storage';\n\nexport type { EvalBuilder } from './evals/builder';\nexport { type Score, type ScorerOptions } from './evals/scorer.types';\n\nimport { createScorer } from './evals/scorer.factory';\nimport { warnScorerDeprecation } from './evals/deprecated';\n\n/** @deprecated Import from 'axiom/ai/evals/scorers' instead. */\nexport const Scorer = ((...args: unknown[]) => {\n  warnScorerDeprecation('axiom/ai/evals');\n  return (createScorer as Function)(...args);\n}) as typeof createScorer;\n\nexport type { Evaluation, Case, Chat, Task } from './evals/eval.types';\n","import { afterAll, beforeAll, describe, inject, it } from 'vitest';\nimport { context, SpanStatusCode, trace, type Context } from '@opentelemetry/api';\nimport { customAlphabet } from 'nanoid';\nimport { withEvalContext, getEvalContext, getConfigScope } from './context/storage';\n\nimport { Attr } from '../otel/semconv/attributes';\nimport type { ResolvedAxiomConfig } from '../config/index';\nimport { startActiveSpan, startSpan, flush, ensureInstrumentationInitialized } from './instrument';\nimport { getGitUserInfo } from './git-info';\nimport type {\n  CollectionRecord,\n  EvalParams,\n  EvalTask,\n  EvaluationReport,\n  EvalCaseReport,\n  RuntimeFlagLog,\n  OutOfScopeFlag,\n  Evaluation,\n  OutOfScopeFlagAccess,\n} from './eval.types';\nimport type { ScoreWithName, ScorerLike, Scorer } from './scorer.types';\nimport { Mean, type Aggregation } from './aggregations';\nimport { EvaluationApiClient, findEvaluationCases } from './eval.service';\nimport { getGlobalFlagOverrides, setGlobalFlagOverrides } from './context/global-flags';\nimport { deepEqual } from '../util/deep-equal';\nimport { dotNotationToNested } from '../util/dot-path';\nimport { AxiomCLIError, errorToString } from '../util/errors';\nimport { tryCatchAsync, toError } from '../util/tryCatch';\nimport type { ValidateName } from '../util/name-validation';\nimport { recordName } from './name-validation-runtime';\n\ndeclare module 'vitest' {\n  interface TestSuiteMeta {\n    evaluation: EvaluationReport;\n  }\n  interface TaskMeta {\n    case: EvalCaseReport;\n    evaluation: EvaluationReport;\n  }\n  export interface ProvidedContext {\n    baseline?: string;\n    debug?: boolean;\n    list?: boolean;\n    overrides?: Record<string, any>;\n    axiomConfig?: ResolvedAxiomConfig;\n    runId: string;\n    consoleUrl?: string;\n  }\n}\n\nconst createVersionId = customAlphabet('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ', 10);\n\ntype RunTaskFailureDetails = {\n  duration: number;\n  outOfScopeFlags: OutOfScopeFlagAccess[];\n  finalFlags: Record<string, any>;\n  overrides?: Record<string, any>;\n};\n\nconst RUN_TASK_FAILURE_DETAILS = Symbol.for('axiom.eval.runTaskFailureDetails');\n\nfunction attachRunTaskFailureDetails(\n  error: unknown,\n  details: RunTaskFailureDetails,\n): Error & { [RUN_TASK_FAILURE_DETAILS]: RunTaskFailureDetails } {\n  const normalized = toError(error) as Error & {\n    [RUN_TASK_FAILURE_DETAILS]?: RunTaskFailureDetails;\n  };\n  normalized[RUN_TASK_FAILURE_DETAILS] = details;\n  return normalized as Error & { [RUN_TASK_FAILURE_DETAILS]: RunTaskFailureDetails };\n}\n\nfunction getRunTaskFailureDetails(error: unknown): RunTaskFailureDetails | undefined {\n  if (typeof error !== 'object' || error === null) {\n    return undefined;\n  }\n  return (error as { [RUN_TASK_FAILURE_DETAILS]?: RunTaskFailureDetails })[\n    RUN_TASK_FAILURE_DETAILS\n  ];\n}\n\n/**\n * Creates and registers an evaluation suite with the given name and parameters.\n *\n * This function sets up a complete evaluation pipeline that will run your {@link EvalTask}\n * against a collection, score the results, and provide detailed {@link EvalCaseReport} reporting.\n *\n *\n * @param name - Human-readable name for the evaluation suite\n * @param params - {@link EvalParams} configuration parameters for the evaluation\n *\n * @example\n * ```typescript\n * import { Eval } from 'axiom/ai/evals';\n *\n * Eval('Text Generation Quality', {\n *   capability: 'capability-name',\n *   data: async () => [\n *     { input: 'Explain photosynthesis', expected: 'Plants convert light to energy...' },\n *     { input: 'What is gravity?', expected: 'Gravity is a fundamental force...' }\n *   ],\n *   task: async ({ input }) => {\n *     const result = await generateText({\n *       model: yourModel,\n *       prompt: input\n *     });\n *     return result.text;\n *   },\n *   scorers: [similarityScorer, factualAccuracyScorer],\n * });\n * ```\n */\nexport function Eval<\n  TInput,\n  TExpected,\n  TOutput,\n  Name extends string = string,\n  Capability extends string = string,\n  Step extends string = string,\n>(\n  name: ValidateName<Name>,\n  params: Omit<EvalParams<TInput, TExpected, TOutput>, 'capability' | 'step'> & {\n    capability: ValidateName<Capability>;\n    step?: ValidateName<Step> | undefined;\n  },\n): void {\n  // Record eval name for validation\n  recordName('eval', name);\n  recordName('capability', params.capability);\n  if (params.step) {\n    recordName('step', params.step);\n  }\n\n  // Record all scorer names for validation\n  if (params.scorers) {\n    for (const scorer of params.scorers) {\n      const scorerName = getScorerName(scorer, '');\n      recordName('scorer', scorerName);\n    }\n  }\n\n  registerEval(name, params as EvalParams<any, any, any>).catch(console.error);\n}\n\n/**\n * Capture full flag configuration filtered by configFlags scope\n */\nfunction captureFlagConfig(configFlags?: string[]): Record<string, any> {\n  if (!configFlags || configFlags.length === 0) {\n    return {};\n  }\n\n  const scope = getConfigScope();\n  const allDefaults = scope?.getAllDefaultFlags?.() ?? {};\n  const overrides = getGlobalFlagOverrides();\n\n  const merged = { ...allDefaults, ...overrides };\n\n  // Filter to only flags in configFlags scope\n  const filtered: Record<string, any> = {};\n  for (const [key, value] of Object.entries(merged)) {\n    const isInScope = configFlags.some((pattern) => key.startsWith(pattern));\n    if (isInScope) {\n      filtered[key] = value;\n    }\n  }\n\n  return dotNotationToNested(filtered);\n}\n\nconst getScorerName = <TScorer extends ScorerLike<any, any, any>>(\n  scorer: TScorer,\n  fallback: string = 'unknown',\n) => {\n  return (scorer as any).name || fallback;\n};\n\nasync function registerEval<\n  TInput extends string | Record<string, any>,\n  TExpected extends string | Record<string, any>,\n  TOutput extends string | Record<string, any>,\n>(evalName: string, opts: EvalParams<TInput, TExpected, TOutput>) {\n  opts.data;\n  const collectionPromise:\n    | readonly CollectionRecord<TInput, TExpected>[]\n    | Promise<readonly CollectionRecord<TInput, TExpected>[]> =\n    typeof opts.data === 'function'\n      ? (\n          opts.data as () =>\n            | readonly CollectionRecord<TInput, TExpected>[]\n            | Promise<readonly CollectionRecord<TInput, TExpected>[]>\n        )()\n      : opts.data;\n  const user = getGitUserInfo();\n\n  // check if user passed a specific baseline id to the CLI\n  const baselineId = inject('baseline');\n  const isDebug = inject('debug');\n  const isList = inject('list');\n  const injectedOverrides = inject('overrides');\n  const axiomConfig = inject('axiomConfig');\n  const runId = inject('runId');\n  const consoleUrl = inject('consoleUrl');\n\n  if (!axiomConfig) {\n    throw new AxiomCLIError('Axiom config not found');\n  }\n\n  const timeoutMs = opts.timeout ?? axiomConfig?.eval.timeoutMs;\n\n  const instrumentationReady = ensureInstrumentationInitialized(axiomConfig, {\n    enabled: !isDebug && !isList,\n  });\n\n  const result = await describe(\n    evalName,\n    async () => {\n      const collection = await collectionPromise;\n\n      const evaluationApiClient = new EvaluationApiClient(axiomConfig, consoleUrl);\n\n      // create a version code\n      const evalVersion = createVersionId();\n      let evalId = ''; // get traceId\n      let suiteStart: number;\n\n      let suiteSpan: ReturnType<typeof startSpan> | undefined;\n      let suiteContext: Context | undefined;\n      let instrumentationError: unknown = undefined;\n      let baseline: Evaluation | null | undefined = undefined;\n\n      // Track out-of-scope flags across all cases for evaluation-level reporting\n      const allOutOfScopeFlags: OutOfScopeFlagAccess[] = [];\n\n      // Track final config snapshot from the last executed case for reporter printing\n      let finalConfigSnapshot:\n        | { flags: Record<string, any>; pickedFlags?: string[]; overrides?: Record<string, any> }\n        | undefined;\n\n      beforeAll(async (suite) => {\n        // Ensure worker process knows CLI overrides\n        if (injectedOverrides && Object.keys(injectedOverrides).length > 0) {\n          try {\n            setGlobalFlagOverrides(injectedOverrides);\n          } catch {}\n        }\n\n        suite.meta.evaluation = {\n          id: evalId,\n          name: evalName,\n          version: evalVersion,\n          runId: runId,\n          orgId: undefined,\n          baseline: baseline ?? undefined,\n          configFlags: opts.configFlags,\n        };\n\n        const [, instrumentationInitError] = await tryCatchAsync(instrumentationReady);\n        if (instrumentationInitError) {\n          instrumentationError = instrumentationInitError;\n        }\n\n        suiteSpan = startSpan(`eval ${evalName}-${evalVersion}`, {\n          attributes: {\n            [Attr.GenAI.Operation.Name]: 'eval',\n            [Attr.Eval.Name]: evalName,\n            [Attr.Eval.Version]: evalVersion,\n            [Attr.Eval.Type]: 'regression', // TODO: where to get experiment type value from?\n            [Attr.Eval.Tags]: [],\n            [Attr.Eval.Collection.ID]: 'custom', // TODO: where to get collection split value from?\n            [Attr.Eval.Collection.Name]: 'custom', // TODO: where to get collection name from?\n            [Attr.Eval.Collection.Size]: collection.length,\n            // capability\n            [Attr.Eval.Capability.Name]: opts.capability,\n            [Attr.Eval.Step.Name]: opts.step ?? undefined,\n            // metadata\n            [Attr.Eval.Metadata]: JSON.stringify(opts.metadata),\n            // run\n            [Attr.Eval.Run.ID]: runId,\n            // user info\n            [Attr.Eval.User.Name]: user?.name,\n            [Attr.Eval.User.Email]: user?.email,\n          },\n        });\n        evalId = suiteSpan.spanContext().traceId;\n        suite.meta.evaluation.id = evalId;\n        suiteSpan.setAttribute(Attr.Eval.ID, evalId);\n        suiteContext = trace.setSpan(context.active(), suiteSpan);\n\n        const flagConfig = captureFlagConfig(opts.configFlags);\n        suite.meta.evaluation.flagConfig = flagConfig;\n        const flagConfigJson = JSON.stringify(flagConfig);\n        suiteSpan.setAttribute(Attr.Eval.Config.Flags, flagConfigJson);\n\n        let createEvalResponse;\n        if (!isDebug && !isList) {\n          createEvalResponse = await evaluationApiClient.createEvaluation({\n            id: evalId,\n            name: evalName,\n            capability: opts.capability,\n            step: opts.step,\n            dataset: axiomConfig.eval.dataset,\n            version: evalVersion,\n            baselineId: baselineId ?? undefined,\n            runId: runId,\n            totalCases: collection.length,\n            config: { overrides: injectedOverrides },\n            configTimeoutMs: timeoutMs,\n            metadata: opts.metadata,\n            status: 'running',\n          });\n        }\n\n        const orgId = createEvalResponse?.data?.orgId;\n        const resolvedBaselineId = createEvalResponse?.data?.baselineId;\n\n        // Load baseline if we got a baselineId from the server\n        if (!isDebug && !isList && !!resolvedBaselineId) {\n          const [baselineResult, baselineError] = await tryCatchAsync(() =>\n            findEvaluationCases(resolvedBaselineId, axiomConfig),\n          );\n          if (baselineError) {\n            console.error(`Failed to load baseline: ${errorToString(baselineError)}`);\n            instrumentationError = instrumentationError || baselineError;\n          } else {\n            baseline = baselineResult;\n          }\n        }\n\n        // Update span with baseline info\n        if (baseline) {\n          suiteSpan.setAttribute(Attr.Eval.Baseline.ID, baseline.id);\n          suiteSpan.setAttribute(Attr.Eval.Baseline.Name, baseline.name);\n          suiteSpan.setAttribute(Attr.Eval.Baseline.Version, baseline.version);\n        }\n\n        suite.meta.evaluation = {\n          id: evalId,\n          name: evalName,\n          version: evalVersion,\n          runId: runId,\n          orgId: orgId ?? undefined,\n          baseline: baseline ?? undefined,\n          configFlags: opts.configFlags,\n          registrationStatus: instrumentationError\n            ? {\n                status: 'failed',\n                error: errorToString(instrumentationError),\n              }\n            : { status: 'success' },\n          trials: opts.trials,\n        };\n\n        suiteStart = performance.now();\n      });\n\n      afterAll(async (suite) => {\n        if (instrumentationError) {\n          throw instrumentationError;\n        }\n\n        const tags: string[] = ['offline'];\n        suiteSpan?.setAttribute(Attr.Eval.Tags, JSON.stringify(tags));\n\n        // Aggregate out-of-scope flags for evaluation-level reporting\n        const flagSummary = new Map<string, OutOfScopeFlag>();\n\n        for (const flag of allOutOfScopeFlags) {\n          if (flagSummary.has(flag.flagPath)) {\n            const existing = flagSummary.get(flag.flagPath)!;\n            existing.count++;\n            existing.firstAccessedAt = Math.min(existing.firstAccessedAt, flag.accessedAt);\n            existing.lastAccessedAt = Math.max(existing.lastAccessedAt, flag.accessedAt);\n          } else {\n            flagSummary.set(flag.flagPath, {\n              flagPath: flag.flagPath,\n              count: 1,\n              firstAccessedAt: flag.accessedAt,\n              lastAccessedAt: flag.accessedAt,\n              stackTrace: flag.stackTrace,\n            });\n          }\n        }\n\n        // Update evaluation report with aggregated out-of-scope flags\n        if (suite.meta.evaluation && suiteSpan) {\n          suite.meta.evaluation.outOfScopeFlags = Array.from(flagSummary.entries()).map(\n            ([_flagPath, stats]) => stats,\n          );\n\n          // Attach end-of-suite config snapshot for reporter printing\n          const allDefaults = getConfigScope()?.getAllDefaultFlags();\n          const pickedFlags = finalConfigSnapshot?.pickedFlags;\n          const overrides = injectedOverrides ?? getGlobalFlagOverrides();\n\n          suite.meta.evaluation.configEnd = {\n            flags: allDefaults,\n            pickedFlags,\n            overrides,\n          };\n        }\n\n        // end root span\n        suiteSpan?.setStatus({ code: SpanStatusCode.OK });\n        suiteSpan?.end();\n\n        // flush traces before updating Evaluation in Axiom\n        const [, flushError] = await tryCatchAsync(flush);\n        if (flushError) {\n          // Update registration status to failed if flush fails\n          if (suite.meta.evaluation) {\n            suite.meta.evaluation.registrationStatus = {\n              status: 'failed',\n              error: errorToString(flushError),\n            };\n          }\n        }\n\n        const durationMs = Math.round(performance.now() - suiteStart);\n\n        const successCases = suite.tasks.filter(\n          (task) => task.meta.case.status === 'success',\n        ).length;\n        const erroredCases = suite.tasks.filter(\n          (task) => task.meta.case.status === 'fail' || task.meta.case.status === 'pending',\n        ).length;\n\n        // signal Axiom that evaluation finished to kick of summary calculations\n        if (!isDebug && !isList) {\n          await evaluationApiClient.updateEvaluation({\n            id: evalId,\n            status: 'completed',\n            totalCases: collection.length,\n            successCases,\n            erroredCases,\n            durationMs,\n          });\n        }\n      });\n\n      type CollectionRecordWithIndex = { index: number } & CollectionRecord<TInput, TExpected>;\n\n      await it.concurrent.for(\n        collection.map((d, index) => ({ ...d, index }) satisfies CollectionRecordWithIndex),\n      )('case', async (data, { task }) => {\n        const start = performance.now();\n        if (!suiteContext) {\n          throw new Error(\n            '[Axiom AI] Suite context not initialized. This is likely a bug – instrumentation should complete before tests run.',\n          );\n        }\n\n        let outOfScopeFlags: OutOfScopeFlagAccess[] = [];\n\n        await startActiveSpan(\n          `case ${data.index}`,\n          {\n            attributes: {\n              [Attr.GenAI.Operation.Name]: 'eval.case',\n              [Attr.Eval.ID]: evalId,\n              [Attr.Eval.Name]: evalName,\n              [Attr.Eval.Version]: evalVersion,\n              [Attr.Eval.Case.Index]: data.index,\n              [Attr.Eval.Case.Input]:\n                typeof data.input === 'string' ? data.input : JSON.stringify(data.input),\n              [Attr.Eval.Case.Expected]:\n                typeof data.expected === 'string' ? data.expected : JSON.stringify(data.expected),\n              [Attr.Eval.Case.Metadata]: data.metadata ? JSON.stringify(data.metadata) : undefined,\n              // user info\n              [Attr.Eval.User.Name]: user?.name,\n              [Attr.Eval.User.Email]: user?.email,\n            },\n          },\n          async (caseSpan) => {\n            const caseContext = trace.setSpan(context.active(), caseSpan);\n            const trials = Math.max(1, opts.trials ?? 1);\n            let intentionalTrialFailureError: Error | undefined;\n            let caseFinalConfigSnapshot:\n              | {\n                  flags: Record<string, any>;\n                  pickedFlags?: string[];\n                  overrides?: Record<string, any>;\n                }\n              | undefined;\n\n            // Set case-level trials attribute\n            caseSpan.setAttribute(Attr.Eval.Case.Trials, trials);\n\n            try {\n              // Accumulators for per-trial scores\n              const perScorerTrials: Record<string, number[]> = {};\n              const trialErrors: Array<string | null> = Array.from({ length: trials }, () => null);\n              const trialFailures: Error[] = [];\n              let lastOutput: TOutput | undefined;\n              let successfulTaskDuration = 0;\n\n              // Run each trial\n              for (let trialIndex = 0; trialIndex < trials; trialIndex++) {\n                try {\n                  await startActiveSpan(\n                    `trial ${trialIndex}`,\n                    {\n                      attributes: {\n                        [Attr.GenAI.Operation.Name]: 'eval.trial',\n                        [Attr.Eval.Trial.Index]: trialIndex,\n                        [Attr.Eval.ID]: evalId,\n                        [Attr.Eval.Name]: evalName,\n                        [Attr.Eval.Version]: evalVersion,\n                      },\n                    },\n                    async (trialSpan) => {\n                      const trialContext = trace.setSpan(context.active(), trialSpan);\n                      try {\n                        const result = await runTask(\n                          trialContext,\n                          {\n                            id: evalId,\n                            version: evalVersion,\n                            name: evalName,\n                          },\n                          {\n                            index: data.index,\n                            input: data.input,\n                            expected: data.expected,\n                            scorers: opts.scorers,\n                            task: opts.task,\n                            metadata: opts.metadata,\n                            configFlags: opts.configFlags,\n                            capability: opts.capability,\n                            step: opts.step,\n                          },\n                        );\n                        const { output, duration } = result;\n                        lastOutput = output;\n                        successfulTaskDuration += duration;\n                        outOfScopeFlags.push(...result.outOfScopeFlags);\n\n                        caseFinalConfigSnapshot = {\n                          flags: result.finalFlags || {},\n                          pickedFlags: opts.configFlags,\n                          overrides: result.overrides,\n                        };\n\n                        // Run scorers inside the trial span\n                        await Promise.all(\n                          opts.scorers.map(async (scorer) => {\n                            const scorerName = getScorerName(scorer);\n                            return startActiveSpan(\n                              `score ${scorerName}`,\n                              {\n                                attributes: {\n                                  [Attr.GenAI.Operation.Name]: 'eval.score',\n                                  [Attr.Eval.ID]: evalId,\n                                  [Attr.Eval.Name]: evalName,\n                                  [Attr.Eval.Version]: evalVersion,\n                                  [Attr.Eval.Trial.Index]: trialIndex,\n                                },\n                              },\n                              async (scorerSpan) => {\n                                const scorerStart = performance.now();\n                                try {\n                                  const [result, scorerError] = await tryCatchAsync(() =>\n                                    scorer({\n                                      input: data.input,\n                                      output: output,\n                                      expected: data.expected,\n                                      trialIndex,\n                                    }),\n                                  );\n\n                                  if (scorerError || !result) {\n                                    const scorerDuration = Math.round(\n                                      performance.now() - scorerStart,\n                                    );\n                                    console.error(\n                                      `ERROR: scorer ${scorerName} failed. Cause: \\n`,\n                                      scorerError,\n                                    );\n                                    const msg = errorToString(scorerError);\n                                    const metadata = {\n                                      duration: scorerDuration,\n                                      startedAt: scorerStart,\n                                      error: msg,\n                                    };\n\n                                    // Count scorer failures as zero so failed trials affect aggregation.\n                                    (perScorerTrials[scorerName] ??= []).push(0);\n\n                                    scorerSpan.setAttributes({\n                                      [Attr.Eval.Score.Name]: scorerName,\n                                      [Attr.Eval.Score.Metadata]: JSON.stringify(metadata),\n                                    });\n\n                                    scorerSpan.setStatus({\n                                      code: SpanStatusCode.ERROR,\n                                      message: msg,\n                                    });\n                                    return;\n                                  }\n\n                                  const scoreDuration = Math.round(performance.now() - scorerStart);\n                                  const scoreValue = result.score as number;\n                                  const metadata = Object.assign(\n                                    { duration: scoreDuration, startedAt: scorerStart },\n                                    result.metadata,\n                                  );\n\n                                  // Collect per-trial score\n                                  (perScorerTrials[scorerName] ??= []).push(scoreValue);\n\n                                  // Get aggregation config for span attributes\n                                  const aggregation: Aggregation =\n                                    (scorer as Scorer).aggregation ?? Mean();\n\n                                  scorerSpan.setAttributes({\n                                    [Attr.Eval.Score.Name]: scorerName,\n                                    [Attr.Eval.Score.Value]: scoreValue,\n                                    [Attr.Eval.Score.Metadata]: JSON.stringify(metadata),\n                                    [Attr.Eval.Score.Aggregation]: aggregation.type,\n                                  });\n\n                                  if (metadata.error) {\n                                    const msg = errorToString(metadata.error);\n                                    scorerSpan.setStatus({\n                                      code: SpanStatusCode.ERROR,\n                                      message: msg,\n                                    });\n                                  }\n                                } finally {\n                                  scorerSpan.end();\n                                }\n                              },\n                              trialContext,\n                            );\n                          }),\n                        );\n                      } catch (error) {\n                        const taskFailureDetails = getRunTaskFailureDetails(error);\n                        const failure = toError(error);\n                        const msg = errorToString(failure);\n                        const spanErrorMessage = failure.message || msg;\n                        trialErrors[trialIndex] = msg;\n                        trialFailures.push(failure);\n\n                        trialSpan.setAttributes({\n                          [Attr.Eval.Trial.Error]: spanErrorMessage,\n                        });\n\n                        for (const scorer of opts.scorers) {\n                          const scorerName = getScorerName(scorer);\n                          (perScorerTrials[scorerName] ??= []).push(0);\n                        }\n\n                        if (taskFailureDetails) {\n                          outOfScopeFlags.push(...taskFailureDetails.outOfScopeFlags);\n                          caseFinalConfigSnapshot = {\n                            flags: taskFailureDetails.finalFlags || {},\n                            pickedFlags: opts.configFlags,\n                            overrides: taskFailureDetails.overrides,\n                          };\n                        }\n\n                        // Re-throw so startActiveSpan records the trial span as ERROR.\n                        throw failure;\n                      }\n                    },\n                    caseContext,\n                  );\n                } catch {\n                  // Continue remaining trials after task-level failures.\n                }\n              }\n\n              // Aggregate scores across trials\n              const scores: Record<string, ScoreWithName> = {};\n              for (const scorer of opts.scorers) {\n                const scorerName = getScorerName(scorer);\n                const trialsArr = perScorerTrials[scorerName] ?? [];\n                const aggregation: Aggregation = (scorer as Scorer).aggregation ?? Mean();\n\n                const aggregatedValue = trialsArr.length > 0 ? aggregation.aggregate(trialsArr) : 0;\n\n                scores[scorerName] = {\n                  name: scorerName,\n                  score: aggregatedValue,\n                  trials: trialsArr,\n                  aggregation: aggregation.type,\n                  threshold: aggregation.threshold,\n                  metadata: {},\n                };\n              }\n\n              const output = lastOutput;\n              const failedTrials = trialFailures.length;\n              const succeededTrials = trials - failedTrials;\n              const trialSummary = {\n                total: trials,\n                succeeded: succeededTrials,\n                failed: failedTrials,\n              };\n\n              caseSpan.setAttribute(Attr.Eval.Case.Scores, JSON.stringify(scores ? scores : {}));\n              if (output !== undefined) {\n                caseSpan.setAttribute(\n                  Attr.Eval.Case.Output,\n                  typeof output === 'string' ? output : JSON.stringify(output),\n                );\n              }\n\n              // set task meta for showing result in vitest report\n              task.meta.case = {\n                index: data.index,\n                name: evalName,\n                expected: data.expected,\n                input: data.input,\n                output: output,\n                metadata: data.metadata,\n                scores,\n                status: 'success',\n                errors: [],\n                trialErrors,\n                trialSummary,\n                duration: successfulTaskDuration,\n                startedAt: start,\n                outOfScopeFlags,\n                pickedFlags: opts.configFlags,\n              };\n\n              if (failedTrials > 0) {\n                const error = new Error(\n                  `Eval case ${data.index} failed with ${failedTrials} trial error(s).`,\n                );\n                intentionalTrialFailureError = error;\n                caseSpan.setStatus({\n                  code: SpanStatusCode.ERROR,\n                  message: error.message,\n                });\n                task.meta.case.status = 'fail';\n                task.meta.case.errors = trialFailures;\n                throw error;\n              }\n\n              // Collect out-of-scope flags for evaluation-level aggregation\n              allOutOfScopeFlags.push(...outOfScopeFlags);\n            } catch (e) {\n              console.log(e);\n              const error = toError(e);\n\n              if (e === intentionalTrialFailureError && task.meta.case) {\n                task.meta.case.status = 'fail';\n                task.meta.case.errors = task.meta.case.errors?.length\n                  ? task.meta.case.errors\n                  : [error];\n                allOutOfScopeFlags.push(...outOfScopeFlags);\n                throw e;\n              }\n\n              const ctx = getEvalContext();\n              const ctxOutOfScopeFlags = ctx.outOfScopeFlags || [];\n              if (ctxOutOfScopeFlags.length > 0) {\n                outOfScopeFlags.push(...ctxOutOfScopeFlags);\n              }\n\n              const ctxFlags = ctx.flags || {};\n              if (!caseFinalConfigSnapshot && Object.keys(ctxFlags).length > 0) {\n                caseFinalConfigSnapshot = {\n                  flags: ctxFlags,\n                  pickedFlags: opts.configFlags,\n                  overrides: ctx.overrides,\n                };\n              }\n\n              // Populate scores with error metadata for all scorers that didn't run\n              const failedScores: Record<string, ScoreWithName> = {};\n              for (const scorer of opts.scorers) {\n                const scorerName = getScorerName(scorer);\n                failedScores[scorerName] = {\n                  name: scorerName,\n                  score: 0,\n                  trials: [],\n                  metadata: {\n                    duration: 0,\n                    startedAt: start,\n                    error: error.message,\n                  },\n                };\n              }\n\n              task.meta.case = {\n                name: evalName,\n                index: data.index,\n                expected: data.expected,\n                input: data.input,\n                output: String(e),\n                metadata: data.metadata,\n                scores: failedScores,\n                status: 'fail',\n                errors: [error],\n                startedAt: start,\n                duration: Math.round(performance.now() - start),\n                outOfScopeFlags,\n                pickedFlags: opts.configFlags,\n              };\n\n              allOutOfScopeFlags.push(...outOfScopeFlags);\n              throw e;\n            } finally {\n              // Compute per-case runtime flags report and attach to span/meta\n              try {\n                const accessedFlags: Record<string, any> = caseFinalConfigSnapshot?.flags || {};\n\n                const accessed = Object.keys(accessedFlags);\n                const allDefaults = getConfigScope()?.getAllDefaultFlags?.() ?? {};\n\n                const runtimeFlags: Record<string, RuntimeFlagLog> = {};\n                for (const key of accessed) {\n                  const value = accessedFlags[key];\n                  if (key in allDefaults) {\n                    const replaced = !deepEqual(value, allDefaults[key]);\n                    if (replaced) {\n                      runtimeFlags[key] = { kind: 'replaced', value, default: allDefaults[key] };\n                    }\n                  } else {\n                    runtimeFlags[key] = { kind: 'introduced', value };\n                  }\n                }\n\n                if (!isDebug && Object.keys(runtimeFlags).length > 0) {\n                  const serialized = JSON.stringify(runtimeFlags);\n                  caseSpan.setAttribute('eval.case.config.runtime_flags', serialized);\n                }\n\n                if (task.meta.case) {\n                  task.meta.case.runtimeFlags = runtimeFlags;\n                }\n              } catch {}\n\n              if (caseFinalConfigSnapshot) {\n                finalConfigSnapshot = caseFinalConfigSnapshot;\n              }\n            }\n          },\n          suiteContext,\n        );\n      });\n    },\n    timeoutMs,\n  );\n\n  return result;\n}\n\nconst joinArrayOfUnknownResults = <T extends string | Record<string, any>>(results: T[]): T => {\n  if (results.length === 0) {\n    return '' as unknown as T;\n  }\n\n  // If all results are strings, concatenate them\n  if (results.every((r) => typeof r === 'string')) {\n    return results.join('') as unknown as T;\n  }\n\n  // If we have objects, return the last one (streaming typically overwrites)\n  return results[results.length - 1];\n};\n\nconst executeTask = async <\n  TInput extends string | Record<string, any>,\n  TExpected extends string | Record<string, any>,\n  TOutput extends string | Record<string, any>,\n>(\n  task: EvalTask<TInput, TExpected, TOutput>,\n  input: TInput,\n  expected: TExpected,\n): Promise<TOutput> => {\n  const taskResultOrStream = await task({ input, expected });\n\n  if (\n    typeof taskResultOrStream === 'object' &&\n    taskResultOrStream &&\n    Symbol.asyncIterator in taskResultOrStream\n  ) {\n    const chunks: TOutput[] = [];\n\n    for await (const chunk of taskResultOrStream) {\n      chunks.push(chunk);\n    }\n\n    return joinArrayOfUnknownResults<TOutput>(chunks as TOutput[]);\n  }\n\n  return taskResultOrStream;\n};\n\nconst runTask = async <\n  TInput extends string | Record<string, any>,\n  TExpected extends string | Record<string, any>,\n  TOutput extends string | Record<string, any>,\n>(\n  caseContext: Context,\n  evaluation: {\n    id: string;\n    name: string;\n    version: string;\n  },\n  opts: {\n    index: number;\n    input: TInput;\n    expected: TExpected | undefined;\n  } & Omit<EvalParams<TInput, TExpected, TOutput>, 'data'>,\n) => {\n  const taskName = opts.task.name ?? 'anonymous';\n\n  return startActiveSpan(\n    `task`,\n    {\n      attributes: {\n        [Attr.GenAI.Operation.Name]: 'eval.task',\n        [Attr.Eval.Task.Name]: taskName,\n        [Attr.Eval.Task.Type]: 'llm_completion', // TODO: How to determine task type?\n        [Attr.Eval.ID]: evaluation.id,\n        [Attr.Eval.Name]: evaluation.name,\n        [Attr.Eval.Version]: evaluation.version,\n      },\n    },\n    async (taskSpan) => {\n      // Initialize evaluation context for flag/fact access\n      const { output, duration, outOfScopeFlags, finalFlags, overrides } = await withEvalContext(\n        { pickedFlags: opts.configFlags },\n        async (): Promise<{\n          output: TOutput;\n          duration: number;\n          outOfScopeFlags: OutOfScopeFlagAccess[];\n          finalFlags: Record<string, any>;\n          overrides?: Record<string, any>;\n        }> => {\n          // TODO: EXPERIMENTS - before we were setting config scope if provided here\n          const start = performance.now();\n          try {\n            const output = await executeTask(opts.task, opts.input, opts.expected!);\n            const duration = Math.round(performance.now() - start);\n            // set task output\n            taskSpan.setAttributes({\n              [Attr.Eval.Task.Output]: typeof output === 'string' ? output : JSON.stringify(output),\n            });\n\n            // Get out-of-scope flags from the evaluation context\n            const ctx = getEvalContext();\n            const outOfScopeFlags = ctx.outOfScopeFlags || [];\n\n            return {\n              output,\n              duration,\n              outOfScopeFlags,\n              finalFlags: ctx.flags || {},\n              overrides: ctx.overrides,\n            };\n          } catch (error) {\n            const ctx = getEvalContext();\n            const duration = Math.round(performance.now() - start);\n            throw attachRunTaskFailureDetails(error, {\n              duration,\n              outOfScopeFlags: ctx.outOfScopeFlags || [],\n              finalFlags: ctx.flags || {},\n              overrides: ctx.overrides,\n            });\n          }\n        },\n      );\n\n      return {\n        output,\n        duration,\n        outOfScopeFlags,\n        finalFlags,\n        overrides,\n      };\n    },\n    caseContext,\n  );\n};\n","import { execSync } from 'node:child_process';\n\nexport function getGitUserInfo() {\n  try {\n    const name = execSync('git config --get user.name').toString().trim();\n    const email = execSync('git config --get user.email').toString().trim();\n    return { name, email };\n  } catch {\n    return null; // Git not installed or not configured\n  }\n}\n","import type { Case, Chat, Evaluation, Task } from './eval.types';\nimport { createFetcher, type Fetcher } from '../utils/fetcher';\nimport type { ResolvedAxiomConfig } from '../config/index';\nimport { resolveAxiomConnection } from '../config/resolver';\nimport { Attr } from '../otel';\nimport { AxiomCLIError } from '../util/errors';\nimport {\n  getCustomOrRegularAttribute,\n  getCustomOrRegularNumber,\n  getCustomOrRegularString,\n} from '../util/traces';\n\nexport interface EvaluationApiConfig {\n  dataset?: string;\n  region?: string;\n  baseUrl?: string;\n  apiUrl?: string;\n  token?: string;\n}\n\nexport type EvaluationStatus = 'running' | 'completed' | 'errored' | 'cancelled';\n\nexport interface EvaluationApiPayloadBase {\n  id: string;\n  name: string;\n  capability: string;\n  step?: string | undefined;\n  dataset: string;\n  baselineId?: string;\n  totalCases?: number;\n  config?: Record<string, unknown>;\n  status: EvaluationStatus;\n  successCases?: number;\n  erroredCases?: number;\n  durationMs?: number;\n  scorerAvgs?: number[];\n  version: string;\n  runId: string;\n  configTimeoutMs: number;\n  metadata?: Record<string, any>;\n}\n\nexport class EvaluationApiClient {\n  private readonly fetcher: Fetcher;\n  constructor(config: ResolvedAxiomConfig, consoleUrl?: string) {\n    const { consoleEndpointUrl, token, orgId } = resolveAxiomConnection(config, consoleUrl);\n\n    this.fetcher = createFetcher({ baseUrl: consoleEndpointUrl, token: token ?? '', orgId });\n  }\n\n  async createEvaluation(evaluation: EvaluationApiPayloadBase) {\n    const resp = await this.fetcher(`/api/v3/evaluations`, {\n      method: 'POST',\n      body: JSON.stringify(evaluation),\n    });\n\n    if (!resp.ok) {\n      const text = await resp.text().catch(() => '');\n      throw new AxiomCLIError(\n        `Failed to create evaluation: ${resp.statusText}${text ? ` - ${text}` : ''}`,\n      );\n    }\n\n    return resp.json();\n  }\n\n  async updateEvaluation(evaluation: Partial<EvaluationApiPayloadBase>) {\n    const resp = await this.fetcher(`/api/v3/evaluations/${evaluation.id}`, {\n      method: 'PATCH',\n      body: JSON.stringify(evaluation),\n    });\n\n    if (!resp.ok) {\n      const text = await resp.text().catch(() => '');\n      throw new AxiomCLIError(\n        `Failed to update evaluation: ${resp.statusText}${text ? ` - ${text}` : ''}`,\n      );\n    }\n\n    // API may return HTTP 200 with an error in the response body\n    const body = await resp.json();\n    if (body.error) {\n      throw new AxiomCLIError(\n        `Failed to update evaluation ${evaluation.id}: ${JSON.stringify(body.error)}`,\n      );\n    }\n\n    return body;\n  }\n}\n\nexport const findEvaluationCases = async (\n  evalId: string,\n  config: ResolvedAxiomConfig,\n): Promise<Evaluation | null> => {\n  const { dataset, edgeUrl, token, orgId } = resolveAxiomConnection(config);\n\n  const apl = `['${dataset}'] | where trace_id == \"${evalId}\" | order by _time`;\n\n  const headers = new Headers({\n    Authorization: `Bearer ${token}`,\n    'Content-Type': 'application/json',\n    ...(orgId ? { 'X-AXIOM-ORG-ID': orgId } : {}),\n  });\n\n  // Use edgeUrl for query operations\n  const resp = await fetch(`${edgeUrl}/v1/datasets/_apl?format=legacy`, {\n    headers: headers,\n    method: 'POST',\n    body: JSON.stringify({ apl }),\n  });\n  const payload = await resp.json();\n\n  if (!resp.ok) {\n    throw new Error(`Failed to query evaluation cases: ${payload.message || resp.statusText}`);\n  }\n\n  return payload.matches.length ? buildSpanTree(payload.matches) : null;\n};\n\ntype DeepPartial<T> = T extends object ? { [P in keyof T]?: DeepPartial<T[P]> } : T;\n\nexport const mapSpanToEval = (span: any): Evaluation => {\n  const flagConfigRaw = getCustomOrRegularAttribute(span.data.attributes, Attr.Eval.Config.Flags);\n  const tagsRaw = getCustomOrRegularAttribute(span.data.attributes, Attr.Eval.Tags);\n\n  const evaluation: DeepPartial<Evaluation> = {\n    id: getCustomOrRegularString(span.data.attributes, Attr.Eval.ID),\n    name: getCustomOrRegularString(span.data.attributes, Attr.Eval.Name),\n    type: getCustomOrRegularString(span.data.attributes, Attr.Eval.Type),\n    version: getCustomOrRegularString(span.data.attributes, Attr.Eval.Version),\n    collection: {\n      name: getCustomOrRegularString(span.data.attributes, Attr.Eval.Collection.Name),\n      size: getCustomOrRegularNumber(span.data.attributes, Attr.Eval.Collection.Size),\n    },\n    baseline: {\n      id: getCustomOrRegularString(span.data.attributes, Attr.Eval.Baseline.ID),\n      name: getCustomOrRegularString(span.data.attributes, Attr.Eval.Baseline.Name),\n    },\n    duration: span.data.duration,\n    status: span.data.status.code,\n    traceId: span.data.trace_id,\n    runAt: span._time,\n    tags: tagsRaw ? (typeof tagsRaw === 'string' ? JSON.parse(tagsRaw) : tagsRaw) : [],\n    user: {\n      name: getCustomOrRegularString(span.data.attributes, Attr.Eval.User.Name),\n      email: getCustomOrRegularString(span.data.attributes, Attr.Eval.User.Email),\n    },\n    cases: [],\n    flagConfig: flagConfigRaw\n      ? typeof flagConfigRaw === 'string'\n        ? JSON.parse(flagConfigRaw)\n        : flagConfigRaw\n      : undefined,\n  };\n\n  // TODO: this is very optimistic!\n  return evaluation as Evaluation;\n};\n\nexport const mapSpanToCase = (item: { _time: string; data: any }): Case => {\n  const data = item.data;\n  // round duration\n  const d = data.duration as string;\n  let duration = '-';\n  if (d.endsWith('s')) {\n    duration = `${Number(d.replace('s', '')).toFixed(2)}s`;\n  } else {\n    duration = d;\n  }\n\n  const scoresRaw = getCustomOrRegularAttribute(data.attributes, Attr.Eval.Case.Scores);\n  const scoresParsed = scoresRaw\n    ? typeof scoresRaw === 'string'\n      ? JSON.parse(scoresRaw)\n      : scoresRaw\n    : {};\n\n  // Normalize scores: convert .score to .value if needed (ScoreWithName uses .score, Case uses .value)\n  // FUTURE: find a better way of handling this\n  const scores: Case['scores'] = {};\n  for (const [name, scoreData] of Object.entries(scoresParsed)) {\n    const s = scoreData as { score?: number; value?: number; metadata?: Record<string, any> };\n    scores[name] = {\n      name,\n      value: s.value ?? s.score ?? 0,\n      metadata: s.metadata ?? {},\n    };\n  }\n\n  const caseData: DeepPartial<Case> = {\n    index: getCustomOrRegularNumber(data.attributes, Attr.Eval.Case.Index),\n    input: getCustomOrRegularString(data.attributes, Attr.Eval.Case.Input),\n    output: getCustomOrRegularString(data.attributes, Attr.Eval.Case.Output),\n    expected: getCustomOrRegularString(data.attributes, Attr.Eval.Case.Expected),\n    duration: duration,\n    status: data.status.code,\n    scores,\n    runAt: item._time,\n    spanId: data.span_id,\n    traceId: data.trace_id,\n  };\n\n  // TODO: this is very optimistic!\n  return caseData as Case;\n};\n\n// compute a root eval with its children spans, results in a usable object of eval, cases, scores and chats\nexport const buildSpanTree = (spans: any[]): Evaluation | null => {\n  if (!spans.length) {\n    return null;\n  }\n\n  // Find the root eval span\n  const evalSpan = spans.find((span) => span.data.attributes.gen_ai.operation.name === 'eval');\n\n  if (!evalSpan) {\n    return null;\n  }\n\n  // Create the root eval structure\n  const rootSpan: Evaluation = mapSpanToEval(evalSpan);\n\n  // Find all case spans and build the tree structure\n  const caseSpans = spans.filter((span) => span.data.name.startsWith('case'));\n\n  for (const caseSpan of caseSpans) {\n    // Convert case data\n    const caseData = mapSpanToCase(caseSpan);\n\n    // Find task spans that belong to this case\n    const taskSpans = spans.filter(\n      (span) =>\n        span.data.name.startsWith('task') && span.data.parent_span_id === caseSpan.data.span_id,\n    );\n\n    if (taskSpans.length > 0) {\n      const taskSpan = taskSpans[0]; // Assuming one task per case\n\n      // Find chat spans that belong to this task\n      const chatSpans = spans.filter(\n        (span) =>\n          span.data.name.startsWith('chat') && span.data.parent_span_id === taskSpan.data.span_id,\n      );\n\n      const chatData: Chat[] = chatSpans.map((chatSpan) => ({\n        operation: getCustomOrRegularString(chatSpan.data.attributes, 'operation') ?? '',\n        capability: getCustomOrRegularString(chatSpan.data.attributes, 'capability') ?? '',\n        step: getCustomOrRegularString(chatSpan.data.attributes, 'step') ?? '',\n        request: {\n          max_token: getCustomOrRegularString(chatSpan.data.attributes, 'request.max_token') ?? '',\n          model: getCustomOrRegularString(chatSpan.data.attributes, 'request.model') ?? '',\n          temperature:\n            getCustomOrRegularNumber(chatSpan.data.attributes, 'request.temperature') ?? 0,\n        },\n        response: {\n          finish_reasons:\n            getCustomOrRegularString(chatSpan.data.attributes, 'response.finish_reasons') ?? '',\n        },\n        usage: {\n          input_tokens:\n            getCustomOrRegularNumber(chatSpan.data.attributes, 'usage.input_tokens') ?? 0,\n          output_tokens:\n            getCustomOrRegularNumber(chatSpan.data.attributes, 'usage.output_tokens') ?? 0,\n        },\n      }));\n\n      // Create task data with chat information\n      const taskData: Task = {\n        name: taskSpan.data.name,\n        output: getCustomOrRegularString(taskSpan.data.attributes, 'output') || '',\n        trial: getCustomOrRegularNumber(taskSpan.data.attributes, 'trial') || 0,\n        type: getCustomOrRegularString(taskSpan.data.attributes, 'type') || '',\n        error: getCustomOrRegularString(taskSpan.data.attributes, 'error') || '',\n        chat: chatData[0] || {\n          operation: '',\n          capability: '',\n          step: '',\n          request: { max_token: '', model: '', temperature: 0 },\n          response: { finish_reasons: '' },\n          usage: { input_tokens: 0, output_tokens: 0 },\n        },\n      };\n\n      caseData.task = taskData;\n    }\n\n    // Find task spans that belong to this case\n    const scoreSpans = spans.filter(\n      (span) =>\n        span.data.attributes.gen_ai.operation.name === 'eval.score' &&\n        span.data.parent_span_id === caseSpan.data.span_id,\n    );\n\n    if (scoreSpans.length > 0) {\n      caseData.scores = {};\n\n      scoreSpans.forEach((score) => {\n        const name = getCustomOrRegularString(score.data.attributes, Attr.Eval.Score.Name) ?? '';\n        const value = getCustomOrRegularNumber(score.data.attributes, Attr.Eval.Score.Value) ?? 0;\n        const metadataRaw = getCustomOrRegularString(\n          score.data.attributes,\n          Attr.Eval.Score.Metadata,\n        );\n        let metadata = {};\n        try {\n          metadata = metadataRaw ? JSON.parse(metadataRaw) : {};\n        } catch {\n          // Ignore error\n        }\n\n        caseData.scores[name] = {\n          name,\n          value,\n          metadata: {\n            error: score.data.attributes.error,\n            ...metadata,\n          },\n        };\n      });\n    }\n\n    rootSpan.cases.push(caseData);\n  }\n\n  rootSpan.cases.sort((a, b) => a.index - b.index);\n\n  return rootSpan;\n};\n","export interface Fetcher {\n  (path: string, options: RequestInit): Promise<Response>;\n}\n\nexport interface FetcherOptions {\n  baseUrl: string;\n  token: string;\n  orgId?: string;\n}\n\nexport const createFetcher = ({\n  baseUrl,\n  token,\n  orgId,\n}: {\n  baseUrl: string;\n  token: string;\n  orgId?: string;\n}): Fetcher => {\n  return (path: string, options: RequestInit) =>\n    fetch(new URL(path, baseUrl).toString(), {\n      ...options,\n      headers: {\n        ...options.headers,\n        'content-type': 'application/json',\n        authorization: `Bearer ${token}`,\n        ...(orgId ? { 'X-AXIOM-ORG-ID': orgId } : {}),\n      },\n    });\n};\n","export function getCustomOrRegularAttribute(obj: unknown, accessKey: string): unknown {\n  if (typeof obj !== 'object' || obj === null) {\n    return undefined;\n  }\n\n  const keyParts = accessKey.split('.');\n  const custom = (obj as Record<string, any>).custom;\n\n  // Try `obj.custom['foo.bar']`\n  if (custom && typeof custom === 'object' && custom !== null && accessKey in custom) {\n    return custom[accessKey];\n  }\n\n  // Try `obj.foo.bar`\n  let current: any = obj;\n  for (const part of keyParts) {\n    if (typeof current !== 'object' || current === null) {\n      return undefined;\n    }\n    current = current[part];\n  }\n\n  return current;\n}\n\nexport function getCustomOrRegularString(obj: unknown, key: string): string | undefined {\n  const value = getCustomOrRegularAttribute(obj, key);\n\n  return typeof value === 'string' ? value : undefined;\n}\n\nexport function getCustomOrRegularNumber(obj: unknown, key: string): number | undefined {\n  const value = getCustomOrRegularAttribute(obj, key);\n\n  if (typeof value === 'number') {\n    return value;\n  }\n\n  if (typeof value === 'string') {\n    const parsed = Number(value);\n\n    return Number.isNaN(parsed) ? undefined : parsed;\n  }\n\n  return undefined;\n}\n","/**\n * Copied from Remeda - https://github.com/remeda/remeda/blob/v2.32.0/packages/remeda/src/isDeepEqual.ts\n */\n\nexport function deepEqual<T>(data: unknown, other: T): data is T {\n  if (data === other) {\n    return true;\n  }\n\n  if (Object.is(data, other)) {\n    // We want to ignore the slight differences between `===` and `Object.is` as\n    // both of them largely define equality from a semantic point-of-view.\n    return true;\n  }\n\n  if (typeof data !== 'object' || typeof other !== 'object') {\n    return false;\n  }\n\n  if (data === null || other === null) {\n    return false;\n  }\n\n  if (Object.getPrototypeOf(data) !== Object.getPrototypeOf(other)) {\n    // If the objects don't share a prototype it's unlikely that they are\n    // semantically equal. It is technically possible to build 2 prototypes that\n    // act the same but are not equal (at the reference level, checked via\n    // `===`) and then create 2 objects that are equal although we would fail on\n    // them. Because this is so unlikely, the optimization we gain here for the\n    // rest of the function by assuming that `other` is of the same type as\n    // `data` is more than worth it.\n    return false;\n  }\n\n  if (Array.isArray(data)) {\n    return isDeepEqualArrays(data, other as unknown as ReadonlyArray<unknown>);\n  }\n\n  if (data instanceof Map) {\n    return isDeepEqualMaps(data, other as unknown as Map<unknown, unknown>);\n  }\n\n  if (data instanceof Set) {\n    return isDeepEqualSets(data, other as unknown as Set<unknown>);\n  }\n\n  if (data instanceof Date) {\n    return data.getTime() === (other as unknown as Date).getTime();\n  }\n\n  if (data instanceof RegExp) {\n    return data.toString() === (other as unknown as RegExp).toString();\n  }\n\n  // At this point we only know that the 2 objects share a prototype and are not\n  // any of the previous types. They could be plain objects (Object.prototype),\n  // they could be classes, they could be other built-ins, or they could be\n  // something weird. We assume that comparing values by keys is enough to judge\n  // their equality.\n\n  if (Object.keys(data).length !== Object.keys(other).length) {\n    return false;\n  }\n\n  for (const [key, value] of Object.entries(data)) {\n    if (!(key in other)) {\n      return false;\n    }\n\n    if (\n      !deepEqual(\n        value,\n        // @ts-expect-error [ts7053] - We already checked that `other` has `key`\n        other[key],\n      )\n    ) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction isDeepEqualArrays(data: ReadonlyArray<unknown>, other: ReadonlyArray<unknown>): boolean {\n  if (data.length !== other.length) {\n    return false;\n  }\n\n  for (const [index, item] of data.entries()) {\n    if (!deepEqual(item, other[index])) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction isDeepEqualMaps(\n  data: ReadonlyMap<unknown, unknown>,\n  other: ReadonlyMap<unknown, unknown>,\n): boolean {\n  if (data.size !== other.size) {\n    return false;\n  }\n\n  for (const [key, value] of data.entries()) {\n    if (!other.has(key)) {\n      return false;\n    }\n\n    if (!deepEqual(value, other.get(key))) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction isDeepEqualSets(data: ReadonlySet<unknown>, other: ReadonlySet<unknown>): boolean {\n  if (data.size !== other.size) {\n    return false;\n  }\n\n  // To ensure we only count each item once we need to \"remember\" which items of\n  // the other set we've already matched against. We do this by creating a copy\n  // of the other set and removing items from it as we find them in the data\n  // set.\n  const otherCopy = [...other];\n\n  for (const dataItem of data) {\n    let isFound = false;\n\n    for (const [index, otherItem] of otherCopy.entries()) {\n      if (deepEqual(dataItem, otherItem)) {\n        isFound = true;\n        otherCopy.splice(index, 1);\n        break;\n      }\n    }\n\n    if (!isFound) {\n      return false;\n    }\n  }\n\n  return true;\n}\n","import { errorToString } from './errors';\n\nexport type TryCatchResult<T, E extends Error = Error> =\n  | [data: T, error: null]\n  | [data: null, error: E | Error];\n\nexport function toError(rawError: unknown, operationName?: string): Error {\n  const processedError = rawError instanceof Error ? rawError : new Error(errorToString(rawError));\n\n  if (operationName) {\n    processedError.message = `Operation \"${operationName}\" failed: ${processedError.message}`;\n  }\n\n  return processedError;\n}\n\nexport function tryCatchSync<T, E extends Error = Error>(\n  fn: () => T,\n  operationName?: string,\n): TryCatchResult<T, E> {\n  try {\n    return [fn(), null];\n  } catch (rawError) {\n    return [null, toError(rawError, operationName)];\n  }\n}\n\nexport async function tryCatchAsync<T, E extends Error = Error>(\n  fn: Promise<T> | (() => T | Promise<T>),\n  operationName?: string,\n): Promise<TryCatchResult<Awaited<T>, E>> {\n  try {\n    const result = typeof fn === 'function' ? fn() : fn;\n    return [await result, null];\n  } catch (rawError) {\n    return [null, toError(rawError, operationName)];\n  }\n}\n\ntype TryCatch = {\n  <T, E extends Error = Error>(\n    fn: Promise<T> | (() => T | Promise<T>),\n    operationName?: string,\n  ): Promise<TryCatchResult<Awaited<T>, E>>;\n  <T, E extends Error = Error>(fn: () => T, operationName?: string): TryCatchResult<T, E>;\n  sync: typeof tryCatchSync;\n  async: typeof tryCatchAsync;\n};\n\nexport const tryCatch = ((fn: unknown, operationName?: string) => {\n  if (typeof fn === 'function') {\n    try {\n      const result = (fn as () => unknown)();\n      if (result instanceof Promise) {\n        return tryCatchAsync(result, operationName);\n      }\n      return [result, null];\n    } catch (rawError) {\n      return [null, toError(rawError, operationName)];\n    }\n  }\n\n  if (fn instanceof Promise) {\n    return tryCatchAsync(fn, operationName);\n  }\n\n  return [fn, null];\n}) as TryCatch;\n\ntryCatch.sync = tryCatchSync;\ntryCatch.async = tryCatchAsync;\n","import type { Aggregation } from './aggregations';\n\nexport type Score<TMetadata extends Record<string, any> = Record<string, any>> = {\n  score: number | boolean | null;\n  metadata?: TMetadata;\n};\n\n// Internal type used when scorer returns Score with name\nexport type ScoreWithName<TMetadata extends Record<string, any> = Record<string, any>> =\n  Score<TMetadata> & {\n    name: string;\n    /** Per-trial scores when running multiple trials */\n    trials?: number[];\n    /** Aggregation type used (e.g., 'mean', 'pass@k') */\n    aggregation?: string;\n    /** Threshold for pass-based aggregations */\n    threshold?: number;\n  };\n\n/**\n * Configuration options for a scorer.\n */\nexport type ScorerOptions = {\n  /**\n   * Aggregation function for combining scores across multiple trials.\n   * Defaults to Mean() if not specified.\n   */\n  aggregation?: Aggregation;\n};\n\n// Loose type - this is what we REQUIRE\n// (we accept scorers with looser requirements than how we define them)\nexport type ScorerLike<\n  TInput = any,\n  TExpected = any,\n  TOutput = any,\n  TExtra extends Record<string, any> = {},\n  TMetadata extends Record<string, any> = Record<string, any>,\n  TReturn extends Score<TMetadata> | Promise<Score<TMetadata>> =\n    | Score<TMetadata>\n    | Promise<Score<TMetadata>>,\n> = (\n  args: {\n    input?: TInput;\n    expected?: TExpected;\n    output: TOutput;\n    /** Current trial index (0-based) when running multiple trials */\n    trialIndex?: number;\n  } & TExtra,\n) => TReturn;\n\n// Strict type - this is how we DEFINE scorers\nexport type Scorer<\n  TInput = any,\n  TExpected = any,\n  TOutput = any,\n  TExtra extends Record<string, any> = {},\n  TMetadata extends Record<string, any> = Record<string, any>,\n  TReturn extends Score<TMetadata> | Promise<Score<TMetadata>> =\n    | Score<TMetadata>\n    | Promise<Score<TMetadata>>,\n> = ScorerLike<TInput, TExpected, TOutput, TExtra, TMetadata, TReturn> & {\n  readonly name: string; // Name property for telemetry\n  readonly aggregation?: Aggregation; // Aggregation config for trials\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;ACAA;AAAA,SAAS,UAAU,WAAW,UAAU,QAAQ,UAAU;AAC1D,SAAS,SAAS,gBAAgB,aAA2B;AAC7D,SAAS,sBAAsB;;;ACF/B;AAAA,SAAS,gBAAgB;AAElB,SAAS,iBAAiB;AAC/B,MAAI;AACF,UAAM,OAAO,SAAS,4BAA4B,EAAE,SAAS,EAAE,KAAK;AACpE,UAAM,QAAQ,SAAS,6BAA6B,EAAE,SAAS,EAAE,KAAK;AACtE,WAAO,EAAE,MAAM,MAAM;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;;;ACVA;;;ACAA;AAUO,IAAM,gBAAgB,CAAC;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AACF,MAIe;AACb,SAAO,CAAC,MAAc,YACpB,MAAM,IAAI,IAAI,MAAM,OAAO,EAAE,SAAS,GAAG;AAAA,IACvC,GAAG;AAAA,IACH,SAAS;AAAA,MACP,GAAG,QAAQ;AAAA,MACX,gBAAgB;AAAA,MAChB,eAAe,UAAU,KAAK;AAAA,MAC9B,GAAI,QAAQ,EAAE,kBAAkB,MAAM,IAAI,CAAC;AAAA,IAC7C;AAAA,EACF,CAAC;AACL;;;AC7BA;AAAO,SAAS,4BAA4B,KAAc,WAA4B;AACpF,MAAI,OAAO,QAAQ,YAAY,QAAQ,MAAM;AAC3C,WAAO;AAAA,EACT;AAEA,QAAM,WAAW,UAAU,MAAM,GAAG;AACpC,QAAM,SAAU,IAA4B;AAG5C,MAAI,UAAU,OAAO,WAAW,YAAY,WAAW,QAAQ,aAAa,QAAQ;AAClF,WAAO,OAAO,SAAS;AAAA,EACzB;AAGA,MAAI,UAAe;AACnB,aAAW,QAAQ,UAAU;AAC3B,QAAI,OAAO,YAAY,YAAY,YAAY,MAAM;AACnD,aAAO;AAAA,IACT;AACA,cAAU,QAAQ,IAAI;AAAA,EACxB;AAEA,SAAO;AACT;AAEO,SAAS,yBAAyB,KAAc,KAAiC;AACtF,QAAM,QAAQ,4BAA4B,KAAK,GAAG;AAElD,SAAO,OAAO,UAAU,WAAW,QAAQ;AAC7C;AAEO,SAAS,yBAAyB,KAAc,KAAiC;AACtF,QAAM,QAAQ,4BAA4B,KAAK,GAAG;AAElD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,SAAS,OAAO,KAAK;AAE3B,WAAO,OAAO,MAAM,MAAM,IAAI,SAAY;AAAA,EAC5C;AAEA,SAAO;AACT;;;AFHO,IAAM,sBAAN,MAA0B;AAAA,EAE/B,YAAY,QAA6B,YAAqB;AAD9D,wBAAiB;AAEf,UAAM,EAAE,oBAAoB,OAAO,MAAM,IAAI,uBAAuB,QAAQ,UAAU;AAEtF,SAAK,UAAU,cAAc,EAAE,SAAS,oBAAoB,OAAO,SAAS,IAAI,MAAM,CAAC;AAAA,EACzF;AAAA,EAEA,MAAM,iBAAiB,YAAsC;AAC3D,UAAM,OAAO,MAAM,KAAK,QAAQ,uBAAuB;AAAA,MACrD,QAAQ;AAAA,MACR,MAAM,KAAK,UAAU,UAAU;AAAA,IACjC,CAAC;AAED,QAAI,CAAC,KAAK,IAAI;AACZ,YAAM,OAAO,MAAM,KAAK,KAAK,EAAE,MAAM,MAAM,EAAE;AAC7C,YAAM,IAAI;AAAA,QACR,gCAAgC,KAAK,UAAU,GAAG,OAAO,MAAM,IAAI,KAAK,EAAE;AAAA,MAC5E;AAAA,IACF;AAEA,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA,EAEA,MAAM,iBAAiB,YAA+C;AACpE,UAAM,OAAO,MAAM,KAAK,QAAQ,uBAAuB,WAAW,EAAE,IAAI;AAAA,MACtE,QAAQ;AAAA,MACR,MAAM,KAAK,UAAU,UAAU;AAAA,IACjC,CAAC;AAED,QAAI,CAAC,KAAK,IAAI;AACZ,YAAM,OAAO,MAAM,KAAK,KAAK,EAAE,MAAM,MAAM,EAAE;AAC7C,YAAM,IAAI;AAAA,QACR,gCAAgC,KAAK,UAAU,GAAG,OAAO,MAAM,IAAI,KAAK,EAAE;AAAA,MAC5E;AAAA,IACF;AAGA,UAAM,OAAO,MAAM,KAAK,KAAK;AAC7B,QAAI,KAAK,OAAO;AACd,YAAM,IAAI;AAAA,QACR,+BAA+B,WAAW,EAAE,KAAK,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,MAC7E;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;AAEO,IAAM,sBAAsB,OACjC,QACA,WAC+B;AAC/B,QAAM,EAAE,SAAS,SAAS,OAAO,MAAM,IAAI,uBAAuB,MAAM;AAExE,QAAM,MAAM,KAAK,OAAO,2BAA2B,MAAM;AAEzD,QAAM,UAAU,IAAI,QAAQ;AAAA,IAC1B,eAAe,UAAU,KAAK;AAAA,IAC9B,gBAAgB;AAAA,IAChB,GAAI,QAAQ,EAAE,kBAAkB,MAAM,IAAI,CAAC;AAAA,EAC7C,CAAC;AAGD,QAAM,OAAO,MAAM,MAAM,GAAG,OAAO,mCAAmC;AAAA,IACpE;AAAA,IACA,QAAQ;AAAA,IACR,MAAM,KAAK,UAAU,EAAE,IAAI,CAAC;AAAA,EAC9B,CAAC;AACD,QAAM,UAAU,MAAM,KAAK,KAAK;AAEhC,MAAI,CAAC,KAAK,IAAI;AACZ,UAAM,IAAI,MAAM,qCAAqC,QAAQ,WAAW,KAAK,UAAU,EAAE;AAAA,EAC3F;AAEA,SAAO,QAAQ,QAAQ,SAAS,cAAc,QAAQ,OAAO,IAAI;AACnE;AAIO,IAAM,gBAAgB,CAAC,SAA0B;AACtD,QAAM,gBAAgB,4BAA4B,KAAK,KAAK,YAAY,KAAK,KAAK,OAAO,KAAK;AAC9F,QAAM,UAAU,4BAA4B,KAAK,KAAK,YAAY,KAAK,KAAK,IAAI;AAEhF,QAAM,aAAsC;AAAA,IAC1C,IAAI,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,EAAE;AAAA,IAC/D,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,IAAI;AAAA,IACnE,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,IAAI;AAAA,IACnE,SAAS,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,OAAO;AAAA,IACzE,YAAY;AAAA,MACV,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW,IAAI;AAAA,MAC9E,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW,IAAI;AAAA,IAChF;AAAA,IACA,UAAU;AAAA,MACR,IAAI,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,SAAS,EAAE;AAAA,MACxE,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,SAAS,IAAI;AAAA,IAC9E;AAAA,IACA,UAAU,KAAK,KAAK;AAAA,IACpB,QAAQ,KAAK,KAAK,OAAO;AAAA,IACzB,SAAS,KAAK,KAAK;AAAA,IACnB,OAAO,KAAK;AAAA,IACZ,MAAM,UAAW,OAAO,YAAY,WAAW,KAAK,MAAM,OAAO,IAAI,UAAW,CAAC;AAAA,IACjF,MAAM;AAAA,MACJ,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,KAAK,IAAI;AAAA,MACxE,OAAO,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,KAAK,KAAK;AAAA,IAC5E;AAAA,IACA,OAAO,CAAC;AAAA,IACR,YAAY,gBACR,OAAO,kBAAkB,WACvB,KAAK,MAAM,aAAa,IACxB,gBACF;AAAA,EACN;AAGA,SAAO;AACT;AAEO,IAAM,gBAAgB,CAAC,SAA6C;AACzE,QAAM,OAAO,KAAK;AAElB,QAAM,IAAI,KAAK;AACf,MAAI,WAAW;AACf,MAAI,EAAE,SAAS,GAAG,GAAG;AACnB,eAAW,GAAG,OAAO,EAAE,QAAQ,KAAK,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,EACrD,OAAO;AACL,eAAW;AAAA,EACb;AAEA,QAAM,YAAY,4BAA4B,KAAK,YAAY,KAAK,KAAK,KAAK,MAAM;AACpF,QAAM,eAAe,YACjB,OAAO,cAAc,WACnB,KAAK,MAAM,SAAS,IACpB,YACF,CAAC;AAIL,QAAM,SAAyB,CAAC;AAChC,aAAW,CAAC,MAAM,SAAS,KAAK,OAAO,QAAQ,YAAY,GAAG;AAC5D,UAAM,IAAI;AACV,WAAO,IAAI,IAAI;AAAA,MACb;AAAA,MACA,OAAO,EAAE,SAAS,EAAE,SAAS;AAAA,MAC7B,UAAU,EAAE,YAAY,CAAC;AAAA,IAC3B;AAAA,EACF;AAEA,QAAM,WAA8B;AAAA,IAClC,OAAO,yBAAyB,KAAK,YAAY,KAAK,KAAK,KAAK,KAAK;AAAA,IACrE,OAAO,yBAAyB,KAAK,YAAY,KAAK,KAAK,KAAK,KAAK;AAAA,IACrE,QAAQ,yBAAyB,KAAK,YAAY,KAAK,KAAK,KAAK,MAAM;AAAA,IACvE,UAAU,yBAAyB,KAAK,YAAY,KAAK,KAAK,KAAK,QAAQ;AAAA,IAC3E;AAAA,IACA,QAAQ,KAAK,OAAO;AAAA,IACpB;AAAA,IACA,OAAO,KAAK;AAAA,IACZ,QAAQ,KAAK;AAAA,IACb,SAAS,KAAK;AAAA,EAChB;AAGA,SAAO;AACT;AAGO,IAAM,gBAAgB,CAAC,UAAoC;AAChE,MAAI,CAAC,MAAM,QAAQ;AACjB,WAAO;AAAA,EACT;AAGA,QAAM,WAAW,MAAM,KAAK,CAAC,SAAS,KAAK,KAAK,WAAW,OAAO,UAAU,SAAS,MAAM;AAE3F,MAAI,CAAC,UAAU;AACb,WAAO;AAAA,EACT;AAGA,QAAM,WAAuB,cAAc,QAAQ;AAGnD,QAAM,YAAY,MAAM,OAAO,CAAC,SAAS,KAAK,KAAK,KAAK,WAAW,MAAM,CAAC;AAE1E,aAAW,YAAY,WAAW;AAEhC,UAAM,WAAW,cAAc,QAAQ;AAGvC,UAAM,YAAY,MAAM;AAAA,MACtB,CAAC,SACC,KAAK,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,KAAK,mBAAmB,SAAS,KAAK;AAAA,IACpF;AAEA,QAAI,UAAU,SAAS,GAAG;AACxB,YAAM,WAAW,UAAU,CAAC;AAG5B,YAAM,YAAY,MAAM;AAAA,QACtB,CAAC,SACC,KAAK,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,KAAK,mBAAmB,SAAS,KAAK;AAAA,MACpF;AAEA,YAAM,WAAmB,UAAU,IAAI,CAAC,cAAc;AAAA,QACpD,WAAW,yBAAyB,SAAS,KAAK,YAAY,WAAW,KAAK;AAAA,QAC9E,YAAY,yBAAyB,SAAS,KAAK,YAAY,YAAY,KAAK;AAAA,QAChF,MAAM,yBAAyB,SAAS,KAAK,YAAY,MAAM,KAAK;AAAA,QACpE,SAAS;AAAA,UACP,WAAW,yBAAyB,SAAS,KAAK,YAAY,mBAAmB,KAAK;AAAA,UACtF,OAAO,yBAAyB,SAAS,KAAK,YAAY,eAAe,KAAK;AAAA,UAC9E,aACE,yBAAyB,SAAS,KAAK,YAAY,qBAAqB,KAAK;AAAA,QACjF;AAAA,QACA,UAAU;AAAA,UACR,gBACE,yBAAyB,SAAS,KAAK,YAAY,yBAAyB,KAAK;AAAA,QACrF;AAAA,QACA,OAAO;AAAA,UACL,cACE,yBAAyB,SAAS,KAAK,YAAY,oBAAoB,KAAK;AAAA,UAC9E,eACE,yBAAyB,SAAS,KAAK,YAAY,qBAAqB,KAAK;AAAA,QACjF;AAAA,MACF,EAAE;AAGF,YAAM,WAAiB;AAAA,QACrB,MAAM,SAAS,KAAK;AAAA,QACpB,QAAQ,yBAAyB,SAAS,KAAK,YAAY,QAAQ,KAAK;AAAA,QACxE,OAAO,yBAAyB,SAAS,KAAK,YAAY,OAAO,KAAK;AAAA,QACtE,MAAM,yBAAyB,SAAS,KAAK,YAAY,MAAM,KAAK;AAAA,QACpE,OAAO,yBAAyB,SAAS,KAAK,YAAY,OAAO,KAAK;AAAA,QACtE,MAAM,SAAS,CAAC,KAAK;AAAA,UACnB,WAAW;AAAA,UACX,YAAY;AAAA,UACZ,MAAM;AAAA,UACN,SAAS,EAAE,WAAW,IAAI,OAAO,IAAI,aAAa,EAAE;AAAA,UACpD,UAAU,EAAE,gBAAgB,GAAG;AAAA,UAC/B,OAAO,EAAE,cAAc,GAAG,eAAe,EAAE;AAAA,QAC7C;AAAA,MACF;AAEA,eAAS,OAAO;AAAA,IAClB;AAGA,UAAM,aAAa,MAAM;AAAA,MACvB,CAAC,SACC,KAAK,KAAK,WAAW,OAAO,UAAU,SAAS,gBAC/C,KAAK,KAAK,mBAAmB,SAAS,KAAK;AAAA,IAC/C;AAEA,QAAI,WAAW,SAAS,GAAG;AACzB,eAAS,SAAS,CAAC;AAEnB,iBAAW,QAAQ,CAAC,UAAU;AAC5B,cAAM,OAAO,yBAAyB,MAAM,KAAK,YAAY,KAAK,KAAK,MAAM,IAAI,KAAK;AACtF,cAAM,QAAQ,yBAAyB,MAAM,KAAK,YAAY,KAAK,KAAK,MAAM,KAAK,KAAK;AACxF,cAAM,cAAc;AAAA,UAClB,MAAM,KAAK;AAAA,UACX,KAAK,KAAK,MAAM;AAAA,QAClB;AACA,YAAI,WAAW,CAAC;AAChB,YAAI;AACF,qBAAW,cAAc,KAAK,MAAM,WAAW,IAAI,CAAC;AAAA,QACtD,QAAQ;AAAA,QAER;AAEA,iBAAS,OAAO,IAAI,IAAI;AAAA,UACtB;AAAA,UACA;AAAA,UACA,UAAU;AAAA,YACR,OAAO,MAAM,KAAK,WAAW;AAAA,YAC7B,GAAG;AAAA,UACL;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAEA,aAAS,MAAM,KAAK,QAAQ;AAAA,EAC9B;AAEA,WAAS,MAAM,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAE/C,SAAO;AACT;;;AGxUA;AAIO,SAAS,UAAa,MAAe,OAAqB;AAC/D,MAAI,SAAS,OAAO;AAClB,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,GAAG,MAAM,KAAK,GAAG;AAG1B,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,SAAS,YAAY,OAAO,UAAU,UAAU;AACzD,WAAO;AAAA,EACT;AAEA,MAAI,SAAS,QAAQ,UAAU,MAAM;AACnC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,eAAe,IAAI,MAAM,OAAO,eAAe,KAAK,GAAG;AAQhE,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAO,kBAAkB,MAAM,KAA0C;AAAA,EAC3E;AAEA,MAAI,gBAAgB,KAAK;AACvB,WAAO,gBAAgB,MAAM,KAAyC;AAAA,EACxE;AAEA,MAAI,gBAAgB,KAAK;AACvB,WAAO,gBAAgB,MAAM,KAAgC;AAAA,EAC/D;AAEA,MAAI,gBAAgB,MAAM;AACxB,WAAO,KAAK,QAAQ,MAAO,MAA0B,QAAQ;AAAA,EAC/D;AAEA,MAAI,gBAAgB,QAAQ;AAC1B,WAAO,KAAK,SAAS,MAAO,MAA4B,SAAS;AAAA,EACnE;AAQA,MAAI,OAAO,KAAK,IAAI,EAAE,WAAW,OAAO,KAAK,KAAK,EAAE,QAAQ;AAC1D,WAAO;AAAA,EACT;AAEA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,IAAI,GAAG;AAC/C,QAAI,EAAE,OAAO,QAAQ;AACnB,aAAO;AAAA,IACT;AAEA,QACE,CAAC;AAAA,MACC;AAAA;AAAA,MAEA,MAAM,GAAG;AAAA,IACX,GACA;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,kBAAkB,MAA8B,OAAwC;AAC/F,MAAI,KAAK,WAAW,MAAM,QAAQ;AAChC,WAAO;AAAA,EACT;AAEA,aAAW,CAAC,OAAO,IAAI,KAAK,KAAK,QAAQ,GAAG;AAC1C,QAAI,CAAC,UAAU,MAAM,MAAM,KAAK,CAAC,GAAG;AAClC,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,gBACP,MACA,OACS;AACT,MAAI,KAAK,SAAS,MAAM,MAAM;AAC5B,WAAO;AAAA,EACT;AAEA,aAAW,CAAC,KAAK,KAAK,KAAK,KAAK,QAAQ,GAAG;AACzC,QAAI,CAAC,MAAM,IAAI,GAAG,GAAG;AACnB,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,UAAU,OAAO,MAAM,IAAI,GAAG,CAAC,GAAG;AACrC,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,gBAAgB,MAA4B,OAAsC;AACzF,MAAI,KAAK,SAAS,MAAM,MAAM;AAC5B,WAAO;AAAA,EACT;AAMA,QAAM,YAAY,CAAC,GAAG,KAAK;AAE3B,aAAW,YAAY,MAAM;AAC3B,QAAI,UAAU;AAEd,eAAW,CAAC,OAAO,SAAS,KAAK,UAAU,QAAQ,GAAG;AACpD,UAAI,UAAU,UAAU,SAAS,GAAG;AAClC,kBAAU;AACV,kBAAU,OAAO,OAAO,CAAC;AACzB;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,SAAS;AACZ,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;;;AClJA;AAMO,SAAS,QAAQ,UAAmB,eAA+B;AACxE,QAAM,iBAAiB,oBAAoB,QAAQ,WAAW,IAAI,MAAM,cAAc,QAAQ,CAAC;AAE/F,MAAI,eAAe;AACjB,mBAAe,UAAU,cAAc,aAAa,aAAa,eAAe,OAAO;AAAA,EACzF;AAEA,SAAO;AACT;AAEO,SAAS,aACd,IACA,eACsB;AACtB,MAAI;AACF,WAAO,CAAC,GAAG,GAAG,IAAI;AAAA,EACpB,SAAS,UAAU;AACjB,WAAO,CAAC,MAAM,QAAQ,UAAU,aAAa,CAAC;AAAA,EAChD;AACF;AAEA,eAAsB,cACpB,IACA,eACwC;AACxC,MAAI;AACF,UAAM,SAAS,OAAO,OAAO,aAAa,GAAG,IAAI;AACjD,WAAO,CAAC,MAAM,QAAQ,IAAI;AAAA,EAC5B,SAAS,UAAU;AACjB,WAAO,CAAC,MAAM,QAAQ,UAAU,aAAa,CAAC;AAAA,EAChD;AACF;AAYO,IAAM,YAAY,CAAC,IAAa,kBAA2B;AAChE,MAAI,OAAO,OAAO,YAAY;AAC5B,QAAI;AACF,YAAM,SAAU,GAAqB;AACrC,UAAI,kBAAkB,SAAS;AAC7B,eAAO,cAAc,QAAQ,aAAa;AAAA,MAC5C;AACA,aAAO,CAAC,QAAQ,IAAI;AAAA,IACtB,SAAS,UAAU;AACjB,aAAO,CAAC,MAAM,QAAQ,UAAU,aAAa,CAAC;AAAA,IAChD;AAAA,EACF;AAEA,MAAI,cAAc,SAAS;AACzB,WAAO,cAAc,IAAI,aAAa;AAAA,EACxC;AAEA,SAAO,CAAC,IAAI,IAAI;AAClB;AAEA,SAAS,OAAO;AAChB,SAAS,QAAQ;;;ANpBjB,IAAM,kBAAkB,eAAe,wCAAwC,EAAE;AASjF,IAAM,2BAA2B,OAAO,IAAI,kCAAkC;AAE9E,SAAS,4BACP,OACA,SAC+D;AAC/D,QAAM,aAAa,QAAQ,KAAK;AAGhC,aAAW,wBAAwB,IAAI;AACvC,SAAO;AACT;AAEA,SAAS,yBAAyB,OAAmD;AACnF,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AACA,SAAQ,MACN,wBACF;AACF;AAiCO,SAAS,KAQd,MACA,QAIM;AAEN,aAAW,QAAQ,IAAI;AACvB,aAAW,cAAc,OAAO,UAAU;AAC1C,MAAI,OAAO,MAAM;AACf,eAAW,QAAQ,OAAO,IAAI;AAAA,EAChC;AAGA,MAAI,OAAO,SAAS;AAClB,eAAW,UAAU,OAAO,SAAS;AACnC,YAAM,aAAa,cAAc,QAAQ,EAAE;AAC3C,iBAAW,UAAU,UAAU;AAAA,IACjC;AAAA,EACF;AAEA,eAAa,MAAM,MAAmC,EAAE,MAAM,QAAQ,KAAK;AAC7E;AAKA,SAAS,kBAAkB,aAA6C;AACtE,MAAI,CAAC,eAAe,YAAY,WAAW,GAAG;AAC5C,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,QAAQ,eAAe;AAC7B,QAAM,cAAc,OAAO,qBAAqB,KAAK,CAAC;AACtD,QAAM,YAAY,uBAAuB;AAEzC,QAAM,SAAS,EAAE,GAAG,aAAa,GAAG,UAAU;AAG9C,QAAM,WAAgC,CAAC;AACvC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,MAAM,GAAG;AACjD,UAAM,YAAY,YAAY,KAAK,CAAC,YAAY,IAAI,WAAW,OAAO,CAAC;AACvE,QAAI,WAAW;AACb,eAAS,GAAG,IAAI;AAAA,IAClB;AAAA,EACF;AAEA,SAAO,oBAAoB,QAAQ;AACrC;AAEA,IAAM,gBAAgB,CACpB,QACA,WAAmB,cAChB;AACH,SAAQ,OAAe,QAAQ;AACjC;AAEA,eAAe,aAIb,UAAkB,MAA8C;AAChE,OAAK;AACL,QAAM,oBAGJ,OAAO,KAAK,SAAS,aAEf,KAAK,KAGL,IACF,KAAK;AACX,QAAM,OAAO,eAAe;AAG5B,QAAM,aAAa,OAAO,UAAU;AACpC,QAAM,UAAU,OAAO,OAAO;AAC9B,QAAM,SAAS,OAAO,MAAM;AAC5B,QAAM,oBAAoB,OAAO,WAAW;AAC5C,QAAM,cAAc,OAAO,aAAa;AACxC,QAAM,QAAQ,OAAO,OAAO;AAC5B,QAAM,aAAa,OAAO,YAAY;AAEtC,MAAI,CAAC,aAAa;AAChB,UAAM,IAAI,cAAc,wBAAwB;AAAA,EAClD;AAEA,QAAM,YAAY,KAAK,WAAW,aAAa,KAAK;AAEpD,QAAM,uBAAuB,iCAAiC,aAAa;AAAA,IACzE,SAAS,CAAC,WAAW,CAAC;AAAA,EACxB,CAAC;AAED,QAAM,SAAS,MAAM;AAAA,IACnB;AAAA,IACA,YAAY;AACV,YAAM,aAAa,MAAM;AAEzB,YAAM,sBAAsB,IAAI,oBAAoB,aAAa,UAAU;AAG3E,YAAM,cAAc,gBAAgB;AACpC,UAAI,SAAS;AACb,UAAI;AAEJ,UAAI;AACJ,UAAI;AACJ,UAAI,uBAAgC;AACpC,UAAI,WAA0C;AAG9C,YAAM,qBAA6C,CAAC;AAGpD,UAAI;AAIJ,gBAAU,OAAO,UAAU;AAEzB,YAAI,qBAAqB,OAAO,KAAK,iBAAiB,EAAE,SAAS,GAAG;AAClE,cAAI;AACF,mCAAuB,iBAAiB;AAAA,UAC1C,QAAQ;AAAA,UAAC;AAAA,QACX;AAEA,cAAM,KAAK,aAAa;AAAA,UACtB,IAAI;AAAA,UACJ,MAAM;AAAA,UACN,SAAS;AAAA,UACT;AAAA,UACA,OAAO;AAAA,UACP,UAAU,YAAY;AAAA,UACtB,aAAa,KAAK;AAAA,QACpB;AAEA,cAAM,CAAC,EAAE,wBAAwB,IAAI,MAAM,cAAc,oBAAoB;AAC7E,YAAI,0BAA0B;AAC5B,iCAAuB;AAAA,QACzB;AAEA,oBAAY,UAAU,QAAQ,QAAQ,IAAI,WAAW,IAAI;AAAA,UACvD,YAAY;AAAA,YACV,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,YAC7B,CAAC,KAAK,KAAK,IAAI,GAAG;AAAA,YAClB,CAAC,KAAK,KAAK,OAAO,GAAG;AAAA,YACrB,CAAC,KAAK,KAAK,IAAI,GAAG;AAAA;AAAA,YAClB,CAAC,KAAK,KAAK,IAAI,GAAG,CAAC;AAAA,YACnB,CAAC,KAAK,KAAK,WAAW,EAAE,GAAG;AAAA;AAAA,YAC3B,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG;AAAA;AAAA,YAC7B,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG,WAAW;AAAA;AAAA,YAExC,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG,KAAK;AAAA,YAClC,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,KAAK,QAAQ;AAAA;AAAA,YAEpC,CAAC,KAAK,KAAK,QAAQ,GAAG,KAAK,UAAU,KAAK,QAAQ;AAAA;AAAA,YAElD,CAAC,KAAK,KAAK,IAAI,EAAE,GAAG;AAAA;AAAA,YAEpB,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM;AAAA,YAC7B,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG,MAAM;AAAA,UAChC;AAAA,QACF,CAAC;AACD,iBAAS,UAAU,YAAY,EAAE;AACjC,cAAM,KAAK,WAAW,KAAK;AAC3B,kBAAU,aAAa,KAAK,KAAK,IAAI,MAAM;AAC3C,uBAAe,MAAM,QAAQ,QAAQ,OAAO,GAAG,SAAS;AAExD,cAAM,aAAa,kBAAkB,KAAK,WAAW;AACrD,cAAM,KAAK,WAAW,aAAa;AACnC,cAAM,iBAAiB,KAAK,UAAU,UAAU;AAChD,kBAAU,aAAa,KAAK,KAAK,OAAO,OAAO,cAAc;AAE7D,YAAI;AACJ,YAAI,CAAC,WAAW,CAAC,QAAQ;AACvB,+BAAqB,MAAM,oBAAoB,iBAAiB;AAAA,YAC9D,IAAI;AAAA,YACJ,MAAM;AAAA,YACN,YAAY,KAAK;AAAA,YACjB,MAAM,KAAK;AAAA,YACX,SAAS,YAAY,KAAK;AAAA,YAC1B,SAAS;AAAA,YACT,YAAY,cAAc;AAAA,YAC1B;AAAA,YACA,YAAY,WAAW;AAAA,YACvB,QAAQ,EAAE,WAAW,kBAAkB;AAAA,YACvC,iBAAiB;AAAA,YACjB,UAAU,KAAK;AAAA,YACf,QAAQ;AAAA,UACV,CAAC;AAAA,QACH;AAEA,cAAM,QAAQ,oBAAoB,MAAM;AACxC,cAAM,qBAAqB,oBAAoB,MAAM;AAGrD,YAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,oBAAoB;AAC/C,gBAAM,CAAC,gBAAgB,aAAa,IAAI,MAAM;AAAA,YAAc,MAC1D,oBAAoB,oBAAoB,WAAW;AAAA,UACrD;AACA,cAAI,eAAe;AACjB,oBAAQ,MAAM,4BAA4B,cAAc,aAAa,CAAC,EAAE;AACxE,mCAAuB,wBAAwB;AAAA,UACjD,OAAO;AACL,uBAAW;AAAA,UACb;AAAA,QACF;AAGA,YAAI,UAAU;AACZ,oBAAU,aAAa,KAAK,KAAK,SAAS,IAAI,SAAS,EAAE;AACzD,oBAAU,aAAa,KAAK,KAAK,SAAS,MAAM,SAAS,IAAI;AAC7D,oBAAU,aAAa,KAAK,KAAK,SAAS,SAAS,SAAS,OAAO;AAAA,QACrE;AAEA,cAAM,KAAK,aAAa;AAAA,UACtB,IAAI;AAAA,UACJ,MAAM;AAAA,UACN,SAAS;AAAA,UACT;AAAA,UACA,OAAO,SAAS;AAAA,UAChB,UAAU,YAAY;AAAA,UACtB,aAAa,KAAK;AAAA,UAClB,oBAAoB,uBAChB;AAAA,YACE,QAAQ;AAAA,YACR,OAAO,cAAc,oBAAoB;AAAA,UAC3C,IACA,EAAE,QAAQ,UAAU;AAAA,UACxB,QAAQ,KAAK;AAAA,QACf;AAEA,qBAAa,YAAY,IAAI;AAAA,MAC/B,CAAC;AAED,eAAS,OAAO,UAAU;AACxB,YAAI,sBAAsB;AACxB,gBAAM;AAAA,QACR;AAEA,cAAM,OAAiB,CAAC,SAAS;AACjC,mBAAW,aAAa,KAAK,KAAK,MAAM,KAAK,UAAU,IAAI,CAAC;AAG5D,cAAM,cAAc,oBAAI,IAA4B;AAEpD,mBAAW,QAAQ,oBAAoB;AACrC,cAAI,YAAY,IAAI,KAAK,QAAQ,GAAG;AAClC,kBAAM,WAAW,YAAY,IAAI,KAAK,QAAQ;AAC9C,qBAAS;AACT,qBAAS,kBAAkB,KAAK,IAAI,SAAS,iBAAiB,KAAK,UAAU;AAC7E,qBAAS,iBAAiB,KAAK,IAAI,SAAS,gBAAgB,KAAK,UAAU;AAAA,UAC7E,OAAO;AACL,wBAAY,IAAI,KAAK,UAAU;AAAA,cAC7B,UAAU,KAAK;AAAA,cACf,OAAO;AAAA,cACP,iBAAiB,KAAK;AAAA,cACtB,gBAAgB,KAAK;AAAA,cACrB,YAAY,KAAK;AAAA,YACnB,CAAC;AAAA,UACH;AAAA,QACF;AAGA,YAAI,MAAM,KAAK,cAAc,WAAW;AACtC,gBAAM,KAAK,WAAW,kBAAkB,MAAM,KAAK,YAAY,QAAQ,CAAC,EAAE;AAAA,YACxE,CAAC,CAAC,WAAW,KAAK,MAAM;AAAA,UAC1B;AAGA,gBAAM,cAAc,eAAe,GAAG,mBAAmB;AACzD,gBAAM,cAAc,qBAAqB;AACzC,gBAAM,YAAY,qBAAqB,uBAAuB;AAE9D,gBAAM,KAAK,WAAW,YAAY;AAAA,YAChC,OAAO;AAAA,YACP;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAGA,mBAAW,UAAU,EAAE,MAAM,eAAe,GAAG,CAAC;AAChD,mBAAW,IAAI;AAGf,cAAM,CAAC,EAAE,UAAU,IAAI,MAAM,cAAc,KAAK;AAChD,YAAI,YAAY;AAEd,cAAI,MAAM,KAAK,YAAY;AACzB,kBAAM,KAAK,WAAW,qBAAqB;AAAA,cACzC,QAAQ;AAAA,cACR,OAAO,cAAc,UAAU;AAAA,YACjC;AAAA,UACF;AAAA,QACF;AAEA,cAAM,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,UAAU;AAE5D,cAAM,eAAe,MAAM,MAAM;AAAA,UAC/B,CAAC,SAAS,KAAK,KAAK,KAAK,WAAW;AAAA,QACtC,EAAE;AACF,cAAM,eAAe,MAAM,MAAM;AAAA,UAC/B,CAAC,SAAS,KAAK,KAAK,KAAK,WAAW,UAAU,KAAK,KAAK,KAAK,WAAW;AAAA,QAC1E,EAAE;AAGF,YAAI,CAAC,WAAW,CAAC,QAAQ;AACvB,gBAAM,oBAAoB,iBAAiB;AAAA,YACzC,IAAI;AAAA,YACJ,QAAQ;AAAA,YACR,YAAY,WAAW;AAAA,YACvB;AAAA,YACA;AAAA,YACA;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AAID,YAAM,GAAG,WAAW;AAAA,QAClB,WAAW,IAAI,CAAC,GAAG,WAAW,EAAE,GAAG,GAAG,MAAM,EAAsC;AAAA,MACpF,EAAE,QAAQ,OAAO,MAAM,EAAE,KAAK,MAAM;AAClC,cAAM,QAAQ,YAAY,IAAI;AAC9B,YAAI,CAAC,cAAc;AACjB,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AAEA,YAAI,kBAA0C,CAAC;AAE/C,cAAM;AAAA,UACJ,QAAQ,KAAK,KAAK;AAAA,UAClB;AAAA,YACE,YAAY;AAAA,cACV,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,cAC7B,CAAC,KAAK,KAAK,EAAE,GAAG;AAAA,cAChB,CAAC,KAAK,KAAK,IAAI,GAAG;AAAA,cAClB,CAAC,KAAK,KAAK,OAAO,GAAG;AAAA,cACrB,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG,KAAK;AAAA,cAC7B,CAAC,KAAK,KAAK,KAAK,KAAK,GACnB,OAAO,KAAK,UAAU,WAAW,KAAK,QAAQ,KAAK,UAAU,KAAK,KAAK;AAAA,cACzE,CAAC,KAAK,KAAK,KAAK,QAAQ,GACtB,OAAO,KAAK,aAAa,WAAW,KAAK,WAAW,KAAK,UAAU,KAAK,QAAQ;AAAA,cAClF,CAAC,KAAK,KAAK,KAAK,QAAQ,GAAG,KAAK,WAAW,KAAK,UAAU,KAAK,QAAQ,IAAI;AAAA;AAAA,cAE3E,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM;AAAA,cAC7B,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG,MAAM;AAAA,YAChC;AAAA,UACF;AAAA,UACA,OAAO,aAAa;AAClB,kBAAM,cAAc,MAAM,QAAQ,QAAQ,OAAO,GAAG,QAAQ;AAC5D,kBAAM,SAAS,KAAK,IAAI,GAAG,KAAK,UAAU,CAAC;AAC3C,gBAAI;AACJ,gBAAI;AASJ,qBAAS,aAAa,KAAK,KAAK,KAAK,QAAQ,MAAM;AAEnD,gBAAI;AAEF,oBAAM,kBAA4C,CAAC;AACnD,oBAAM,cAAoC,MAAM,KAAK,EAAE,QAAQ,OAAO,GAAG,MAAM,IAAI;AACnF,oBAAM,gBAAyB,CAAC;AAChC,kBAAI;AACJ,kBAAI,yBAAyB;AAG7B,uBAAS,aAAa,GAAG,aAAa,QAAQ,cAAc;AAC1D,oBAAI;AACF,wBAAM;AAAA,oBACJ,SAAS,UAAU;AAAA,oBACnB;AAAA,sBACE,YAAY;AAAA,wBACV,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,wBAC7B,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG;AAAA,wBACzB,CAAC,KAAK,KAAK,EAAE,GAAG;AAAA,wBAChB,CAAC,KAAK,KAAK,IAAI,GAAG;AAAA,wBAClB,CAAC,KAAK,KAAK,OAAO,GAAG;AAAA,sBACvB;AAAA,oBACF;AAAA,oBACA,OAAO,cAAc;AACnB,4BAAM,eAAe,MAAM,QAAQ,QAAQ,OAAO,GAAG,SAAS;AAC9D,0BAAI;AACF,8BAAMA,UAAS,MAAM;AAAA,0BACnB;AAAA,0BACA;AAAA,4BACE,IAAI;AAAA,4BACJ,SAAS;AAAA,4BACT,MAAM;AAAA,0BACR;AAAA,0BACA;AAAA,4BACE,OAAO,KAAK;AAAA,4BACZ,OAAO,KAAK;AAAA,4BACZ,UAAU,KAAK;AAAA,4BACf,SAAS,KAAK;AAAA,4BACd,MAAM,KAAK;AAAA,4BACX,UAAU,KAAK;AAAA,4BACf,aAAa,KAAK;AAAA,4BAClB,YAAY,KAAK;AAAA,4BACjB,MAAM,KAAK;AAAA,0BACb;AAAA,wBACF;AACA,8BAAM,EAAE,QAAAC,SAAQ,SAAS,IAAID;AAC7B,qCAAaC;AACb,kDAA0B;AAC1B,wCAAgB,KAAK,GAAGD,QAAO,eAAe;AAE9C,kDAA0B;AAAA,0BACxB,OAAOA,QAAO,cAAc,CAAC;AAAA,0BAC7B,aAAa,KAAK;AAAA,0BAClB,WAAWA,QAAO;AAAA,wBACpB;AAGA,8BAAM,QAAQ;AAAA,0BACZ,KAAK,QAAQ,IAAI,OAAO,WAAW;AACjC,kCAAM,aAAa,cAAc,MAAM;AACvC,mCAAO;AAAA,8BACL,SAAS,UAAU;AAAA,8BACnB;AAAA,gCACE,YAAY;AAAA,kCACV,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,kCAC7B,CAAC,KAAK,KAAK,EAAE,GAAG;AAAA,kCAChB,CAAC,KAAK,KAAK,IAAI,GAAG;AAAA,kCAClB,CAAC,KAAK,KAAK,OAAO,GAAG;AAAA,kCACrB,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG;AAAA,gCAC3B;AAAA,8BACF;AAAA,8BACA,OAAO,eAAe;AACpB,sCAAM,cAAc,YAAY,IAAI;AACpC,oCAAI;AACF,wCAAM,CAACA,SAAQ,WAAW,IAAI,MAAM;AAAA,oCAAc,MAChD,OAAO;AAAA,sCACL,OAAO,KAAK;AAAA,sCACZ,QAAQC;AAAA,sCACR,UAAU,KAAK;AAAA,sCACf;AAAA,oCACF,CAAC;AAAA,kCACH;AAEA,sCAAI,eAAe,CAACD,SAAQ;AAC1B,0CAAM,iBAAiB,KAAK;AAAA,sCAC1B,YAAY,IAAI,IAAI;AAAA,oCACtB;AACA,4CAAQ;AAAA,sCACN,iBAAiB,UAAU;AAAA;AAAA,sCAC3B;AAAA,oCACF;AACA,0CAAM,MAAM,cAAc,WAAW;AACrC,0CAAME,YAAW;AAAA,sCACf,UAAU;AAAA,sCACV,WAAW;AAAA,sCACX,OAAO;AAAA,oCACT;AAGA,qCAAC,8DAAgC,CAAC,IAAG,KAAK,CAAC;AAE3C,+CAAW,cAAc;AAAA,sCACvB,CAAC,KAAK,KAAK,MAAM,IAAI,GAAG;AAAA,sCACxB,CAAC,KAAK,KAAK,MAAM,QAAQ,GAAG,KAAK,UAAUA,SAAQ;AAAA,oCACrD,CAAC;AAED,+CAAW,UAAU;AAAA,sCACnB,MAAM,eAAe;AAAA,sCACrB,SAAS;AAAA,oCACX,CAAC;AACD;AAAA,kCACF;AAEA,wCAAM,gBAAgB,KAAK,MAAM,YAAY,IAAI,IAAI,WAAW;AAChE,wCAAM,aAAaF,QAAO;AAC1B,wCAAM,WAAW,OAAO;AAAA,oCACtB,EAAE,UAAU,eAAe,WAAW,YAAY;AAAA,oCAClDA,QAAO;AAAA,kCACT;AAGA,mCAAC,8DAAgC,CAAC,IAAG,KAAK,UAAU;AAGpD,wCAAM,cACH,OAAkB,eAAe,KAAK;AAEzC,6CAAW,cAAc;AAAA,oCACvB,CAAC,KAAK,KAAK,MAAM,IAAI,GAAG;AAAA,oCACxB,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG;AAAA,oCACzB,CAAC,KAAK,KAAK,MAAM,QAAQ,GAAG,KAAK,UAAU,QAAQ;AAAA,oCACnD,CAAC,KAAK,KAAK,MAAM,WAAW,GAAG,YAAY;AAAA,kCAC7C,CAAC;AAED,sCAAI,SAAS,OAAO;AAClB,0CAAM,MAAM,cAAc,SAAS,KAAK;AACxC,+CAAW,UAAU;AAAA,sCACnB,MAAM,eAAe;AAAA,sCACrB,SAAS;AAAA,oCACX,CAAC;AAAA,kCACH;AAAA,gCACF,UAAE;AACA,6CAAW,IAAI;AAAA,gCACjB;AAAA,8BACF;AAAA,8BACA;AAAA,4BACF;AAAA,0BACF,CAAC;AAAA,wBACH;AAAA,sBACF,SAAS,OAAO;AACd,8BAAM,qBAAqB,yBAAyB,KAAK;AACzD,8BAAM,UAAU,QAAQ,KAAK;AAC7B,8BAAM,MAAM,cAAc,OAAO;AACjC,8BAAM,mBAAmB,QAAQ,WAAW;AAC5C,oCAAY,UAAU,IAAI;AAC1B,sCAAc,KAAK,OAAO;AAE1B,kCAAU,cAAc;AAAA,0BACtB,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG;AAAA,wBAC3B,CAAC;AAED,mCAAW,UAAU,KAAK,SAAS;AACjC,gCAAM,aAAa,cAAc,MAAM;AACvC,2BAAC,8DAAgC,CAAC,IAAG,KAAK,CAAC;AAAA,wBAC7C;AAEA,4BAAI,oBAAoB;AACtB,0CAAgB,KAAK,GAAG,mBAAmB,eAAe;AAC1D,oDAA0B;AAAA,4BACxB,OAAO,mBAAmB,cAAc,CAAC;AAAA,4BACzC,aAAa,KAAK;AAAA,4BAClB,WAAW,mBAAmB;AAAA,0BAChC;AAAA,wBACF;AAGA,8BAAM;AAAA,sBACR;AAAA,oBACF;AAAA,oBACA;AAAA,kBACF;AAAA,gBACF,QAAQ;AAAA,gBAER;AAAA,cACF;AAGA,oBAAM,SAAwC,CAAC;AAC/C,yBAAW,UAAU,KAAK,SAAS;AACjC,sBAAM,aAAa,cAAc,MAAM;AACvC,sBAAM,YAAY,gBAAgB,UAAU,KAAK,CAAC;AAClD,sBAAM,cAA4B,OAAkB,eAAe,KAAK;AAExE,sBAAM,kBAAkB,UAAU,SAAS,IAAI,YAAY,UAAU,SAAS,IAAI;AAElF,uBAAO,UAAU,IAAI;AAAA,kBACnB,MAAM;AAAA,kBACN,OAAO;AAAA,kBACP,QAAQ;AAAA,kBACR,aAAa,YAAY;AAAA,kBACzB,WAAW,YAAY;AAAA,kBACvB,UAAU,CAAC;AAAA,gBACb;AAAA,cACF;AAEA,oBAAM,SAAS;AACf,oBAAM,eAAe,cAAc;AACnC,oBAAM,kBAAkB,SAAS;AACjC,oBAAM,eAAe;AAAA,gBACnB,OAAO;AAAA,gBACP,WAAW;AAAA,gBACX,QAAQ;AAAA,cACV;AAEA,uBAAS,aAAa,KAAK,KAAK,KAAK,QAAQ,KAAK,UAAU,SAAS,SAAS,CAAC,CAAC,CAAC;AACjF,kBAAI,WAAW,QAAW;AACxB,yBAAS;AAAA,kBACP,KAAK,KAAK,KAAK;AAAA,kBACf,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,gBAC7D;AAAA,cACF;AAGA,mBAAK,KAAK,OAAO;AAAA,gBACf,OAAO,KAAK;AAAA,gBACZ,MAAM;AAAA,gBACN,UAAU,KAAK;AAAA,gBACf,OAAO,KAAK;AAAA,gBACZ;AAAA,gBACA,UAAU,KAAK;AAAA,gBACf;AAAA,gBACA,QAAQ;AAAA,gBACR,QAAQ,CAAC;AAAA,gBACT;AAAA,gBACA;AAAA,gBACA,UAAU;AAAA,gBACV,WAAW;AAAA,gBACX;AAAA,gBACA,aAAa,KAAK;AAAA,cACpB;AAEA,kBAAI,eAAe,GAAG;AACpB,sBAAM,QAAQ,IAAI;AAAA,kBAChB,aAAa,KAAK,KAAK,gBAAgB,YAAY;AAAA,gBACrD;AACA,+CAA+B;AAC/B,yBAAS,UAAU;AAAA,kBACjB,MAAM,eAAe;AAAA,kBACrB,SAAS,MAAM;AAAA,gBACjB,CAAC;AACD,qBAAK,KAAK,KAAK,SAAS;AACxB,qBAAK,KAAK,KAAK,SAAS;AACxB,sBAAM;AAAA,cACR;AAGA,iCAAmB,KAAK,GAAG,eAAe;AAAA,YAC5C,SAAS,GAAG;AACV,sBAAQ,IAAI,CAAC;AACb,oBAAM,QAAQ,QAAQ,CAAC;AAEvB,kBAAI,MAAM,gCAAgC,KAAK,KAAK,MAAM;AACxD,qBAAK,KAAK,KAAK,SAAS;AACxB,qBAAK,KAAK,KAAK,SAAS,KAAK,KAAK,KAAK,QAAQ,SAC3C,KAAK,KAAK,KAAK,SACf,CAAC,KAAK;AACV,mCAAmB,KAAK,GAAG,eAAe;AAC1C,sBAAM;AAAA,cACR;AAEA,oBAAM,MAAM,eAAe;AAC3B,oBAAM,qBAAqB,IAAI,mBAAmB,CAAC;AACnD,kBAAI,mBAAmB,SAAS,GAAG;AACjC,gCAAgB,KAAK,GAAG,kBAAkB;AAAA,cAC5C;AAEA,oBAAM,WAAW,IAAI,SAAS,CAAC;AAC/B,kBAAI,CAAC,2BAA2B,OAAO,KAAK,QAAQ,EAAE,SAAS,GAAG;AAChE,0CAA0B;AAAA,kBACxB,OAAO;AAAA,kBACP,aAAa,KAAK;AAAA,kBAClB,WAAW,IAAI;AAAA,gBACjB;AAAA,cACF;AAGA,oBAAM,eAA8C,CAAC;AACrD,yBAAW,UAAU,KAAK,SAAS;AACjC,sBAAM,aAAa,cAAc,MAAM;AACvC,6BAAa,UAAU,IAAI;AAAA,kBACzB,MAAM;AAAA,kBACN,OAAO;AAAA,kBACP,QAAQ,CAAC;AAAA,kBACT,UAAU;AAAA,oBACR,UAAU;AAAA,oBACV,WAAW;AAAA,oBACX,OAAO,MAAM;AAAA,kBACf;AAAA,gBACF;AAAA,cACF;AAEA,mBAAK,KAAK,OAAO;AAAA,gBACf,MAAM;AAAA,gBACN,OAAO,KAAK;AAAA,gBACZ,UAAU,KAAK;AAAA,gBACf,OAAO,KAAK;AAAA,gBACZ,QAAQ,OAAO,CAAC;AAAA,gBAChB,UAAU,KAAK;AAAA,gBACf,QAAQ;AAAA,gBACR,QAAQ;AAAA,gBACR,QAAQ,CAAC,KAAK;AAAA,gBACd,WAAW;AAAA,gBACX,UAAU,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;AAAA,gBAC9C;AAAA,gBACA,aAAa,KAAK;AAAA,cACpB;AAEA,iCAAmB,KAAK,GAAG,eAAe;AAC1C,oBAAM;AAAA,YACR,UAAE;AAEA,kBAAI;AACF,sBAAM,gBAAqC,yBAAyB,SAAS,CAAC;AAE9E,sBAAM,WAAW,OAAO,KAAK,aAAa;AAC1C,sBAAM,cAAc,eAAe,GAAG,qBAAqB,KAAK,CAAC;AAEjE,sBAAM,eAA+C,CAAC;AACtD,2BAAW,OAAO,UAAU;AAC1B,wBAAM,QAAQ,cAAc,GAAG;AAC/B,sBAAI,OAAO,aAAa;AACtB,0BAAM,WAAW,CAAC,UAAU,OAAO,YAAY,GAAG,CAAC;AACnD,wBAAI,UAAU;AACZ,mCAAa,GAAG,IAAI,EAAE,MAAM,YAAY,OAAO,SAAS,YAAY,GAAG,EAAE;AAAA,oBAC3E;AAAA,kBACF,OAAO;AACL,iCAAa,GAAG,IAAI,EAAE,MAAM,cAAc,MAAM;AAAA,kBAClD;AAAA,gBACF;AAEA,oBAAI,CAAC,WAAW,OAAO,KAAK,YAAY,EAAE,SAAS,GAAG;AACpD,wBAAM,aAAa,KAAK,UAAU,YAAY;AAC9C,2BAAS,aAAa,kCAAkC,UAAU;AAAA,gBACpE;AAEA,oBAAI,KAAK,KAAK,MAAM;AAClB,uBAAK,KAAK,KAAK,eAAe;AAAA,gBAChC;AAAA,cACF,QAAQ;AAAA,cAAC;AAET,kBAAI,yBAAyB;AAC3B,sCAAsB;AAAA,cACxB;AAAA,YACF;AAAA,UACF;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA;AAAA,EACF;AAEA,SAAO;AACT;AAEA,IAAM,4BAA4B,CAAyC,YAAoB;AAC7F,MAAI,QAAQ,WAAW,GAAG;AACxB,WAAO;AAAA,EACT;AAGA,MAAI,QAAQ,MAAM,CAAC,MAAM,OAAO,MAAM,QAAQ,GAAG;AAC/C,WAAO,QAAQ,KAAK,EAAE;AAAA,EACxB;AAGA,SAAO,QAAQ,QAAQ,SAAS,CAAC;AACnC;AAEA,IAAM,cAAc,OAKlB,MACA,OACA,aACqB;AACrB,QAAM,qBAAqB,MAAM,KAAK,EAAE,OAAO,SAAS,CAAC;AAEzD,MACE,OAAO,uBAAuB,YAC9B,sBACA,OAAO,iBAAiB,oBACxB;AACA,UAAM,SAAoB,CAAC;AAE3B,qBAAiB,SAAS,oBAAoB;AAC5C,aAAO,KAAK,KAAK;AAAA,IACnB;AAEA,WAAO,0BAAmC,MAAmB;AAAA,EAC/D;AAEA,SAAO;AACT;AAEA,IAAM,UAAU,OAKd,aACA,YAKA,SAKG;AACH,QAAM,WAAW,KAAK,KAAK,QAAQ;AAEnC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,MACE,YAAY;AAAA,QACV,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,QAC7B,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG;AAAA,QACvB,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG;AAAA;AAAA,QACvB,CAAC,KAAK,KAAK,EAAE,GAAG,WAAW;AAAA,QAC3B,CAAC,KAAK,KAAK,IAAI,GAAG,WAAW;AAAA,QAC7B,CAAC,KAAK,KAAK,OAAO,GAAG,WAAW;AAAA,MAClC;AAAA,IACF;AAAA,IACA,OAAO,aAAa;AAElB,YAAM,EAAE,QAAQ,UAAU,iBAAiB,YAAY,UAAU,IAAI,MAAM;AAAA,QACzE,EAAE,aAAa,KAAK,YAAY;AAAA,QAChC,YAMM;AAEJ,gBAAM,QAAQ,YAAY,IAAI;AAC9B,cAAI;AACF,kBAAMC,UAAS,MAAM,YAAY,KAAK,MAAM,KAAK,OAAO,KAAK,QAAS;AACtE,kBAAME,YAAW,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;AAErD,qBAAS,cAAc;AAAA,cACrB,CAAC,KAAK,KAAK,KAAK,MAAM,GAAG,OAAOF,YAAW,WAAWA,UAAS,KAAK,UAAUA,OAAM;AAAA,YACtF,CAAC;AAGD,kBAAM,MAAM,eAAe;AAC3B,kBAAMG,mBAAkB,IAAI,mBAAmB,CAAC;AAEhD,mBAAO;AAAA,cACL,QAAAH;AAAA,cACA,UAAAE;AAAA,cACA,iBAAAC;AAAA,cACA,YAAY,IAAI,SAAS,CAAC;AAAA,cAC1B,WAAW,IAAI;AAAA,YACjB;AAAA,UACF,SAAS,OAAO;AACd,kBAAM,MAAM,eAAe;AAC3B,kBAAMD,YAAW,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;AACrD,kBAAM,4BAA4B,OAAO;AAAA,cACvC,UAAAA;AAAA,cACA,iBAAiB,IAAI,mBAAmB,CAAC;AAAA,cACzC,YAAY,IAAI,SAAS,CAAC;AAAA,cAC1B,WAAW,IAAI;AAAA,YACjB,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;;;AOp9BA;;;ARaO,IAAM,UAAU,IAAI,SAAoB;AAC7C,wBAAsB,gBAAgB;AACtC,SAAQ,aAA0B,GAAG,IAAI;AAC3C;","names":["result","output","metadata","duration","outOfScopeFlags"]}
+{"version":3,"sources":["../src/evals.ts","../src/evals/eval.ts","../src/evals/git-info.ts","../src/evals/eval.service.ts","../src/utils/fetcher.ts","../src/util/traces.ts","../src/util/deep-equal.ts","../src/util/tryCatch.ts","../src/evals/scorer.types.ts"],"sourcesContent":["export { Eval } from './evals/eval';\nexport type { EvalTask, EvalParams } from './evals/eval.types';\nexport { AxiomReporter } from './evals/reporter';\n\nexport type { EvalContextData } from './evals/context/storage';\n\nexport type { EvalBuilder } from './evals/builder';\nexport { type Score, type ScorerOptions } from './evals/scorer.types';\n\nimport { createScorer } from './evals/scorer.factory';\nimport { warnScorerDeprecation } from './evals/deprecated';\n\n/** @deprecated Import from 'axiom/ai/evals/scorers' instead. */\nexport const Scorer = ((...args: unknown[]) => {\n  warnScorerDeprecation('axiom/ai/evals');\n  return (createScorer as Function)(...args);\n}) as typeof createScorer;\n\nexport type { Evaluation, Case, Chat, Task } from './evals/eval.types';\n","import { afterAll, beforeAll, describe, inject, it } from 'vitest';\nimport { context, SpanStatusCode, trace, type Context } from '@opentelemetry/api';\nimport { customAlphabet } from 'nanoid';\nimport { withEvalContext, getEvalContext, getConfigScope } from './context/storage';\n\nimport { Attr } from '../otel/semconv/attributes';\nimport type { ResolvedAxiomConfig } from '../config/index';\nimport { startActiveSpan, startSpan, flush, ensureInstrumentationInitialized } from './instrument';\nimport { getGitUserInfo } from './git-info';\nimport type {\n  CollectionRecord,\n  EvalParams,\n  EvalTask,\n  EvaluationReport,\n  EvalCaseReport,\n  RuntimeFlagLog,\n  OutOfScopeFlag,\n  Evaluation,\n  OutOfScopeFlagAccess,\n} from './eval.types';\nimport type { ScoreWithName, ScorerLike, Scorer } from './scorer.types';\nimport { Mean, type Aggregation } from './aggregations';\nimport { EvaluationApiClient, findEvaluationCases } from './eval.service';\nimport { getGlobalFlagOverrides, setGlobalFlagOverrides } from './context/global-flags';\nimport { deepEqual } from '../util/deep-equal';\nimport { dotNotationToNested } from '../util/dot-path';\nimport { AxiomCLIError, errorToString } from '../util/errors';\nimport { tryCatchAsync, toError } from '../util/tryCatch';\nimport type { ValidateName } from '../util/name-validation';\nimport { recordName } from './name-validation-runtime';\n\ndeclare module 'vitest' {\n  interface TestSuiteMeta {\n    evaluation: EvaluationReport;\n  }\n  interface TaskMeta {\n    case: EvalCaseReport;\n    evaluation: EvaluationReport;\n  }\n  export interface ProvidedContext {\n    baseline?: string;\n    debug?: boolean;\n    list?: boolean;\n    overrides?: Record<string, any>;\n    axiomConfig?: ResolvedAxiomConfig;\n    runId: string;\n    consoleUrl?: string;\n  }\n}\n\nconst createVersionId = customAlphabet('0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ', 10);\n\ntype RunTaskFailureDetails = {\n  duration: number;\n  outOfScopeFlags: OutOfScopeFlagAccess[];\n  finalFlags: Record<string, any>;\n  overrides?: Record<string, any>;\n};\n\nconst RUN_TASK_FAILURE_DETAILS = Symbol.for('axiom.eval.runTaskFailureDetails');\n\nfunction attachRunTaskFailureDetails(\n  error: unknown,\n  details: RunTaskFailureDetails,\n): Error & { [RUN_TASK_FAILURE_DETAILS]: RunTaskFailureDetails } {\n  const normalized = toError(error) as Error & {\n    [RUN_TASK_FAILURE_DETAILS]?: RunTaskFailureDetails;\n  };\n  normalized[RUN_TASK_FAILURE_DETAILS] = details;\n  return normalized as Error & { [RUN_TASK_FAILURE_DETAILS]: RunTaskFailureDetails };\n}\n\nfunction getRunTaskFailureDetails(error: unknown): RunTaskFailureDetails | undefined {\n  if (typeof error !== 'object' || error === null) {\n    return undefined;\n  }\n  return (error as { [RUN_TASK_FAILURE_DETAILS]?: RunTaskFailureDetails })[\n    RUN_TASK_FAILURE_DETAILS\n  ];\n}\n\n/**\n * Creates and registers an evaluation suite with the given name and parameters.\n *\n * This function sets up a complete evaluation pipeline that will run your {@link EvalTask}\n * against a collection, score the results, and provide detailed {@link EvalCaseReport} reporting.\n *\n *\n * @param name - Human-readable name for the evaluation suite\n * @param params - {@link EvalParams} configuration parameters for the evaluation\n *\n * @example\n * ```typescript\n * import { Eval } from 'axiom/ai/evals';\n *\n * Eval('Text Generation Quality', {\n *   capability: 'capability-name',\n *   data: async () => [\n *     { input: 'Explain photosynthesis', expected: 'Plants convert light to energy...' },\n *     { input: 'What is gravity?', expected: 'Gravity is a fundamental force...' }\n *   ],\n *   task: async ({ input }) => {\n *     const result = await generateText({\n *       model: yourModel,\n *       prompt: input\n *     });\n *     return result.text;\n *   },\n *   scorers: [similarityScorer, factualAccuracyScorer],\n * });\n * ```\n */\nexport function Eval<\n  TInput,\n  TExpected,\n  TOutput,\n  Name extends string = string,\n  Capability extends string = string,\n  Step extends string = string,\n>(\n  name: ValidateName<Name>,\n  params: Omit<EvalParams<TInput, TExpected, TOutput>, 'capability' | 'step'> & {\n    capability: ValidateName<Capability>;\n    step?: ValidateName<Step> | undefined;\n  },\n): void {\n  // Record eval name for validation\n  recordName('eval', name);\n  recordName('capability', params.capability);\n  if (params.step) {\n    recordName('step', params.step);\n  }\n\n  // Record all scorer names for validation\n  if (params.scorers) {\n    for (const scorer of params.scorers) {\n      const scorerName = getScorerName(scorer, '');\n      recordName('scorer', scorerName);\n    }\n  }\n\n  registerEval(name, params as EvalParams<any, any, any>).catch(console.error);\n}\n\n/**\n * Capture full flag configuration filtered by configFlags scope\n */\nfunction captureFlagConfig(configFlags?: string[]): Record<string, any> {\n  if (!configFlags || configFlags.length === 0) {\n    return {};\n  }\n\n  const scope = getConfigScope();\n  const allDefaults = scope?.getAllDefaultFlags?.() ?? {};\n  const overrides = getGlobalFlagOverrides();\n\n  const merged = { ...allDefaults, ...overrides };\n\n  // Filter to only flags in configFlags scope\n  const filtered: Record<string, any> = {};\n  for (const [key, value] of Object.entries(merged)) {\n    const isInScope = configFlags.some((pattern) => key.startsWith(pattern));\n    if (isInScope) {\n      filtered[key] = value;\n    }\n  }\n\n  return dotNotationToNested(filtered);\n}\n\nconst getScorerName = <TScorer extends ScorerLike<any, any, any>>(\n  scorer: TScorer,\n  fallback: string = 'unknown',\n) => {\n  return (scorer as any).name || fallback;\n};\n\nasync function registerEval<\n  TInput extends string | Record<string, any>,\n  TExpected extends string | Record<string, any>,\n  TOutput extends string | Record<string, any>,\n>(evalName: string, opts: EvalParams<TInput, TExpected, TOutput>) {\n  opts.data;\n  const collectionPromise:\n    | readonly CollectionRecord<TInput, TExpected>[]\n    | Promise<readonly CollectionRecord<TInput, TExpected>[]> =\n    typeof opts.data === 'function'\n      ? (\n          opts.data as () =>\n            | readonly CollectionRecord<TInput, TExpected>[]\n            | Promise<readonly CollectionRecord<TInput, TExpected>[]>\n        )()\n      : opts.data;\n  const user = getGitUserInfo();\n\n  // check if user passed a specific baseline id to the CLI\n  const baselineId = inject('baseline');\n  const isDebug = inject('debug');\n  const isList = inject('list');\n  const injectedOverrides = inject('overrides');\n  const axiomConfig = inject('axiomConfig');\n  const runId = inject('runId');\n  const consoleUrl = inject('consoleUrl');\n\n  if (!axiomConfig) {\n    throw new AxiomCLIError('Axiom config not found');\n  }\n\n  const timeoutMs = opts.timeout ?? axiomConfig?.eval.timeoutMs;\n\n  const instrumentationReady = ensureInstrumentationInitialized(axiomConfig, {\n    enabled: !isDebug && !isList,\n  });\n\n  const result = await describe(\n    evalName,\n    async () => {\n      const collection = await collectionPromise;\n\n      const evaluationApiClient = new EvaluationApiClient(axiomConfig, consoleUrl);\n\n      // create a version code\n      const evalVersion = createVersionId();\n      let evalId = ''; // get traceId\n      let suiteStart: number;\n\n      let suiteSpan: ReturnType<typeof startSpan> | undefined;\n      let suiteContext: Context | undefined;\n      let instrumentationError: unknown = undefined;\n      let baseline: Evaluation | null | undefined = undefined;\n\n      // Track out-of-scope flags across all cases for evaluation-level reporting\n      const allOutOfScopeFlags: OutOfScopeFlagAccess[] = [];\n\n      // Track final config snapshot from the last executed case for reporter printing\n      let finalConfigSnapshot:\n        | { flags: Record<string, any>; pickedFlags?: string[]; overrides?: Record<string, any> }\n        | undefined;\n\n      beforeAll(async (suite) => {\n        // Ensure worker process knows CLI overrides\n        if (injectedOverrides && Object.keys(injectedOverrides).length > 0) {\n          try {\n            setGlobalFlagOverrides(injectedOverrides);\n          } catch {}\n        }\n\n        suite.meta.evaluation = {\n          id: evalId,\n          name: evalName,\n          version: evalVersion,\n          runId: runId,\n          orgId: undefined,\n          baseline: baseline ?? undefined,\n          configFlags: opts.configFlags,\n        };\n\n        const [, instrumentationInitError] = await tryCatchAsync(instrumentationReady);\n        if (instrumentationInitError) {\n          instrumentationError = instrumentationInitError;\n        }\n\n        suiteSpan = startSpan(`eval ${evalName}-${evalVersion}`, {\n          attributes: {\n            [Attr.GenAI.Operation.Name]: 'eval',\n            [Attr.Eval.Name]: evalName,\n            [Attr.Eval.Version]: evalVersion,\n            [Attr.Eval.Type]: 'regression', // TODO: where to get experiment type value from?\n            [Attr.Eval.Tags]: JSON.stringify(['offline']),\n            [Attr.Eval.Collection.ID]: 'custom', // TODO: where to get collection split value from?\n            [Attr.Eval.Collection.Name]: 'custom', // TODO: where to get collection name from?\n            [Attr.Eval.Collection.Size]: collection.length,\n            // capability\n            [Attr.Eval.Capability.Name]: opts.capability,\n            [Attr.Eval.Step.Name]: opts.step ?? undefined,\n            // metadata\n            [Attr.Eval.Metadata]: JSON.stringify(opts.metadata),\n            // run\n            [Attr.Eval.Run.ID]: runId,\n            // user info\n            [Attr.Eval.User.Name]: user?.name,\n            [Attr.Eval.User.Email]: user?.email,\n          },\n        });\n        evalId = suiteSpan.spanContext().traceId;\n        suite.meta.evaluation.id = evalId;\n        suiteSpan.setAttribute(Attr.Eval.ID, evalId);\n        suiteContext = trace.setSpan(context.active(), suiteSpan);\n\n        const flagConfig = captureFlagConfig(opts.configFlags);\n        suite.meta.evaluation.flagConfig = flagConfig;\n        const flagConfigJson = JSON.stringify(flagConfig);\n        suiteSpan.setAttribute(Attr.Eval.Config.Flags, flagConfigJson);\n\n        let createEvalResponse;\n        if (!isDebug && !isList) {\n          createEvalResponse = await evaluationApiClient.createEvaluation({\n            id: evalId,\n            name: evalName,\n            capability: opts.capability,\n            step: opts.step,\n            dataset: axiomConfig.eval.dataset,\n            version: evalVersion,\n            baselineId: baselineId ?? undefined,\n            runId: runId,\n            totalCases: collection.length,\n            config: { overrides: injectedOverrides },\n            configTimeoutMs: timeoutMs,\n            metadata: opts.metadata,\n            status: 'running',\n          });\n        }\n\n        const orgId = createEvalResponse?.data?.orgId;\n        const resolvedBaselineId = createEvalResponse?.data?.baselineId;\n\n        // Load baseline if we got a baselineId from the server\n        if (!isDebug && !isList && !!resolvedBaselineId) {\n          const [baselineResult, baselineError] = await tryCatchAsync(() =>\n            findEvaluationCases(resolvedBaselineId, axiomConfig),\n          );\n          if (baselineError) {\n            console.error(`Failed to load baseline: ${errorToString(baselineError)}`);\n            instrumentationError = instrumentationError || baselineError;\n          } else {\n            baseline = baselineResult;\n          }\n        }\n\n        // Update span with baseline info\n        if (baseline) {\n          suiteSpan.setAttribute(Attr.Eval.Baseline.ID, baseline.id);\n          suiteSpan.setAttribute(Attr.Eval.Baseline.Name, baseline.name);\n          suiteSpan.setAttribute(Attr.Eval.Baseline.Version, baseline.version);\n        }\n\n        suite.meta.evaluation = {\n          id: evalId,\n          name: evalName,\n          version: evalVersion,\n          runId: runId,\n          orgId: orgId ?? undefined,\n          baseline: baseline ?? undefined,\n          configFlags: opts.configFlags,\n          registrationStatus: instrumentationError\n            ? {\n                status: 'failed',\n                error: errorToString(instrumentationError),\n              }\n            : { status: 'success' },\n          trials: opts.trials,\n        };\n\n        suiteStart = performance.now();\n      });\n\n      afterAll(async (suite) => {\n        if (instrumentationError) {\n          throw instrumentationError;\n        }\n\n        // Aggregate out-of-scope flags for evaluation-level reporting\n        const flagSummary = new Map<string, OutOfScopeFlag>();\n\n        for (const flag of allOutOfScopeFlags) {\n          if (flagSummary.has(flag.flagPath)) {\n            const existing = flagSummary.get(flag.flagPath)!;\n            existing.count++;\n            existing.firstAccessedAt = Math.min(existing.firstAccessedAt, flag.accessedAt);\n            existing.lastAccessedAt = Math.max(existing.lastAccessedAt, flag.accessedAt);\n          } else {\n            flagSummary.set(flag.flagPath, {\n              flagPath: flag.flagPath,\n              count: 1,\n              firstAccessedAt: flag.accessedAt,\n              lastAccessedAt: flag.accessedAt,\n              stackTrace: flag.stackTrace,\n            });\n          }\n        }\n\n        // Update evaluation report with aggregated out-of-scope flags\n        if (suite.meta.evaluation && suiteSpan) {\n          suite.meta.evaluation.outOfScopeFlags = Array.from(flagSummary.entries()).map(\n            ([_flagPath, stats]) => stats,\n          );\n\n          // Attach end-of-suite config snapshot for reporter printing\n          const allDefaults = getConfigScope()?.getAllDefaultFlags();\n          const pickedFlags = finalConfigSnapshot?.pickedFlags;\n          const overrides = injectedOverrides ?? getGlobalFlagOverrides();\n\n          suite.meta.evaluation.configEnd = {\n            flags: allDefaults,\n            pickedFlags,\n            overrides,\n          };\n        }\n\n        // end root span\n        suiteSpan?.setStatus({ code: SpanStatusCode.OK });\n        suiteSpan?.end();\n\n        // flush traces before updating Evaluation in Axiom\n        const [, flushError] = await tryCatchAsync(flush);\n        if (flushError) {\n          // Update registration status to failed if flush fails\n          if (suite.meta.evaluation) {\n            suite.meta.evaluation.registrationStatus = {\n              status: 'failed',\n              error: errorToString(flushError),\n            };\n          }\n        }\n\n        const durationMs = Math.round(performance.now() - suiteStart);\n\n        const successCases = suite.tasks.filter(\n          (task) => task.meta.case.status === 'success',\n        ).length;\n        const erroredCases = suite.tasks.filter(\n          (task) => task.meta.case.status === 'fail' || task.meta.case.status === 'pending',\n        ).length;\n\n        // signal Axiom that evaluation finished to kick of summary calculations\n        if (!isDebug && !isList) {\n          await evaluationApiClient.updateEvaluation({\n            id: evalId,\n            status: 'completed',\n            totalCases: collection.length,\n            successCases,\n            erroredCases,\n            durationMs,\n          });\n        }\n      });\n\n      type CollectionRecordWithIndex = { index: number } & CollectionRecord<TInput, TExpected>;\n\n      await it.concurrent.for(\n        collection.map((d, index) => ({ ...d, index }) satisfies CollectionRecordWithIndex),\n      )('case', async (data, { task }) => {\n        const start = performance.now();\n        if (!suiteContext) {\n          throw new Error(\n            '[Axiom AI] Suite context not initialized. This is likely a bug – instrumentation should complete before tests run.',\n          );\n        }\n\n        let outOfScopeFlags: OutOfScopeFlagAccess[] = [];\n\n        await startActiveSpan(\n          `case ${data.index}`,\n          {\n            attributes: {\n              [Attr.GenAI.Operation.Name]: 'eval.case',\n              [Attr.Eval.ID]: evalId,\n              [Attr.Eval.Name]: evalName,\n              [Attr.Eval.Version]: evalVersion,\n              [Attr.Eval.Case.Index]: data.index,\n              [Attr.Eval.Case.Input]:\n                typeof data.input === 'string' ? data.input : JSON.stringify(data.input),\n              [Attr.Eval.Case.Expected]:\n                typeof data.expected === 'string' ? data.expected : JSON.stringify(data.expected),\n              [Attr.Eval.Case.Metadata]: data.metadata ? JSON.stringify(data.metadata) : undefined,\n              // user info\n              [Attr.Eval.User.Name]: user?.name,\n              [Attr.Eval.User.Email]: user?.email,\n            },\n          },\n          async (caseSpan) => {\n            const caseContext = trace.setSpan(context.active(), caseSpan);\n            const trials = Math.max(1, opts.trials ?? 1);\n            let intentionalTrialFailureError: Error | undefined;\n            let caseFinalConfigSnapshot:\n              | {\n                  flags: Record<string, any>;\n                  pickedFlags?: string[];\n                  overrides?: Record<string, any>;\n                }\n              | undefined;\n\n            // Set case-level trials attribute\n            caseSpan.setAttribute(Attr.Eval.Case.Trials, trials);\n\n            try {\n              // Accumulators for per-trial scores\n              const perScorerTrials: Record<string, number[]> = {};\n              const trialErrors: Array<string | null> = Array.from({ length: trials }, () => null);\n              const trialFailures: Error[] = [];\n              let lastOutput: TOutput | undefined;\n              let successfulTaskDuration = 0;\n\n              // Run each trial\n              for (let trialIndex = 0; trialIndex < trials; trialIndex++) {\n                try {\n                  await startActiveSpan(\n                    `trial ${trialIndex}`,\n                    {\n                      attributes: {\n                        [Attr.GenAI.Operation.Name]: 'eval.trial',\n                        [Attr.Eval.Trial.Index]: trialIndex,\n                        [Attr.Eval.ID]: evalId,\n                        [Attr.Eval.Name]: evalName,\n                        [Attr.Eval.Version]: evalVersion,\n                      },\n                    },\n                    async (trialSpan) => {\n                      const trialContext = trace.setSpan(context.active(), trialSpan);\n                      try {\n                        const result = await runTask(\n                          trialContext,\n                          {\n                            id: evalId,\n                            version: evalVersion,\n                            name: evalName,\n                          },\n                          {\n                            index: data.index,\n                            input: data.input,\n                            expected: data.expected,\n                            scorers: opts.scorers,\n                            task: opts.task,\n                            metadata: opts.metadata,\n                            configFlags: opts.configFlags,\n                            capability: opts.capability,\n                            step: opts.step,\n                          },\n                        );\n                        const { output, duration } = result;\n                        lastOutput = output;\n                        successfulTaskDuration += duration;\n                        outOfScopeFlags.push(...result.outOfScopeFlags);\n\n                        caseFinalConfigSnapshot = {\n                          flags: result.finalFlags || {},\n                          pickedFlags: opts.configFlags,\n                          overrides: result.overrides,\n                        };\n\n                        // Run scorers inside the trial span\n                        await Promise.all(\n                          opts.scorers.map(async (scorer) => {\n                            const scorerName = getScorerName(scorer);\n                            return startActiveSpan(\n                              `score ${scorerName}`,\n                              {\n                                attributes: {\n                                  [Attr.GenAI.Operation.Name]: 'eval.score',\n                                  [Attr.Eval.Tags]: JSON.stringify(['offline']),\n                                  [Attr.Eval.ID]: evalId,\n                                  [Attr.Eval.Name]: evalName,\n                                  [Attr.Eval.Version]: evalVersion,\n                                  [Attr.Eval.Trial.Index]: trialIndex,\n                                },\n                              },\n                              async (scorerSpan) => {\n                                const scorerStart = performance.now();\n                                try {\n                                  const [result, scorerError] = await tryCatchAsync(() =>\n                                    scorer({\n                                      input: data.input,\n                                      output: output,\n                                      expected: data.expected,\n                                      trialIndex,\n                                    }),\n                                  );\n\n                                  if (scorerError || !result) {\n                                    const scorerDuration = Math.round(\n                                      performance.now() - scorerStart,\n                                    );\n                                    console.error(\n                                      `ERROR: scorer ${scorerName} failed. Cause: \\n`,\n                                      scorerError,\n                                    );\n                                    const msg = errorToString(scorerError);\n                                    const metadata = {\n                                      duration: scorerDuration,\n                                      startedAt: scorerStart,\n                                      error: msg,\n                                    };\n\n                                    // Count scorer failures as zero so failed trials affect aggregation.\n                                    (perScorerTrials[scorerName] ??= []).push(0);\n\n                                    scorerSpan.setAttributes({\n                                      [Attr.Eval.Score.Name]: scorerName,\n                                      [Attr.Eval.Score.Metadata]: JSON.stringify(metadata),\n                                    });\n\n                                    scorerSpan.setStatus({\n                                      code: SpanStatusCode.ERROR,\n                                      message: msg,\n                                    });\n                                    return;\n                                  }\n\n                                  const scoreDuration = Math.round(performance.now() - scorerStart);\n                                  const scoreValue = result.score as number;\n                                  const metadata = Object.assign(\n                                    { duration: scoreDuration, startedAt: scorerStart },\n                                    result.metadata,\n                                  );\n\n                                  // Collect per-trial score\n                                  (perScorerTrials[scorerName] ??= []).push(scoreValue);\n\n                                  // Get aggregation config for span attributes\n                                  const aggregation: Aggregation =\n                                    (scorer as Scorer).aggregation ?? Mean();\n\n                                  scorerSpan.setAttributes({\n                                    [Attr.Eval.Score.Name]: scorerName,\n                                    [Attr.Eval.Score.Value]: scoreValue,\n                                    [Attr.Eval.Score.Metadata]: JSON.stringify(metadata),\n                                    [Attr.Eval.Score.Aggregation]: aggregation.type,\n                                  });\n\n                                  if (metadata.error) {\n                                    const msg = errorToString(metadata.error);\n                                    scorerSpan.setStatus({\n                                      code: SpanStatusCode.ERROR,\n                                      message: msg,\n                                    });\n                                  }\n                                } finally {\n                                  scorerSpan.end();\n                                }\n                              },\n                              trialContext,\n                            );\n                          }),\n                        );\n                      } catch (error) {\n                        const taskFailureDetails = getRunTaskFailureDetails(error);\n                        const failure = toError(error);\n                        const msg = errorToString(failure);\n                        const spanErrorMessage = failure.message || msg;\n                        trialErrors[trialIndex] = msg;\n                        trialFailures.push(failure);\n\n                        trialSpan.setAttributes({\n                          [Attr.Eval.Trial.Error]: spanErrorMessage,\n                        });\n\n                        for (const scorer of opts.scorers) {\n                          const scorerName = getScorerName(scorer);\n                          (perScorerTrials[scorerName] ??= []).push(0);\n                        }\n\n                        if (taskFailureDetails) {\n                          outOfScopeFlags.push(...taskFailureDetails.outOfScopeFlags);\n                          caseFinalConfigSnapshot = {\n                            flags: taskFailureDetails.finalFlags || {},\n                            pickedFlags: opts.configFlags,\n                            overrides: taskFailureDetails.overrides,\n                          };\n                        }\n\n                        // Re-throw so startActiveSpan records the trial span as ERROR.\n                        throw failure;\n                      }\n                    },\n                    caseContext,\n                  );\n                } catch {\n                  // Continue remaining trials after task-level failures.\n                }\n              }\n\n              // Aggregate scores across trials\n              const scores: Record<string, ScoreWithName> = {};\n              for (const scorer of opts.scorers) {\n                const scorerName = getScorerName(scorer);\n                const trialsArr = perScorerTrials[scorerName] ?? [];\n                const aggregation: Aggregation = (scorer as Scorer).aggregation ?? Mean();\n\n                const aggregatedValue = trialsArr.length > 0 ? aggregation.aggregate(trialsArr) : 0;\n\n                scores[scorerName] = {\n                  name: scorerName,\n                  score: aggregatedValue,\n                  trials: trialsArr,\n                  aggregation: aggregation.type,\n                  threshold: aggregation.threshold,\n                  metadata: {},\n                };\n              }\n\n              const output = lastOutput;\n              const failedTrials = trialFailures.length;\n              const succeededTrials = trials - failedTrials;\n              const trialSummary = {\n                total: trials,\n                succeeded: succeededTrials,\n                failed: failedTrials,\n              };\n\n              caseSpan.setAttribute(Attr.Eval.Case.Scores, JSON.stringify(scores ? scores : {}));\n              if (output !== undefined) {\n                caseSpan.setAttribute(\n                  Attr.Eval.Case.Output,\n                  typeof output === 'string' ? output : JSON.stringify(output),\n                );\n              }\n\n              // set task meta for showing result in vitest report\n              task.meta.case = {\n                index: data.index,\n                name: evalName,\n                expected: data.expected,\n                input: data.input,\n                output: output,\n                metadata: data.metadata,\n                scores,\n                status: 'success',\n                errors: [],\n                trialErrors,\n                trialSummary,\n                duration: successfulTaskDuration,\n                startedAt: start,\n                outOfScopeFlags,\n                pickedFlags: opts.configFlags,\n              };\n\n              if (failedTrials > 0) {\n                const error = new Error(\n                  `Eval case ${data.index} failed with ${failedTrials} trial error(s).`,\n                );\n                intentionalTrialFailureError = error;\n                caseSpan.setStatus({\n                  code: SpanStatusCode.ERROR,\n                  message: error.message,\n                });\n                task.meta.case.status = 'fail';\n                task.meta.case.errors = trialFailures;\n                throw error;\n              }\n\n              // Collect out-of-scope flags for evaluation-level aggregation\n              allOutOfScopeFlags.push(...outOfScopeFlags);\n            } catch (e) {\n              console.log(e);\n              const error = toError(e);\n\n              if (e === intentionalTrialFailureError && task.meta.case) {\n                task.meta.case.status = 'fail';\n                task.meta.case.errors = task.meta.case.errors?.length\n                  ? task.meta.case.errors\n                  : [error];\n                allOutOfScopeFlags.push(...outOfScopeFlags);\n                throw e;\n              }\n\n              const ctx = getEvalContext();\n              const ctxOutOfScopeFlags = ctx.outOfScopeFlags || [];\n              if (ctxOutOfScopeFlags.length > 0) {\n                outOfScopeFlags.push(...ctxOutOfScopeFlags);\n              }\n\n              const ctxFlags = ctx.flags || {};\n              if (!caseFinalConfigSnapshot && Object.keys(ctxFlags).length > 0) {\n                caseFinalConfigSnapshot = {\n                  flags: ctxFlags,\n                  pickedFlags: opts.configFlags,\n                  overrides: ctx.overrides,\n                };\n              }\n\n              // Populate scores with error metadata for all scorers that didn't run\n              const failedScores: Record<string, ScoreWithName> = {};\n              for (const scorer of opts.scorers) {\n                const scorerName = getScorerName(scorer);\n                failedScores[scorerName] = {\n                  name: scorerName,\n                  score: 0,\n                  trials: [],\n                  metadata: {\n                    duration: 0,\n                    startedAt: start,\n                    error: error.message,\n                  },\n                };\n              }\n\n              task.meta.case = {\n                name: evalName,\n                index: data.index,\n                expected: data.expected,\n                input: data.input,\n                output: String(e),\n                metadata: data.metadata,\n                scores: failedScores,\n                status: 'fail',\n                errors: [error],\n                startedAt: start,\n                duration: Math.round(performance.now() - start),\n                outOfScopeFlags,\n                pickedFlags: opts.configFlags,\n              };\n\n              allOutOfScopeFlags.push(...outOfScopeFlags);\n              throw e;\n            } finally {\n              // Compute per-case runtime flags report and attach to span/meta\n              try {\n                const accessedFlags: Record<string, any> = caseFinalConfigSnapshot?.flags || {};\n\n                const accessed = Object.keys(accessedFlags);\n                const allDefaults = getConfigScope()?.getAllDefaultFlags?.() ?? {};\n\n                const runtimeFlags: Record<string, RuntimeFlagLog> = {};\n                for (const key of accessed) {\n                  const value = accessedFlags[key];\n                  if (key in allDefaults) {\n                    const replaced = !deepEqual(value, allDefaults[key]);\n                    if (replaced) {\n                      runtimeFlags[key] = { kind: 'replaced', value, default: allDefaults[key] };\n                    }\n                  } else {\n                    runtimeFlags[key] = { kind: 'introduced', value };\n                  }\n                }\n\n                if (!isDebug && Object.keys(runtimeFlags).length > 0) {\n                  const serialized = JSON.stringify(runtimeFlags);\n                  caseSpan.setAttribute('eval.case.config.runtime_flags', serialized);\n                }\n\n                if (task.meta.case) {\n                  task.meta.case.runtimeFlags = runtimeFlags;\n                }\n              } catch {}\n\n              if (caseFinalConfigSnapshot) {\n                finalConfigSnapshot = caseFinalConfigSnapshot;\n              }\n            }\n          },\n          suiteContext,\n        );\n      });\n    },\n    timeoutMs,\n  );\n\n  return result;\n}\n\nconst joinArrayOfUnknownResults = <T extends string | Record<string, any>>(results: T[]): T => {\n  if (results.length === 0) {\n    return '' as unknown as T;\n  }\n\n  // If all results are strings, concatenate them\n  if (results.every((r) => typeof r === 'string')) {\n    return results.join('') as unknown as T;\n  }\n\n  // If we have objects, return the last one (streaming typically overwrites)\n  return results[results.length - 1];\n};\n\nconst executeTask = async <\n  TInput extends string | Record<string, any>,\n  TExpected extends string | Record<string, any>,\n  TOutput extends string | Record<string, any>,\n>(\n  task: EvalTask<TInput, TExpected, TOutput>,\n  input: TInput,\n  expected: TExpected,\n): Promise<TOutput> => {\n  const taskResultOrStream = await task({ input, expected });\n\n  if (\n    typeof taskResultOrStream === 'object' &&\n    taskResultOrStream &&\n    Symbol.asyncIterator in taskResultOrStream\n  ) {\n    const chunks: TOutput[] = [];\n\n    for await (const chunk of taskResultOrStream) {\n      chunks.push(chunk);\n    }\n\n    return joinArrayOfUnknownResults<TOutput>(chunks as TOutput[]);\n  }\n\n  return taskResultOrStream;\n};\n\nconst runTask = async <\n  TInput extends string | Record<string, any>,\n  TExpected extends string | Record<string, any>,\n  TOutput extends string | Record<string, any>,\n>(\n  caseContext: Context,\n  evaluation: {\n    id: string;\n    name: string;\n    version: string;\n  },\n  opts: {\n    index: number;\n    input: TInput;\n    expected: TExpected | undefined;\n  } & Omit<EvalParams<TInput, TExpected, TOutput>, 'data'>,\n) => {\n  const taskName = opts.task.name ?? 'anonymous';\n\n  return startActiveSpan(\n    `task`,\n    {\n      attributes: {\n        [Attr.GenAI.Operation.Name]: 'eval.task',\n        [Attr.Eval.Task.Name]: taskName,\n        [Attr.Eval.Task.Type]: 'llm_completion', // TODO: How to determine task type?\n        [Attr.Eval.ID]: evaluation.id,\n        [Attr.Eval.Name]: evaluation.name,\n        [Attr.Eval.Version]: evaluation.version,\n      },\n    },\n    async (taskSpan) => {\n      // Initialize evaluation context for flag/fact access\n      const { output, duration, outOfScopeFlags, finalFlags, overrides } = await withEvalContext(\n        { pickedFlags: opts.configFlags },\n        async (): Promise<{\n          output: TOutput;\n          duration: number;\n          outOfScopeFlags: OutOfScopeFlagAccess[];\n          finalFlags: Record<string, any>;\n          overrides?: Record<string, any>;\n        }> => {\n          // TODO: EXPERIMENTS - before we were setting config scope if provided here\n          const start = performance.now();\n          try {\n            const output = await executeTask(opts.task, opts.input, opts.expected!);\n            const duration = Math.round(performance.now() - start);\n            // set task output\n            taskSpan.setAttributes({\n              [Attr.Eval.Task.Output]: typeof output === 'string' ? output : JSON.stringify(output),\n            });\n\n            // Get out-of-scope flags from the evaluation context\n            const ctx = getEvalContext();\n            const outOfScopeFlags = ctx.outOfScopeFlags || [];\n\n            return {\n              output,\n              duration,\n              outOfScopeFlags,\n              finalFlags: ctx.flags || {},\n              overrides: ctx.overrides,\n            };\n          } catch (error) {\n            const ctx = getEvalContext();\n            const duration = Math.round(performance.now() - start);\n            throw attachRunTaskFailureDetails(error, {\n              duration,\n              outOfScopeFlags: ctx.outOfScopeFlags || [],\n              finalFlags: ctx.flags || {},\n              overrides: ctx.overrides,\n            });\n          }\n        },\n      );\n\n      return {\n        output,\n        duration,\n        outOfScopeFlags,\n        finalFlags,\n        overrides,\n      };\n    },\n    caseContext,\n  );\n};\n","import { execSync } from 'node:child_process';\n\nexport function getGitUserInfo() {\n  try {\n    const name = execSync('git config --get user.name').toString().trim();\n    const email = execSync('git config --get user.email').toString().trim();\n    return { name, email };\n  } catch {\n    return null; // Git not installed or not configured\n  }\n}\n","import type { Case, Chat, Evaluation, Task } from './eval.types';\nimport { createFetcher, type Fetcher } from '../utils/fetcher';\nimport type { ResolvedAxiomConfig } from '../config/index';\nimport { resolveAxiomConnection } from '../config/resolver';\nimport { Attr } from '../otel';\nimport { AxiomCLIError } from '../util/errors';\nimport {\n  getCustomOrRegularAttribute,\n  getCustomOrRegularNumber,\n  getCustomOrRegularString,\n} from '../util/traces';\n\nexport interface EvaluationApiConfig {\n  dataset?: string;\n  region?: string;\n  baseUrl?: string;\n  apiUrl?: string;\n  token?: string;\n}\n\nexport type EvaluationStatus = 'running' | 'completed' | 'errored' | 'cancelled';\n\nexport interface EvaluationApiPayloadBase {\n  id: string;\n  name: string;\n  capability: string;\n  step?: string | undefined;\n  dataset: string;\n  baselineId?: string;\n  totalCases?: number;\n  config?: Record<string, unknown>;\n  status: EvaluationStatus;\n  successCases?: number;\n  erroredCases?: number;\n  durationMs?: number;\n  scorerAvgs?: number[];\n  version: string;\n  runId: string;\n  configTimeoutMs: number;\n  metadata?: Record<string, any>;\n}\n\nexport class EvaluationApiClient {\n  private readonly fetcher: Fetcher;\n  constructor(config: ResolvedAxiomConfig, consoleUrl?: string) {\n    const { consoleEndpointUrl, token, orgId } = resolveAxiomConnection(config, consoleUrl);\n\n    this.fetcher = createFetcher({ baseUrl: consoleEndpointUrl, token: token ?? '', orgId });\n  }\n\n  async createEvaluation(evaluation: EvaluationApiPayloadBase) {\n    const resp = await this.fetcher(`/api/v3/evaluations`, {\n      method: 'POST',\n      body: JSON.stringify(evaluation),\n    });\n\n    if (!resp.ok) {\n      const text = await resp.text().catch(() => '');\n      throw new AxiomCLIError(\n        `Failed to create evaluation: ${resp.statusText}${text ? ` - ${text}` : ''}`,\n      );\n    }\n\n    return resp.json();\n  }\n\n  async updateEvaluation(evaluation: Partial<EvaluationApiPayloadBase>) {\n    const resp = await this.fetcher(`/api/v3/evaluations/${evaluation.id}`, {\n      method: 'PATCH',\n      body: JSON.stringify(evaluation),\n    });\n\n    if (!resp.ok) {\n      const text = await resp.text().catch(() => '');\n      throw new AxiomCLIError(\n        `Failed to update evaluation: ${resp.statusText}${text ? ` - ${text}` : ''}`,\n      );\n    }\n\n    // API may return HTTP 200 with an error in the response body\n    const body = await resp.json();\n    if (body.error) {\n      throw new AxiomCLIError(\n        `Failed to update evaluation ${evaluation.id}: ${JSON.stringify(body.error)}`,\n      );\n    }\n\n    return body;\n  }\n}\n\nexport const findEvaluationCases = async (\n  evalId: string,\n  config: ResolvedAxiomConfig,\n): Promise<Evaluation | null> => {\n  const { dataset, edgeUrl, token, orgId } = resolveAxiomConnection(config);\n\n  const apl = `['${dataset}'] | where trace_id == \"${evalId}\" | order by _time`;\n\n  const headers = new Headers({\n    Authorization: `Bearer ${token}`,\n    'Content-Type': 'application/json',\n    ...(orgId ? { 'X-AXIOM-ORG-ID': orgId } : {}),\n  });\n\n  // Use edgeUrl for query operations\n  const resp = await fetch(`${edgeUrl}/v1/datasets/_apl?format=legacy`, {\n    headers: headers,\n    method: 'POST',\n    body: JSON.stringify({ apl }),\n  });\n  const payload = await resp.json();\n\n  if (!resp.ok) {\n    throw new Error(`Failed to query evaluation cases: ${payload.message || resp.statusText}`);\n  }\n\n  return payload.matches.length ? buildSpanTree(payload.matches) : null;\n};\n\ntype DeepPartial<T> = T extends object ? { [P in keyof T]?: DeepPartial<T[P]> } : T;\n\nexport const mapSpanToEval = (span: any): Evaluation => {\n  const flagConfigRaw = getCustomOrRegularAttribute(span.data.attributes, Attr.Eval.Config.Flags);\n  const tagsRaw = getCustomOrRegularAttribute(span.data.attributes, Attr.Eval.Tags);\n\n  const evaluation: DeepPartial<Evaluation> = {\n    id: getCustomOrRegularString(span.data.attributes, Attr.Eval.ID),\n    name: getCustomOrRegularString(span.data.attributes, Attr.Eval.Name),\n    type: getCustomOrRegularString(span.data.attributes, Attr.Eval.Type),\n    version: getCustomOrRegularString(span.data.attributes, Attr.Eval.Version),\n    collection: {\n      name: getCustomOrRegularString(span.data.attributes, Attr.Eval.Collection.Name),\n      size: getCustomOrRegularNumber(span.data.attributes, Attr.Eval.Collection.Size),\n    },\n    baseline: {\n      id: getCustomOrRegularString(span.data.attributes, Attr.Eval.Baseline.ID),\n      name: getCustomOrRegularString(span.data.attributes, Attr.Eval.Baseline.Name),\n    },\n    duration: span.data.duration,\n    status: span.data.status.code,\n    traceId: span.data.trace_id,\n    runAt: span._time,\n    tags: tagsRaw ? (typeof tagsRaw === 'string' ? JSON.parse(tagsRaw) : tagsRaw) : [],\n    user: {\n      name: getCustomOrRegularString(span.data.attributes, Attr.Eval.User.Name),\n      email: getCustomOrRegularString(span.data.attributes, Attr.Eval.User.Email),\n    },\n    cases: [],\n    flagConfig: flagConfigRaw\n      ? typeof flagConfigRaw === 'string'\n        ? JSON.parse(flagConfigRaw)\n        : flagConfigRaw\n      : undefined,\n  };\n\n  // TODO: this is very optimistic!\n  return evaluation as Evaluation;\n};\n\nexport const mapSpanToCase = (item: { _time: string; data: any }): Case => {\n  const data = item.data;\n  // round duration\n  const d = data.duration as string;\n  let duration = '-';\n  if (d.endsWith('s')) {\n    duration = `${Number(d.replace('s', '')).toFixed(2)}s`;\n  } else {\n    duration = d;\n  }\n\n  const scoresRaw = getCustomOrRegularAttribute(data.attributes, Attr.Eval.Case.Scores);\n  const scoresParsed = scoresRaw\n    ? typeof scoresRaw === 'string'\n      ? JSON.parse(scoresRaw)\n      : scoresRaw\n    : {};\n\n  // Normalize scores: convert .score to .value if needed (ScoreWithName uses .score, Case uses .value)\n  // FUTURE: find a better way of handling this\n  const scores: Case['scores'] = {};\n  for (const [name, scoreData] of Object.entries(scoresParsed)) {\n    const s = scoreData as { score?: number; value?: number; metadata?: Record<string, any> };\n    scores[name] = {\n      name,\n      value: s.value ?? s.score ?? 0,\n      metadata: s.metadata ?? {},\n    };\n  }\n\n  const caseData: DeepPartial<Case> = {\n    index: getCustomOrRegularNumber(data.attributes, Attr.Eval.Case.Index),\n    input: getCustomOrRegularString(data.attributes, Attr.Eval.Case.Input),\n    output: getCustomOrRegularString(data.attributes, Attr.Eval.Case.Output),\n    expected: getCustomOrRegularString(data.attributes, Attr.Eval.Case.Expected),\n    duration: duration,\n    status: data.status.code,\n    scores,\n    runAt: item._time,\n    spanId: data.span_id,\n    traceId: data.trace_id,\n  };\n\n  // TODO: this is very optimistic!\n  return caseData as Case;\n};\n\n// compute a root eval with its children spans, results in a usable object of eval, cases, scores and chats\nexport const buildSpanTree = (spans: any[]): Evaluation | null => {\n  if (!spans.length) {\n    return null;\n  }\n\n  // Find the root eval span\n  const evalSpan = spans.find((span) => span.data.attributes.gen_ai.operation.name === 'eval');\n\n  if (!evalSpan) {\n    return null;\n  }\n\n  // Create the root eval structure\n  const rootSpan: Evaluation = mapSpanToEval(evalSpan);\n\n  // Find all case spans and build the tree structure\n  const caseSpans = spans.filter((span) => span.data.name.startsWith('case'));\n\n  for (const caseSpan of caseSpans) {\n    // Convert case data\n    const caseData = mapSpanToCase(caseSpan);\n\n    // Find task spans that belong to this case\n    const taskSpans = spans.filter(\n      (span) =>\n        span.data.name.startsWith('task') && span.data.parent_span_id === caseSpan.data.span_id,\n    );\n\n    if (taskSpans.length > 0) {\n      const taskSpan = taskSpans[0]; // Assuming one task per case\n\n      // Find chat spans that belong to this task\n      const chatSpans = spans.filter(\n        (span) =>\n          span.data.name.startsWith('chat') && span.data.parent_span_id === taskSpan.data.span_id,\n      );\n\n      const chatData: Chat[] = chatSpans.map((chatSpan) => ({\n        operation: getCustomOrRegularString(chatSpan.data.attributes, 'operation') ?? '',\n        capability: getCustomOrRegularString(chatSpan.data.attributes, 'capability') ?? '',\n        step: getCustomOrRegularString(chatSpan.data.attributes, 'step') ?? '',\n        request: {\n          max_token: getCustomOrRegularString(chatSpan.data.attributes, 'request.max_token') ?? '',\n          model: getCustomOrRegularString(chatSpan.data.attributes, 'request.model') ?? '',\n          temperature:\n            getCustomOrRegularNumber(chatSpan.data.attributes, 'request.temperature') ?? 0,\n        },\n        response: {\n          finish_reasons:\n            getCustomOrRegularString(chatSpan.data.attributes, 'response.finish_reasons') ?? '',\n        },\n        usage: {\n          input_tokens:\n            getCustomOrRegularNumber(chatSpan.data.attributes, 'usage.input_tokens') ?? 0,\n          output_tokens:\n            getCustomOrRegularNumber(chatSpan.data.attributes, 'usage.output_tokens') ?? 0,\n        },\n      }));\n\n      // Create task data with chat information\n      const taskData: Task = {\n        name: taskSpan.data.name,\n        output: getCustomOrRegularString(taskSpan.data.attributes, 'output') || '',\n        trial: getCustomOrRegularNumber(taskSpan.data.attributes, 'trial') || 0,\n        type: getCustomOrRegularString(taskSpan.data.attributes, 'type') || '',\n        error: getCustomOrRegularString(taskSpan.data.attributes, 'error') || '',\n        chat: chatData[0] || {\n          operation: '',\n          capability: '',\n          step: '',\n          request: { max_token: '', model: '', temperature: 0 },\n          response: { finish_reasons: '' },\n          usage: { input_tokens: 0, output_tokens: 0 },\n        },\n      };\n\n      caseData.task = taskData;\n    }\n\n    // Find task spans that belong to this case\n    const scoreSpans = spans.filter(\n      (span) =>\n        span.data.attributes.gen_ai.operation.name === 'eval.score' &&\n        span.data.parent_span_id === caseSpan.data.span_id,\n    );\n\n    if (scoreSpans.length > 0) {\n      caseData.scores = {};\n\n      scoreSpans.forEach((score) => {\n        const name = getCustomOrRegularString(score.data.attributes, Attr.Eval.Score.Name) ?? '';\n        const value = getCustomOrRegularNumber(score.data.attributes, Attr.Eval.Score.Value) ?? 0;\n        const metadataRaw = getCustomOrRegularString(\n          score.data.attributes,\n          Attr.Eval.Score.Metadata,\n        );\n        let metadata = {};\n        try {\n          metadata = metadataRaw ? JSON.parse(metadataRaw) : {};\n        } catch {\n          // Ignore error\n        }\n\n        caseData.scores[name] = {\n          name,\n          value,\n          metadata: {\n            error: score.data.attributes.error,\n            ...metadata,\n          },\n        };\n      });\n    }\n\n    rootSpan.cases.push(caseData);\n  }\n\n  rootSpan.cases.sort((a, b) => a.index - b.index);\n\n  return rootSpan;\n};\n","export interface Fetcher {\n  (path: string, options: RequestInit): Promise<Response>;\n}\n\nexport interface FetcherOptions {\n  baseUrl: string;\n  token: string;\n  orgId?: string;\n}\n\nexport const createFetcher = ({\n  baseUrl,\n  token,\n  orgId,\n}: {\n  baseUrl: string;\n  token: string;\n  orgId?: string;\n}): Fetcher => {\n  return (path: string, options: RequestInit) =>\n    fetch(new URL(path, baseUrl).toString(), {\n      ...options,\n      headers: {\n        ...options.headers,\n        'content-type': 'application/json',\n        authorization: `Bearer ${token}`,\n        ...(orgId ? { 'X-AXIOM-ORG-ID': orgId } : {}),\n      },\n    });\n};\n","export function getCustomOrRegularAttribute(obj: unknown, accessKey: string): unknown {\n  if (typeof obj !== 'object' || obj === null) {\n    return undefined;\n  }\n\n  const keyParts = accessKey.split('.');\n  const custom = (obj as Record<string, any>).custom;\n\n  // Try `obj.custom['foo.bar']`\n  if (custom && typeof custom === 'object' && custom !== null && accessKey in custom) {\n    return custom[accessKey];\n  }\n\n  // Try `obj.foo.bar`\n  let current: any = obj;\n  for (const part of keyParts) {\n    if (typeof current !== 'object' || current === null) {\n      return undefined;\n    }\n    current = current[part];\n  }\n\n  return current;\n}\n\nexport function getCustomOrRegularString(obj: unknown, key: string): string | undefined {\n  const value = getCustomOrRegularAttribute(obj, key);\n\n  return typeof value === 'string' ? value : undefined;\n}\n\nexport function getCustomOrRegularNumber(obj: unknown, key: string): number | undefined {\n  const value = getCustomOrRegularAttribute(obj, key);\n\n  if (typeof value === 'number') {\n    return value;\n  }\n\n  if (typeof value === 'string') {\n    const parsed = Number(value);\n\n    return Number.isNaN(parsed) ? undefined : parsed;\n  }\n\n  return undefined;\n}\n","/**\n * Copied from Remeda - https://github.com/remeda/remeda/blob/v2.32.0/packages/remeda/src/isDeepEqual.ts\n */\n\nexport function deepEqual<T>(data: unknown, other: T): data is T {\n  if (data === other) {\n    return true;\n  }\n\n  if (Object.is(data, other)) {\n    // We want to ignore the slight differences between `===` and `Object.is` as\n    // both of them largely define equality from a semantic point-of-view.\n    return true;\n  }\n\n  if (typeof data !== 'object' || typeof other !== 'object') {\n    return false;\n  }\n\n  if (data === null || other === null) {\n    return false;\n  }\n\n  if (Object.getPrototypeOf(data) !== Object.getPrototypeOf(other)) {\n    // If the objects don't share a prototype it's unlikely that they are\n    // semantically equal. It is technically possible to build 2 prototypes that\n    // act the same but are not equal (at the reference level, checked via\n    // `===`) and then create 2 objects that are equal although we would fail on\n    // them. Because this is so unlikely, the optimization we gain here for the\n    // rest of the function by assuming that `other` is of the same type as\n    // `data` is more than worth it.\n    return false;\n  }\n\n  if (Array.isArray(data)) {\n    return isDeepEqualArrays(data, other as unknown as ReadonlyArray<unknown>);\n  }\n\n  if (data instanceof Map) {\n    return isDeepEqualMaps(data, other as unknown as Map<unknown, unknown>);\n  }\n\n  if (data instanceof Set) {\n    return isDeepEqualSets(data, other as unknown as Set<unknown>);\n  }\n\n  if (data instanceof Date) {\n    return data.getTime() === (other as unknown as Date).getTime();\n  }\n\n  if (data instanceof RegExp) {\n    return data.toString() === (other as unknown as RegExp).toString();\n  }\n\n  // At this point we only know that the 2 objects share a prototype and are not\n  // any of the previous types. They could be plain objects (Object.prototype),\n  // they could be classes, they could be other built-ins, or they could be\n  // something weird. We assume that comparing values by keys is enough to judge\n  // their equality.\n\n  if (Object.keys(data).length !== Object.keys(other).length) {\n    return false;\n  }\n\n  for (const [key, value] of Object.entries(data)) {\n    if (!(key in other)) {\n      return false;\n    }\n\n    if (\n      !deepEqual(\n        value,\n        // @ts-expect-error [ts7053] - We already checked that `other` has `key`\n        other[key],\n      )\n    ) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction isDeepEqualArrays(data: ReadonlyArray<unknown>, other: ReadonlyArray<unknown>): boolean {\n  if (data.length !== other.length) {\n    return false;\n  }\n\n  for (const [index, item] of data.entries()) {\n    if (!deepEqual(item, other[index])) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction isDeepEqualMaps(\n  data: ReadonlyMap<unknown, unknown>,\n  other: ReadonlyMap<unknown, unknown>,\n): boolean {\n  if (data.size !== other.size) {\n    return false;\n  }\n\n  for (const [key, value] of data.entries()) {\n    if (!other.has(key)) {\n      return false;\n    }\n\n    if (!deepEqual(value, other.get(key))) {\n      return false;\n    }\n  }\n\n  return true;\n}\n\nfunction isDeepEqualSets(data: ReadonlySet<unknown>, other: ReadonlySet<unknown>): boolean {\n  if (data.size !== other.size) {\n    return false;\n  }\n\n  // To ensure we only count each item once we need to \"remember\" which items of\n  // the other set we've already matched against. We do this by creating a copy\n  // of the other set and removing items from it as we find them in the data\n  // set.\n  const otherCopy = [...other];\n\n  for (const dataItem of data) {\n    let isFound = false;\n\n    for (const [index, otherItem] of otherCopy.entries()) {\n      if (deepEqual(dataItem, otherItem)) {\n        isFound = true;\n        otherCopy.splice(index, 1);\n        break;\n      }\n    }\n\n    if (!isFound) {\n      return false;\n    }\n  }\n\n  return true;\n}\n","import { errorToString } from './errors';\n\nexport type TryCatchResult<T, E extends Error = Error> =\n  | [data: T, error: null]\n  | [data: null, error: E | Error];\n\nexport function toError(rawError: unknown, operationName?: string): Error {\n  const processedError = rawError instanceof Error ? rawError : new Error(errorToString(rawError));\n\n  if (operationName) {\n    processedError.message = `Operation \"${operationName}\" failed: ${processedError.message}`;\n  }\n\n  return processedError;\n}\n\nexport function tryCatchSync<T, E extends Error = Error>(\n  fn: () => T,\n  operationName?: string,\n): TryCatchResult<T, E> {\n  try {\n    return [fn(), null];\n  } catch (rawError) {\n    return [null, toError(rawError, operationName)];\n  }\n}\n\nexport async function tryCatchAsync<T, E extends Error = Error>(\n  fn: Promise<T> | (() => T | Promise<T>),\n  operationName?: string,\n): Promise<TryCatchResult<Awaited<T>, E>> {\n  try {\n    const result = typeof fn === 'function' ? fn() : fn;\n    return [await result, null];\n  } catch (rawError) {\n    return [null, toError(rawError, operationName)];\n  }\n}\n\ntype TryCatch = {\n  <T, E extends Error = Error>(\n    fn: Promise<T> | (() => T | Promise<T>),\n    operationName?: string,\n  ): Promise<TryCatchResult<Awaited<T>, E>>;\n  <T, E extends Error = Error>(fn: () => T, operationName?: string): TryCatchResult<T, E>;\n  sync: typeof tryCatchSync;\n  async: typeof tryCatchAsync;\n};\n\nexport const tryCatch = ((fn: unknown, operationName?: string) => {\n  if (typeof fn === 'function') {\n    try {\n      const result = (fn as () => unknown)();\n      if (result instanceof Promise) {\n        return tryCatchAsync(result, operationName);\n      }\n      return [result, null];\n    } catch (rawError) {\n      return [null, toError(rawError, operationName)];\n    }\n  }\n\n  if (fn instanceof Promise) {\n    return tryCatchAsync(fn, operationName);\n  }\n\n  return [fn, null];\n}) as TryCatch;\n\ntryCatch.sync = tryCatchSync;\ntryCatch.async = tryCatchAsync;\n","import type { Aggregation } from './aggregations';\n\nexport type Score<TMetadata extends Record<string, any> = Record<string, any>> = {\n  score: number | boolean | null;\n  metadata?: TMetadata;\n};\n\n// Internal type used when scorer returns Score with name\nexport type ScoreWithName<TMetadata extends Record<string, any> = Record<string, any>> =\n  Score<TMetadata> & {\n    name: string;\n    /** Per-trial scores when running multiple trials */\n    trials?: number[];\n    /** Aggregation type used (e.g., 'mean', 'pass@k') */\n    aggregation?: string;\n    /** Threshold for pass-based aggregations */\n    threshold?: number;\n  };\n\n/**\n * Configuration options for a scorer.\n */\nexport type ScorerOptions = {\n  /**\n   * Aggregation function for combining scores across multiple trials.\n   * Defaults to Mean() if not specified.\n   */\n  aggregation?: Aggregation;\n};\n\n// Loose type - this is what we REQUIRE\n// (we accept scorers with looser requirements than how we define them)\nexport type ScorerLike<\n  TInput = any,\n  TExpected = any,\n  TOutput = any,\n  TExtra extends Record<string, any> = {},\n  TMetadata extends Record<string, any> = Record<string, any>,\n  TReturn extends Score<TMetadata> | Promise<Score<TMetadata>> =\n    | Score<TMetadata>\n    | Promise<Score<TMetadata>>,\n> = (\n  args: {\n    input?: TInput;\n    expected?: TExpected;\n    output: TOutput;\n    /** Current trial index (0-based) when running multiple trials */\n    trialIndex?: number;\n  } & TExtra,\n) => TReturn;\n\n// Strict type - this is how we DEFINE scorers\nexport type Scorer<\n  TInput = any,\n  TExpected = any,\n  TOutput = any,\n  TExtra extends Record<string, any> = {},\n  TMetadata extends Record<string, any> = Record<string, any>,\n  TReturn extends Score<TMetadata> | Promise<Score<TMetadata>> =\n    | Score<TMetadata>\n    | Promise<Score<TMetadata>>,\n> = ScorerLike<TInput, TExpected, TOutput, TExtra, TMetadata, TReturn> & {\n  readonly name: string; // Name property for telemetry\n  readonly aggregation?: Aggregation; // Aggregation config for trials\n};\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA;;;ACAA;AAAA,SAAS,UAAU,WAAW,UAAU,QAAQ,UAAU;AAC1D,SAAS,SAAS,gBAAgB,aAA2B;AAC7D,SAAS,sBAAsB;;;ACF/B;AAAA,SAAS,gBAAgB;AAElB,SAAS,iBAAiB;AAC/B,MAAI;AACF,UAAM,OAAO,SAAS,4BAA4B,EAAE,SAAS,EAAE,KAAK;AACpE,UAAM,QAAQ,SAAS,6BAA6B,EAAE,SAAS,EAAE,KAAK;AACtE,WAAO,EAAE,MAAM,MAAM;AAAA,EACvB,QAAQ;AACN,WAAO;AAAA,EACT;AACF;;;ACVA;;;ACAA;AAUO,IAAM,gBAAgB,CAAC;AAAA,EAC5B;AAAA,EACA;AAAA,EACA;AACF,MAIe;AACb,SAAO,CAAC,MAAc,YACpB,MAAM,IAAI,IAAI,MAAM,OAAO,EAAE,SAAS,GAAG;AAAA,IACvC,GAAG;AAAA,IACH,SAAS;AAAA,MACP,GAAG,QAAQ;AAAA,MACX,gBAAgB;AAAA,MAChB,eAAe,UAAU,KAAK;AAAA,MAC9B,GAAI,QAAQ,EAAE,kBAAkB,MAAM,IAAI,CAAC;AAAA,IAC7C;AAAA,EACF,CAAC;AACL;;;AC7BA;AAAO,SAAS,4BAA4B,KAAc,WAA4B;AACpF,MAAI,OAAO,QAAQ,YAAY,QAAQ,MAAM;AAC3C,WAAO;AAAA,EACT;AAEA,QAAM,WAAW,UAAU,MAAM,GAAG;AACpC,QAAM,SAAU,IAA4B;AAG5C,MAAI,UAAU,OAAO,WAAW,YAAY,WAAW,QAAQ,aAAa,QAAQ;AAClF,WAAO,OAAO,SAAS;AAAA,EACzB;AAGA,MAAI,UAAe;AACnB,aAAW,QAAQ,UAAU;AAC3B,QAAI,OAAO,YAAY,YAAY,YAAY,MAAM;AACnD,aAAO;AAAA,IACT;AACA,cAAU,QAAQ,IAAI;AAAA,EACxB;AAEA,SAAO;AACT;AAEO,SAAS,yBAAyB,KAAc,KAAiC;AACtF,QAAM,QAAQ,4BAA4B,KAAK,GAAG;AAElD,SAAO,OAAO,UAAU,WAAW,QAAQ;AAC7C;AAEO,SAAS,yBAAyB,KAAc,KAAiC;AACtF,QAAM,QAAQ,4BAA4B,KAAK,GAAG;AAElD,MAAI,OAAO,UAAU,UAAU;AAC7B,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,UAAU,UAAU;AAC7B,UAAM,SAAS,OAAO,KAAK;AAE3B,WAAO,OAAO,MAAM,MAAM,IAAI,SAAY;AAAA,EAC5C;AAEA,SAAO;AACT;;;AFHO,IAAM,sBAAN,MAA0B;AAAA,EAE/B,YAAY,QAA6B,YAAqB;AAD9D,wBAAiB;AAEf,UAAM,EAAE,oBAAoB,OAAO,MAAM,IAAI,uBAAuB,QAAQ,UAAU;AAEtF,SAAK,UAAU,cAAc,EAAE,SAAS,oBAAoB,OAAO,SAAS,IAAI,MAAM,CAAC;AAAA,EACzF;AAAA,EAEA,MAAM,iBAAiB,YAAsC;AAC3D,UAAM,OAAO,MAAM,KAAK,QAAQ,uBAAuB;AAAA,MACrD,QAAQ;AAAA,MACR,MAAM,KAAK,UAAU,UAAU;AAAA,IACjC,CAAC;AAED,QAAI,CAAC,KAAK,IAAI;AACZ,YAAM,OAAO,MAAM,KAAK,KAAK,EAAE,MAAM,MAAM,EAAE;AAC7C,YAAM,IAAI;AAAA,QACR,gCAAgC,KAAK,UAAU,GAAG,OAAO,MAAM,IAAI,KAAK,EAAE;AAAA,MAC5E;AAAA,IACF;AAEA,WAAO,KAAK,KAAK;AAAA,EACnB;AAAA,EAEA,MAAM,iBAAiB,YAA+C;AACpE,UAAM,OAAO,MAAM,KAAK,QAAQ,uBAAuB,WAAW,EAAE,IAAI;AAAA,MACtE,QAAQ;AAAA,MACR,MAAM,KAAK,UAAU,UAAU;AAAA,IACjC,CAAC;AAED,QAAI,CAAC,KAAK,IAAI;AACZ,YAAM,OAAO,MAAM,KAAK,KAAK,EAAE,MAAM,MAAM,EAAE;AAC7C,YAAM,IAAI;AAAA,QACR,gCAAgC,KAAK,UAAU,GAAG,OAAO,MAAM,IAAI,KAAK,EAAE;AAAA,MAC5E;AAAA,IACF;AAGA,UAAM,OAAO,MAAM,KAAK,KAAK;AAC7B,QAAI,KAAK,OAAO;AACd,YAAM,IAAI;AAAA,QACR,+BAA+B,WAAW,EAAE,KAAK,KAAK,UAAU,KAAK,KAAK,CAAC;AAAA,MAC7E;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;AAEO,IAAM,sBAAsB,OACjC,QACA,WAC+B;AAC/B,QAAM,EAAE,SAAS,SAAS,OAAO,MAAM,IAAI,uBAAuB,MAAM;AAExE,QAAM,MAAM,KAAK,OAAO,2BAA2B,MAAM;AAEzD,QAAM,UAAU,IAAI,QAAQ;AAAA,IAC1B,eAAe,UAAU,KAAK;AAAA,IAC9B,gBAAgB;AAAA,IAChB,GAAI,QAAQ,EAAE,kBAAkB,MAAM,IAAI,CAAC;AAAA,EAC7C,CAAC;AAGD,QAAM,OAAO,MAAM,MAAM,GAAG,OAAO,mCAAmC;AAAA,IACpE;AAAA,IACA,QAAQ;AAAA,IACR,MAAM,KAAK,UAAU,EAAE,IAAI,CAAC;AAAA,EAC9B,CAAC;AACD,QAAM,UAAU,MAAM,KAAK,KAAK;AAEhC,MAAI,CAAC,KAAK,IAAI;AACZ,UAAM,IAAI,MAAM,qCAAqC,QAAQ,WAAW,KAAK,UAAU,EAAE;AAAA,EAC3F;AAEA,SAAO,QAAQ,QAAQ,SAAS,cAAc,QAAQ,OAAO,IAAI;AACnE;AAIO,IAAM,gBAAgB,CAAC,SAA0B;AACtD,QAAM,gBAAgB,4BAA4B,KAAK,KAAK,YAAY,KAAK,KAAK,OAAO,KAAK;AAC9F,QAAM,UAAU,4BAA4B,KAAK,KAAK,YAAY,KAAK,KAAK,IAAI;AAEhF,QAAM,aAAsC;AAAA,IAC1C,IAAI,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,EAAE;AAAA,IAC/D,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,IAAI;AAAA,IACnE,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,IAAI;AAAA,IACnE,SAAS,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,OAAO;AAAA,IACzE,YAAY;AAAA,MACV,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW,IAAI;AAAA,MAC9E,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,WAAW,IAAI;AAAA,IAChF;AAAA,IACA,UAAU;AAAA,MACR,IAAI,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,SAAS,EAAE;AAAA,MACxE,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,SAAS,IAAI;AAAA,IAC9E;AAAA,IACA,UAAU,KAAK,KAAK;AAAA,IACpB,QAAQ,KAAK,KAAK,OAAO;AAAA,IACzB,SAAS,KAAK,KAAK;AAAA,IACnB,OAAO,KAAK;AAAA,IACZ,MAAM,UAAW,OAAO,YAAY,WAAW,KAAK,MAAM,OAAO,IAAI,UAAW,CAAC;AAAA,IACjF,MAAM;AAAA,MACJ,MAAM,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,KAAK,IAAI;AAAA,MACxE,OAAO,yBAAyB,KAAK,KAAK,YAAY,KAAK,KAAK,KAAK,KAAK;AAAA,IAC5E;AAAA,IACA,OAAO,CAAC;AAAA,IACR,YAAY,gBACR,OAAO,kBAAkB,WACvB,KAAK,MAAM,aAAa,IACxB,gBACF;AAAA,EACN;AAGA,SAAO;AACT;AAEO,IAAM,gBAAgB,CAAC,SAA6C;AACzE,QAAM,OAAO,KAAK;AAElB,QAAM,IAAI,KAAK;AACf,MAAI,WAAW;AACf,MAAI,EAAE,SAAS,GAAG,GAAG;AACnB,eAAW,GAAG,OAAO,EAAE,QAAQ,KAAK,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;AAAA,EACrD,OAAO;AACL,eAAW;AAAA,EACb;AAEA,QAAM,YAAY,4BAA4B,KAAK,YAAY,KAAK,KAAK,KAAK,MAAM;AACpF,QAAM,eAAe,YACjB,OAAO,cAAc,WACnB,KAAK,MAAM,SAAS,IACpB,YACF,CAAC;AAIL,QAAM,SAAyB,CAAC;AAChC,aAAW,CAAC,MAAM,SAAS,KAAK,OAAO,QAAQ,YAAY,GAAG;AAC5D,UAAM,IAAI;AACV,WAAO,IAAI,IAAI;AAAA,MACb;AAAA,MACA,OAAO,EAAE,SAAS,EAAE,SAAS;AAAA,MAC7B,UAAU,EAAE,YAAY,CAAC;AAAA,IAC3B;AAAA,EACF;AAEA,QAAM,WAA8B;AAAA,IAClC,OAAO,yBAAyB,KAAK,YAAY,KAAK,KAAK,KAAK,KAAK;AAAA,IACrE,OAAO,yBAAyB,KAAK,YAAY,KAAK,KAAK,KAAK,KAAK;AAAA,IACrE,QAAQ,yBAAyB,KAAK,YAAY,KAAK,KAAK,KAAK,MAAM;AAAA,IACvE,UAAU,yBAAyB,KAAK,YAAY,KAAK,KAAK,KAAK,QAAQ;AAAA,IAC3E;AAAA,IACA,QAAQ,KAAK,OAAO;AAAA,IACpB;AAAA,IACA,OAAO,KAAK;AAAA,IACZ,QAAQ,KAAK;AAAA,IACb,SAAS,KAAK;AAAA,EAChB;AAGA,SAAO;AACT;AAGO,IAAM,gBAAgB,CAAC,UAAoC;AAChE,MAAI,CAAC,MAAM,QAAQ;AACjB,WAAO;AAAA,EACT;AAGA,QAAM,WAAW,MAAM,KAAK,CAAC,SAAS,KAAK,KAAK,WAAW,OAAO,UAAU,SAAS,MAAM;AAE3F,MAAI,CAAC,UAAU;AACb,WAAO;AAAA,EACT;AAGA,QAAM,WAAuB,cAAc,QAAQ;AAGnD,QAAM,YAAY,MAAM,OAAO,CAAC,SAAS,KAAK,KAAK,KAAK,WAAW,MAAM,CAAC;AAE1E,aAAW,YAAY,WAAW;AAEhC,UAAM,WAAW,cAAc,QAAQ;AAGvC,UAAM,YAAY,MAAM;AAAA,MACtB,CAAC,SACC,KAAK,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,KAAK,mBAAmB,SAAS,KAAK;AAAA,IACpF;AAEA,QAAI,UAAU,SAAS,GAAG;AACxB,YAAM,WAAW,UAAU,CAAC;AAG5B,YAAM,YAAY,MAAM;AAAA,QACtB,CAAC,SACC,KAAK,KAAK,KAAK,WAAW,MAAM,KAAK,KAAK,KAAK,mBAAmB,SAAS,KAAK;AAAA,MACpF;AAEA,YAAM,WAAmB,UAAU,IAAI,CAAC,cAAc;AAAA,QACpD,WAAW,yBAAyB,SAAS,KAAK,YAAY,WAAW,KAAK;AAAA,QAC9E,YAAY,yBAAyB,SAAS,KAAK,YAAY,YAAY,KAAK;AAAA,QAChF,MAAM,yBAAyB,SAAS,KAAK,YAAY,MAAM,KAAK;AAAA,QACpE,SAAS;AAAA,UACP,WAAW,yBAAyB,SAAS,KAAK,YAAY,mBAAmB,KAAK;AAAA,UACtF,OAAO,yBAAyB,SAAS,KAAK,YAAY,eAAe,KAAK;AAAA,UAC9E,aACE,yBAAyB,SAAS,KAAK,YAAY,qBAAqB,KAAK;AAAA,QACjF;AAAA,QACA,UAAU;AAAA,UACR,gBACE,yBAAyB,SAAS,KAAK,YAAY,yBAAyB,KAAK;AAAA,QACrF;AAAA,QACA,OAAO;AAAA,UACL,cACE,yBAAyB,SAAS,KAAK,YAAY,oBAAoB,KAAK;AAAA,UAC9E,eACE,yBAAyB,SAAS,KAAK,YAAY,qBAAqB,KAAK;AAAA,QACjF;AAAA,MACF,EAAE;AAGF,YAAM,WAAiB;AAAA,QACrB,MAAM,SAAS,KAAK;AAAA,QACpB,QAAQ,yBAAyB,SAAS,KAAK,YAAY,QAAQ,KAAK;AAAA,QACxE,OAAO,yBAAyB,SAAS,KAAK,YAAY,OAAO,KAAK;AAAA,QACtE,MAAM,yBAAyB,SAAS,KAAK,YAAY,MAAM,KAAK;AAAA,QACpE,OAAO,yBAAyB,SAAS,KAAK,YAAY,OAAO,KAAK;AAAA,QACtE,MAAM,SAAS,CAAC,KAAK;AAAA,UACnB,WAAW;AAAA,UACX,YAAY;AAAA,UACZ,MAAM;AAAA,UACN,SAAS,EAAE,WAAW,IAAI,OAAO,IAAI,aAAa,EAAE;AAAA,UACpD,UAAU,EAAE,gBAAgB,GAAG;AAAA,UAC/B,OAAO,EAAE,cAAc,GAAG,eAAe,EAAE;AAAA,QAC7C;AAAA,MACF;AAEA,eAAS,OAAO;AAAA,IAClB;AAGA,UAAM,aAAa,MAAM;AAAA,MACvB,CAAC,SACC,KAAK,KAAK,WAAW,OAAO,UAAU,SAAS,gBAC/C,KAAK,KAAK,mBAAmB,SAAS,KAAK;AAAA,IAC/C;AAEA,QAAI,WAAW,SAAS,GAAG;AACzB,eAAS,SAAS,CAAC;AAEnB,iBAAW,QAAQ,CAAC,UAAU;AAC5B,cAAM,OAAO,yBAAyB,MAAM,KAAK,YAAY,KAAK,KAAK,MAAM,IAAI,KAAK;AACtF,cAAM,QAAQ,yBAAyB,MAAM,KAAK,YAAY,KAAK,KAAK,MAAM,KAAK,KAAK;AACxF,cAAM,cAAc;AAAA,UAClB,MAAM,KAAK;AAAA,UACX,KAAK,KAAK,MAAM;AAAA,QAClB;AACA,YAAI,WAAW,CAAC;AAChB,YAAI;AACF,qBAAW,cAAc,KAAK,MAAM,WAAW,IAAI,CAAC;AAAA,QACtD,QAAQ;AAAA,QAER;AAEA,iBAAS,OAAO,IAAI,IAAI;AAAA,UACtB;AAAA,UACA;AAAA,UACA,UAAU;AAAA,YACR,OAAO,MAAM,KAAK,WAAW;AAAA,YAC7B,GAAG;AAAA,UACL;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAEA,aAAS,MAAM,KAAK,QAAQ;AAAA,EAC9B;AAEA,WAAS,MAAM,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAE/C,SAAO;AACT;;;AGxUA;AAIO,SAAS,UAAa,MAAe,OAAqB;AAC/D,MAAI,SAAS,OAAO;AAClB,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,GAAG,MAAM,KAAK,GAAG;AAG1B,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,SAAS,YAAY,OAAO,UAAU,UAAU;AACzD,WAAO;AAAA,EACT;AAEA,MAAI,SAAS,QAAQ,UAAU,MAAM;AACnC,WAAO;AAAA,EACT;AAEA,MAAI,OAAO,eAAe,IAAI,MAAM,OAAO,eAAe,KAAK,GAAG;AAQhE,WAAO;AAAA,EACT;AAEA,MAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAO,kBAAkB,MAAM,KAA0C;AAAA,EAC3E;AAEA,MAAI,gBAAgB,KAAK;AACvB,WAAO,gBAAgB,MAAM,KAAyC;AAAA,EACxE;AAEA,MAAI,gBAAgB,KAAK;AACvB,WAAO,gBAAgB,MAAM,KAAgC;AAAA,EAC/D;AAEA,MAAI,gBAAgB,MAAM;AACxB,WAAO,KAAK,QAAQ,MAAO,MAA0B,QAAQ;AAAA,EAC/D;AAEA,MAAI,gBAAgB,QAAQ;AAC1B,WAAO,KAAK,SAAS,MAAO,MAA4B,SAAS;AAAA,EACnE;AAQA,MAAI,OAAO,KAAK,IAAI,EAAE,WAAW,OAAO,KAAK,KAAK,EAAE,QAAQ;AAC1D,WAAO;AAAA,EACT;AAEA,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,IAAI,GAAG;AAC/C,QAAI,EAAE,OAAO,QAAQ;AACnB,aAAO;AAAA,IACT;AAEA,QACE,CAAC;AAAA,MACC;AAAA;AAAA,MAEA,MAAM,GAAG;AAAA,IACX,GACA;AACA,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,kBAAkB,MAA8B,OAAwC;AAC/F,MAAI,KAAK,WAAW,MAAM,QAAQ;AAChC,WAAO;AAAA,EACT;AAEA,aAAW,CAAC,OAAO,IAAI,KAAK,KAAK,QAAQ,GAAG;AAC1C,QAAI,CAAC,UAAU,MAAM,MAAM,KAAK,CAAC,GAAG;AAClC,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,gBACP,MACA,OACS;AACT,MAAI,KAAK,SAAS,MAAM,MAAM;AAC5B,WAAO;AAAA,EACT;AAEA,aAAW,CAAC,KAAK,KAAK,KAAK,KAAK,QAAQ,GAAG;AACzC,QAAI,CAAC,MAAM,IAAI,GAAG,GAAG;AACnB,aAAO;AAAA,IACT;AAEA,QAAI,CAAC,UAAU,OAAO,MAAM,IAAI,GAAG,CAAC,GAAG;AACrC,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,gBAAgB,MAA4B,OAAsC;AACzF,MAAI,KAAK,SAAS,MAAM,MAAM;AAC5B,WAAO;AAAA,EACT;AAMA,QAAM,YAAY,CAAC,GAAG,KAAK;AAE3B,aAAW,YAAY,MAAM;AAC3B,QAAI,UAAU;AAEd,eAAW,CAAC,OAAO,SAAS,KAAK,UAAU,QAAQ,GAAG;AACpD,UAAI,UAAU,UAAU,SAAS,GAAG;AAClC,kBAAU;AACV,kBAAU,OAAO,OAAO,CAAC;AACzB;AAAA,MACF;AAAA,IACF;AAEA,QAAI,CAAC,SAAS;AACZ,aAAO;AAAA,IACT;AAAA,EACF;AAEA,SAAO;AACT;;;AClJA;AAMO,SAAS,QAAQ,UAAmB,eAA+B;AACxE,QAAM,iBAAiB,oBAAoB,QAAQ,WAAW,IAAI,MAAM,cAAc,QAAQ,CAAC;AAE/F,MAAI,eAAe;AACjB,mBAAe,UAAU,cAAc,aAAa,aAAa,eAAe,OAAO;AAAA,EACzF;AAEA,SAAO;AACT;AAEO,SAAS,aACd,IACA,eACsB;AACtB,MAAI;AACF,WAAO,CAAC,GAAG,GAAG,IAAI;AAAA,EACpB,SAAS,UAAU;AACjB,WAAO,CAAC,MAAM,QAAQ,UAAU,aAAa,CAAC;AAAA,EAChD;AACF;AAEA,eAAsB,cACpB,IACA,eACwC;AACxC,MAAI;AACF,UAAM,SAAS,OAAO,OAAO,aAAa,GAAG,IAAI;AACjD,WAAO,CAAC,MAAM,QAAQ,IAAI;AAAA,EAC5B,SAAS,UAAU;AACjB,WAAO,CAAC,MAAM,QAAQ,UAAU,aAAa,CAAC;AAAA,EAChD;AACF;AAYO,IAAM,YAAY,CAAC,IAAa,kBAA2B;AAChE,MAAI,OAAO,OAAO,YAAY;AAC5B,QAAI;AACF,YAAM,SAAU,GAAqB;AACrC,UAAI,kBAAkB,SAAS;AAC7B,eAAO,cAAc,QAAQ,aAAa;AAAA,MAC5C;AACA,aAAO,CAAC,QAAQ,IAAI;AAAA,IACtB,SAAS,UAAU;AACjB,aAAO,CAAC,MAAM,QAAQ,UAAU,aAAa,CAAC;AAAA,IAChD;AAAA,EACF;AAEA,MAAI,cAAc,SAAS;AACzB,WAAO,cAAc,IAAI,aAAa;AAAA,EACxC;AAEA,SAAO,CAAC,IAAI,IAAI;AAClB;AAEA,SAAS,OAAO;AAChB,SAAS,QAAQ;;;ANpBjB,IAAM,kBAAkB,eAAe,wCAAwC,EAAE;AASjF,IAAM,2BAA2B,OAAO,IAAI,kCAAkC;AAE9E,SAAS,4BACP,OACA,SAC+D;AAC/D,QAAM,aAAa,QAAQ,KAAK;AAGhC,aAAW,wBAAwB,IAAI;AACvC,SAAO;AACT;AAEA,SAAS,yBAAyB,OAAmD;AACnF,MAAI,OAAO,UAAU,YAAY,UAAU,MAAM;AAC/C,WAAO;AAAA,EACT;AACA,SAAQ,MACN,wBACF;AACF;AAiCO,SAAS,KAQd,MACA,QAIM;AAEN,aAAW,QAAQ,IAAI;AACvB,aAAW,cAAc,OAAO,UAAU;AAC1C,MAAI,OAAO,MAAM;AACf,eAAW,QAAQ,OAAO,IAAI;AAAA,EAChC;AAGA,MAAI,OAAO,SAAS;AAClB,eAAW,UAAU,OAAO,SAAS;AACnC,YAAM,aAAa,cAAc,QAAQ,EAAE;AAC3C,iBAAW,UAAU,UAAU;AAAA,IACjC;AAAA,EACF;AAEA,eAAa,MAAM,MAAmC,EAAE,MAAM,QAAQ,KAAK;AAC7E;AAKA,SAAS,kBAAkB,aAA6C;AACtE,MAAI,CAAC,eAAe,YAAY,WAAW,GAAG;AAC5C,WAAO,CAAC;AAAA,EACV;AAEA,QAAM,QAAQ,eAAe;AAC7B,QAAM,cAAc,OAAO,qBAAqB,KAAK,CAAC;AACtD,QAAM,YAAY,uBAAuB;AAEzC,QAAM,SAAS,EAAE,GAAG,aAAa,GAAG,UAAU;AAG9C,QAAM,WAAgC,CAAC;AACvC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,MAAM,GAAG;AACjD,UAAM,YAAY,YAAY,KAAK,CAAC,YAAY,IAAI,WAAW,OAAO,CAAC;AACvE,QAAI,WAAW;AACb,eAAS,GAAG,IAAI;AAAA,IAClB;AAAA,EACF;AAEA,SAAO,oBAAoB,QAAQ;AACrC;AAEA,IAAM,gBAAgB,CACpB,QACA,WAAmB,cAChB;AACH,SAAQ,OAAe,QAAQ;AACjC;AAEA,eAAe,aAIb,UAAkB,MAA8C;AAChE,OAAK;AACL,QAAM,oBAGJ,OAAO,KAAK,SAAS,aAEf,KAAK,KAGL,IACF,KAAK;AACX,QAAM,OAAO,eAAe;AAG5B,QAAM,aAAa,OAAO,UAAU;AACpC,QAAM,UAAU,OAAO,OAAO;AAC9B,QAAM,SAAS,OAAO,MAAM;AAC5B,QAAM,oBAAoB,OAAO,WAAW;AAC5C,QAAM,cAAc,OAAO,aAAa;AACxC,QAAM,QAAQ,OAAO,OAAO;AAC5B,QAAM,aAAa,OAAO,YAAY;AAEtC,MAAI,CAAC,aAAa;AAChB,UAAM,IAAI,cAAc,wBAAwB;AAAA,EAClD;AAEA,QAAM,YAAY,KAAK,WAAW,aAAa,KAAK;AAEpD,QAAM,uBAAuB,iCAAiC,aAAa;AAAA,IACzE,SAAS,CAAC,WAAW,CAAC;AAAA,EACxB,CAAC;AAED,QAAM,SAAS,MAAM;AAAA,IACnB;AAAA,IACA,YAAY;AACV,YAAM,aAAa,MAAM;AAEzB,YAAM,sBAAsB,IAAI,oBAAoB,aAAa,UAAU;AAG3E,YAAM,cAAc,gBAAgB;AACpC,UAAI,SAAS;AACb,UAAI;AAEJ,UAAI;AACJ,UAAI;AACJ,UAAI,uBAAgC;AACpC,UAAI,WAA0C;AAG9C,YAAM,qBAA6C,CAAC;AAGpD,UAAI;AAIJ,gBAAU,OAAO,UAAU;AAEzB,YAAI,qBAAqB,OAAO,KAAK,iBAAiB,EAAE,SAAS,GAAG;AAClE,cAAI;AACF,mCAAuB,iBAAiB;AAAA,UAC1C,QAAQ;AAAA,UAAC;AAAA,QACX;AAEA,cAAM,KAAK,aAAa;AAAA,UACtB,IAAI;AAAA,UACJ,MAAM;AAAA,UACN,SAAS;AAAA,UACT;AAAA,UACA,OAAO;AAAA,UACP,UAAU,YAAY;AAAA,UACtB,aAAa,KAAK;AAAA,QACpB;AAEA,cAAM,CAAC,EAAE,wBAAwB,IAAI,MAAM,cAAc,oBAAoB;AAC7E,YAAI,0BAA0B;AAC5B,iCAAuB;AAAA,QACzB;AAEA,oBAAY,UAAU,QAAQ,QAAQ,IAAI,WAAW,IAAI;AAAA,UACvD,YAAY;AAAA,YACV,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,YAC7B,CAAC,KAAK,KAAK,IAAI,GAAG;AAAA,YAClB,CAAC,KAAK,KAAK,OAAO,GAAG;AAAA,YACrB,CAAC,KAAK,KAAK,IAAI,GAAG;AAAA;AAAA,YAClB,CAAC,KAAK,KAAK,IAAI,GAAG,KAAK,UAAU,CAAC,SAAS,CAAC;AAAA,YAC5C,CAAC,KAAK,KAAK,WAAW,EAAE,GAAG;AAAA;AAAA,YAC3B,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG;AAAA;AAAA,YAC7B,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG,WAAW;AAAA;AAAA,YAExC,CAAC,KAAK,KAAK,WAAW,IAAI,GAAG,KAAK;AAAA,YAClC,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,KAAK,QAAQ;AAAA;AAAA,YAEpC,CAAC,KAAK,KAAK,QAAQ,GAAG,KAAK,UAAU,KAAK,QAAQ;AAAA;AAAA,YAElD,CAAC,KAAK,KAAK,IAAI,EAAE,GAAG;AAAA;AAAA,YAEpB,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM;AAAA,YAC7B,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG,MAAM;AAAA,UAChC;AAAA,QACF,CAAC;AACD,iBAAS,UAAU,YAAY,EAAE;AACjC,cAAM,KAAK,WAAW,KAAK;AAC3B,kBAAU,aAAa,KAAK,KAAK,IAAI,MAAM;AAC3C,uBAAe,MAAM,QAAQ,QAAQ,OAAO,GAAG,SAAS;AAExD,cAAM,aAAa,kBAAkB,KAAK,WAAW;AACrD,cAAM,KAAK,WAAW,aAAa;AACnC,cAAM,iBAAiB,KAAK,UAAU,UAAU;AAChD,kBAAU,aAAa,KAAK,KAAK,OAAO,OAAO,cAAc;AAE7D,YAAI;AACJ,YAAI,CAAC,WAAW,CAAC,QAAQ;AACvB,+BAAqB,MAAM,oBAAoB,iBAAiB;AAAA,YAC9D,IAAI;AAAA,YACJ,MAAM;AAAA,YACN,YAAY,KAAK;AAAA,YACjB,MAAM,KAAK;AAAA,YACX,SAAS,YAAY,KAAK;AAAA,YAC1B,SAAS;AAAA,YACT,YAAY,cAAc;AAAA,YAC1B;AAAA,YACA,YAAY,WAAW;AAAA,YACvB,QAAQ,EAAE,WAAW,kBAAkB;AAAA,YACvC,iBAAiB;AAAA,YACjB,UAAU,KAAK;AAAA,YACf,QAAQ;AAAA,UACV,CAAC;AAAA,QACH;AAEA,cAAM,QAAQ,oBAAoB,MAAM;AACxC,cAAM,qBAAqB,oBAAoB,MAAM;AAGrD,YAAI,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC,oBAAoB;AAC/C,gBAAM,CAAC,gBAAgB,aAAa,IAAI,MAAM;AAAA,YAAc,MAC1D,oBAAoB,oBAAoB,WAAW;AAAA,UACrD;AACA,cAAI,eAAe;AACjB,oBAAQ,MAAM,4BAA4B,cAAc,aAAa,CAAC,EAAE;AACxE,mCAAuB,wBAAwB;AAAA,UACjD,OAAO;AACL,uBAAW;AAAA,UACb;AAAA,QACF;AAGA,YAAI,UAAU;AACZ,oBAAU,aAAa,KAAK,KAAK,SAAS,IAAI,SAAS,EAAE;AACzD,oBAAU,aAAa,KAAK,KAAK,SAAS,MAAM,SAAS,IAAI;AAC7D,oBAAU,aAAa,KAAK,KAAK,SAAS,SAAS,SAAS,OAAO;AAAA,QACrE;AAEA,cAAM,KAAK,aAAa;AAAA,UACtB,IAAI;AAAA,UACJ,MAAM;AAAA,UACN,SAAS;AAAA,UACT;AAAA,UACA,OAAO,SAAS;AAAA,UAChB,UAAU,YAAY;AAAA,UACtB,aAAa,KAAK;AAAA,UAClB,oBAAoB,uBAChB;AAAA,YACE,QAAQ;AAAA,YACR,OAAO,cAAc,oBAAoB;AAAA,UAC3C,IACA,EAAE,QAAQ,UAAU;AAAA,UACxB,QAAQ,KAAK;AAAA,QACf;AAEA,qBAAa,YAAY,IAAI;AAAA,MAC/B,CAAC;AAED,eAAS,OAAO,UAAU;AACxB,YAAI,sBAAsB;AACxB,gBAAM;AAAA,QACR;AAGA,cAAM,cAAc,oBAAI,IAA4B;AAEpD,mBAAW,QAAQ,oBAAoB;AACrC,cAAI,YAAY,IAAI,KAAK,QAAQ,GAAG;AAClC,kBAAM,WAAW,YAAY,IAAI,KAAK,QAAQ;AAC9C,qBAAS;AACT,qBAAS,kBAAkB,KAAK,IAAI,SAAS,iBAAiB,KAAK,UAAU;AAC7E,qBAAS,iBAAiB,KAAK,IAAI,SAAS,gBAAgB,KAAK,UAAU;AAAA,UAC7E,OAAO;AACL,wBAAY,IAAI,KAAK,UAAU;AAAA,cAC7B,UAAU,KAAK;AAAA,cACf,OAAO;AAAA,cACP,iBAAiB,KAAK;AAAA,cACtB,gBAAgB,KAAK;AAAA,cACrB,YAAY,KAAK;AAAA,YACnB,CAAC;AAAA,UACH;AAAA,QACF;AAGA,YAAI,MAAM,KAAK,cAAc,WAAW;AACtC,gBAAM,KAAK,WAAW,kBAAkB,MAAM,KAAK,YAAY,QAAQ,CAAC,EAAE;AAAA,YACxE,CAAC,CAAC,WAAW,KAAK,MAAM;AAAA,UAC1B;AAGA,gBAAM,cAAc,eAAe,GAAG,mBAAmB;AACzD,gBAAM,cAAc,qBAAqB;AACzC,gBAAM,YAAY,qBAAqB,uBAAuB;AAE9D,gBAAM,KAAK,WAAW,YAAY;AAAA,YAChC,OAAO;AAAA,YACP;AAAA,YACA;AAAA,UACF;AAAA,QACF;AAGA,mBAAW,UAAU,EAAE,MAAM,eAAe,GAAG,CAAC;AAChD,mBAAW,IAAI;AAGf,cAAM,CAAC,EAAE,UAAU,IAAI,MAAM,cAAc,KAAK;AAChD,YAAI,YAAY;AAEd,cAAI,MAAM,KAAK,YAAY;AACzB,kBAAM,KAAK,WAAW,qBAAqB;AAAA,cACzC,QAAQ;AAAA,cACR,OAAO,cAAc,UAAU;AAAA,YACjC;AAAA,UACF;AAAA,QACF;AAEA,cAAM,aAAa,KAAK,MAAM,YAAY,IAAI,IAAI,UAAU;AAE5D,cAAM,eAAe,MAAM,MAAM;AAAA,UAC/B,CAAC,SAAS,KAAK,KAAK,KAAK,WAAW;AAAA,QACtC,EAAE;AACF,cAAM,eAAe,MAAM,MAAM;AAAA,UAC/B,CAAC,SAAS,KAAK,KAAK,KAAK,WAAW,UAAU,KAAK,KAAK,KAAK,WAAW;AAAA,QAC1E,EAAE;AAGF,YAAI,CAAC,WAAW,CAAC,QAAQ;AACvB,gBAAM,oBAAoB,iBAAiB;AAAA,YACzC,IAAI;AAAA,YACJ,QAAQ;AAAA,YACR,YAAY,WAAW;AAAA,YACvB;AAAA,YACA;AAAA,YACA;AAAA,UACF,CAAC;AAAA,QACH;AAAA,MACF,CAAC;AAID,YAAM,GAAG,WAAW;AAAA,QAClB,WAAW,IAAI,CAAC,GAAG,WAAW,EAAE,GAAG,GAAG,MAAM,EAAsC;AAAA,MACpF,EAAE,QAAQ,OAAO,MAAM,EAAE,KAAK,MAAM;AAClC,cAAM,QAAQ,YAAY,IAAI;AAC9B,YAAI,CAAC,cAAc;AACjB,gBAAM,IAAI;AAAA,YACR;AAAA,UACF;AAAA,QACF;AAEA,YAAI,kBAA0C,CAAC;AAE/C,cAAM;AAAA,UACJ,QAAQ,KAAK,KAAK;AAAA,UAClB;AAAA,YACE,YAAY;AAAA,cACV,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,cAC7B,CAAC,KAAK,KAAK,EAAE,GAAG;AAAA,cAChB,CAAC,KAAK,KAAK,IAAI,GAAG;AAAA,cAClB,CAAC,KAAK,KAAK,OAAO,GAAG;AAAA,cACrB,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG,KAAK;AAAA,cAC7B,CAAC,KAAK,KAAK,KAAK,KAAK,GACnB,OAAO,KAAK,UAAU,WAAW,KAAK,QAAQ,KAAK,UAAU,KAAK,KAAK;AAAA,cACzE,CAAC,KAAK,KAAK,KAAK,QAAQ,GACtB,OAAO,KAAK,aAAa,WAAW,KAAK,WAAW,KAAK,UAAU,KAAK,QAAQ;AAAA,cAClF,CAAC,KAAK,KAAK,KAAK,QAAQ,GAAG,KAAK,WAAW,KAAK,UAAU,KAAK,QAAQ,IAAI;AAAA;AAAA,cAE3E,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM;AAAA,cAC7B,CAAC,KAAK,KAAK,KAAK,KAAK,GAAG,MAAM;AAAA,YAChC;AAAA,UACF;AAAA,UACA,OAAO,aAAa;AAClB,kBAAM,cAAc,MAAM,QAAQ,QAAQ,OAAO,GAAG,QAAQ;AAC5D,kBAAM,SAAS,KAAK,IAAI,GAAG,KAAK,UAAU,CAAC;AAC3C,gBAAI;AACJ,gBAAI;AASJ,qBAAS,aAAa,KAAK,KAAK,KAAK,QAAQ,MAAM;AAEnD,gBAAI;AAEF,oBAAM,kBAA4C,CAAC;AACnD,oBAAM,cAAoC,MAAM,KAAK,EAAE,QAAQ,OAAO,GAAG,MAAM,IAAI;AACnF,oBAAM,gBAAyB,CAAC;AAChC,kBAAI;AACJ,kBAAI,yBAAyB;AAG7B,uBAAS,aAAa,GAAG,aAAa,QAAQ,cAAc;AAC1D,oBAAI;AACF,wBAAM;AAAA,oBACJ,SAAS,UAAU;AAAA,oBACnB;AAAA,sBACE,YAAY;AAAA,wBACV,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,wBAC7B,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG;AAAA,wBACzB,CAAC,KAAK,KAAK,EAAE,GAAG;AAAA,wBAChB,CAAC,KAAK,KAAK,IAAI,GAAG;AAAA,wBAClB,CAAC,KAAK,KAAK,OAAO,GAAG;AAAA,sBACvB;AAAA,oBACF;AAAA,oBACA,OAAO,cAAc;AACnB,4BAAM,eAAe,MAAM,QAAQ,QAAQ,OAAO,GAAG,SAAS;AAC9D,0BAAI;AACF,8BAAMA,UAAS,MAAM;AAAA,0BACnB;AAAA,0BACA;AAAA,4BACE,IAAI;AAAA,4BACJ,SAAS;AAAA,4BACT,MAAM;AAAA,0BACR;AAAA,0BACA;AAAA,4BACE,OAAO,KAAK;AAAA,4BACZ,OAAO,KAAK;AAAA,4BACZ,UAAU,KAAK;AAAA,4BACf,SAAS,KAAK;AAAA,4BACd,MAAM,KAAK;AAAA,4BACX,UAAU,KAAK;AAAA,4BACf,aAAa,KAAK;AAAA,4BAClB,YAAY,KAAK;AAAA,4BACjB,MAAM,KAAK;AAAA,0BACb;AAAA,wBACF;AACA,8BAAM,EAAE,QAAAC,SAAQ,SAAS,IAAID;AAC7B,qCAAaC;AACb,kDAA0B;AAC1B,wCAAgB,KAAK,GAAGD,QAAO,eAAe;AAE9C,kDAA0B;AAAA,0BACxB,OAAOA,QAAO,cAAc,CAAC;AAAA,0BAC7B,aAAa,KAAK;AAAA,0BAClB,WAAWA,QAAO;AAAA,wBACpB;AAGA,8BAAM,QAAQ;AAAA,0BACZ,KAAK,QAAQ,IAAI,OAAO,WAAW;AACjC,kCAAM,aAAa,cAAc,MAAM;AACvC,mCAAO;AAAA,8BACL,SAAS,UAAU;AAAA,8BACnB;AAAA,gCACE,YAAY;AAAA,kCACV,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,kCAC7B,CAAC,KAAK,KAAK,IAAI,GAAG,KAAK,UAAU,CAAC,SAAS,CAAC;AAAA,kCAC5C,CAAC,KAAK,KAAK,EAAE,GAAG;AAAA,kCAChB,CAAC,KAAK,KAAK,IAAI,GAAG;AAAA,kCAClB,CAAC,KAAK,KAAK,OAAO,GAAG;AAAA,kCACrB,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG;AAAA,gCAC3B;AAAA,8BACF;AAAA,8BACA,OAAO,eAAe;AACpB,sCAAM,cAAc,YAAY,IAAI;AACpC,oCAAI;AACF,wCAAM,CAACA,SAAQ,WAAW,IAAI,MAAM;AAAA,oCAAc,MAChD,OAAO;AAAA,sCACL,OAAO,KAAK;AAAA,sCACZ,QAAQC;AAAA,sCACR,UAAU,KAAK;AAAA,sCACf;AAAA,oCACF,CAAC;AAAA,kCACH;AAEA,sCAAI,eAAe,CAACD,SAAQ;AAC1B,0CAAM,iBAAiB,KAAK;AAAA,sCAC1B,YAAY,IAAI,IAAI;AAAA,oCACtB;AACA,4CAAQ;AAAA,sCACN,iBAAiB,UAAU;AAAA;AAAA,sCAC3B;AAAA,oCACF;AACA,0CAAM,MAAM,cAAc,WAAW;AACrC,0CAAME,YAAW;AAAA,sCACf,UAAU;AAAA,sCACV,WAAW;AAAA,sCACX,OAAO;AAAA,oCACT;AAGA,qCAAC,8DAAgC,CAAC,IAAG,KAAK,CAAC;AAE3C,+CAAW,cAAc;AAAA,sCACvB,CAAC,KAAK,KAAK,MAAM,IAAI,GAAG;AAAA,sCACxB,CAAC,KAAK,KAAK,MAAM,QAAQ,GAAG,KAAK,UAAUA,SAAQ;AAAA,oCACrD,CAAC;AAED,+CAAW,UAAU;AAAA,sCACnB,MAAM,eAAe;AAAA,sCACrB,SAAS;AAAA,oCACX,CAAC;AACD;AAAA,kCACF;AAEA,wCAAM,gBAAgB,KAAK,MAAM,YAAY,IAAI,IAAI,WAAW;AAChE,wCAAM,aAAaF,QAAO;AAC1B,wCAAM,WAAW,OAAO;AAAA,oCACtB,EAAE,UAAU,eAAe,WAAW,YAAY;AAAA,oCAClDA,QAAO;AAAA,kCACT;AAGA,mCAAC,8DAAgC,CAAC,IAAG,KAAK,UAAU;AAGpD,wCAAM,cACH,OAAkB,eAAe,KAAK;AAEzC,6CAAW,cAAc;AAAA,oCACvB,CAAC,KAAK,KAAK,MAAM,IAAI,GAAG;AAAA,oCACxB,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG;AAAA,oCACzB,CAAC,KAAK,KAAK,MAAM,QAAQ,GAAG,KAAK,UAAU,QAAQ;AAAA,oCACnD,CAAC,KAAK,KAAK,MAAM,WAAW,GAAG,YAAY;AAAA,kCAC7C,CAAC;AAED,sCAAI,SAAS,OAAO;AAClB,0CAAM,MAAM,cAAc,SAAS,KAAK;AACxC,+CAAW,UAAU;AAAA,sCACnB,MAAM,eAAe;AAAA,sCACrB,SAAS;AAAA,oCACX,CAAC;AAAA,kCACH;AAAA,gCACF,UAAE;AACA,6CAAW,IAAI;AAAA,gCACjB;AAAA,8BACF;AAAA,8BACA;AAAA,4BACF;AAAA,0BACF,CAAC;AAAA,wBACH;AAAA,sBACF,SAAS,OAAO;AACd,8BAAM,qBAAqB,yBAAyB,KAAK;AACzD,8BAAM,UAAU,QAAQ,KAAK;AAC7B,8BAAM,MAAM,cAAc,OAAO;AACjC,8BAAM,mBAAmB,QAAQ,WAAW;AAC5C,oCAAY,UAAU,IAAI;AAC1B,sCAAc,KAAK,OAAO;AAE1B,kCAAU,cAAc;AAAA,0BACtB,CAAC,KAAK,KAAK,MAAM,KAAK,GAAG;AAAA,wBAC3B,CAAC;AAED,mCAAW,UAAU,KAAK,SAAS;AACjC,gCAAM,aAAa,cAAc,MAAM;AACvC,2BAAC,8DAAgC,CAAC,IAAG,KAAK,CAAC;AAAA,wBAC7C;AAEA,4BAAI,oBAAoB;AACtB,0CAAgB,KAAK,GAAG,mBAAmB,eAAe;AAC1D,oDAA0B;AAAA,4BACxB,OAAO,mBAAmB,cAAc,CAAC;AAAA,4BACzC,aAAa,KAAK;AAAA,4BAClB,WAAW,mBAAmB;AAAA,0BAChC;AAAA,wBACF;AAGA,8BAAM;AAAA,sBACR;AAAA,oBACF;AAAA,oBACA;AAAA,kBACF;AAAA,gBACF,QAAQ;AAAA,gBAER;AAAA,cACF;AAGA,oBAAM,SAAwC,CAAC;AAC/C,yBAAW,UAAU,KAAK,SAAS;AACjC,sBAAM,aAAa,cAAc,MAAM;AACvC,sBAAM,YAAY,gBAAgB,UAAU,KAAK,CAAC;AAClD,sBAAM,cAA4B,OAAkB,eAAe,KAAK;AAExE,sBAAM,kBAAkB,UAAU,SAAS,IAAI,YAAY,UAAU,SAAS,IAAI;AAElF,uBAAO,UAAU,IAAI;AAAA,kBACnB,MAAM;AAAA,kBACN,OAAO;AAAA,kBACP,QAAQ;AAAA,kBACR,aAAa,YAAY;AAAA,kBACzB,WAAW,YAAY;AAAA,kBACvB,UAAU,CAAC;AAAA,gBACb;AAAA,cACF;AAEA,oBAAM,SAAS;AACf,oBAAM,eAAe,cAAc;AACnC,oBAAM,kBAAkB,SAAS;AACjC,oBAAM,eAAe;AAAA,gBACnB,OAAO;AAAA,gBACP,WAAW;AAAA,gBACX,QAAQ;AAAA,cACV;AAEA,uBAAS,aAAa,KAAK,KAAK,KAAK,QAAQ,KAAK,UAAU,SAAS,SAAS,CAAC,CAAC,CAAC;AACjF,kBAAI,WAAW,QAAW;AACxB,yBAAS;AAAA,kBACP,KAAK,KAAK,KAAK;AAAA,kBACf,OAAO,WAAW,WAAW,SAAS,KAAK,UAAU,MAAM;AAAA,gBAC7D;AAAA,cACF;AAGA,mBAAK,KAAK,OAAO;AAAA,gBACf,OAAO,KAAK;AAAA,gBACZ,MAAM;AAAA,gBACN,UAAU,KAAK;AAAA,gBACf,OAAO,KAAK;AAAA,gBACZ;AAAA,gBACA,UAAU,KAAK;AAAA,gBACf;AAAA,gBACA,QAAQ;AAAA,gBACR,QAAQ,CAAC;AAAA,gBACT;AAAA,gBACA;AAAA,gBACA,UAAU;AAAA,gBACV,WAAW;AAAA,gBACX;AAAA,gBACA,aAAa,KAAK;AAAA,cACpB;AAEA,kBAAI,eAAe,GAAG;AACpB,sBAAM,QAAQ,IAAI;AAAA,kBAChB,aAAa,KAAK,KAAK,gBAAgB,YAAY;AAAA,gBACrD;AACA,+CAA+B;AAC/B,yBAAS,UAAU;AAAA,kBACjB,MAAM,eAAe;AAAA,kBACrB,SAAS,MAAM;AAAA,gBACjB,CAAC;AACD,qBAAK,KAAK,KAAK,SAAS;AACxB,qBAAK,KAAK,KAAK,SAAS;AACxB,sBAAM;AAAA,cACR;AAGA,iCAAmB,KAAK,GAAG,eAAe;AAAA,YAC5C,SAAS,GAAG;AACV,sBAAQ,IAAI,CAAC;AACb,oBAAM,QAAQ,QAAQ,CAAC;AAEvB,kBAAI,MAAM,gCAAgC,KAAK,KAAK,MAAM;AACxD,qBAAK,KAAK,KAAK,SAAS;AACxB,qBAAK,KAAK,KAAK,SAAS,KAAK,KAAK,KAAK,QAAQ,SAC3C,KAAK,KAAK,KAAK,SACf,CAAC,KAAK;AACV,mCAAmB,KAAK,GAAG,eAAe;AAC1C,sBAAM;AAAA,cACR;AAEA,oBAAM,MAAM,eAAe;AAC3B,oBAAM,qBAAqB,IAAI,mBAAmB,CAAC;AACnD,kBAAI,mBAAmB,SAAS,GAAG;AACjC,gCAAgB,KAAK,GAAG,kBAAkB;AAAA,cAC5C;AAEA,oBAAM,WAAW,IAAI,SAAS,CAAC;AAC/B,kBAAI,CAAC,2BAA2B,OAAO,KAAK,QAAQ,EAAE,SAAS,GAAG;AAChE,0CAA0B;AAAA,kBACxB,OAAO;AAAA,kBACP,aAAa,KAAK;AAAA,kBAClB,WAAW,IAAI;AAAA,gBACjB;AAAA,cACF;AAGA,oBAAM,eAA8C,CAAC;AACrD,yBAAW,UAAU,KAAK,SAAS;AACjC,sBAAM,aAAa,cAAc,MAAM;AACvC,6BAAa,UAAU,IAAI;AAAA,kBACzB,MAAM;AAAA,kBACN,OAAO;AAAA,kBACP,QAAQ,CAAC;AAAA,kBACT,UAAU;AAAA,oBACR,UAAU;AAAA,oBACV,WAAW;AAAA,oBACX,OAAO,MAAM;AAAA,kBACf;AAAA,gBACF;AAAA,cACF;AAEA,mBAAK,KAAK,OAAO;AAAA,gBACf,MAAM;AAAA,gBACN,OAAO,KAAK;AAAA,gBACZ,UAAU,KAAK;AAAA,gBACf,OAAO,KAAK;AAAA,gBACZ,QAAQ,OAAO,CAAC;AAAA,gBAChB,UAAU,KAAK;AAAA,gBACf,QAAQ;AAAA,gBACR,QAAQ;AAAA,gBACR,QAAQ,CAAC,KAAK;AAAA,gBACd,WAAW;AAAA,gBACX,UAAU,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;AAAA,gBAC9C;AAAA,gBACA,aAAa,KAAK;AAAA,cACpB;AAEA,iCAAmB,KAAK,GAAG,eAAe;AAC1C,oBAAM;AAAA,YACR,UAAE;AAEA,kBAAI;AACF,sBAAM,gBAAqC,yBAAyB,SAAS,CAAC;AAE9E,sBAAM,WAAW,OAAO,KAAK,aAAa;AAC1C,sBAAM,cAAc,eAAe,GAAG,qBAAqB,KAAK,CAAC;AAEjE,sBAAM,eAA+C,CAAC;AACtD,2BAAW,OAAO,UAAU;AAC1B,wBAAM,QAAQ,cAAc,GAAG;AAC/B,sBAAI,OAAO,aAAa;AACtB,0BAAM,WAAW,CAAC,UAAU,OAAO,YAAY,GAAG,CAAC;AACnD,wBAAI,UAAU;AACZ,mCAAa,GAAG,IAAI,EAAE,MAAM,YAAY,OAAO,SAAS,YAAY,GAAG,EAAE;AAAA,oBAC3E;AAAA,kBACF,OAAO;AACL,iCAAa,GAAG,IAAI,EAAE,MAAM,cAAc,MAAM;AAAA,kBAClD;AAAA,gBACF;AAEA,oBAAI,CAAC,WAAW,OAAO,KAAK,YAAY,EAAE,SAAS,GAAG;AACpD,wBAAM,aAAa,KAAK,UAAU,YAAY;AAC9C,2BAAS,aAAa,kCAAkC,UAAU;AAAA,gBACpE;AAEA,oBAAI,KAAK,KAAK,MAAM;AAClB,uBAAK,KAAK,KAAK,eAAe;AAAA,gBAChC;AAAA,cACF,QAAQ;AAAA,cAAC;AAET,kBAAI,yBAAyB;AAC3B,sCAAsB;AAAA,cACxB;AAAA,YACF;AAAA,UACF;AAAA,UACA;AAAA,QACF;AAAA,MACF,CAAC;AAAA,IACH;AAAA,IACA;AAAA,EACF;AAEA,SAAO;AACT;AAEA,IAAM,4BAA4B,CAAyC,YAAoB;AAC7F,MAAI,QAAQ,WAAW,GAAG;AACxB,WAAO;AAAA,EACT;AAGA,MAAI,QAAQ,MAAM,CAAC,MAAM,OAAO,MAAM,QAAQ,GAAG;AAC/C,WAAO,QAAQ,KAAK,EAAE;AAAA,EACxB;AAGA,SAAO,QAAQ,QAAQ,SAAS,CAAC;AACnC;AAEA,IAAM,cAAc,OAKlB,MACA,OACA,aACqB;AACrB,QAAM,qBAAqB,MAAM,KAAK,EAAE,OAAO,SAAS,CAAC;AAEzD,MACE,OAAO,uBAAuB,YAC9B,sBACA,OAAO,iBAAiB,oBACxB;AACA,UAAM,SAAoB,CAAC;AAE3B,qBAAiB,SAAS,oBAAoB;AAC5C,aAAO,KAAK,KAAK;AAAA,IACnB;AAEA,WAAO,0BAAmC,MAAmB;AAAA,EAC/D;AAEA,SAAO;AACT;AAEA,IAAM,UAAU,OAKd,aACA,YAKA,SAKG;AACH,QAAM,WAAW,KAAK,KAAK,QAAQ;AAEnC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,MACE,YAAY;AAAA,QACV,CAAC,KAAK,MAAM,UAAU,IAAI,GAAG;AAAA,QAC7B,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG;AAAA,QACvB,CAAC,KAAK,KAAK,KAAK,IAAI,GAAG;AAAA;AAAA,QACvB,CAAC,KAAK,KAAK,EAAE,GAAG,WAAW;AAAA,QAC3B,CAAC,KAAK,KAAK,IAAI,GAAG,WAAW;AAAA,QAC7B,CAAC,KAAK,KAAK,OAAO,GAAG,WAAW;AAAA,MAClC;AAAA,IACF;AAAA,IACA,OAAO,aAAa;AAElB,YAAM,EAAE,QAAQ,UAAU,iBAAiB,YAAY,UAAU,IAAI,MAAM;AAAA,QACzE,EAAE,aAAa,KAAK,YAAY;AAAA,QAChC,YAMM;AAEJ,gBAAM,QAAQ,YAAY,IAAI;AAC9B,cAAI;AACF,kBAAMC,UAAS,MAAM,YAAY,KAAK,MAAM,KAAK,OAAO,KAAK,QAAS;AACtE,kBAAME,YAAW,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;AAErD,qBAAS,cAAc;AAAA,cACrB,CAAC,KAAK,KAAK,KAAK,MAAM,GAAG,OAAOF,YAAW,WAAWA,UAAS,KAAK,UAAUA,OAAM;AAAA,YACtF,CAAC;AAGD,kBAAM,MAAM,eAAe;AAC3B,kBAAMG,mBAAkB,IAAI,mBAAmB,CAAC;AAEhD,mBAAO;AAAA,cACL,QAAAH;AAAA,cACA,UAAAE;AAAA,cACA,iBAAAC;AAAA,cACA,YAAY,IAAI,SAAS,CAAC;AAAA,cAC1B,WAAW,IAAI;AAAA,YACjB;AAAA,UACF,SAAS,OAAO;AACd,kBAAM,MAAM,eAAe;AAC3B,kBAAMD,YAAW,KAAK,MAAM,YAAY,IAAI,IAAI,KAAK;AACrD,kBAAM,4BAA4B,OAAO;AAAA,cACvC,UAAAA;AAAA,cACA,iBAAiB,IAAI,mBAAmB,CAAC;AAAA,cACzC,YAAY,IAAI,SAAS,CAAC;AAAA,cAC1B,WAAW,IAAI;AAAA,YACjB,CAAC;AAAA,UACH;AAAA,QACF;AAAA,MACF;AAEA,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,IACA;AAAA,EACF;AACF;;;AOl9BA;;;ARaO,IAAM,UAAU,IAAI,SAAoB;AAC7C,wBAAsB,gBAAgB;AACtC,SAAQ,aAA0B,GAAG,IAAI;AAC3C;","names":["result","output","metadata","duration","outOfScopeFlags"]}

package/dist/index.cjs

@@ -5990,7 +5990,7 @@ var import_api2 = require("@opentelemetry/api");
 // package.json
 var package_default = {
   name: "axiom",
-  version: "0.44.0",
+  version: "0.45.0",
   type: "module",
   author: "Axiom, Inc.",
   contributors: [
@@ -8576,17 +8576,32 @@ var import_api8 = require("@opentelemetry/api");
 // src/online-evals/executor.ts
 init_cjs_shims();
 var import_api7 = require("@opentelemetry/api");
+
+// src/evals/normalize-score.ts
+init_cjs_shims();
+function normalizeBooleanScore(score, metadata) {
+  if (typeof score !== "boolean") {
+    return { score, metadata };
+  }
+  return {
+    score: score ? 1 : 0,
+    metadata: { ...metadata, [Attr.Eval.Score.IsBoolean]: true }
+  };
+}
+
+// src/online-evals/executor.ts
 function setScorerSpanAttrs(scorerSpan, scorerName, result, meta) {
+  const { score: scoreValue, metadata } = normalizeBooleanScore(result.score, result.metadata);
   const attrs = {
     [Attr.GenAI.Operation.Name]: "eval.score",
     [Attr.Eval.Score.Name]: scorerName,
     [Attr.Eval.Tags]: JSON.stringify(["online"]),
-    [Attr.Eval.Score.Value]: result.score ?? void 0,
+    [Attr.Eval.Score.Value]: scoreValue ?? void 0,
     [Attr.Eval.Capability.Name]: meta?.capability,
     [Attr.Eval.Step.Name]: meta?.step
   };
-  if (result.metadata && Object.keys(result.metadata).length > 0) {
-    attrs[Attr.Eval.Score.Metadata] = JSON.stringify(result.metadata);
+  if (metadata && Object.keys(metadata).length > 0) {
+    attrs[Attr.Eval.Score.Metadata] = JSON.stringify(metadata);
   }
   scorerSpan.setAttributes(attrs);
 }
@@ -8690,15 +8705,16 @@ function onlineEval(meta, options) {
   if (options.scorers.length === 0) {
     return Promise.resolve({});
   }
-  const linkSpanContext = meta.link ?? import_api8.trace.getSpan(import_api8.context.active())?.spanContext();
-  return executeOnlineEvalInternal(meta, options, linkSpanContext);
+  const rawLinks = meta.links ?? import_api8.trace.getSpan(import_api8.context.active())?.spanContext();
+  const linkContexts = rawLinks ? Array.isArray(rawLinks) ? rawLinks : [rawLinks] : [];
+  return executeOnlineEvalInternal(meta, options, linkContexts);
 }
-async function executeOnlineEvalInternal(meta, options, linkSpanContext) {
+async function executeOnlineEvalInternal(meta, options, linkContexts) {
   const tracer = getGlobalTracer();
   const spanName = meta.step ? `eval ${meta.capability}/${meta.step}` : `eval ${meta.capability}`;
   const evalSpan = tracer.startSpan(
     spanName,
-    linkSpanContext ? { links: [{ context: linkSpanContext }] } : {}
+    linkContexts.length > 0 ? { links: linkContexts.map((ctx) => ({ context: ctx })) } : {}
   );
   const evalAttrs = {
     [Attr.GenAI.Operation.Name]: "eval",
@@ -8767,7 +8783,17 @@ async function executeOnlineEvalInternal(meta, options, linkSpanContext) {
     }
     const failedCount = Object.values(results).filter((result) => result.error).length;
     const ranCount = Object.keys(results).length;
+    const scoresSummary = {};
+    for (const [name, result] of Object.entries(results)) {
+      scoresSummary[name] = {
+        name: result.name,
+        score: result.score,
+        ...result.metadata && Object.keys(result.metadata).length > 0 && { metadata: result.metadata },
+        ...result.error && { error: result.error }
+      };
+    }
     evalSpan.setAttributes({
+      [Attr.Eval.Case.Scores]: JSON.stringify(scoresSummary),
       "axiom.eval.online.scorers.total": normalizedScorers.length,
       "axiom.eval.online.scorers.ran": ranCount,
       "axiom.eval.online.scorers.sampled_out": sampledOutCount,
@@ -8822,23 +8848,9 @@ function createScorer(name, fn, options) {
       return { score: res };
     }
     if (typeof res === "boolean") {
-      return {
-        score: res ? 1 : 0,
-        metadata: {
-          [Attr.Eval.Score.IsBoolean]: true
-        }
-      };
-    }
-    if (typeof res.score === "boolean") {
-      return {
-        score: res.score ? 1 : 0,
-        metadata: {
-          ...res.metadata,
-          [Attr.Eval.Score.IsBoolean]: true
-        }
-      };
+      return normalizeBooleanScore(res);
     }
-    return res;
+    return normalizeBooleanScore(res.score, res.metadata);
   };
   const scorer = (args) => {
     const res = fn(args);